diff --git a/.circleci/config.yml b/.circleci/config.yml index 55a3da4f..d6440de1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3,10 +3,17 @@ version: 2 defaults: &defaults working_directory: /go/src/github.com/tendermint/tendermint docker: - - image: circleci/golang:1.10.3 + - image: circleci/golang:1.11.4 environment: GOBIN: /tmp/workspace/bin +docs_update_config: &docs_update_config + working_directory: ~/repo + docker: + - image: tendermint/docs_deployment + environment: + AWS_REGION: us-east-1 + jobs: setup_dependencies: <<: *defaults @@ -41,10 +48,10 @@ jobs: key: v3-pkg-cache paths: - /go/pkg - # - save_cache: - # key: v3-tree-{{ .Environment.CIRCLE_SHA1 }} - # paths: - # - /go/src/github.com/tendermint/tendermint + - save_cache: + key: v3-tree-{{ .Environment.CIRCLE_SHA1 }} + paths: + - /go/src/github.com/tendermint/tendermint build_slate: <<: *defaults @@ -53,23 +60,8 @@ jobs: at: /tmp/workspace - restore_cache: key: v3-pkg-cache - # https://discuss.circleci.com/t/saving-cache-stopped-working-warning-skipping-this-step-disabled-in-configuration/24423/2 - # - restore_cache: - # key: v3-tree-{{ .Environment.CIRCLE_SHA1 }} - - checkout - - run: - name: tools - command: | - export PATH="$GOBIN:$PATH" - make get_tools - - run: - name: dependencies - command: | - export PATH="$GOBIN:$PATH" - make get_vendor_deps - - run: mkdir -p $GOPATH/src/github.com/tendermint - - run: ln -sf /home/circleci/project $GOPATH/src/github.com/tendermint/tendermint - + - restore_cache: + key: v3-tree-{{ .Environment.CIRCLE_SHA1 }} - run: name: slate docs command: | @@ -84,28 +76,14 @@ jobs: at: /tmp/workspace - restore_cache: key: v3-pkg-cache - # - restore_cache: - # key: v3-tree-{{ .Environment.CIRCLE_SHA1 }} - - checkout - - run: - name: tools - command: | - export PATH="$GOBIN:$PATH" - make get_tools - - run: - name: dependencies - command: | - export PATH="$GOBIN:$PATH" - make get_vendor_deps - - run: mkdir -p $GOPATH/src/github.com/tendermint - - run: ln -sf /home/circleci/project $GOPATH/src/github.com/tendermint/tendermint - + - restore_cache: + key: v3-tree-{{ .Environment.CIRCLE_SHA1 }} - run: name: metalinter command: | set -ex export PATH="$GOBIN:$PATH" - make metalinter + make lint - run: name: check_dep command: | @@ -120,22 +98,8 @@ jobs: at: /tmp/workspace - restore_cache: key: v3-pkg-cache - # - restore_cache: - # key: v3-tree-{{ .Environment.CIRCLE_SHA1 }} - - checkout - - run: - name: tools - command: | - export PATH="$GOBIN:$PATH" - make get_tools - - run: - name: dependencies - command: | - export PATH="$GOBIN:$PATH" - make get_vendor_deps - - run: mkdir -p $GOPATH/src/github.com/tendermint - - run: ln -sf /home/circleci/project $GOPATH/src/github.com/tendermint/tendermint - + - restore_cache: + key: v3-tree-{{ .Environment.CIRCLE_SHA1 }} - run: name: Run abci apps tests command: | @@ -151,22 +115,8 @@ jobs: at: /tmp/workspace - restore_cache: key: v3-pkg-cache - # - restore_cache: - # key: v3-tree-{{ .Environment.CIRCLE_SHA1 }} - - checkout - - run: - name: tools - command: | - export PATH="$GOBIN:$PATH" - make get_tools - - run: - name: dependencies - command: | - export PATH="$GOBIN:$PATH" - make get_vendor_deps - - run: mkdir -p $GOPATH/src/github.com/tendermint - - run: ln -sf /home/circleci/project $GOPATH/src/github.com/tendermint/tendermint - + - restore_cache: + key: v3-tree-{{ .Environment.CIRCLE_SHA1 }} - run: name: Run abci-cli tests command: | @@ -180,22 +130,8 @@ jobs: at: /tmp/workspace - restore_cache: key: v3-pkg-cache - # - restore_cache: - # key: v3-tree-{{ .Environment.CIRCLE_SHA1 }} - - checkout - - run: - name: tools - command: | - export PATH="$GOBIN:$PATH" - make get_tools - - run: - name: dependencies - command: | - export PATH="$GOBIN:$PATH" - make get_vendor_deps - - run: mkdir -p $GOPATH/src/github.com/tendermint - - run: ln -sf /home/circleci/project $GOPATH/src/github.com/tendermint/tendermint - + - restore_cache: + key: v3-tree-{{ .Environment.CIRCLE_SHA1 }} - run: sudo apt-get update && sudo apt-get install -y --no-install-recommends bsdmainutils - run: name: Run tests @@ -209,22 +145,8 @@ jobs: at: /tmp/workspace - restore_cache: key: v3-pkg-cache - # - restore_cache: - # key: v3-tree-{{ .Environment.CIRCLE_SHA1 }} - - checkout - - run: - name: tools - command: | - export PATH="$GOBIN:$PATH" - make get_tools - - run: - name: dependencies - command: | - export PATH="$GOBIN:$PATH" - make get_vendor_deps - - run: mkdir -p $GOPATH/src/github.com/tendermint - - run: ln -sf /home/circleci/project $GOPATH/src/github.com/tendermint/tendermint - + - restore_cache: + key: v3-tree-{{ .Environment.CIRCLE_SHA1 }} - run: mkdir -p /tmp/logs - run: name: Run tests @@ -232,7 +154,7 @@ jobs: for pkg in $(go list github.com/tendermint/tendermint/... | circleci tests split --split-by=timings); do id=$(basename "$pkg") - GOCACHE=off go test -timeout 5m -race -coverprofile=/tmp/workspace/profiles/$id.out -covermode=atomic "$pkg" | tee "/tmp/logs/$id-$RANDOM.log" + GOCACHE=off go test -v -timeout 5m -race -coverprofile=/tmp/workspace/profiles/$id.out -covermode=atomic "$pkg" | tee "/tmp/logs/$id-$RANDOM.log" done - persist_to_workspace: root: /tmp/workspace @@ -248,22 +170,8 @@ jobs: at: /tmp/workspace - restore_cache: key: v3-pkg-cache - # - restore_cache: - # key: v3-tree-{{ .Environment.CIRCLE_SHA1 }} - - checkout - - run: - name: tools - command: | - export PATH="$GOBIN:$PATH" - make get_tools - - run: - name: dependencies - command: | - export PATH="$GOBIN:$PATH" - make get_vendor_deps - - run: mkdir -p $GOPATH/src/github.com/tendermint - - run: ln -sf /home/circleci/project $GOPATH/src/github.com/tendermint/tendermint - + - restore_cache: + key: v3-tree-{{ .Environment.CIRCLE_SHA1 }} - run: name: Run tests command: bash test/persist/test_failure_indices.sh @@ -284,9 +192,7 @@ jobs: name: run localnet and exit on failure command: | set -x - make get_tools - make get_vendor_deps - make build-linux + docker run --rm -v "$PWD":/go/src/github.com/tendermint/tendermint -w /go/src/github.com/tendermint/tendermint golang:1.11.4 make build-linux make localnet-start & ./scripts/localnet-blocks-test.sh 40 5 10 localhost @@ -309,22 +215,10 @@ jobs: steps: - attach_workspace: at: /tmp/workspace - # - restore_cache: - # key: v3-tree-{{ .Environment.CIRCLE_SHA1 }} - - checkout - - run: - name: tools - command: | - export PATH="$GOBIN:$PATH" - make get_tools - - run: - name: dependencies - command: | - export PATH="$GOBIN:$PATH" - make get_vendor_deps - - run: mkdir -p $GOPATH/src/github.com/tendermint - - run: ln -sf /home/circleci/project $GOPATH/src/github.com/tendermint/tendermint - + - restore_cache: + key: v3-pkg-cache + - restore_cache: + key: v3-tree-{{ .Environment.CIRCLE_SHA1 }} - run: name: gather command: | @@ -338,10 +232,25 @@ jobs: name: upload command: bash .circleci/codecov.sh -f coverage.txt + deploy_docs: + <<: *docs_update_config + steps: + - checkout + - run: + name: Trigger website build + command: | + chamber exec tendermint -- start_website_build + workflows: version: 2 test-suite: jobs: + - deploy_docs: + filters: + branches: + only: + - master + - develop - setup_dependencies - lint: requires: diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 00000000..4cae2f76 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,61 @@ +run: + deadline: 1m + +linters: + enable-all: true + disable: + - gocyclo + - golint + - maligned + - errcheck + - staticcheck + - dupl + - ineffassign + - interfacer + - unconvert + - goconst + - unparam + - nakedret + - lll + - gochecknoglobals + - govet + - gocritic + - gosec + - gochecknoinits + - scopelint + - stylecheck + +# linters-settings: +# govet: +# check-shadowing: true +# golint: +# min-confidence: 0 +# gocyclo: +# min-complexity: 10 +# maligned: +# suggest-new: true +# dupl: +# threshold: 100 +# goconst: +# min-len: 2 +# min-occurrences: 2 +# depguard: +# list-type: blacklist +# packages: +# # logging is allowed only by logutils.Log, logrus +# # is allowed to use only in logutils package +# - github.com/sirupsen/logrus +# misspell: +# locale: US +# lll: +# line-length: 140 +# goimports: +# local-prefixes: github.com/golangci/golangci-lint +# gocritic: +# enabled-tags: +# - performance +# - style +# - experimental +# disabled-checks: +# - wrapperFunc +# - commentFormatting # https://github.com/go-critic/go-critic/issues/755 diff --git a/CHANGELOG.md b/CHANGELOG.md index 6032fc20..a0e736bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,614 @@ # Changelog +## v0.29.2 + +*February 7th, 2019* + +Special thanks to external contributors on this release: +@ackratos, @rickyyangz + +**Note**: This release contains security sensitive patches in the `p2p` and +`crypto` packages: +- p2p: + - Partial fix for MITM attacks on the p2p connection. MITM conditions may + still exist. See \#3010. +- crypto: + - Eliminate our fork of `btcd` and use the `btcd/btcec` library directly for + native secp256k1 signing. Note we still modify the signature encoding to + prevent malleability. + - Support the libsecp256k1 library via CGo through the `go-ethereum/crypto/secp256k1` package. + - Eliminate MixEntropy functions + +### BREAKING CHANGES: + +* Go API + - [crypto] [\#3278](https://github.com/tendermint/tendermint/issues/3278) Remove + MixEntropy functions + - [types] [\#3245](https://github.com/tendermint/tendermint/issues/3245) Commit uses `type CommitSig Vote` instead of `Vote` directly. + In preparation for removing redundant fields from the commit [\#1648](https://github.com/tendermint/tendermint/issues/1648) + +### IMPROVEMENTS: +- [consensus] [\#3246](https://github.com/tendermint/tendermint/issues/3246) Better logging and notes on recovery for corrupted WAL file +- [crypto] [\#3163](https://github.com/tendermint/tendermint/issues/3163) Use ethereum's libsecp256k1 go-wrapper for signatures when cgo is available +- [crypto] [\#3162](https://github.com/tendermint/tendermint/issues/3162) Wrap btcd instead of forking it to keep up with fixes (used if cgo is not available) +- [makefile] [\#3233](https://github.com/tendermint/tendermint/issues/3233) Use golangci-lint instead of go-metalinter +- [tools] [\#3218](https://github.com/tendermint/tendermint/issues/3218) Add go-deadlock tool to help detect deadlocks +- [tools] [\#3106](https://github.com/tendermint/tendermint/issues/3106) Add tm-signer-harness test harness for remote signers +- [tests] [\#3258](https://github.com/tendermint/tendermint/issues/3258) Fixed a bunch of non-deterministic test failures + +### BUG FIXES: +- [node] [\#3186](https://github.com/tendermint/tendermint/issues/3186) EventBus and indexerService should be started before first block (for replay last block on handshake) execution (@ackratos) +- [p2p] [\#3232](https://github.com/tendermint/tendermint/issues/3232) Fix infinite loop leading to addrbook deadlock for seed nodes +- [p2p] [\#3247](https://github.com/tendermint/tendermint/issues/3247) Fix panic in SeedMode when calling FlushStop and OnStop + concurrently +- [p2p] [\#3040](https://github.com/tendermint/tendermint/issues/3040) Fix MITM on secret connection by checking low-order points +- [privval] [\#3258](https://github.com/tendermint/tendermint/issues/3258) Fix race between sign requests and ping requests in socket + +## v0.29.1 + +*January 24, 2019* + +Special thanks to external contributors on this release: +@infinytum, @gauthamzz + +This release contains two important fixes: one for p2p layer where we sometimes +were not closing connections and one for consensus layer where consensus with +no empty blocks (`create_empty_blocks = false`) could halt. + +Friendly reminder, we have a [bug bounty +program](https://hackerone.com/tendermint). + +### IMPROVEMENTS: +- [pex] [\#3037](https://github.com/tendermint/tendermint/issues/3037) Only log "Reached max attempts to dial" once +- [rpc] [\#3159](https://github.com/tendermint/tendermint/issues/3159) Expose + `triggered_timeout_commit` in the `/dump_consensus_state` + +### BUG FIXES: +- [consensus] [\#3199](https://github.com/tendermint/tendermint/issues/3199) Fix consensus halt with no empty blocks from not resetting triggeredTimeoutCommit +- [p2p] [\#2967](https://github.com/tendermint/tendermint/issues/2967) Fix file descriptor leak + +## v0.29.0 + +*January 21, 2019* + +Special thanks to external contributors on this release: +@bradyjoestar, @kunaldhariwal, @gauthamzz, @hrharder + +This release is primarily about making some breaking changes to +the Block protocol version before Cosmos launch, and to fixing more issues +in the proposer selection algorithm discovered on Cosmos testnets. + +The Block protocol changes include using a standard Merkle tree format (RFC 6962), +fixing some inconsistencies between field orders in Vote and Proposal structs, +and constraining the hash of the ConsensusParams to include only a few fields. + +The proposer selection algorithm saw significant progress, +including a [formal proof by @cwgoes for the base-case in Idris](https://github.com/cwgoes/tm-proposer-idris) +and a [much more detailed specification (still in progress) by +@ancazamfir](https://github.com/tendermint/tendermint/pull/3140). + +Fixes to the proposer selection algorithm include normalizing the proposer +priorities to mitigate the effects of large changes to the validator set. +That said, we just discovered [another bug](https://github.com/tendermint/tendermint/issues/3181), +which will be fixed in the next breaking release. + +While we are trying to stabilize the Block protocol to preserve compatibility +with old chains, there may be some final changes yet to come before Cosmos +launch as we continue to audit and test the software. + +Friendly reminder, we have a [bug bounty +program](https://hackerone.com/tendermint). + +### BREAKING CHANGES: + +* CLI/RPC/Config + +* Apps + - [state] [\#3049](https://github.com/tendermint/tendermint/issues/3049) Total voting power of the validator set is upper bounded by + `MaxInt64 / 8`. Apps must ensure they do not return changes to the validator + set that cause this maximum to be exceeded. + +* Go API + - [node] [\#3082](https://github.com/tendermint/tendermint/issues/3082) MetricsProvider now requires you to pass a chain ID + - [types] [\#2713](https://github.com/tendermint/tendermint/issues/2713) Rename `TxProof.LeafHash` to `TxProof.Leaf` + - [crypto/merkle] [\#2713](https://github.com/tendermint/tendermint/issues/2713) `SimpleProof.Verify` takes a `leaf` instead of a + `leafHash` and performs the hashing itself + +* Blockchain Protocol + * [crypto/merkle] [\#2713](https://github.com/tendermint/tendermint/issues/2713) Merkle trees now match the RFC 6962 specification + * [types] [\#3078](https://github.com/tendermint/tendermint/issues/3078) Re-order Timestamp and BlockID in CanonicalVote so it's + consistent with CanonicalProposal (BlockID comes + first) + * [types] [\#3165](https://github.com/tendermint/tendermint/issues/3165) Hash of ConsensusParams only includes BlockSize.MaxBytes and + BlockSize.MaxGas + +* P2P Protocol + - [consensus] [\#3049](https://github.com/tendermint/tendermint/issues/3049) Normalize priorities to not exceed `2*TotalVotingPower` to mitigate unfair proposer selection + heavily preferring earlier joined validators in the case of an early bonded large validator unbonding + +### FEATURES: + +### IMPROVEMENTS: +- [rpc] [\#3065](https://github.com/tendermint/tendermint/issues/3065) Return maxPerPage (100), not defaultPerPage (30) if `per_page` is greater than the max 100. +- [instrumentation] [\#3082](https://github.com/tendermint/tendermint/issues/3082) Add `chain_id` label for all metrics + +### BUG FIXES: +- [crypto] [\#3164](https://github.com/tendermint/tendermint/issues/3164) Update `btcd` fork for rare signRFC6979 bug +- [lite] [\#3171](https://github.com/tendermint/tendermint/issues/3171) Fix verifying large validator set changes +- [log] [\#3125](https://github.com/tendermint/tendermint/issues/3125) Fix year format +- [mempool] [\#3168](https://github.com/tendermint/tendermint/issues/3168) Limit tx size to fit in the max reactor msg size +- [scripts] [\#3147](https://github.com/tendermint/tendermint/issues/3147) Fix json2wal for large block parts (@bradyjoestar) + +## v0.28.1 + +*January 18th, 2019* + +Special thanks to external contributors on this release: +@HaoyangLiu + +Friendly reminder, we have a [bug bounty +program](https://hackerone.com/tendermint). + +### BUG FIXES: +- [consensus] Fix consensus halt from proposing blocks with too much evidence + +## v0.28.0 + +*January 16th, 2019* + +Special thanks to external contributors on this release: +@fmauricios, @gianfelipe93, @husio, @needkane, @srmo, @yutianwu + +This release is primarily about upgrades to the `privval` system - +separating the `priv_validator.json` into distinct config and data files, and +refactoring the socket validator to support reconnections. + +**Note:** Please backup your existing `priv_validator.json` before using this +version. + +See [UPGRADING.md](UPGRADING.md) for more details. + +### BREAKING CHANGES: + +* CLI/RPC/Config + - [cli] Removed `--proxy_app=dummy` option. Use `kvstore` (`persistent_kvstore`) instead. + - [cli] Renamed `--proxy_app=nilapp` to `--proxy_app=noop`. + - [config] [\#2992](https://github.com/tendermint/tendermint/issues/2992) `allow_duplicate_ip` is now set to false + - [privval] [\#1181](https://github.com/tendermint/tendermint/issues/1181) Split `priv_validator.json` into immutable (`config/priv_validator_key.json`) and mutable (`data/priv_validator_state.json`) parts (@yutianwu) + - [privval] [\#2926](https://github.com/tendermint/tendermint/issues/2926) Split up `PubKeyMsg` into `PubKeyRequest` and `PubKeyResponse` to be consistent with other message types + - [privval] [\#2923](https://github.com/tendermint/tendermint/issues/2923) Listen for unix socket connections instead of dialing them + +* Apps + +* Go API + - [types] [\#2981](https://github.com/tendermint/tendermint/issues/2981) Remove `PrivValidator.GetAddress()` + +* Blockchain Protocol + +* P2P Protocol + +### FEATURES: +- [rpc] [\#3052](https://github.com/tendermint/tendermint/issues/3052) Include peer's remote IP in `/net_info` + +### IMPROVEMENTS: +- [consensus] [\#3086](https://github.com/tendermint/tendermint/issues/3086) Log peerID on ignored votes (@srmo) +- [docs] [\#3061](https://github.com/tendermint/tendermint/issues/3061) Added specification for signing consensus msgs at + ./docs/spec/consensus/signing.md +- [privval] [\#2948](https://github.com/tendermint/tendermint/issues/2948) Memoize pubkey so it's only requested once on startup +- [privval] [\#2923](https://github.com/tendermint/tendermint/issues/2923) Retry RemoteSigner connections on error + +### BUG FIXES: + +- [build] [\#3085](https://github.com/tendermint/tendermint/issues/3085) Fix `Version` field in build scripts (@husio) +- [crypto/multisig] [\#3102](https://github.com/tendermint/tendermint/issues/3102) Fix multisig keys address length +- [crypto/encoding] [\#3101](https://github.com/tendermint/tendermint/issues/3101) Fix `PubKeyMultisigThreshold` unmarshalling into `crypto.PubKey` interface +- [p2p/conn] [\#3111](https://github.com/tendermint/tendermint/issues/3111) Make SecretConnection thread safe +- [rpc] [\#3053](https://github.com/tendermint/tendermint/issues/3053) Fix internal error in `/tx_search` when results are empty + (@gianfelipe93) +- [types] [\#2926](https://github.com/tendermint/tendermint/issues/2926) Do not panic if retrieving the privval's public key fails + +## v0.27.4 + +*December 21st, 2018* + +### BUG FIXES: + +- [mempool] [\#3036](https://github.com/tendermint/tendermint/issues/3036) Fix + LRU cache by popping the least recently used item when the cache is full, + not the most recently used one! + +## v0.27.3 + +*December 16th, 2018* + +### BREAKING CHANGES: + +* Go API + - [dep] [\#3027](https://github.com/tendermint/tendermint/issues/3027) Revert to mainline Go crypto library, eliminating the modified + `bcrypt.GenerateFromPassword` + +## v0.27.2 + +*December 16th, 2018* + +### IMPROVEMENTS: + +- [node] [\#3025](https://github.com/tendermint/tendermint/issues/3025) Validate NodeInfo addresses on startup. + +### BUG FIXES: + +- [p2p] [\#3025](https://github.com/tendermint/tendermint/pull/3025) Revert to using defers in addrbook. Fixes deadlocks in pex and consensus upon invalid ExternalAddr/ListenAddr configuration. + +## v0.27.1 + +*December 15th, 2018* + +Special thanks to external contributors on this release: +@danil-lashin, @hleb-albau, @james-ray, @leo-xinwang + +### FEATURES: +- [rpc] [\#2964](https://github.com/tendermint/tendermint/issues/2964) Add `UnconfirmedTxs(limit)` and `NumUnconfirmedTxs()` methods to HTTP/Local clients (@danil-lashin) +- [docs] [\#3004](https://github.com/tendermint/tendermint/issues/3004) Enable full-text search on docs pages + +### IMPROVEMENTS: +- [consensus] [\#2971](https://github.com/tendermint/tendermint/issues/2971) Return error if ValidatorSet is empty after InitChain + (@leo-xinwang) +- [ci/cd] [\#3005](https://github.com/tendermint/tendermint/issues/3005) Updated CircleCI job to trigger website build when docs are updated +- [docs] Various updates + +### BUG FIXES: +- [cmd] [\#2983](https://github.com/tendermint/tendermint/issues/2983) `testnet` command always sets `addr_book_strict = false` +- [config] [\#2980](https://github.com/tendermint/tendermint/issues/2980) Fix CORS options formatting +- [kv indexer] [\#2912](https://github.com/tendermint/tendermint/issues/2912) Don't ignore key when executing CONTAINS +- [mempool] [\#2961](https://github.com/tendermint/tendermint/issues/2961) Call `notifyTxsAvailable` if there're txs left after committing a block, but recheck=false +- [mempool] [\#2994](https://github.com/tendermint/tendermint/issues/2994) Reject txs with negative GasWanted +- [p2p] [\#2990](https://github.com/tendermint/tendermint/issues/2990) Fix a bug where seeds don't disconnect from a peer after 3h +- [consensus] [\#3006](https://github.com/tendermint/tendermint/issues/3006) Save state after InitChain only when stateHeight is also 0 (@james-ray) + +## v0.27.0 + +*December 5th, 2018* + +Special thanks to external contributors on this release: +@danil-lashin, @srmo + +Special thanks to @dlguddus for discovering a [major +issue](https://github.com/tendermint/tendermint/issues/2718#issuecomment-440888677) +in the proposer selection algorithm. + +Friendly reminder, we have a [bug bounty +program](https://hackerone.com/tendermint). + +This release is primarily about fixes to the proposer selection algorithm +in preparation for the [Cosmos Game of +Stakes](https://blog.cosmos.network/the-game-of-stakes-is-open-for-registration-83a404746ee6). +It also makes use of the `ConsensusParams.Validator.PubKeyTypes` to restrict the +key types that can be used by validators, and removes the `Heartbeat` consensus +message. + +### BREAKING CHANGES: + +* CLI/RPC/Config + - [rpc] [\#2932](https://github.com/tendermint/tendermint/issues/2932) Rename `accum` to `proposer_priority` + +* Go API + - [db] [\#2913](https://github.com/tendermint/tendermint/pull/2913) + ReverseIterator API change: start < end, and end is exclusive. + - [types] [\#2932](https://github.com/tendermint/tendermint/issues/2932) Rename `Validator.Accum` to `Validator.ProposerPriority` + +* Blockchain Protocol + - [state] [\#2714](https://github.com/tendermint/tendermint/issues/2714) Validators can now only use pubkeys allowed within + ConsensusParams.Validator.PubKeyTypes + +* P2P Protocol + - [consensus] [\#2871](https://github.com/tendermint/tendermint/issues/2871) + Remove *ProposalHeartbeat* message as it serves no real purpose (@srmo) + - [state] Fixes for proposer selection: + - [\#2785](https://github.com/tendermint/tendermint/issues/2785) Accum for new validators is `-1.125*totalVotingPower` instead of 0 + - [\#2941](https://github.com/tendermint/tendermint/issues/2941) val.Accum is preserved during ValidatorSet.Update to avoid being + reset to 0 + +### IMPROVEMENTS: + +- [state] [\#2929](https://github.com/tendermint/tendermint/issues/2929) Minor refactor of updateState logic (@danil-lashin) +- [node] [\#2959](https://github.com/tendermint/tendermint/issues/2959) Allow node to start even if software's BlockProtocol is + different from state's BlockProtocol +- [pex] [\#2959](https://github.com/tendermint/tendermint/issues/2959) Pex reactor logger uses `module=pex` + +### BUG FIXES: + +- [p2p] [\#2968](https://github.com/tendermint/tendermint/issues/2968) Panic on transport error rather than continuing to run but not + accept new connections +- [p2p] [\#2969](https://github.com/tendermint/tendermint/issues/2969) Fix mismatch in peer count between `/net_info` and the prometheus + metrics +- [rpc] [\#2408](https://github.com/tendermint/tendermint/issues/2408) `/broadcast_tx_commit`: Fix "interface conversion: interface {} in nil, not EventDataTx" panic (could happen if somebody sent a tx using `/broadcast_tx_commit` while Tendermint was being stopped) +- [state] [\#2785](https://github.com/tendermint/tendermint/issues/2785) Fix accum for new validators to be `-1.125*totalVotingPower` + instead of 0, forcing them to wait before becoming the proposer. Also: + - do not batch clip + - keep accums averaged near 0 +- [txindex/kv] [\#2925](https://github.com/tendermint/tendermint/issues/2925) Don't return false positives when range searching for a prefix of a tag value +- [types] [\#2938](https://github.com/tendermint/tendermint/issues/2938) Fix regression in v0.26.4 where we panic on empty + genDoc.Validators +- [types] [\#2941](https://github.com/tendermint/tendermint/issues/2941) Preserve val.Accum during ValidatorSet.Update to avoid it being + reset to 0 every time a validator is updated + +## v0.26.4 + +*November 27th, 2018* + +Special thanks to external contributors on this release: +@ackratos, @goolAdapter, @james-ray, @joe-bowman, @kostko, +@nagarajmanjunath, @tomtau + +Friendly reminder, we have a [bug bounty +program](https://hackerone.com/tendermint). + +### FEATURES: + +- [rpc] [\#2747](https://github.com/tendermint/tendermint/issues/2747) Enable subscription to tags emitted from `BeginBlock`/`EndBlock` (@kostko) +- [types] [\#2747](https://github.com/tendermint/tendermint/issues/2747) Add `ResultBeginBlock` and `ResultEndBlock` fields to `EventDataNewBlock` + and `EventDataNewBlockHeader` to support subscriptions (@kostko) +- [types] [\#2918](https://github.com/tendermint/tendermint/issues/2918) Add Marshal, MarshalTo, Unmarshal methods to various structs + to support Protobuf compatibility (@nagarajmanjunath) + +### IMPROVEMENTS: + +- [config] [\#2877](https://github.com/tendermint/tendermint/issues/2877) Add `blocktime_iota` to the config.toml (@ackratos) + - NOTE: this should be a ConsensusParam, not part of the config, and will be + removed from the config at a later date + ([\#2920](https://github.com/tendermint/tendermint/issues/2920). +- [mempool] [\#2882](https://github.com/tendermint/tendermint/issues/2882) Add txs from Update to cache +- [mempool] [\#2891](https://github.com/tendermint/tendermint/issues/2891) Remove local int64 counter from being stored in every tx +- [node] [\#2866](https://github.com/tendermint/tendermint/issues/2866) Add ability to instantiate IPCVal (@joe-bowman) + +### BUG FIXES: + +- [blockchain] [\#2731](https://github.com/tendermint/tendermint/issues/2731) Retry both blocks if either is bad to avoid getting stuck during fast sync (@goolAdapter) +- [consensus] [\#2893](https://github.com/tendermint/tendermint/issues/2893) Use genDoc.Validators instead of state.NextValidators on replay when appHeight==0 (@james-ray) +- [log] [\#2868](https://github.com/tendermint/tendermint/issues/2868) Fix `module=main` setting overriding all others + - NOTE: this changes the default logging behaviour to be much less verbose. + Set `log_level="info"` to restore the previous behaviour. +- [rpc] [\#2808](https://github.com/tendermint/tendermint/issues/2808) Fix `accum` field in `/validators` by calling `IncrementAccum` if necessary +- [rpc] [\#2811](https://github.com/tendermint/tendermint/issues/2811) Allow integer IDs in JSON-RPC requests (@tomtau) +- [txindex/kv] [\#2759](https://github.com/tendermint/tendermint/issues/2759) Fix tx.height range queries +- [txindex/kv] [\#2775](https://github.com/tendermint/tendermint/issues/2775) Order tx results by index if height is the same +- [txindex/kv] [\#2908](https://github.com/tendermint/tendermint/issues/2908) Don't return false positives when searching for a prefix of a tag value + +## v0.26.3 + +*November 17th, 2018* + +Special thanks to external contributors on this release: +@danil-lashin, @kevlubkcm, @krhubert, @srmo + +Friendly reminder, we have a [bug bounty +program](https://hackerone.com/tendermint). + +### BREAKING CHANGES: + +* Go API + - [rpc] [\#2791](https://github.com/tendermint/tendermint/issues/2791) Functions that start HTTP servers are now blocking: + - Impacts `StartHTTPServer`, `StartHTTPAndTLSServer`, and `StartGRPCServer` + - These functions now take a `net.Listener` instead of an address + - [rpc] [\#2767](https://github.com/tendermint/tendermint/issues/2767) Subscribing to events + `NewRound` and `CompleteProposal` return new types `EventDataNewRound` and + `EventDataCompleteProposal`, respectively, instead of the generic `EventDataRoundState`. (@kevlubkcm) + +### FEATURES: + +- [log] [\#2843](https://github.com/tendermint/tendermint/issues/2843) New `log_format` config option, which can be set to 'plain' for colored + text or 'json' for JSON output +- [types] [\#2767](https://github.com/tendermint/tendermint/issues/2767) New event types EventDataNewRound (with ProposerInfo) and EventDataCompleteProposal (with BlockID). (@kevlubkcm) + +### IMPROVEMENTS: + +- [dep] [\#2844](https://github.com/tendermint/tendermint/issues/2844) Dependencies are no longer pinned to an exact version in the + Gopkg.toml: + - Serialization libs are allowed to vary by patch release + - Other libs are allowed to vary by minor release +- [p2p] [\#2857](https://github.com/tendermint/tendermint/issues/2857) "Send failed" is logged at debug level instead of error. +- [rpc] [\#2780](https://github.com/tendermint/tendermint/issues/2780) Add read and write timeouts to HTTP servers +- [state] [\#2848](https://github.com/tendermint/tendermint/issues/2848) Make "Update to validators" msg value pretty (@danil-lashin) + +### BUG FIXES: +- [consensus] [\#2819](https://github.com/tendermint/tendermint/issues/2819) Don't send proposalHearbeat if not a validator +- [docs] [\#2859](https://github.com/tendermint/tendermint/issues/2859) Fix ConsensusParams details in spec +- [libs/autofile] [\#2760](https://github.com/tendermint/tendermint/issues/2760) Comment out autofile permissions check - should fix + running Tendermint on Windows +- [p2p] [\#2869](https://github.com/tendermint/tendermint/issues/2869) Set connection config properly instead of always using default +- [p2p/pex] [\#2802](https://github.com/tendermint/tendermint/issues/2802) Seed mode fixes: + - Only disconnect from inbound peers + - Use FlushStop instead of Sleep to ensure all messages are sent before + disconnecting + +## v0.26.2 + +*November 15th, 2018* + +Special thanks to external contributors on this release: @hleb-albau, @zhuzeyu + +Friendly reminder, we have a [bug bounty program](https://hackerone.com/tendermint). + +### FEATURES: + +- [rpc] [\#2582](https://github.com/tendermint/tendermint/issues/2582) Enable CORS on RPC API (@hleb-albau) + +### BUG FIXES: + +- [abci] [\#2748](https://github.com/tendermint/tendermint/issues/2748) Unlock mutex in localClient so even when app panics (e.g. during CheckTx), consensus continue working +- [abci] [\#2748](https://github.com/tendermint/tendermint/issues/2748) Fix DATA RACE in localClient +- [amino] [\#2822](https://github.com/tendermint/tendermint/issues/2822) Update to v0.14.1 to support compiling on 32-bit platforms +- [rpc] [\#2748](https://github.com/tendermint/tendermint/issues/2748) Drain channel before calling Unsubscribe(All) in `/broadcast_tx_commit` + +## v0.26.1 + +*November 11, 2018* + +Special thanks to external contributors on this release: @katakonst + +Friendly reminder, we have a [bug bounty program](https://hackerone.com/tendermint). + +### IMPROVEMENTS: + +- [consensus] [\#2704](https://github.com/tendermint/tendermint/issues/2704) Simplify valid POL round logic +- [docs] [\#2749](https://github.com/tendermint/tendermint/issues/2749) Deduplicate some ABCI docs +- [mempool] More detailed log messages + - [\#2724](https://github.com/tendermint/tendermint/issues/2724) + - [\#2762](https://github.com/tendermint/tendermint/issues/2762) + +### BUG FIXES: + +- [autofile] [\#2703](https://github.com/tendermint/tendermint/issues/2703) Do not panic when checking Head size +- [crypto/merkle] [\#2756](https://github.com/tendermint/tendermint/issues/2756) Fix crypto/merkle ProofOperators.Verify to check bounds on keypath parts. +- [mempool] fix a bug where we create a WAL despite `wal_dir` being empty +- [p2p] [\#2771](https://github.com/tendermint/tendermint/issues/2771) Fix `peer-id` label name to `peer_id` in prometheus metrics +- [p2p] [\#2797](https://github.com/tendermint/tendermint/pull/2797) Fix IDs in peer NodeInfo and require them for addresses + in AddressBook +- [p2p] [\#2797](https://github.com/tendermint/tendermint/pull/2797) Do not close conn immediately after sending pex addrs in seed mode. Partial fix for [\#2092](https://github.com/tendermint/tendermint/issues/2092). + +## v0.26.0 + +*November 2, 2018* + +Special thanks to external contributors on this release: +@bradyjoestar, @connorwstein, @goolAdapter, @HaoyangLiu, +@james-ray, @overbool, @phymbert, @Slamper, @Uzair1995, @yutianwu. + +Special thanks to @Slamper for a series of bug reports in our [bug bounty +program](https://hackerone.com/tendermint) which are fixed in this release. + +This release is primarily about adding Version fields to various data structures, +optimizing consensus messages for signing and verification in +restricted environments (like HSMs and the Ethereum Virtual Machine), and +aligning the consensus code with the [specification](https://arxiv.org/abs/1807.04938). +It also includes our first take at a generalized merkle proof system, and +changes the length of hashes used for hashing data structures from 20 to 32 +bytes. + +See the [UPGRADING.md](UPGRADING.md#v0.26.0) for details on upgrading to the new +version. + +Please note that we are still making breaking changes to the protocols. +While the new Version fields should help us to keep the software backwards compatible +even while upgrading the protocols, we cannot guarantee that new releases will +be compatible with old chains just yet. We expect there will be another breaking +release or two before the Cosmos Hub launch, but we will otherwise be paying +increasing attention to backwards compatibility. Thanks for bearing with us! + +### BREAKING CHANGES: + +* CLI/RPC/Config + * [config] [\#2232](https://github.com/tendermint/tendermint/issues/2232) Timeouts are now strings like "3s" and "100ms", not ints + * [config] [\#2505](https://github.com/tendermint/tendermint/issues/2505) Remove Mempool.RecheckEmpty (it was effectively useless anyways) + * [config] [\#2490](https://github.com/tendermint/tendermint/issues/2490) `mempool.wal` is disabled by default + * [privval] [\#2459](https://github.com/tendermint/tendermint/issues/2459) Split `SocketPVMsg`s implementations into Request and Response, where the Response may contain a error message (returned by the remote signer) + * [state] [\#2644](https://github.com/tendermint/tendermint/issues/2644) Add Version field to State, breaking the format of State as + encoded on disk. + * [rpc] [\#2298](https://github.com/tendermint/tendermint/issues/2298) `/abci_query` takes `prove` argument instead of `trusted` and switches the default + behaviour to `prove=false` + * [rpc] [\#2654](https://github.com/tendermint/tendermint/issues/2654) Remove all `node_info.other.*_version` fields in `/status` and + `/net_info` + * [rpc] [\#2636](https://github.com/tendermint/tendermint/issues/2636) Remove + `_params` suffix from fields in `consensus_params`. + +* Apps + * [abci] [\#2298](https://github.com/tendermint/tendermint/issues/2298) ResponseQuery.Proof is now a structured merkle.Proof, not just + arbitrary bytes + * [abci] [\#2644](https://github.com/tendermint/tendermint/issues/2644) Add Version to Header and shift all fields by one + * [abci] [\#2662](https://github.com/tendermint/tendermint/issues/2662) Bump the field numbers for some `ResponseInfo` fields to make room for + `AppVersion` + * [abci] [\#2636](https://github.com/tendermint/tendermint/issues/2636) Updates to ConsensusParams + * Remove `Params` suffix from field names + * Add `Params` suffix to message types + * Add new field and type, `Validator ValidatorParams`, to control what types of validator keys are allowed. + +* Go API + * [config] [\#2232](https://github.com/tendermint/tendermint/issues/2232) Timeouts are time.Duration, not ints + * [crypto/merkle & lite] [\#2298](https://github.com/tendermint/tendermint/issues/2298) Various changes to accomodate General Merkle trees + * [crypto/merkle] [\#2595](https://github.com/tendermint/tendermint/issues/2595) Remove all Hasher objects in favor of byte slices + * [crypto/merkle] [\#2635](https://github.com/tendermint/tendermint/issues/2635) merkle.SimpleHashFromTwoHashes is no longer exported + * [node] [\#2479](https://github.com/tendermint/tendermint/issues/2479) Remove node.RunForever + * [rpc/client] [\#2298](https://github.com/tendermint/tendermint/issues/2298) `ABCIQueryOptions.Trusted` -> `ABCIQueryOptions.Prove` + * [types] [\#2298](https://github.com/tendermint/tendermint/issues/2298) Remove `Index` and `Total` fields from `TxProof`. + * [types] [\#2598](https://github.com/tendermint/tendermint/issues/2598) + `VoteTypeXxx` are now of type `SignedMsgType byte` and named `XxxType`, eg. + `PrevoteType`, `PrecommitType`. + * [types] [\#2636](https://github.com/tendermint/tendermint/issues/2636) Rename fields in ConsensusParams to remove `Params` suffixes + * [types] [\#2735](https://github.com/tendermint/tendermint/issues/2735) Simplify Proposal message to align with spec + +* Blockchain Protocol + * [crypto/tmhash] [\#2732](https://github.com/tendermint/tendermint/issues/2732) TMHASH is now full 32-byte SHA256 + * All hashes in the block header and Merkle trees are now 32-bytes + * PubKey Addresses are still only 20-bytes + * [state] [\#2587](https://github.com/tendermint/tendermint/issues/2587) Require block.Time of the fist block to be genesis time + * [state] [\#2644](https://github.com/tendermint/tendermint/issues/2644) Require block.Version to match state.Version + * [types] Update SignBytes for `Vote`/`Proposal`/`Heartbeat`: + * [\#2459](https://github.com/tendermint/tendermint/issues/2459) Use amino encoding instead of JSON in `SignBytes`. + * [\#2598](https://github.com/tendermint/tendermint/issues/2598) Reorder fields and use fixed sized encoding. + * [\#2598](https://github.com/tendermint/tendermint/issues/2598) Change `Type` field from `string` to `byte` and use new + `SignedMsgType` to enumerate. + * [types] [\#2730](https://github.com/tendermint/tendermint/issues/2730) Use + same order for fields in `Vote` as in the SignBytes + * [types] [\#2732](https://github.com/tendermint/tendermint/issues/2732) Remove the address field from the validator hash + * [types] [\#2644](https://github.com/tendermint/tendermint/issues/2644) Add Version struct to Header + * [types] [\#2609](https://github.com/tendermint/tendermint/issues/2609) ConsensusParams.Hash() is the hash of the amino encoded + struct instead of the Merkle tree of the fields + * [types] [\#2670](https://github.com/tendermint/tendermint/issues/2670) Header.Hash() builds Merkle tree out of fields in the same + order they appear in the header, instead of sorting by field name + * [types] [\#2682](https://github.com/tendermint/tendermint/issues/2682) Use proto3 `varint` encoding for ints that are usually unsigned (instead of zigzag encoding). + * [types] [\#2636](https://github.com/tendermint/tendermint/issues/2636) Add Validator field to ConsensusParams + (Used to control which pubkey types validators can use, by abci type). + +* P2P Protocol + * [consensus] [\#2652](https://github.com/tendermint/tendermint/issues/2652) + Replace `CommitStepMessage` with `NewValidBlockMessage` + * [consensus] [\#2735](https://github.com/tendermint/tendermint/issues/2735) Simplify `Proposal` message to align with spec + * [consensus] [\#2730](https://github.com/tendermint/tendermint/issues/2730) + Add `Type` field to `Proposal` and use same order of fields as in the + SignBytes for both `Proposal` and `Vote` + * [p2p] [\#2654](https://github.com/tendermint/tendermint/issues/2654) Add `ProtocolVersion` struct with protocol versions to top of + DefaultNodeInfo and require `ProtocolVersion.Block` to match during peer handshake + + +### FEATURES: +- [abci] [\#2557](https://github.com/tendermint/tendermint/issues/2557) Add `Codespace` field to `Response{CheckTx, DeliverTx, Query}` +- [abci] [\#2662](https://github.com/tendermint/tendermint/issues/2662) Add `BlockVersion` and `P2PVersion` to `RequestInfo` +- [crypto/merkle] [\#2298](https://github.com/tendermint/tendermint/issues/2298) General Merkle Proof scheme for chaining various types of Merkle trees together +- [docs/architecture] [\#1181](https://github.com/tendermint/tendermint/issues/1181) S +plit immutable and mutable parts of priv_validator.json + +### IMPROVEMENTS: +- Additional Metrics + - [consensus] [\#2169](https://github.com/cosmos/cosmos-sdk/issues/2169) + - [p2p] [\#2169](https://github.com/cosmos/cosmos-sdk/issues/2169) +- [config] [\#2232](https://github.com/tendermint/tendermint/issues/2232) Added ValidateBasic method, which performs basic checks +- [crypto/ed25519] [\#2558](https://github.com/tendermint/tendermint/issues/2558) Switch to use latest `golang.org/x/crypto` through our fork at + github.com/tendermint/crypto +- [libs/log] [\#2707](https://github.com/tendermint/tendermint/issues/2707) Add year to log format (@yutianwu) +- [tools] [\#2238](https://github.com/tendermint/tendermint/issues/2238) Binary dependencies are now locked to a specific git commit + +### BUG FIXES: +- [\#2711](https://github.com/tendermint/tendermint/issues/2711) Validate all incoming reactor messages. Fixes various bugs due to negative ints. +- [autofile] [\#2428](https://github.com/tendermint/tendermint/issues/2428) Group.RotateFile need call Flush() before rename (@goolAdapter) +- [common] [\#2533](https://github.com/tendermint/tendermint/issues/2533) Fixed a bug in the `BitArray.Or` method +- [common] [\#2506](https://github.com/tendermint/tendermint/issues/2506) Fixed a bug in the `BitArray.Sub` method (@james-ray) +- [common] [\#2534](https://github.com/tendermint/tendermint/issues/2534) Fix `BitArray.PickRandom` to choose uniformly from true bits +- [consensus] [\#1690](https://github.com/tendermint/tendermint/issues/1690) Wait for + timeoutPrecommit before starting next round +- [consensus] [\#1745](https://github.com/tendermint/tendermint/issues/1745) Wait for + Proposal or timeoutProposal before entering prevote +- [consensus] [\#2642](https://github.com/tendermint/tendermint/issues/2642) Only propose ValidBlock, not LockedBlock +- [consensus] [\#2642](https://github.com/tendermint/tendermint/issues/2642) Initialized ValidRound and LockedRound to -1 +- [consensus] [\#1637](https://github.com/tendermint/tendermint/issues/1637) Limit the amount of evidence that can be included in a + block +- [consensus] [\#2652](https://github.com/tendermint/tendermint/issues/2652) Ensure valid block property with faulty proposer +- [evidence] [\#2515](https://github.com/tendermint/tendermint/issues/2515) Fix db iter leak (@goolAdapter) +- [libs/event] [\#2518](https://github.com/tendermint/tendermint/issues/2518) Fix event concurrency flaw (@goolAdapter) +- [node] [\#2434](https://github.com/tendermint/tendermint/issues/2434) Make node respond to signal interrupts while sleeping for genesis time +- [state] [\#2616](https://github.com/tendermint/tendermint/issues/2616) Pass nil to NewValidatorSet() when genesis file's Validators field is nil +- [p2p] [\#2555](https://github.com/tendermint/tendermint/issues/2555) Fix p2p switch FlushThrottle value (@goolAdapter) +- [p2p] [\#2668](https://github.com/tendermint/tendermint/issues/2668) Reconnect to originally dialed address (not self-reported address) for persistent peers + ## v0.25.0 *September 22, 2018* @@ -164,8 +773,8 @@ BUG FIXES: *August 22nd, 2018* BUG FIXES: -- [libs/autofile] \#2261 Fix log rotation so it actually happens. - - Fixes issues with consensus WAL growing unbounded ala \#2259 +- [libs/autofile] [\#2261](https://github.com/tendermint/tendermint/issues/2261) Fix log rotation so it actually happens. + - Fixes issues with consensus WAL growing unbounded ala [\#2259](https://github.com/tendermint/tendermint/issues/2259) ## 0.23.0 diff --git a/CHANGELOG_PENDING.md b/CHANGELOG_PENDING.md index bdc2a731..4548eb1e 100644 --- a/CHANGELOG_PENDING.md +++ b/CHANGELOG_PENDING.md @@ -1,59 +1,13 @@ -# Pending +## v0.30 + +** Special thanks to external contributors on this release: -@goolAdapter, @bradyjoestar -BREAKING CHANGES: +### BREAKING CHANGES: -* CLI/RPC/Config - * [config] \#2232 timeouts as time.Duration, not ints - * [config] \#2505 Remove Mempool.RecheckEmpty (it was effectively useless anyways) - * [config] `mempool.wal` is disabled by default - * [rpc] \#2298 `/abci_query` takes `prove` argument instead of `trusted` and switches the default - behaviour to `prove=false` - * [privval] \#2459 Split `SocketPVMsg`s implementations into Request and Response, where the Response may contain a error message (returned by the remote signer) +### FEATURES: -* Apps - * [abci] \#2298 ResponseQuery.Proof is now a structured merkle.Proof, not just - arbitrary bytes +### IMPROVEMENTS: -* Go API - * [node] Remove node.RunForever - * [config] \#2232 timeouts as time.Duration, not ints - * [rpc/client] \#2298 `ABCIQueryOptions.Trusted` -> `ABCIQueryOptions.Prove` - * [types] \#2298 Remove `Index` and `Total` fields from `TxProof`. - * [crypto/merkle & lite] \#2298 Various changes to accomodate General Merkle trees - * [crypto/merkle] \#2595 Remove all Hasher objects in favor of byte slices - -* Blockchain Protocol - * [types] \#2459 `Vote`/`Proposal`/`Heartbeat` use amino encoding instead of JSON in `SignBytes`. - * [types] \#2512 Remove the pubkey field from the validator hash - -* P2P Protocol - -FEATURES: -- [crypto/merkle] \#2298 General Merkle Proof scheme for chaining various types of Merkle trees together -- [abci] \#2557 Add `Codespace` field to `Response{CheckTx, DeliverTx, Query}` - -IMPROVEMENTS: -- [consensus] [\#2169](https://github.com/cosmos/cosmos-sdk/issues/2169) add additional metrics -- [p2p] [\#2169](https://github.com/cosmos/cosmos-sdk/issues/2169) add additional metrics -- [config] \#2232 added ValidateBasic method, which performs basic checks -- [crypto/ed25519] \#2558 Switch to use latest `golang.org/x/crypto` through our fork at - github.com/tendermint/crypto -- [tools] \#2238 Binary dependencies are now locked to a specific git commit -- [crypto] \#2099 make crypto random use chacha, and have forward secrecy of generated randomness - -BUG FIXES: -- [autofile] \#2428 Group.RotateFile need call Flush() before rename (@goolAdapter) -- [node] \#2434 Make node respond to signal interrupts while sleeping for genesis time -- [consensus] [\#1690](https://github.com/tendermint/tendermint/issues/1690) wait for -timeoutPrecommit before starting next round -- [evidence] \#2515 fix db iter leak (@goolAdapter) -- [common/bit_array] Fixed a bug in the `Or` function -- [common/bit_array] Fixed a bug in the `Sub` function (@james-ray) -- [common] \#2534 Make bit array's PickRandom choose uniformly from true bits -- [consensus] \#1637 Limit the amount of evidence that can be included in a - block -- [p2p] \#2555 fix p2p switch FlushThrottle value (@goolAdapter) -- [libs/event] \#2518 fix event concurrency flaw (@goolAdapter) +### BUG FIXES: diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index d47c0f15..7088fca4 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -6,7 +6,7 @@ This code of conduct applies to all projects run by the Tendermint/COSMOS team a # Conduct -## Contact: adrian@tendermint.com +## Contact: conduct@tendermint.com * We are committed to providing a friendly, safe and welcoming environment for all, regardless of level of experience, gender, gender identity and expression, sexual orientation, disability, personal appearance, body size, race, ethnicity, age, religion, nationality, or other similar characteristic. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3500732f..3dab3b8a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -27,8 +27,8 @@ Of course, replace `ebuchman` with your git handle. To pull in updates from the origin repo, run - * `git fetch upstream` - * `git rebase upstream/master` (or whatever branch you want) + * `git fetch upstream` + * `git rebase upstream/master` (or whatever branch you want) Please don't make Pull Requests to `master`. @@ -50,6 +50,11 @@ as apps, tools, and the core, should use dep. Run `dep status` to get a list of vendor dependencies that may not be up-to-date. +When updating dependencies, please only update the particular dependencies you +need. Instead of running `dep ensure -update`, which will update anything, +specify exactly the dependency you want to update, eg. +`dep ensure -update github.com/tendermint/go-amino`. + ## Vagrant If you are a [Vagrant](https://www.vagrantup.com/) user, you can get started @@ -64,43 +69,74 @@ vagrant ssh make test ``` -## Testing +## Changelog -All repos should be hooked up to [CircleCI](https://circleci.com/). +Every fix, improvement, feature, or breaking change should be made in a +pull-request that includes an update to the `CHANGELOG_PENDING.md` file. -If they have `.go` files in the root directory, they will be automatically -tested by circle using `go test -v -race ./...`. If not, they will need a -`circle.yml`. Ideally, every repo has a `Makefile` that defines `make test` and -includes its continuous integration status using a badge in the `README.md`. +Changelog entries should be formatted as follows: + +``` +- [module] \#xxx Some description about the change (@contributor) +``` + +Here, `module` is the part of the code that changed (typically a +top-level Go package), `xxx` is the pull-request number, and `contributor` +is the author/s of the change. + +It's also acceptable for `xxx` to refer to the relevent issue number, but pull-request +numbers are preferred. +Note this means pull-requests should be opened first so the changelog can then +be updated with the pull-request's number. +There is no need to include the full link, as this will be added +automatically during release. But please include the backslash and pound, eg. `\#2313`. + +Changelog entries should be ordered alphabetically according to the +`module`, and numerically according to the pull-request number. + +Changes with multiple classifications should be doubly included (eg. a bug fix +that is also a breaking change should be recorded under both). + +Breaking changes are further subdivided according to the APIs/users they impact. +Any change that effects multiple APIs/users should be recorded multiply - for +instance, a change to the `Blockchain Protocol` that removes a field from the +header should also be recorded under `CLI/RPC/Config` since the field will be +removed from the header in rpc responses as well. ## Branching Model and Release -User-facing repos should adhere to the branching model: http://nvie.com/posts/a-successful-git-branching-model/. -That is, these repos should be well versioned, and any merge to master requires a version bump and tagged release. - -Libraries need not follow the model strictly, but would be wise to, -especially `go-p2p` and `go-rpc`, as their versions are referenced in tendermint core. +All repos should adhere to the branching model: http://nvie.com/posts/a-successful-git-branching-model/. +This means that all pull-requests should be made against develop. Any merge to +master constitutes a tagged release. ### Development Procedure: - the latest state of development is on `develop` - `develop` must never fail `make test` -- no --force onto `develop` (except when reverting a broken commit, which should seldom happen) +- never --force onto `develop` (except when reverting a broken commit, which should seldom happen) - create a development branch either on github.com/tendermint/tendermint, or your fork (using `git remote add origin`) -- before submitting a pull request, begin `git rebase` on top of `develop` +- make changes and update the `CHANGELOG_PENDING.md` to record your change +- before submitting a pull request, run `git rebase` on top of the latest `develop` ### Pull Merge Procedure: -- ensure pull branch is rebased on develop +- ensure pull branch is based on a recent develop - run `make test` to ensure that all tests pass - merge pull request -- the `unstable` branch may be used to aggregate pull merges before testing once -- push master may request that pull requests be rebased on top of `unstable` +- the `unstable` branch may be used to aggregate pull merges before fixing tests ### Release Procedure: - start on `develop` - run integration tests (see `test_integrations` in Makefile) -- prepare changelog/release issue +- prepare changelog: + - copy `CHANGELOG_PENDING.md` to top of `CHANGELOG.md` + - run `python ./scripts/linkify_changelog.py CHANGELOG.md` to add links for + all issues + - run `bash ./scripts/authors.sh` to get a list of authors since the latest + release, and add the github aliases of external contributors to the top of + the changelog. To lookup an alias from an email, try `bash + ./scripts/authors.sh ` + - reset the `CHANGELOG_PENDING.md` - bump versions -- push to release-vX.X.X to run the extended integration tests on the CI +- push to release/vX.X.X to run the extended integration tests on the CI - merge to master - merge master back to develop @@ -115,3 +151,13 @@ especially `go-p2p` and `go-rpc`, as their versions are referenced in tendermint - merge hotfix-vX.X.X to master - merge hotfix-vX.X.X to develop - delete the hotfix-vX.X.X branch + + +## Testing + +All repos should be hooked up to [CircleCI](https://circleci.com/). + +If they have `.go` files in the root directory, they will be automatically +tested by circle using `go test -v -race ./...`. If not, they will need a +`circle.yml`. Ideally, every repo has a `Makefile` that defines `make test` and +includes its continuous integration status using a badge in the `README.md`. diff --git a/DOCKER/build.sh b/DOCKER/build.sh index ee617cc6..2aa42a6c 100755 --- a/DOCKER/build.sh +++ b/DOCKER/build.sh @@ -3,7 +3,7 @@ set -e # Get the tag from the version, or try to figure it out. if [ -z "$TAG" ]; then - TAG=$(awk -F\" '/Version =/ { print $2; exit }' < ../version/version.go) + TAG=$(awk -F\" '/TMCoreSemVer =/ { print $2; exit }' < ../version/version.go) fi if [ -z "$TAG" ]; then echo "Please specify a tag." diff --git a/DOCKER/push.sh b/DOCKER/push.sh index 32741dce..f228406d 100755 --- a/DOCKER/push.sh +++ b/DOCKER/push.sh @@ -3,7 +3,7 @@ set -e # Get the tag from the version, or try to figure it out. if [ -z "$TAG" ]; then - TAG=$(awk -F\" '/Version =/ { print $2; exit }' < ../version/version.go) + TAG=$(awk -F\" '/TMCoreSemVer =/ { print $2; exit }' < ../version/version.go) fi if [ -z "$TAG" ]; then echo "Please specify a tag." diff --git a/Gopkg.lock b/Gopkg.lock index 0f70bb2f..9e3d5d8a 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -10,12 +10,11 @@ revision = "3a771d992973f24aa725d07868b467d1ddfceafb" [[projects]] - branch = "master" - digest = "1:2c00f064ba355903866cbfbf3f7f4c0fe64af6638cc7d1b8bdcf3181bc67f1d8" + digest = "1:093bf93a65962e8191e3e8cd8fc6c363f83d43caca9739c906531ba7210a9904" name = "github.com/btcsuite/btcd" packages = ["btcec"] pruneopts = "UT" - revision = "f5e261fc9ec3437697fb31d8b38453c293204b29" + revision = "ed77733ec07dfc8a513741138419b8d9d3de9d2d" [[projects]] digest = "1:1d8e1cb71c33a9470bbbae09bfec09db43c6bf358dfcae13cd8807c4e2a9a2bf" @@ -28,19 +27,20 @@ revision = "d4cc87b860166d00d6b5b9e0d3b3d71d6088d4d4" [[projects]] - digest = "1:a2c1d0e43bd3baaa071d1b9ed72c27d78169b2b269f71c105ac4ba34b1be4a39" + digest = "1:ffe9824d294da03b391f44e1ae8281281b4afc1bdaa9588c9097785e3af10cec" name = "github.com/davecgh/go-spew" packages = ["spew"] pruneopts = "UT" - revision = "346938d642f2ec3594ed81d874461961cd0faa76" - version = "v1.1.0" + revision = "8991bc29aa16c548c550c7ff78260e27b9ab7c73" + version = "v1.1.1" [[projects]] - digest = "1:c7644c73a3d23741fdba8a99b1464e021a224b7e205be497271a8003a15ca41b" - name = "github.com/ebuchman/fail-test" - packages = ["."] - pruneopts = "UT" - revision = "95f809107225be108efcf10a3509e4ea6ceef3c4" + digest = "1:b42be5a3601f833e0b9f2d6625d887ec1309764bfcac3d518f3db425dcd4ec5c" + name = "github.com/ethereum/go-ethereum" + packages = ["crypto/secp256k1"] + pruneopts = "T" + revision = "9dc5d1a915ac0e0bd8429d6ac41df50eec91de5f" + version = "v1.8.21" [[projects]] digest = "1:544229a3ca0fb2dd5ebc2896d3d2ff7ce096d9751635301e44e37e761349ee70" @@ -83,12 +83,12 @@ version = "v0.3.0" [[projects]] - digest = "1:c4a2528ccbcabf90f9f3c464a5fc9e302d592861bbfd0b7135a7de8a943d0406" + digest = "1:586ea76dbd0374d6fb649a91d70d652b7fe0ccffb8910a77468e7702e7901f3d" name = "github.com/go-stack/stack" packages = ["."] pruneopts = "UT" - revision = "259ab82a6cad3992b4e21ff5cac294ccb06474bc" - version = "v1.7.0" + revision = "2fee6af1a9795aafbe0253a0cfbdf668e1fb8a9a" + version = "v1.8.0" [[projects]] digest = "1:35621fe20f140f05a0c4ef662c26c0ab4ee50bca78aa30fe87d33120bd28165e" @@ -136,8 +136,7 @@ version = "v1.2.0" [[projects]] - branch = "master" - digest = "1:12247a2e99a060cc692f6680e5272c8adf0b8f572e6bce0d7095e624c958a240" + digest = "1:ea40c24cdbacd054a6ae9de03e62c5f252479b96c716375aace5c120d68647c8" name = "github.com/hashicorp/hcl" packages = [ ".", @@ -151,7 +150,8 @@ "json/token", ] pruneopts = "UT" - revision = "ef8a98b0bbce4a65b5aa4c368430a80ddc533168" + revision = "8cb6e5b959231cc1119e43259c4a608f9c51a241" + version = "v1.0.0" [[projects]] digest = "1:870d441fe217b8e689d7949fef6e43efbc787e50f200cb1e70dbca9204a1d6be" @@ -193,12 +193,12 @@ version = "v1.0.1" [[projects]] - branch = "master" - digest = "1:5ab79470a1d0fb19b041a624415612f8236b3c06070161a910562f2b2d064355" + digest = "1:53bc4cd4914cd7cd52139990d5170d6dc99067ae31c56530621b18b35fc30318" name = "github.com/mitchellh/mapstructure" packages = ["."] pruneopts = "UT" - revision = "f15292f7a699fcc1a38a80977f80a046874ba8ac" + revision = "3536a929edddb9a5b34bd6861dc4a9647cb459fe" + version = "v1.1.2" [[projects]] digest = "1:95741de3af260a92cc5c7f3f3061e85273f5a81b5db20d4bd68da74bd521675e" @@ -225,14 +225,16 @@ version = "v1.0.0" [[projects]] - digest = "1:c1a04665f9613e082e1209cf288bf64f4068dcd6c87a64bf1c4ff006ad422ba0" + digest = "1:26663fafdea73a38075b07e8e9d82fc0056379d2be8bb4e13899e8fda7c7dd23" name = "github.com/prometheus/client_golang" packages = [ "prometheus", + "prometheus/internal", "prometheus/promhttp", ] pruneopts = "UT" - revision = "ae27198cdd90bf12cd134ad79d1366a6cf49f632" + revision = "abad2d1bd44235a26707c172eab6bca5bf2dbad3" + version = "v0.9.1" [[projects]] branch = "master" @@ -244,7 +246,7 @@ [[projects]] branch = "master" - digest = "1:63b68062b8968092eb86bedc4e68894bd096ea6b24920faca8b9dcf451f54bb5" + digest = "1:db712fde5d12d6cdbdf14b777f0c230f4ff5ab0be8e35b239fc319953ed577a4" name = "github.com/prometheus/common" packages = [ "expfmt", @@ -252,11 +254,11 @@ "model", ] pruneopts = "UT" - revision = "c7de2306084e37d54b8be01f3541a8464345e9a5" + revision = "7e9e6cabbd393fc208072eedef99188d0ce788b6" [[projects]] branch = "master" - digest = "1:8c49953a1414305f2ff5465147ee576dd705487c35b15918fcd4efdc0cb7a290" + digest = "1:ef74914912f99c79434d9c09658274678bc85080ebe3ab32bec3940ebce5e1fc" name = "github.com/prometheus/procfs" packages = [ ".", @@ -265,7 +267,7 @@ "xfs", ] pruneopts = "UT" - revision = "05ee40e3a273f7245e8777337fc7b46e533a9a92" + revision = "185b4288413d2a0dd0806f78c90dde719829e5ae" [[projects]] digest = "1:c4556a44e350b50a490544d9b06e9fba9c286c21d6c0e47f54f3a9214597298c" @@ -275,23 +277,31 @@ revision = "e2704e165165ec55d062f5919b4b29494e9fa790" [[projects]] - digest = "1:bd1ae00087d17c5a748660b8e89e1043e1e5479d0fea743352cda2f8dd8c4f84" + digest = "1:b0c25f00bad20d783d259af2af8666969e2fc343fa0dc9efe52936bbd67fb758" + name = "github.com/rs/cors" + packages = ["."] + pruneopts = "UT" + revision = "9a47f48565a795472d43519dd49aac781f3034fb" + version = "v1.6.0" + +[[projects]] + digest = "1:6a4a11ba764a56d2758899ec6f3848d24698d48442ebce85ee7a3f63284526cd" name = "github.com/spf13/afero" packages = [ ".", "mem", ] pruneopts = "UT" - revision = "787d034dfe70e44075ccc060d346146ef53270ad" - version = "v1.1.1" + revision = "d40851caa0d747393da1ffb28f7f9d8b4eeffebd" + version = "v1.1.2" [[projects]] - digest = "1:516e71bed754268937f57d4ecb190e01958452336fa73dbac880894164e91c1f" + digest = "1:08d65904057412fc0270fc4812a1c90c594186819243160dc779a402d4b6d0bc" name = "github.com/spf13/cast" packages = ["."] pruneopts = "UT" - revision = "8965335b8c7107321228e3e3702cab9832751bac" - version = "v1.2.0" + revision = "8c9545af88b134710ab1cd196795e7f2388358d7" + version = "v1.3.0" [[projects]] digest = "1:7ffc0983035bc7e297da3688d9fe19d60a420e9c38bef23f845c53788ed6a05e" @@ -302,20 +312,20 @@ version = "v0.0.1" [[projects]] - branch = "master" - digest = "1:080e5f630945ad754f4b920e60b4d3095ba0237ebf88dc462eb28002932e3805" + digest = "1:68ea4e23713989dc20b1bded5d9da2c5f9be14ff9885beef481848edd18c26cb" name = "github.com/spf13/jwalterweatherman" packages = ["."] pruneopts = "UT" - revision = "7c0cea34c8ece3fbeb2b27ab9b59511d360fb394" + revision = "4a4406e478ca629068e7768fc33f3f044173c0a6" + version = "v1.0.0" [[projects]] - digest = "1:9424f440bba8f7508b69414634aef3b2b3a877e522d8a4624692412805407bb7" + digest = "1:c1b1102241e7f645bc8e0c22ae352e8f0dc6484b6cb4d132fa9f24174e0119e2" name = "github.com/spf13/pflag" packages = ["."] pruneopts = "UT" - revision = "583c0c0531f06d5278b7d917446061adc344b5cd" - version = "v1.0.1" + revision = "298182f68c66c05229eb03ac171abe6e309ee79a" + version = "v1.0.3" [[projects]] digest = "1:f8e1a678a2571e265f4bf91a3e5e32aa6b1474a55cb0ea849750cc177b664d96" @@ -338,7 +348,7 @@ [[projects]] branch = "master" - digest = "1:b3cfb8d82b1601a846417c3f31c03a7961862cb2c98dcf0959c473843e6d9a2b" + digest = "1:59483b8e8183f10ab21a85ba1f4cbb4a2335d48891801f79ed7b9499f44d383c" name = "github.com/syndtr/goleveldb" packages = [ "leveldb", @@ -355,25 +365,18 @@ "leveldb/util", ] pruneopts = "UT" - revision = "c4c61651e9e37fa117f53c5a906d3b63090d8445" + revision = "6b91fda63f2e36186f1c9d0e48578defb69c5d43" [[projects]] - digest = "1:605b6546f3f43745695298ec2d342d3e952b6d91cdf9f349bea9315f677d759f" - name = "github.com/tendermint/btcd" - packages = ["btcec"] - pruneopts = "UT" - revision = "e5840949ff4fff0c56f9b6a541e22b63581ea9df" - -[[projects]] - digest = "1:e0a2a4be1e20c305badc2b0a7a9ab7fef6da500763bec23ab81df3b5f9eec9ee" + digest = "1:ad9c4c1a4e7875330b1f62906f2830f043a23edb5db997e3a5ac5d3e6eadf80a" name = "github.com/tendermint/go-amino" packages = ["."] pruneopts = "UT" - revision = "a8328986c1608950fa5d3d1c0472cccc4f8fc02c" - version = "v0.12.0-rc0" + revision = "dc14acf9ef15f85828bfbc561ed9dd9d2a284885" + version = "v0.14.1" [[projects]] - digest = "1:72b71e3a29775e5752ed7a8012052a3dee165e27ec18cedddae5288058f09acf" + digest = "1:00d2b3e64cdc3fa69aa250dfbe4cc38c4837d4f37e62279be2ae52107ffbbb44" name = "golang.org/x/crypto" packages = [ "bcrypt", @@ -394,8 +397,7 @@ "salsa20/salsa", ] pruneopts = "UT" - revision = "3764759f34a542a3aef74d6b02e35be7ab893bba" - source = "github.com/tendermint/crypto" + revision = "505ab145d0a99da450461ae2c1a9f6cd10d1f447" [[projects]] digest = "1:d36f55a999540d29b6ea3c2ea29d71c76b1d9853fdcd3e5c5cb4836f2ba118f1" @@ -415,14 +417,14 @@ [[projects]] branch = "master" - digest = "1:bb0fe59917bdd5b89f49b9a8b26e5f465e325d9223b3a8e32254314bdf51e0f1" + digest = "1:6f86e2f2e2217cd4d74dec6786163cf80e4d2b99adb341ecc60a45113b844dca" name = "golang.org/x/sys" packages = [ "cpu", "unix", ] pruneopts = "UT" - revision = "3dc4335d56c789b04b0ba99b7a37249d9b614314" + revision = "7e31e0c00fa05cb5fbf4347b585621d6709e19a4" [[projects]] digest = "1:a2ab62866c75542dd18d2b069fec854577a20211d7c0ea6ae746072a1dccdd18" @@ -449,11 +451,11 @@ [[projects]] branch = "master" - digest = "1:077c1c599507b3b3e9156d17d36e1e61928ee9b53a5b420f10f28ebd4a0b275c" + digest = "1:56b0bca90b7e5d1facf5fbdacba23e4e0ce069d25381b8e2f70ef1e7ebfb9c1a" name = "google.golang.org/genproto" packages = ["googleapis/rpc/status"] pruneopts = "UT" - revision = "daca94659cb50e9f37c1b834680f2e46358f10b0" + revision = "b69ba1387ce2108ac9bc8e8e5e5a46e7d5c72313" [[projects]] digest = "1:2dab32a43451e320e49608ff4542fdfc653c95dcc35d0065ec9c6c3dd540ed74" @@ -501,9 +503,10 @@ analyzer-name = "dep" analyzer-version = 1 input-imports = [ + "github.com/btcsuite/btcd/btcec", "github.com/btcsuite/btcutil/base58", "github.com/btcsuite/btcutil/bech32", - "github.com/ebuchman/fail-test", + "github.com/ethereum/go-ethereum/crypto/secp256k1", "github.com/fortytw2/leaktest", "github.com/go-kit/kit/log", "github.com/go-kit/kit/log/level", @@ -524,6 +527,7 @@ "github.com/prometheus/client_golang/prometheus", "github.com/prometheus/client_golang/prometheus/promhttp", "github.com/rcrowley/go-metrics", + "github.com/rs/cors", "github.com/spf13/cobra", "github.com/spf13/viper", "github.com/stretchr/testify/assert", @@ -532,7 +536,6 @@ "github.com/syndtr/goleveldb/leveldb/errors", "github.com/syndtr/goleveldb/leveldb/iterator", "github.com/syndtr/goleveldb/leveldb/opt", - "github.com/tendermint/btcd/btcec", "github.com/tendermint/go-amino", "golang.org/x/crypto/bcrypt", "golang.org/x/crypto/chacha20poly1305", diff --git a/Gopkg.toml b/Gopkg.toml index 07ff3c53..e5d6b8da 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -20,84 +20,94 @@ # unused-packages = true # ########################################################### -# NOTE: All packages should be pinned to specific versions. -# Packages without releases must pin to a commit. - +# Allow only patch releases for serialization libraries [[constraint]] - name = "github.com/go-kit/kit" - version = "=0.6.0" + name = "github.com/tendermint/go-amino" + version = "~0.14.1" [[constraint]] name = "github.com/gogo/protobuf" - version = "=1.1.1" + version = "~1.1.1" [[constraint]] name = "github.com/golang/protobuf" - version = "=1.1.0" + version = "~1.1.0" + +# Allow only minor releases for other libraries +[[constraint]] + name = "github.com/go-kit/kit" + version = "^0.6.0" [[constraint]] name = "github.com/gorilla/websocket" - version = "=1.2.0" + version = "^1.2.0" + +[[constraint]] + name = "github.com/rs/cors" + version = "^1.6.0" [[constraint]] name = "github.com/pkg/errors" - version = "=0.8.0" + version = "^0.8.0" [[constraint]] name = "github.com/spf13/cobra" - version = "=0.0.1" + version = "^0.0.1" [[constraint]] name = "github.com/spf13/viper" - version = "=1.0.0" + version = "^1.0.0" [[constraint]] name = "github.com/stretchr/testify" - version = "=1.2.1" - -[[constraint]] - name = "github.com/tendermint/go-amino" - version = "v0.12.0-rc0" + version = "^1.2.1" [[constraint]] name = "google.golang.org/grpc" - version = "=1.13.0" + version = "^1.13.0" [[constraint]] name = "github.com/fortytw2/leaktest" - version = "=1.2.0" + version = "^1.2.0" + +[[constraint]] + name = "github.com/prometheus/client_golang" + version = "^0.9.1" + +# we use the secp256k1 implementation: +[[constraint]] + name = "github.com/ethereum/go-ethereum" + version = "^v1.8.21" + + # Prevent dep from pruning build scripts and codegen templates + # note: this leaves the whole go-ethereum package in vendor + # can be removed when https://github.com/golang/dep/issues/1847 is resolved + [[prune.project]] + name = "github.com/ethereum/go-ethereum" + unused-packages = false ################################### ## Some repos dont have releases. ## Pin to revision +[[constraint]] + name = "github.com/btcsuite/btcd" + revision = "ed77733ec07dfc8a513741138419b8d9d3de9d2d" + [[constraint]] name = "golang.org/x/crypto" - source = "github.com/tendermint/crypto" - revision = "3764759f34a542a3aef74d6b02e35be7ab893bba" + revision = "505ab145d0a99da450461ae2c1a9f6cd10d1f447" [[override]] name = "github.com/jmhodges/levigo" revision = "c42d9e0ca023e2198120196f842701bb4c55d7b9" -[[constraint]] - name = "github.com/ebuchman/fail-test" - revision = "95f809107225be108efcf10a3509e4ea6ceef3c4" - # last revision used by go-crypto [[constraint]] name = "github.com/btcsuite/btcutil" revision = "d4cc87b860166d00d6b5b9e0d3b3d71d6088d4d4" -[[constraint]] - name = "github.com/tendermint/btcd" - revision = "e5840949ff4fff0c56f9b6a541e22b63581ea9df" - -# Haven't made a release since 2016. -[[constraint]] - name = "github.com/prometheus/client_golang" - revision = "ae27198cdd90bf12cd134ad79d1366a6cf49f632" [[constraint]] name = "github.com/rcrowley/go-metrics" diff --git a/Makefile b/Makefile index 0b78574b..a1ba06aa 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ GOTOOLS = \ github.com/mitchellh/gox \ github.com/golang/dep/cmd/dep \ - github.com/alecthomas/gometalinter \ + github.com/golangci/golangci-lint/cmd/golangci-lint \ github.com/gogo/protobuf/protoc-gen-gogo \ github.com/square/certstrap GOBIN?=${GOPATH}/bin @@ -11,13 +11,10 @@ INCLUDE = -I=. -I=${GOPATH}/src -I=${GOPATH}/src/github.com/gogo/protobuf/protob BUILD_TAGS?='tendermint' BUILD_FLAGS = -ldflags "-X github.com/tendermint/tendermint/version.GitCommit=`git rev-parse --short=8 HEAD`" -LINT_FLAGS = --exclude '.*\.pb\.go' --exclude 'vendor/*' --vendor --deadline=600s - all: check build test install check: check_tools get_vendor_deps - ######################################## ### Build Tendermint @@ -33,10 +30,13 @@ build_race: install: CGO_ENABLED=0 go install $(BUILD_FLAGS) -tags $(BUILD_TAGS) ./cmd/tendermint +install_c: + CGO_ENABLED=1 go install $(BUILD_FLAGS) -tags "$(BUILD_TAGS) gcc" ./cmd/tendermint + ######################################## ### Protobuf -protoc_all: protoc_libs protoc_merkle protoc_abci protoc_grpc +protoc_all: protoc_libs protoc_merkle protoc_abci protoc_grpc protoc_proto3types %.pb.go: %.proto ## If you get the following error, @@ -52,6 +52,8 @@ protoc_all: protoc_libs protoc_merkle protoc_abci protoc_grpc # see protobuf section above protoc_abci: abci/types/types.pb.go +protoc_proto3types: types/proto3/block.pb.go + build_abci: @go build -i ./abci/cmd/... @@ -77,8 +79,6 @@ check_tools: get_tools: @echo "--> Installing tools" ./scripts/get_tools.sh - @echo "--> Downloading linters (this may take awhile)" - $(GOPATH)/src/github.com/alecthomas/gometalinter/scripts/install.sh -b $(GOBIN) update_tools: @echo "--> Updating tools" @@ -220,6 +220,22 @@ test_race: @echo "--> Running go test --race" @GOCACHE=off go test -p 1 -v -race $(PACKAGES) +# uses https://github.com/sasha-s/go-deadlock/ to detect potential deadlocks +test_with_deadlock: + make set_with_deadlock + make test + make cleanup_after_test_with_deadlock + +set_with_deadlock: + find . -name "*.go" | grep -v "vendor/" | xargs -n 1 sed -i.bak 's/sync.RWMutex/deadlock.RWMutex/' + find . -name "*.go" | grep -v "vendor/" | xargs -n 1 sed -i.bak 's/sync.Mutex/deadlock.Mutex/' + find . -name "*.go" | grep -v "vendor/" | xargs -n 1 goimports -w + +# cleanes up after you ran test_with_deadlock +cleanup_after_test_with_deadlock: + find . -name "*.go" | grep -v "vendor/" | xargs -n 1 sed -i.bak 's/deadlock.RWMutex/sync.RWMutex/' + find . -name "*.go" | grep -v "vendor/" | xargs -n 1 sed -i.bak 's/deadlock.Mutex/sync.Mutex/' + find . -name "*.go" | grep -v "vendor/" | xargs -n 1 goimports -w ######################################## ### Formatting, linting, and vetting @@ -227,38 +243,9 @@ test_race: fmt: @go fmt ./... -metalinter: +lint: @echo "--> Running linter" - @gometalinter $(LINT_FLAGS) --disable-all \ - --enable=deadcode \ - --enable=gosimple \ - --enable=misspell \ - --enable=safesql \ - ./... - #--enable=gas \ - #--enable=maligned \ - #--enable=dupl \ - #--enable=errcheck \ - #--enable=goconst \ - #--enable=gocyclo \ - #--enable=goimports \ - #--enable=golint \ <== comments on anything exported - #--enable=gotype \ - #--enable=ineffassign \ - #--enable=interfacer \ - #--enable=megacheck \ - #--enable=staticcheck \ - #--enable=structcheck \ - #--enable=unconvert \ - #--enable=unparam \ - #--enable=unused \ - #--enable=varcheck \ - #--enable=vet \ - #--enable=vetshadow \ - -metalinter_all: - @echo "--> Running linter (all)" - gometalinter $(LINT_FLAGS) --enable-all --disable=lll ./... + @golangci-lint run DESTINATION = ./index.html.md @@ -282,12 +269,11 @@ build-docker: ### Local testnet using docker # Build linux binary on other platforms -build-linux: +build-linux: get_tools get_vendor_deps GOOS=linux GOARCH=amd64 $(MAKE) build build-docker-localnode: - cd networks/local - make + @cd networks/local && make # Run a 4-node testnet locally localnet-start: localnet-stop @@ -325,4 +311,4 @@ build-slate: # To avoid unintended conflicts with file names, always add to .PHONY # unless there is a reason not to. # https://www.gnu.org/software/make/manual/html_node/Phony-Targets.html -.PHONY: check build build_race build_abci dist install install_abci check_dep check_tools get_tools update_tools get_vendor_deps draw_deps get_protoc protoc_abci protoc_libs gen_certs clean_certs grpc_dbserver test_cover test_apps test_persistence test_p2p test test_race test_integrations test_release test100 vagrant_test fmt rpc-docs build-linux localnet-start localnet-stop build-docker build-docker-localnode sentry-start sentry-config sentry-stop build-slate protoc_grpc protoc_all +.PHONY: check build build_race build_abci dist install install_abci check_dep check_tools get_tools update_tools get_vendor_deps draw_deps get_protoc protoc_abci protoc_libs gen_certs clean_certs grpc_dbserver test_cover test_apps test_persistence test_p2p test test_race test_integrations test_release test100 vagrant_test fmt rpc-docs build-linux localnet-start localnet-stop build-docker build-docker-localnode sentry-start sentry-config sentry-stop build-slate protoc_grpc protoc_all build_c install_c test_with_deadlock cleanup_after_test_with_deadlock lint diff --git a/PHILOSOPHY.md b/PHILOSOPHY.md new file mode 100644 index 00000000..cf79710f --- /dev/null +++ b/PHILOSOPHY.md @@ -0,0 +1,158 @@ +## Design goals + +The design goals for Tendermint (and the SDK and related libraries) are: + + * Simplicity and Legibility + * Parallel performance, namely ability to utilize multicore architecture + * Ability to evolve the codebase bug-free + * Debuggability + * Complete correctness that considers all edge cases, esp in concurrency + * Future-proof modular architecture, message protocol, APIs, and encapsulation + + +### Justification + +Legibility is key to maintaining bug-free software as it evolves toward more +optimizations, more ease of debugging, and additional features. + +It is too easy to introduce bugs over time by replacing lines of code with +those that may panic, which means ideally locks are unlocked by defer +statements. + +For example, + +```go +func (obj *MyObj) something() { + mtx.Lock() + obj.something = other + mtx.Unlock() +} +``` + +It is too easy to refactor the codebase in the future to replace `other` with +`other.String()` for example, and this may introduce a bug that causes a +deadlock. So as much as reasonably possible, we need to be using defer +statements, even though it introduces additional overhead. + +If it is necessary to optimize the unlocking of mutex locks, the solution is +more modularity via smaller functions, so that defer'd unlocks are scoped +within a smaller function. + +Similarly, idiomatic for-loops should always be preferred over those that use +custom counters, because it is too easy to evolve the body of a for-loop to +become more complicated over time, and it becomes more and more difficult to +assess the correctness of such a for-loop by visual inspection. + + +### On performance + +It doesn't matter whether there are alternative implementations that are 2x or +3x more performant, when the software doesn't work, deadlocks, or if bugs +cannot be debugged. By taking advantage of multicore concurrency, the +Tendermint implementation will at least be an order of magnitude within the +range of what is theoretically possible. The design philosophy of Tendermint, +and the choice of Go as implementation language, is designed to make Tendermint +implementation the standard specification for concurrent BFT software. + +By focusing on the message protocols (e.g. ABCI, p2p messages), and +encapsulation e.g. IAVL module, (relatively) independent reactors, we are both +implementing a standard implementation to be used as the specification for +future implementations in more optimizable languages like Rust, Java, and C++; +as well as creating sufficiently performant software. Tendermint Core will +never be as fast as future implementations of the Tendermint Spec, because Go +isn't designed to be as fast as possible. The advantage of using Go is that we +can develop the whole stack of modular components **faster** than in other +languages. + +Furthermore, the real bottleneck is in the application layer, and it isn't +necessary to support more than a sufficiently decentralized set of validators +(e.g. 100 ~ 300 validators is sufficient, with delegated bonded PoS). + +Instead of optimizing Tendermint performance down to the metal, lets focus on +optimizing on other matters, namely ability to push feature complete software +that works well enough, can be debugged and maintained, and can serve as a spec +for future implementations. + + +### On encapsulation + +In order to create maintainable, forward-optimizable software, it is critical +to develop well-encapsulated objects that have well understood properties, and +to re-use these easy-to-use-correctly components as building blocks for further +encapsulated meta-objects. + +For example, mutexes are cheap enough for Tendermint's design goals when there +isn't goroutine contention, so it is encouraged to create concurrency safe +structures with struct-level mutexes. If they are used in the context of +non-concurrent logic, then the performance is good enough. If they are used in +the context of concurrent logic, then it will still perform correctly. + +Examples of this design principle can be seen in the types.ValidatorSet struct, +and the cmn.Rand struct. It's one single struct declaration that can be used +in both concurrent and non-concurrent logic, and due to its well encapsulation, +it's easy to get the usage of the mutex right. + +#### example: cmn.Rand: + +`The default Source is safe for concurrent use by multiple goroutines, but +Sources created by NewSource are not`. The reason why the default +package-level source is safe for concurrent use is because it is protected (see +`lockedSource` in https://golang.org/src/math/rand/rand.go). + +But we shouldn't rely on the global source, we should be creating our own +Rand/Source instances and using them, especially for determinism in testing. +So it is reasonable to have cmn.Rand be protected by a mutex. Whether we want +our own implementation of Rand is another question, but the answer there is +also in the affirmative. Sometimes you want to know where Rand is being used +in your code, so it becomes a simple matter of dropping in a log statement to +inject inspectability into Rand usage. Also, it is nice to be able to extend +the functionality of Rand with custom methods. For these reasons, and for the +reasons which is outlined in this design philosophy document, we should +continue to use the cmn.Rand object, with mutex protection. + +Another key aspect of good encapsulation is the choice of exposed vs unexposed +methods. It should be clear to the reader of the code, which methods are +intended to be used in what context, and what safe usage is. Part of this is +solved by hiding methods via unexported methods. Another part of this is +naming conventions on the methods (e.g. underscores) with good documentation, +and code organization. If there are too many exposed methods and it isn't +clear what methods have what side effects, then there is something wrong about +the design of abstractions that should be revisited. + + +### On concurrency + +In order for Tendermint to remain relevant in the years to come, it is vital +for Tendermint to take advantage of multicore architectures. Due to the nature +of the problem, namely consensus across a concurrent p2p gossip network, and to +handle RPC requests for a large number of consuming subscribers, it is +unavoidable for Tendermint development to require expertise in concurrency +design, especially when it comes to the reactor design, and also for RPC +request handling. + + +## Guidelines + +Here are some guidelines for designing for (sufficient) performance and concurrency: + + * Mutex locks are cheap enough when there isn't contention. + * Do not optimize code without analytical or observed proof that it is in a hot path. + * Don't over-use channels when mutex locks w/ encapsulation are sufficient. + * The need to drain channels are often a hint of unconsidered edge cases. + * The creation of O(N) one-off goroutines is generally technical debt that + needs to get addressed sooner than later. Avoid creating too many +goroutines as a patch around incomplete concurrency design, or at least be +aware of the debt and do not invest in the debt. On the other hand, Tendermint +is designed to have a limited number of peers (e.g. 10 or 20), so the creation +of O(C) goroutines per O(P) peers is still O(C\*P=constant). + * Use defer statements to unlock as much as possible. If you want to unlock sooner, + try to create more modular functions that do make use of defer statements. + +## Matras + +* Premature optimization kills +* Readability is paramount +* Beautiful is better than fast. +* In the face of ambiguity, refuse the temptation to guess. +* In the face of bugs, refuse the temptation to cover the bug. +* There should be one-- and preferably only one --obvious way to do it. diff --git a/README.md b/README.md index 069f9f13..9251e3ca 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ # Tendermint [Byzantine-Fault Tolerant](https://en.wikipedia.org/wiki/Byzantine_fault_tolerance) -[State Machine Replication](https://en.wikipedia.org/wiki/State_machine_replication). -Or [Blockchain](https://en.wikipedia.org/wiki/Blockchain_(database)) for short. +[State Machines](https://en.wikipedia.org/wiki/State_machine_replication). +Or [Blockchain](https://en.wikipedia.org/wiki/Blockchain_(database)), for short. [![version](https://img.shields.io/github/tag/tendermint/tendermint.svg)](https://github.com/tendermint/tendermint/releases/latest) [![API Reference]( @@ -36,12 +36,12 @@ We are also still making breaking changes to the protocol and the APIs. Thus, we tag the releases as *alpha software*. In any case, if you intend to run Tendermint in production, -please [contact us](https://riot.im/app/#/room/#tendermint:matrix.org) :) +please [contact us](mailto:partners@tendermint.com) and [join the chat](https://riot.im/app/#/room/#tendermint:matrix.org). ## Security To report a security vulnerability, see our [bug bounty -program](https://tendermint.com/security). +program](https://hackerone.com/tendermint) For examples of the kinds of bugs we're looking for, see [SECURITY.md](SECURITY.md) @@ -49,61 +49,43 @@ For examples of the kinds of bugs we're looking for, see [SECURITY.md](SECURITY. Requirement|Notes ---|--- -Go version | Go1.10 or higher +Go version | Go1.11.4 or higher -## Install +## Documentation + +Complete documentation can be found on the [website](https://tendermint.com/docs/). + +### Install See the [install instructions](/docs/introduction/install.md) -## Quick Start +### Quick Start -- [Single node](/docs/tendermint-core/using-tendermint.md) -- [Local cluster using docker-compose](/networks/local) +- [Single node](/docs/introduction/quick-start.md) +- [Local cluster using docker-compose](/docs/networks/docker-compose.md) - [Remote cluster using terraform and ansible](/docs/networks/terraform-and-ansible.md) - [Join the Cosmos testnet](https://cosmos.network/testnet) -## Resources - -### Tendermint Core - -For details about the blockchain data structures and the p2p protocols, see the -the [Tendermint specification](/docs/spec). - -For details on using the software, see the [documentation](/docs/) which is also -hosted at: https://tendermint.com/docs/ - -### Tools - -Benchmarking and monitoring is provided by `tm-bench` and `tm-monitor`, respectively. -Their code is found [here](/tools) and these binaries need to be built seperately. -Additional documentation is found [here](/docs/tools). - -### Sub-projects - -* [Amino](http://github.com/tendermint/go-amino), a reflection-based improvement on proto3 -* [IAVL](http://github.com/tendermint/iavl), Merkleized IAVL+ Tree implementation - -### Applications - -* [Cosmos SDK](http://github.com/cosmos/cosmos-sdk); a cryptocurrency application framework -* [Ethermint](http://github.com/cosmos/ethermint); Ethereum on Tendermint -* [Many more](https://tendermint.com/ecosystem) - -### Research - -* [Master's Thesis on Tendermint](https://atrium.lib.uoguelph.ca/xmlui/handle/10214/9769) -* [Original Whitepaper](https://tendermint.com/static/docs/tendermint.pdf) -* [Blog](https://blog.cosmos.network/tendermint/home) - ## Contributing -Yay open source! Please see our [contributing guidelines](CONTRIBUTING.md). +Please abide by the [Code of Conduct](CODE_OF_CONDUCT.md) in all interactions, +and the [contributing guidelines](CONTRIBUTING.md) when submitting code. + +Join the larger community on the [forum](https://forum.cosmos.network/) and the [chat](https://riot.im/app/#/room/#tendermint:matrix.org). + +To learn more about the structure of the software, watch the [Developer +Sessions](https://www.youtube.com/playlist?list=PLdQIb0qr3pnBbG5ZG-0gr3zM86_s8Rpqv) +and read some [Architectural +Decision Records](https://github.com/tendermint/tendermint/tree/master/docs/architecture). + +Learn more by reading the code and comparing it to the +[specification](https://github.com/tendermint/tendermint/tree/develop/docs/spec). ## Versioning -### SemVer +### Semantic Versioning -Tendermint uses [SemVer](http://semver.org/) to determine when and how the version changes. +Tendermint uses [Semantic Versioning](http://semver.org/) to determine when and how the version changes. According to SemVer, anything in the public API can change at any time before version 1.0.0 To provide some stability to Tendermint users in these 0.X.X days, the MINOR version is used @@ -114,6 +96,7 @@ include the in-process Go APIs. That said, breaking changes in the following packages will be documented in the CHANGELOG even if they don't lead to MINOR version bumps: +- crypto - types - rpc/client - config @@ -140,8 +123,40 @@ data into the new chain. However, any bump in the PATCH version should be compatible with existing histories (if not please open an [issue](https://github.com/tendermint/tendermint/issues)). -For more information on upgrading, see [here](./UPGRADING.md) +For more information on upgrading, see [UPGRADING.md](./UPGRADING.md) -## Code of Conduct +## Resources + +### Tendermint Core + +For details about the blockchain data structures and the p2p protocols, see the +[Tendermint specification](/docs/spec). + +For details on using the software, see the [documentation](/docs/) which is also +hosted at: https://tendermint.com/docs/ + +### Tools + +Benchmarking and monitoring is provided by `tm-bench` and `tm-monitor`, respectively. +Their code is found [here](/tools) and these binaries need to be built seperately. +Additional documentation is found [here](/docs/tools). + +### Sub-projects + +* [Amino](http://github.com/tendermint/go-amino), reflection-based proto3, with + interfaces +* [IAVL](http://github.com/tendermint/iavl), Merkleized IAVL+ Tree implementation + +### Applications + +* [Cosmos SDK](http://github.com/cosmos/cosmos-sdk); a cryptocurrency application framework +* [Ethermint](http://github.com/cosmos/ethermint); Ethereum on Tendermint +* [Many more](https://tendermint.com/ecosystem) + +### Research + +* [The latest gossip on BFT consensus](https://arxiv.org/abs/1807.04938) +* [Master's Thesis on Tendermint](https://atrium.lib.uoguelph.ca/xmlui/handle/10214/9769) +* [Original Whitepaper](https://tendermint.com/static/docs/tendermint.pdf) +* [Blog](https://blog.cosmos.network/tendermint/home) -Please read, understand and adhere to our [code of conduct](CODE_OF_CONDUCT.md). diff --git a/SECURITY.md b/SECURITY.md index 8b979378..8a373a29 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -1,7 +1,8 @@ # Security As part of our [Coordinated Vulnerability Disclosure -Policy](https://tendermint.com/security), we operate a bug bounty. +Policy](https://tendermint.com/security), we operate a [bug +bounty](https://hackerone.com/tendermint). See the policy for more details on submissions and rewards. Here is a list of examples of the kinds of bugs we're most interested in: diff --git a/UPGRADING.md b/UPGRADING.md index 81e56e58..dd35ff26 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -3,6 +3,198 @@ This guide provides steps to be followed when you upgrade your applications to a newer version of Tendermint Core. +## v0.29.0 + +This release contains some breaking changes to the block and p2p protocols, +and will not be compatible with any previous versions of the software, primarily +due to changes in how various data structures are hashed. + +Any implementations of Tendermint blockchain verification, including lite clients, +will need to be updated. For specific details: +- [Merkle tree](./docs/spec/blockchain/encoding.md#merkle-trees) +- [ConsensusParams](./docs/spec/blockchain/state.md#consensusparams) + +There was also a small change to field ordering in the vote struct. Any +implementations of an out-of-process validator (like a Key-Management Server) +will need to be updated. For specific details: +- [Vote](https://github.com/tendermint/tendermint/blob/develop/docs/spec/consensus/signing.md#votes) + +Finally, the proposer selection algorithm continues to evolve. See the +[work-in-progress +specification](https://github.com/tendermint/tendermint/pull/3140). + +For everything else, please see the [CHANGELOG](./CHANGELOG.md#v0.29.0). + +## v0.28.0 + +This release breaks the format for the `priv_validator.json` file +and the protocol used for the external validator process. +It is compatible with v0.27.0 blockchains (neither the BlockProtocol nor the +P2PProtocol have changed). + +Please read carefully for details about upgrading. + +**Note:** Backup your `config/priv_validator.json` +before proceeding. + +### `priv_validator.json` + +The `config/priv_validator.json` is now two files: +`config/priv_validator_key.json` and `data/priv_validator_state.json`. +The former contains the key material, the later contains the details on the last +message signed. + +When running v0.28.0 for the first time, it will back up any pre-existing +`priv_validator.json` file and proceed to split it into the two new files. +Upgrading should happen automatically without problem. + +To upgrade manually, use the provided `privValUpgrade.go` script, with exact paths for the old +`priv_validator.json` and the locations for the two new files. It's recomended +to use the default paths, of `config/priv_validator_key.json` and +`data/priv_validator_state.json`, respectively: + +``` +go run scripts/privValUpgrade.go +``` + +### External validator signers + +The Unix and TCP implementations of the remote signing validator +have been consolidated into a single implementation. +Thus in both cases, the external process is expected to dial +Tendermint. This is different from how Unix sockets used to work, where +Tendermint dialed the external process. + +The `PubKeyMsg` was also split into separate `Request` and `Response` types +for consistency with other messages. + +Note that the TCP sockets don't yet use a persistent key, +so while they're encrypted, they can't yet be properly authenticated. +See [#3105](https://github.com/tendermint/tendermint/issues/3105). +Note the Unix socket has neither encryption nor authentication, but will +add a shared-secret in [#3099](https://github.com/tendermint/tendermint/issues/3099). + +## v0.27.0 + +This release contains some breaking changes to the block and p2p protocols, +but does not change any core data structures, so it should be compatible with +existing blockchains from the v0.26 series that only used Ed25519 validator keys. +Blockchains using Secp256k1 for validators will not be compatible. This is due +to the fact that we now enforce which key types validators can use as a +consensus param. The default is Ed25519, and Secp256k1 must be activated +explicitly. + +It is recommended to upgrade all nodes at once to avoid incompatibilities at the +peer layer - namely, the heartbeat consensus message has been removed (only +relevant if `create_empty_blocks=false` or `create_empty_blocks_interval > 0`), +and the proposer selection algorithm has changed. Since proposer information is +never included in the blockchain, this change only affects the peer layer. + +### Go API Changes + +#### libs/db + +The ReverseIterator API has changed the meaning of `start` and `end`. +Before, iteration was from `start` to `end`, where +`start > end`. Now, iteration is from `end` to `start`, where `start < end`. +The iterator also excludes `end`. This change allows a simplified and more +intuitive logic, aligning the semantic meaning of `start` and `end` in the +`Iterator` and `ReverseIterator`. + +### Applications + +This release enforces a new consensus parameter, the +ValidatorParams.PubKeyTypes. Applications must ensure that they only return +validator updates with the allowed PubKeyTypes. If a validator update includes a +pubkey type that is not included in the ConsensusParams.Validator.PubKeyTypes, +block execution will fail and the consensus will halt. + +By default, only Ed25519 pubkeys may be used for validators. Enabling +Secp256k1 requires explicit modification of the ConsensusParams. +Please update your application accordingly (ie. restrict validators to only be +able to use Ed25519 keys, or explicitly add additional key types to the genesis +file). + +## v0.26.0 + +This release contains a lot of changes to core data types and protocols. It is not +compatible to the old versions and there is no straight forward way to update +old data to be compatible with the new version. + +To reset the state do: + +``` +$ tendermint unsafe_reset_all +``` + +Here we summarize some other notable changes to be mindful of. + +### Config Changes + +All timeouts must be changed from integers to strings with their duration, for +instance `flush_throttle_timeout = 100` would be changed to +`flush_throttle_timeout = "100ms"` and `timeout_propose = 3000` would be changed +to `timeout_propose = "3s"`. + +### RPC Changes + +The default behaviour of `/abci_query` has been changed to not return a proof, +and the name of the parameter that controls this has been changed from `trusted` +to `prove`. To get proofs with your queries, ensure you set `prove=true`. + +Various version fields like `amino_version`, `p2p_version`, `consensus_version`, +and `rpc_version` have been removed from the `node_info.other` and are +consolidated under the tendermint semantic version (ie. `node_info.version`) and +the new `block` and `p2p` protocol versions under `node_info.protocol_version`. + +### ABCI Changes + +Field numbers were bumped in the `Header` and `ResponseInfo` messages to make +room for new `version` fields. It should be straight forward to recompile the +protobuf file for these changes. + +#### Proofs + +The `ResponseQuery.Proof` field is now structured as a `[]ProofOp` to support +generalized Merkle tree constructions where the leaves of one Merkle tree are +the root of another. If you don't need this functionality, and you used to +return `` here, you should instead return a single `ProofOp` with +just the `Data` field set: + +``` +[]ProofOp{ + ProofOp{ + Data: , + } +} +``` + +For more information, see: + +- [ADR-026](https://github.com/tendermint/tendermint/blob/30519e8361c19f4bf320ef4d26288ebc621ad725/docs/architecture/adr-026-general-merkle-proof.md) +- [Relevant ABCI + documentation](https://github.com/tendermint/tendermint/blob/30519e8361c19f4bf320ef4d26288ebc621ad725/docs/spec/abci/apps.md#query-proofs) +- [Description of + keys](https://github.com/tendermint/tendermint/blob/30519e8361c19f4bf320ef4d26288ebc621ad725/crypto/merkle/proof_key_path.go#L14) + +### Go API Changes + +#### crypto/merkle + +The `merkle.Hasher` interface was removed. Functions which used to take `Hasher` +now simply take `[]byte`. This means that any objects being Merklized should be +serialized before they are passed in. + +#### node + +The `node.RunForever` function was removed. Signal handling and running forever +should instead be explicitly configured by the caller. See how we do it +[here](https://github.com/tendermint/tendermint/blob/30519e8361c19f4bf320ef4d26288ebc621ad725/cmd/tendermint/commands/run_node.go#L60). + +### Other + +All hashes, except for public key addresses, are now 32-bytes. + ## v0.25.0 This release has minimal impact. diff --git a/abci/README.md b/abci/README.md index 63b43e54..110ad40e 100644 --- a/abci/README.md +++ b/abci/README.md @@ -1,7 +1,5 @@ # Application BlockChain Interface (ABCI) -[![CircleCI](https://circleci.com/gh/tendermint/abci.svg?style=svg)](https://circleci.com/gh/tendermint/abci) - Blockchains are systems for multi-master state machine replication. **ABCI** is an interface that defines the boundary between the replication engine (the blockchain), and the state machine (the application). @@ -12,160 +10,28 @@ Previously, the ABCI was referred to as TMSP. The community has provided a number of addtional implementations, see the [Tendermint Ecosystem](https://tendermint.com/ecosystem) + +## Installation & Usage + +To get up and running quickly, see the [getting started guide](../docs/app-dev/getting-started.md) along with the [abci-cli documentation](../docs/app-dev/abci-cli.md) which will go through the examples found in the [examples](./example/) directory. + ## Specification A detailed description of the ABCI methods and message types is contained in: -- [A prose specification](specification.md) -- [A protobuf file](https://github.com/tendermint/tendermint/blob/master/abci/types/types.proto) -- [A Go interface](https://github.com/tendermint/tendermint/blob/master/abci/types/application.go). - -For more background information on ABCI, motivations, and tendermint, please visit [the documentation](https://tendermint.com/docs/). -The two guides to focus on are the `Application Development Guide` and `Using ABCI-CLI`. - +- [The main spec](../docs/spec/abci/abci.md) +- [A protobuf file](./types/types.proto) +- [A Go interface](./types/application.go) ## Protocol Buffers -To compile the protobuf file, run: +To compile the protobuf file, run (from the root of the repo): ``` -cd $GOPATH/src/github.com/tendermint/tendermint/; make protoc_abci +make protoc_abci ``` See `protoc --help` and [the Protocol Buffers site](https://developers.google.com/protocol-buffers) -for details on compiling for other languages. Note we also include a [GRPC](http://www.grpc.io/docs) +for details on compiling for other languages. Note we also include a [GRPC](https://www.grpc.io/docs) service definition. -## Install ABCI-CLI - -The `abci-cli` is a simple tool for debugging ABCI servers and running some -example apps. To install it: - -``` -mkdir -p $GOPATH/src/github.com/tendermint -cd $GOPATH/src/github.com/tendermint -git clone https://github.com/tendermint/tendermint.git -cd tendermint -make get_tools -make get_vendor_deps -make install_abci -``` - -## Implementation - -We provide three implementations of the ABCI in Go: - -- Golang in-process -- ABCI-socket -- GRPC - -Note the GRPC version is maintained primarily to simplify onboarding and prototyping and is not receiving the same -attention to security and performance as the others - -### In Process - -The simplest implementation just uses function calls within Go. -This means ABCI applications written in Golang can be compiled with TendermintCore and run as a single binary. - -See the [examples](#examples) below for more information. - -### Socket (TSP) - -ABCI is best implemented as a streaming protocol. -The socket implementation provides for asynchronous, ordered message passing over unix or tcp. -Messages are serialized using Protobuf3 and length-prefixed with a [signed Varint](https://developers.google.com/protocol-buffers/docs/encoding?csw=1#signed-integers) - -For example, if the Protobuf3 encoded ABCI message is `0xDEADBEEF` (4 bytes), the length-prefixed message is `0x08DEADBEEF`, since `0x08` is the signed varint -encoding of `4`. If the Protobuf3 encoded ABCI message is 65535 bytes long, the length-prefixed message would be like `0xFEFF07...`. - -Note the benefit of using this `varint` encoding over the old version (where integers were encoded as `` is that -it is the standard way to encode integers in Protobuf. It is also generally shorter. - -### GRPC - -GRPC is an rpc framework native to Protocol Buffers with support in many languages. -Implementing the ABCI using GRPC can allow for faster prototyping, but is expected to be much slower than -the ordered, asynchronous socket protocol. The implementation has also not received as much testing or review. - -Note the length-prefixing used in the socket implementation does not apply for GRPC. - -## Usage - -The `abci-cli` tool wraps an ABCI client and can be used for probing/testing an ABCI server. -For instance, `abci-cli test` will run a test sequence against a listening server running the Counter application (see below). -It can also be used to run some example applications. -See [the documentation](https://tendermint.com/docs/) for more details. - -### Examples - -Check out the variety of example applications in the [example directory](example/). -It also contains the code refered to by the `counter` and `kvstore` apps; these apps come -built into the `abci-cli` binary. - -#### Counter - -The `abci-cli counter` application illustrates nonce checking in transactions. It's code looks like: - -```golang -func cmdCounter(cmd *cobra.Command, args []string) error { - - app := counter.NewCounterApplication(flagSerial) - - logger := log.NewTMLogger(log.NewSyncWriter(os.Stdout)) - - // Start the listener - srv, err := server.NewServer(flagAddrC, flagAbci, app) - if err != nil { - return err - } - srv.SetLogger(logger.With("module", "abci-server")) - if err := srv.Start(); err != nil { - return err - } - - // Wait forever - cmn.TrapSignal(func() { - // Cleanup - srv.Stop() - }) - return nil -} -``` - -and can be found in [this file](cmd/abci-cli/abci-cli.go). - -#### kvstore - -The `abci-cli kvstore` application, which illustrates a simple key-value Merkle tree - -```golang -func cmdKVStore(cmd *cobra.Command, args []string) error { - logger := log.NewTMLogger(log.NewSyncWriter(os.Stdout)) - - // Create the application - in memory or persisted to disk - var app types.Application - if flagPersist == "" { - app = kvstore.NewKVStoreApplication() - } else { - app = kvstore.NewPersistentKVStoreApplication(flagPersist) - app.(*kvstore.PersistentKVStoreApplication).SetLogger(logger.With("module", "kvstore")) - } - - // Start the listener - srv, err := server.NewServer(flagAddrD, flagAbci, app) - if err != nil { - return err - } - srv.SetLogger(logger.With("module", "abci-server")) - if err := srv.Start(); err != nil { - return err - } - - // Wait forever - cmn.TrapSignal(func() { - // Cleanup - srv.Stop() - }) - return nil -} -``` diff --git a/abci/client/client.go b/abci/client/client.go index 55858810..e1eea5d4 100644 --- a/abci/client/client.go +++ b/abci/client/client.go @@ -105,8 +105,8 @@ func (reqRes *ReqRes) SetCallback(cb func(res *types.Response)) { return } - defer reqRes.mtx.Unlock() reqRes.cb = cb + reqRes.mtx.Unlock() } func (reqRes *ReqRes) GetCallback() func(*types.Response) { diff --git a/abci/client/grpc_client.go b/abci/client/grpc_client.go index 4f37b17b..94aabc5e 100644 --- a/abci/client/grpc_client.go +++ b/abci/client/grpc_client.go @@ -54,7 +54,7 @@ RETRY_LOOP: if cli.mustConnect { return err } - cli.Logger.Error(fmt.Sprintf("abci.grpcClient failed to connect to %v. Retrying...\n", cli.addr)) + cli.Logger.Error(fmt.Sprintf("abci.grpcClient failed to connect to %v. Retrying...\n", cli.addr), "err", err) time.Sleep(time.Second * dialRetryIntervalSeconds) continue RETRY_LOOP } @@ -111,8 +111,8 @@ func (cli *grpcClient) Error() error { // NOTE: callback may get internally generated flush responses. func (cli *grpcClient) SetResponseCallback(resCb Callback) { cli.mtx.Lock() - defer cli.mtx.Unlock() cli.resCb = resCb + cli.mtx.Unlock() } //---------------------------------------- diff --git a/abci/client/local_client.go b/abci/client/local_client.go index 3ac3b6af..d0e50c33 100644 --- a/abci/client/local_client.go +++ b/abci/client/local_client.go @@ -9,8 +9,13 @@ import ( var _ Client = (*localClient)(nil) +// NOTE: use defer to unlock mutex because Application might panic (e.g., in +// case of malicious tx or query). It only makes sense for publicly exposed +// methods like CheckTx (/broadcast_tx_* RPC endpoint) or Query (/abci_query +// RPC endpoint), but defers are used everywhere for the sake of consistency. type localClient struct { cmn.BaseService + mtx *sync.Mutex types.Application Callback @@ -30,8 +35,8 @@ func NewLocalClient(mtx *sync.Mutex, app types.Application) *localClient { func (app *localClient) SetResponseCallback(cb Callback) { app.mtx.Lock() - defer app.mtx.Unlock() app.Callback = cb + app.mtx.Unlock() } // TODO: change types.Application to include Error()? @@ -45,6 +50,9 @@ func (app *localClient) FlushAsync() *ReqRes { } func (app *localClient) EchoAsync(msg string) *ReqRes { + app.mtx.Lock() + defer app.mtx.Unlock() + return app.callback( types.ToRequestEcho(msg), types.ToResponseEcho(msg), @@ -53,8 +61,9 @@ func (app *localClient) EchoAsync(msg string) *ReqRes { func (app *localClient) InfoAsync(req types.RequestInfo) *ReqRes { app.mtx.Lock() + defer app.mtx.Unlock() + res := app.Application.Info(req) - app.mtx.Unlock() return app.callback( types.ToRequestInfo(req), types.ToResponseInfo(res), @@ -63,8 +72,9 @@ func (app *localClient) InfoAsync(req types.RequestInfo) *ReqRes { func (app *localClient) SetOptionAsync(req types.RequestSetOption) *ReqRes { app.mtx.Lock() + defer app.mtx.Unlock() + res := app.Application.SetOption(req) - app.mtx.Unlock() return app.callback( types.ToRequestSetOption(req), types.ToResponseSetOption(res), @@ -73,8 +83,9 @@ func (app *localClient) SetOptionAsync(req types.RequestSetOption) *ReqRes { func (app *localClient) DeliverTxAsync(tx []byte) *ReqRes { app.mtx.Lock() + defer app.mtx.Unlock() + res := app.Application.DeliverTx(tx) - app.mtx.Unlock() return app.callback( types.ToRequestDeliverTx(tx), types.ToResponseDeliverTx(res), @@ -83,8 +94,9 @@ func (app *localClient) DeliverTxAsync(tx []byte) *ReqRes { func (app *localClient) CheckTxAsync(tx []byte) *ReqRes { app.mtx.Lock() + defer app.mtx.Unlock() + res := app.Application.CheckTx(tx) - app.mtx.Unlock() return app.callback( types.ToRequestCheckTx(tx), types.ToResponseCheckTx(res), @@ -93,8 +105,9 @@ func (app *localClient) CheckTxAsync(tx []byte) *ReqRes { func (app *localClient) QueryAsync(req types.RequestQuery) *ReqRes { app.mtx.Lock() + defer app.mtx.Unlock() + res := app.Application.Query(req) - app.mtx.Unlock() return app.callback( types.ToRequestQuery(req), types.ToResponseQuery(res), @@ -103,8 +116,9 @@ func (app *localClient) QueryAsync(req types.RequestQuery) *ReqRes { func (app *localClient) CommitAsync() *ReqRes { app.mtx.Lock() + defer app.mtx.Unlock() + res := app.Application.Commit() - app.mtx.Unlock() return app.callback( types.ToRequestCommit(), types.ToResponseCommit(res), @@ -113,19 +127,20 @@ func (app *localClient) CommitAsync() *ReqRes { func (app *localClient) InitChainAsync(req types.RequestInitChain) *ReqRes { app.mtx.Lock() + defer app.mtx.Unlock() + res := app.Application.InitChain(req) - reqRes := app.callback( + return app.callback( types.ToRequestInitChain(req), types.ToResponseInitChain(res), ) - app.mtx.Unlock() - return reqRes } func (app *localClient) BeginBlockAsync(req types.RequestBeginBlock) *ReqRes { app.mtx.Lock() + defer app.mtx.Unlock() + res := app.Application.BeginBlock(req) - app.mtx.Unlock() return app.callback( types.ToRequestBeginBlock(req), types.ToResponseBeginBlock(res), @@ -134,8 +149,9 @@ func (app *localClient) BeginBlockAsync(req types.RequestBeginBlock) *ReqRes { func (app *localClient) EndBlockAsync(req types.RequestEndBlock) *ReqRes { app.mtx.Lock() + defer app.mtx.Unlock() + res := app.Application.EndBlock(req) - app.mtx.Unlock() return app.callback( types.ToRequestEndBlock(req), types.ToResponseEndBlock(res), @@ -154,64 +170,73 @@ func (app *localClient) EchoSync(msg string) (*types.ResponseEcho, error) { func (app *localClient) InfoSync(req types.RequestInfo) (*types.ResponseInfo, error) { app.mtx.Lock() + defer app.mtx.Unlock() + res := app.Application.Info(req) - app.mtx.Unlock() return &res, nil } func (app *localClient) SetOptionSync(req types.RequestSetOption) (*types.ResponseSetOption, error) { app.mtx.Lock() + defer app.mtx.Unlock() + res := app.Application.SetOption(req) - app.mtx.Unlock() return &res, nil } func (app *localClient) DeliverTxSync(tx []byte) (*types.ResponseDeliverTx, error) { app.mtx.Lock() + defer app.mtx.Unlock() + res := app.Application.DeliverTx(tx) - app.mtx.Unlock() return &res, nil } func (app *localClient) CheckTxSync(tx []byte) (*types.ResponseCheckTx, error) { app.mtx.Lock() + defer app.mtx.Unlock() + res := app.Application.CheckTx(tx) - app.mtx.Unlock() return &res, nil } func (app *localClient) QuerySync(req types.RequestQuery) (*types.ResponseQuery, error) { app.mtx.Lock() + defer app.mtx.Unlock() + res := app.Application.Query(req) - app.mtx.Unlock() return &res, nil } func (app *localClient) CommitSync() (*types.ResponseCommit, error) { app.mtx.Lock() + defer app.mtx.Unlock() + res := app.Application.Commit() - app.mtx.Unlock() return &res, nil } func (app *localClient) InitChainSync(req types.RequestInitChain) (*types.ResponseInitChain, error) { app.mtx.Lock() + defer app.mtx.Unlock() + res := app.Application.InitChain(req) - app.mtx.Unlock() return &res, nil } func (app *localClient) BeginBlockSync(req types.RequestBeginBlock) (*types.ResponseBeginBlock, error) { app.mtx.Lock() + defer app.mtx.Unlock() + res := app.Application.BeginBlock(req) - app.mtx.Unlock() return &res, nil } func (app *localClient) EndBlockSync(req types.RequestEndBlock) (*types.ResponseEndBlock, error) { app.mtx.Lock() + defer app.mtx.Unlock() + res := app.Application.EndBlock(req) - app.mtx.Unlock() return &res, nil } diff --git a/abci/client/socket_client.go b/abci/client/socket_client.go index affea1a9..56267660 100644 --- a/abci/client/socket_client.go +++ b/abci/client/socket_client.go @@ -67,7 +67,7 @@ RETRY_LOOP: if cli.mustConnect { return err } - cli.Logger.Error(fmt.Sprintf("abci.socketClient failed to connect to %v. Retrying...", cli.addr)) + cli.Logger.Error(fmt.Sprintf("abci.socketClient failed to connect to %v. Retrying...", cli.addr), "err", err) time.Sleep(time.Second * dialRetryIntervalSeconds) continue RETRY_LOOP } @@ -118,8 +118,8 @@ func (cli *socketClient) Error() error { // NOTE: callback may get internally generated flush responses. func (cli *socketClient) SetResponseCallback(resCb Callback) { cli.mtx.Lock() - defer cli.mtx.Unlock() cli.resCb = resCb + cli.mtx.Unlock() } //---------------------------------------- diff --git a/abci/cmd/abci-cli/abci-cli.go b/abci/cmd/abci-cli/abci-cli.go index 50972ec3..cc3f9c45 100644 --- a/abci/cmd/abci-cli/abci-cli.go +++ b/abci/cmd/abci-cli/abci-cli.go @@ -58,7 +58,7 @@ var RootCmd = &cobra.Command{ PersistentPreRunE: func(cmd *cobra.Command, args []string) error { switch cmd.Use { - case "counter", "kvstore", "dummy": // for the examples apps, don't pre-run + case "counter", "kvstore": // for the examples apps, don't pre-run return nil case "version": // skip running for version command return nil @@ -127,10 +127,6 @@ func addCounterFlags() { counterCmd.PersistentFlags().BoolVarP(&flagSerial, "serial", "", false, "enforce incrementing (serial) transactions") } -func addDummyFlags() { - dummyCmd.PersistentFlags().StringVarP(&flagPersist, "persist", "", "", "directory to use for a database") -} - func addKVStoreFlags() { kvstoreCmd.PersistentFlags().StringVarP(&flagPersist, "persist", "", "", "directory to use for a database") } @@ -152,10 +148,6 @@ func addCommands() { // examples addCounterFlags() RootCmd.AddCommand(counterCmd) - // deprecated, left for backwards compatibility - addDummyFlags() - RootCmd.AddCommand(dummyCmd) - // replaces dummy, see issue #196 addKVStoreFlags() RootCmd.AddCommand(kvstoreCmd) } @@ -291,18 +283,6 @@ var counterCmd = &cobra.Command{ }, } -// deprecated, left for backwards compatibility -var dummyCmd = &cobra.Command{ - Use: "dummy", - Deprecated: "use: [abci-cli kvstore] instead", - Short: "ABCI demo example", - Long: "ABCI demo example", - Args: cobra.ExactArgs(0), - RunE: func(cmd *cobra.Command, args []string) error { - return cmdKVStore(cmd, args) - }, -} - var kvstoreCmd = &cobra.Command{ Use: "kvstore", Short: "ABCI demo example", diff --git a/abci/example/kvstore/kvstore.go b/abci/example/kvstore/kvstore.go index 9523bf74..955baefb 100644 --- a/abci/example/kvstore/kvstore.go +++ b/abci/example/kvstore/kvstore.go @@ -10,11 +10,14 @@ import ( "github.com/tendermint/tendermint/abci/types" cmn "github.com/tendermint/tendermint/libs/common" dbm "github.com/tendermint/tendermint/libs/db" + "github.com/tendermint/tendermint/version" ) var ( stateKey = []byte("stateKey") kvPairPrefixKey = []byte("kvPairKey:") + + ProtocolVersion version.Protocol = 0x1 ) type State struct { @@ -65,7 +68,11 @@ func NewKVStoreApplication() *KVStoreApplication { } func (app *KVStoreApplication) Info(req types.RequestInfo) (resInfo types.ResponseInfo) { - return types.ResponseInfo{Data: fmt.Sprintf("{\"size\":%v}", app.state.Size)} + return types.ResponseInfo{ + Data: fmt.Sprintf("{\"size\":%v}", app.state.Size), + Version: version.ABCIVersion, + AppVersion: ProtocolVersion.Uint64(), + } } // tx is either "key=value" or just arbitrary bytes diff --git a/abci/types/messages_test.go b/abci/types/messages_test.go index 14bc5718..762111b6 100644 --- a/abci/types/messages_test.go +++ b/abci/types/messages_test.go @@ -83,7 +83,7 @@ func TestWriteReadMessage2(t *testing.T) { Log: phrase, GasWanted: 10, Tags: []cmn.KVPair{ - cmn.KVPair{Key: []byte("abc"), Value: []byte("def")}, + {Key: []byte("abc"), Value: []byte("def")}, }, }, // TODO: add the rest diff --git a/abci/types/types.pb.go b/abci/types/types.pb.go index 1ec51602..c867dffc 100644 --- a/abci/types/types.pb.go +++ b/abci/types/types.pb.go @@ -61,7 +61,7 @@ func (m *Request) Reset() { *m = Request{} } func (m *Request) String() string { return proto.CompactTextString(m) } func (*Request) ProtoMessage() {} func (*Request) Descriptor() ([]byte, []int) { - return fileDescriptor_types_4a7ab597ee120b05, []int{0} + return fileDescriptor_types_5b877df1938afe10, []int{0} } func (m *Request) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -483,7 +483,7 @@ func (m *RequestEcho) Reset() { *m = RequestEcho{} } func (m *RequestEcho) String() string { return proto.CompactTextString(m) } func (*RequestEcho) ProtoMessage() {} func (*RequestEcho) Descriptor() ([]byte, []int) { - return fileDescriptor_types_4a7ab597ee120b05, []int{1} + return fileDescriptor_types_5b877df1938afe10, []int{1} } func (m *RequestEcho) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -529,7 +529,7 @@ func (m *RequestFlush) Reset() { *m = RequestFlush{} } func (m *RequestFlush) String() string { return proto.CompactTextString(m) } func (*RequestFlush) ProtoMessage() {} func (*RequestFlush) Descriptor() ([]byte, []int) { - return fileDescriptor_types_4a7ab597ee120b05, []int{2} + return fileDescriptor_types_5b877df1938afe10, []int{2} } func (m *RequestFlush) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -560,6 +560,8 @@ var xxx_messageInfo_RequestFlush proto.InternalMessageInfo type RequestInfo struct { Version string `protobuf:"bytes,1,opt,name=version,proto3" json:"version,omitempty"` + BlockVersion uint64 `protobuf:"varint,2,opt,name=block_version,json=blockVersion,proto3" json:"block_version,omitempty"` + P2PVersion uint64 `protobuf:"varint,3,opt,name=p2p_version,json=p2pVersion,proto3" json:"p2p_version,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -569,7 +571,7 @@ func (m *RequestInfo) Reset() { *m = RequestInfo{} } func (m *RequestInfo) String() string { return proto.CompactTextString(m) } func (*RequestInfo) ProtoMessage() {} func (*RequestInfo) Descriptor() ([]byte, []int) { - return fileDescriptor_types_4a7ab597ee120b05, []int{3} + return fileDescriptor_types_5b877df1938afe10, []int{3} } func (m *RequestInfo) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -605,6 +607,20 @@ func (m *RequestInfo) GetVersion() string { return "" } +func (m *RequestInfo) GetBlockVersion() uint64 { + if m != nil { + return m.BlockVersion + } + return 0 +} + +func (m *RequestInfo) GetP2PVersion() uint64 { + if m != nil { + return m.P2PVersion + } + return 0 +} + // nondeterministic type RequestSetOption struct { Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` @@ -618,7 +634,7 @@ func (m *RequestSetOption) Reset() { *m = RequestSetOption{} } func (m *RequestSetOption) String() string { return proto.CompactTextString(m) } func (*RequestSetOption) ProtoMessage() {} func (*RequestSetOption) Descriptor() ([]byte, []int) { - return fileDescriptor_types_4a7ab597ee120b05, []int{4} + return fileDescriptor_types_5b877df1938afe10, []int{4} } func (m *RequestSetOption) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -676,7 +692,7 @@ func (m *RequestInitChain) Reset() { *m = RequestInitChain{} } func (m *RequestInitChain) String() string { return proto.CompactTextString(m) } func (*RequestInitChain) ProtoMessage() {} func (*RequestInitChain) Descriptor() ([]byte, []int) { - return fileDescriptor_types_4a7ab597ee120b05, []int{5} + return fileDescriptor_types_5b877df1938afe10, []int{5} } func (m *RequestInitChain) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -754,7 +770,7 @@ func (m *RequestQuery) Reset() { *m = RequestQuery{} } func (m *RequestQuery) String() string { return proto.CompactTextString(m) } func (*RequestQuery) ProtoMessage() {} func (*RequestQuery) Descriptor() ([]byte, []int) { - return fileDescriptor_types_4a7ab597ee120b05, []int{6} + return fileDescriptor_types_5b877df1938afe10, []int{6} } func (m *RequestQuery) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -811,7 +827,6 @@ func (m *RequestQuery) GetProve() bool { return false } -// NOTE: validators here have empty pubkeys. type RequestBeginBlock struct { Hash []byte `protobuf:"bytes,1,opt,name=hash,proto3" json:"hash,omitempty"` Header Header `protobuf:"bytes,2,opt,name=header" json:"header"` @@ -826,7 +841,7 @@ func (m *RequestBeginBlock) Reset() { *m = RequestBeginBlock{} } func (m *RequestBeginBlock) String() string { return proto.CompactTextString(m) } func (*RequestBeginBlock) ProtoMessage() {} func (*RequestBeginBlock) Descriptor() ([]byte, []int) { - return fileDescriptor_types_4a7ab597ee120b05, []int{7} + return fileDescriptor_types_5b877df1938afe10, []int{7} } func (m *RequestBeginBlock) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -894,7 +909,7 @@ func (m *RequestCheckTx) Reset() { *m = RequestCheckTx{} } func (m *RequestCheckTx) String() string { return proto.CompactTextString(m) } func (*RequestCheckTx) ProtoMessage() {} func (*RequestCheckTx) Descriptor() ([]byte, []int) { - return fileDescriptor_types_4a7ab597ee120b05, []int{8} + return fileDescriptor_types_5b877df1938afe10, []int{8} } func (m *RequestCheckTx) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -941,7 +956,7 @@ func (m *RequestDeliverTx) Reset() { *m = RequestDeliverTx{} } func (m *RequestDeliverTx) String() string { return proto.CompactTextString(m) } func (*RequestDeliverTx) ProtoMessage() {} func (*RequestDeliverTx) Descriptor() ([]byte, []int) { - return fileDescriptor_types_4a7ab597ee120b05, []int{9} + return fileDescriptor_types_5b877df1938afe10, []int{9} } func (m *RequestDeliverTx) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -988,7 +1003,7 @@ func (m *RequestEndBlock) Reset() { *m = RequestEndBlock{} } func (m *RequestEndBlock) String() string { return proto.CompactTextString(m) } func (*RequestEndBlock) ProtoMessage() {} func (*RequestEndBlock) Descriptor() ([]byte, []int) { - return fileDescriptor_types_4a7ab597ee120b05, []int{10} + return fileDescriptor_types_5b877df1938afe10, []int{10} } func (m *RequestEndBlock) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1034,7 +1049,7 @@ func (m *RequestCommit) Reset() { *m = RequestCommit{} } func (m *RequestCommit) String() string { return proto.CompactTextString(m) } func (*RequestCommit) ProtoMessage() {} func (*RequestCommit) Descriptor() ([]byte, []int) { - return fileDescriptor_types_4a7ab597ee120b05, []int{11} + return fileDescriptor_types_5b877df1938afe10, []int{11} } func (m *RequestCommit) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1087,7 +1102,7 @@ func (m *Response) Reset() { *m = Response{} } func (m *Response) String() string { return proto.CompactTextString(m) } func (*Response) ProtoMessage() {} func (*Response) Descriptor() ([]byte, []int) { - return fileDescriptor_types_4a7ab597ee120b05, []int{12} + return fileDescriptor_types_5b877df1938afe10, []int{12} } func (m *Response) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1540,7 +1555,7 @@ func (m *ResponseException) Reset() { *m = ResponseException{} } func (m *ResponseException) String() string { return proto.CompactTextString(m) } func (*ResponseException) ProtoMessage() {} func (*ResponseException) Descriptor() ([]byte, []int) { - return fileDescriptor_types_4a7ab597ee120b05, []int{13} + return fileDescriptor_types_5b877df1938afe10, []int{13} } func (m *ResponseException) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1587,7 +1602,7 @@ func (m *ResponseEcho) Reset() { *m = ResponseEcho{} } func (m *ResponseEcho) String() string { return proto.CompactTextString(m) } func (*ResponseEcho) ProtoMessage() {} func (*ResponseEcho) Descriptor() ([]byte, []int) { - return fileDescriptor_types_4a7ab597ee120b05, []int{14} + return fileDescriptor_types_5b877df1938afe10, []int{14} } func (m *ResponseEcho) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1633,7 +1648,7 @@ func (m *ResponseFlush) Reset() { *m = ResponseFlush{} } func (m *ResponseFlush) String() string { return proto.CompactTextString(m) } func (*ResponseFlush) ProtoMessage() {} func (*ResponseFlush) Descriptor() ([]byte, []int) { - return fileDescriptor_types_4a7ab597ee120b05, []int{15} + return fileDescriptor_types_5b877df1938afe10, []int{15} } func (m *ResponseFlush) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1665,8 +1680,9 @@ var xxx_messageInfo_ResponseFlush proto.InternalMessageInfo type ResponseInfo struct { Data string `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"` Version string `protobuf:"bytes,2,opt,name=version,proto3" json:"version,omitempty"` - LastBlockHeight int64 `protobuf:"varint,3,opt,name=last_block_height,json=lastBlockHeight,proto3" json:"last_block_height,omitempty"` - LastBlockAppHash []byte `protobuf:"bytes,4,opt,name=last_block_app_hash,json=lastBlockAppHash,proto3" json:"last_block_app_hash,omitempty"` + AppVersion uint64 `protobuf:"varint,3,opt,name=app_version,json=appVersion,proto3" json:"app_version,omitempty"` + LastBlockHeight int64 `protobuf:"varint,4,opt,name=last_block_height,json=lastBlockHeight,proto3" json:"last_block_height,omitempty"` + LastBlockAppHash []byte `protobuf:"bytes,5,opt,name=last_block_app_hash,json=lastBlockAppHash,proto3" json:"last_block_app_hash,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1676,7 +1692,7 @@ func (m *ResponseInfo) Reset() { *m = ResponseInfo{} } func (m *ResponseInfo) String() string { return proto.CompactTextString(m) } func (*ResponseInfo) ProtoMessage() {} func (*ResponseInfo) Descriptor() ([]byte, []int) { - return fileDescriptor_types_4a7ab597ee120b05, []int{16} + return fileDescriptor_types_5b877df1938afe10, []int{16} } func (m *ResponseInfo) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1719,6 +1735,13 @@ func (m *ResponseInfo) GetVersion() string { return "" } +func (m *ResponseInfo) GetAppVersion() uint64 { + if m != nil { + return m.AppVersion + } + return 0 +} + func (m *ResponseInfo) GetLastBlockHeight() int64 { if m != nil { return m.LastBlockHeight @@ -1748,7 +1771,7 @@ func (m *ResponseSetOption) Reset() { *m = ResponseSetOption{} } func (m *ResponseSetOption) String() string { return proto.CompactTextString(m) } func (*ResponseSetOption) ProtoMessage() {} func (*ResponseSetOption) Descriptor() ([]byte, []int) { - return fileDescriptor_types_4a7ab597ee120b05, []int{17} + return fileDescriptor_types_5b877df1938afe10, []int{17} } func (m *ResponseSetOption) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1810,7 +1833,7 @@ func (m *ResponseInitChain) Reset() { *m = ResponseInitChain{} } func (m *ResponseInitChain) String() string { return proto.CompactTextString(m) } func (*ResponseInitChain) ProtoMessage() {} func (*ResponseInitChain) Descriptor() ([]byte, []int) { - return fileDescriptor_types_4a7ab597ee120b05, []int{18} + return fileDescriptor_types_5b877df1938afe10, []int{18} } func (m *ResponseInitChain) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1873,7 +1896,7 @@ func (m *ResponseQuery) Reset() { *m = ResponseQuery{} } func (m *ResponseQuery) String() string { return proto.CompactTextString(m) } func (*ResponseQuery) ProtoMessage() {} func (*ResponseQuery) Descriptor() ([]byte, []int) { - return fileDescriptor_types_4a7ab597ee120b05, []int{19} + return fileDescriptor_types_5b877df1938afe10, []int{19} } func (m *ResponseQuery) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1976,7 +1999,7 @@ func (m *ResponseBeginBlock) Reset() { *m = ResponseBeginBlock{} } func (m *ResponseBeginBlock) String() string { return proto.CompactTextString(m) } func (*ResponseBeginBlock) ProtoMessage() {} func (*ResponseBeginBlock) Descriptor() ([]byte, []int) { - return fileDescriptor_types_4a7ab597ee120b05, []int{20} + return fileDescriptor_types_5b877df1938afe10, []int{20} } func (m *ResponseBeginBlock) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2030,7 +2053,7 @@ func (m *ResponseCheckTx) Reset() { *m = ResponseCheckTx{} } func (m *ResponseCheckTx) String() string { return proto.CompactTextString(m) } func (*ResponseCheckTx) ProtoMessage() {} func (*ResponseCheckTx) Descriptor() ([]byte, []int) { - return fileDescriptor_types_4a7ab597ee120b05, []int{21} + return fileDescriptor_types_5b877df1938afe10, []int{21} } func (m *ResponseCheckTx) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2133,7 +2156,7 @@ func (m *ResponseDeliverTx) Reset() { *m = ResponseDeliverTx{} } func (m *ResponseDeliverTx) String() string { return proto.CompactTextString(m) } func (*ResponseDeliverTx) ProtoMessage() {} func (*ResponseDeliverTx) Descriptor() ([]byte, []int) { - return fileDescriptor_types_4a7ab597ee120b05, []int{22} + return fileDescriptor_types_5b877df1938afe10, []int{22} } func (m *ResponseDeliverTx) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2231,7 +2254,7 @@ func (m *ResponseEndBlock) Reset() { *m = ResponseEndBlock{} } func (m *ResponseEndBlock) String() string { return proto.CompactTextString(m) } func (*ResponseEndBlock) ProtoMessage() {} func (*ResponseEndBlock) Descriptor() ([]byte, []int) { - return fileDescriptor_types_4a7ab597ee120b05, []int{23} + return fileDescriptor_types_5b877df1938afe10, []int{23} } func (m *ResponseEndBlock) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2293,7 +2316,7 @@ func (m *ResponseCommit) Reset() { *m = ResponseCommit{} } func (m *ResponseCommit) String() string { return proto.CompactTextString(m) } func (*ResponseCommit) ProtoMessage() {} func (*ResponseCommit) Descriptor() ([]byte, []int) { - return fileDescriptor_types_4a7ab597ee120b05, []int{24} + return fileDescriptor_types_5b877df1938afe10, []int{24} } func (m *ResponseCommit) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2332,18 +2355,19 @@ func (m *ResponseCommit) GetData() []byte { // ConsensusParams contains all consensus-relevant parameters // that can be adjusted by the abci app type ConsensusParams struct { - BlockSize *BlockSize `protobuf:"bytes,1,opt,name=block_size,json=blockSize" json:"block_size,omitempty"` - EvidenceParams *EvidenceParams `protobuf:"bytes,2,opt,name=evidence_params,json=evidenceParams" json:"evidence_params,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + BlockSize *BlockSizeParams `protobuf:"bytes,1,opt,name=block_size,json=blockSize" json:"block_size,omitempty"` + Evidence *EvidenceParams `protobuf:"bytes,2,opt,name=evidence" json:"evidence,omitempty"` + Validator *ValidatorParams `protobuf:"bytes,3,opt,name=validator" json:"validator,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *ConsensusParams) Reset() { *m = ConsensusParams{} } func (m *ConsensusParams) String() string { return proto.CompactTextString(m) } func (*ConsensusParams) ProtoMessage() {} func (*ConsensusParams) Descriptor() ([]byte, []int) { - return fileDescriptor_types_4a7ab597ee120b05, []int{25} + return fileDescriptor_types_5b877df1938afe10, []int{25} } func (m *ConsensusParams) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2372,22 +2396,29 @@ func (m *ConsensusParams) XXX_DiscardUnknown() { var xxx_messageInfo_ConsensusParams proto.InternalMessageInfo -func (m *ConsensusParams) GetBlockSize() *BlockSize { +func (m *ConsensusParams) GetBlockSize() *BlockSizeParams { if m != nil { return m.BlockSize } return nil } -func (m *ConsensusParams) GetEvidenceParams() *EvidenceParams { +func (m *ConsensusParams) GetEvidence() *EvidenceParams { if m != nil { - return m.EvidenceParams + return m.Evidence + } + return nil +} + +func (m *ConsensusParams) GetValidator() *ValidatorParams { + if m != nil { + return m.Validator } return nil } // BlockSize contains limits on the block size. -type BlockSize struct { +type BlockSizeParams struct { // Note: must be greater than 0 MaxBytes int64 `protobuf:"varint,1,opt,name=max_bytes,json=maxBytes,proto3" json:"max_bytes,omitempty"` // Note: must be greater or equal to -1 @@ -2397,18 +2428,18 @@ type BlockSize struct { XXX_sizecache int32 `json:"-"` } -func (m *BlockSize) Reset() { *m = BlockSize{} } -func (m *BlockSize) String() string { return proto.CompactTextString(m) } -func (*BlockSize) ProtoMessage() {} -func (*BlockSize) Descriptor() ([]byte, []int) { - return fileDescriptor_types_4a7ab597ee120b05, []int{26} +func (m *BlockSizeParams) Reset() { *m = BlockSizeParams{} } +func (m *BlockSizeParams) String() string { return proto.CompactTextString(m) } +func (*BlockSizeParams) ProtoMessage() {} +func (*BlockSizeParams) Descriptor() ([]byte, []int) { + return fileDescriptor_types_5b877df1938afe10, []int{26} } -func (m *BlockSize) XXX_Unmarshal(b []byte) error { +func (m *BlockSizeParams) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } -func (m *BlockSize) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { +func (m *BlockSizeParams) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { - return xxx_messageInfo_BlockSize.Marshal(b, m, deterministic) + return xxx_messageInfo_BlockSizeParams.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalTo(b) @@ -2418,26 +2449,26 @@ func (m *BlockSize) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return b[:n], nil } } -func (dst *BlockSize) XXX_Merge(src proto.Message) { - xxx_messageInfo_BlockSize.Merge(dst, src) +func (dst *BlockSizeParams) XXX_Merge(src proto.Message) { + xxx_messageInfo_BlockSizeParams.Merge(dst, src) } -func (m *BlockSize) XXX_Size() int { +func (m *BlockSizeParams) XXX_Size() int { return m.Size() } -func (m *BlockSize) XXX_DiscardUnknown() { - xxx_messageInfo_BlockSize.DiscardUnknown(m) +func (m *BlockSizeParams) XXX_DiscardUnknown() { + xxx_messageInfo_BlockSizeParams.DiscardUnknown(m) } -var xxx_messageInfo_BlockSize proto.InternalMessageInfo +var xxx_messageInfo_BlockSizeParams proto.InternalMessageInfo -func (m *BlockSize) GetMaxBytes() int64 { +func (m *BlockSizeParams) GetMaxBytes() int64 { if m != nil { return m.MaxBytes } return 0 } -func (m *BlockSize) GetMaxGas() int64 { +func (m *BlockSizeParams) GetMaxGas() int64 { if m != nil { return m.MaxGas } @@ -2457,7 +2488,7 @@ func (m *EvidenceParams) Reset() { *m = EvidenceParams{} } func (m *EvidenceParams) String() string { return proto.CompactTextString(m) } func (*EvidenceParams) ProtoMessage() {} func (*EvidenceParams) Descriptor() ([]byte, []int) { - return fileDescriptor_types_4a7ab597ee120b05, []int{27} + return fileDescriptor_types_5b877df1938afe10, []int{27} } func (m *EvidenceParams) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2493,6 +2524,54 @@ func (m *EvidenceParams) GetMaxAge() int64 { return 0 } +// ValidatorParams contains limits on validators. +type ValidatorParams struct { + PubKeyTypes []string `protobuf:"bytes,1,rep,name=pub_key_types,json=pubKeyTypes" json:"pub_key_types,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ValidatorParams) Reset() { *m = ValidatorParams{} } +func (m *ValidatorParams) String() string { return proto.CompactTextString(m) } +func (*ValidatorParams) ProtoMessage() {} +func (*ValidatorParams) Descriptor() ([]byte, []int) { + return fileDescriptor_types_5b877df1938afe10, []int{28} +} +func (m *ValidatorParams) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *ValidatorParams) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_ValidatorParams.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalTo(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (dst *ValidatorParams) XXX_Merge(src proto.Message) { + xxx_messageInfo_ValidatorParams.Merge(dst, src) +} +func (m *ValidatorParams) XXX_Size() int { + return m.Size() +} +func (m *ValidatorParams) XXX_DiscardUnknown() { + xxx_messageInfo_ValidatorParams.DiscardUnknown(m) +} + +var xxx_messageInfo_ValidatorParams proto.InternalMessageInfo + +func (m *ValidatorParams) GetPubKeyTypes() []string { + if m != nil { + return m.PubKeyTypes + } + return nil +} + type LastCommitInfo struct { Round int32 `protobuf:"varint,1,opt,name=round,proto3" json:"round,omitempty"` Votes []VoteInfo `protobuf:"bytes,2,rep,name=votes" json:"votes"` @@ -2505,7 +2584,7 @@ func (m *LastCommitInfo) Reset() { *m = LastCommitInfo{} } func (m *LastCommitInfo) String() string { return proto.CompactTextString(m) } func (*LastCommitInfo) ProtoMessage() {} func (*LastCommitInfo) Descriptor() ([]byte, []int) { - return fileDescriptor_types_4a7ab597ee120b05, []int{28} + return fileDescriptor_types_5b877df1938afe10, []int{29} } func (m *LastCommitInfo) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2550,25 +2629,26 @@ func (m *LastCommitInfo) GetVotes() []VoteInfo { type Header struct { // basic block info - ChainID string `protobuf:"bytes,1,opt,name=chain_id,json=chainId,proto3" json:"chain_id,omitempty"` - Height int64 `protobuf:"varint,2,opt,name=height,proto3" json:"height,omitempty"` - Time time.Time `protobuf:"bytes,3,opt,name=time,stdtime" json:"time"` - NumTxs int64 `protobuf:"varint,4,opt,name=num_txs,json=numTxs,proto3" json:"num_txs,omitempty"` - TotalTxs int64 `protobuf:"varint,5,opt,name=total_txs,json=totalTxs,proto3" json:"total_txs,omitempty"` + Version Version `protobuf:"bytes,1,opt,name=version" json:"version"` + ChainID string `protobuf:"bytes,2,opt,name=chain_id,json=chainId,proto3" json:"chain_id,omitempty"` + Height int64 `protobuf:"varint,3,opt,name=height,proto3" json:"height,omitempty"` + Time time.Time `protobuf:"bytes,4,opt,name=time,stdtime" json:"time"` + NumTxs int64 `protobuf:"varint,5,opt,name=num_txs,json=numTxs,proto3" json:"num_txs,omitempty"` + TotalTxs int64 `protobuf:"varint,6,opt,name=total_txs,json=totalTxs,proto3" json:"total_txs,omitempty"` // prev block info - LastBlockId BlockID `protobuf:"bytes,6,opt,name=last_block_id,json=lastBlockId" json:"last_block_id"` + LastBlockId BlockID `protobuf:"bytes,7,opt,name=last_block_id,json=lastBlockId" json:"last_block_id"` // hashes of block data - LastCommitHash []byte `protobuf:"bytes,7,opt,name=last_commit_hash,json=lastCommitHash,proto3" json:"last_commit_hash,omitempty"` - DataHash []byte `protobuf:"bytes,8,opt,name=data_hash,json=dataHash,proto3" json:"data_hash,omitempty"` + LastCommitHash []byte `protobuf:"bytes,8,opt,name=last_commit_hash,json=lastCommitHash,proto3" json:"last_commit_hash,omitempty"` + DataHash []byte `protobuf:"bytes,9,opt,name=data_hash,json=dataHash,proto3" json:"data_hash,omitempty"` // hashes from the app output from the prev block - ValidatorsHash []byte `protobuf:"bytes,9,opt,name=validators_hash,json=validatorsHash,proto3" json:"validators_hash,omitempty"` - NextValidatorsHash []byte `protobuf:"bytes,10,opt,name=next_validators_hash,json=nextValidatorsHash,proto3" json:"next_validators_hash,omitempty"` - ConsensusHash []byte `protobuf:"bytes,11,opt,name=consensus_hash,json=consensusHash,proto3" json:"consensus_hash,omitempty"` - AppHash []byte `protobuf:"bytes,12,opt,name=app_hash,json=appHash,proto3" json:"app_hash,omitempty"` - LastResultsHash []byte `protobuf:"bytes,13,opt,name=last_results_hash,json=lastResultsHash,proto3" json:"last_results_hash,omitempty"` + ValidatorsHash []byte `protobuf:"bytes,10,opt,name=validators_hash,json=validatorsHash,proto3" json:"validators_hash,omitempty"` + NextValidatorsHash []byte `protobuf:"bytes,11,opt,name=next_validators_hash,json=nextValidatorsHash,proto3" json:"next_validators_hash,omitempty"` + ConsensusHash []byte `protobuf:"bytes,12,opt,name=consensus_hash,json=consensusHash,proto3" json:"consensus_hash,omitempty"` + AppHash []byte `protobuf:"bytes,13,opt,name=app_hash,json=appHash,proto3" json:"app_hash,omitempty"` + LastResultsHash []byte `protobuf:"bytes,14,opt,name=last_results_hash,json=lastResultsHash,proto3" json:"last_results_hash,omitempty"` // consensus info - EvidenceHash []byte `protobuf:"bytes,14,opt,name=evidence_hash,json=evidenceHash,proto3" json:"evidence_hash,omitempty"` - ProposerAddress []byte `protobuf:"bytes,15,opt,name=proposer_address,json=proposerAddress,proto3" json:"proposer_address,omitempty"` + EvidenceHash []byte `protobuf:"bytes,15,opt,name=evidence_hash,json=evidenceHash,proto3" json:"evidence_hash,omitempty"` + ProposerAddress []byte `protobuf:"bytes,16,opt,name=proposer_address,json=proposerAddress,proto3" json:"proposer_address,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -2578,7 +2658,7 @@ func (m *Header) Reset() { *m = Header{} } func (m *Header) String() string { return proto.CompactTextString(m) } func (*Header) ProtoMessage() {} func (*Header) Descriptor() ([]byte, []int) { - return fileDescriptor_types_4a7ab597ee120b05, []int{29} + return fileDescriptor_types_5b877df1938afe10, []int{30} } func (m *Header) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2607,6 +2687,13 @@ func (m *Header) XXX_DiscardUnknown() { var xxx_messageInfo_Header proto.InternalMessageInfo +func (m *Header) GetVersion() Version { + if m != nil { + return m.Version + } + return Version{} +} + func (m *Header) GetChainID() string { if m != nil { return m.ChainID @@ -2712,6 +2799,61 @@ func (m *Header) GetProposerAddress() []byte { return nil } +type Version struct { + Block uint64 `protobuf:"varint,1,opt,name=Block,proto3" json:"Block,omitempty"` + App uint64 `protobuf:"varint,2,opt,name=App,proto3" json:"App,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Version) Reset() { *m = Version{} } +func (m *Version) String() string { return proto.CompactTextString(m) } +func (*Version) ProtoMessage() {} +func (*Version) Descriptor() ([]byte, []int) { + return fileDescriptor_types_5b877df1938afe10, []int{31} +} +func (m *Version) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *Version) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_Version.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalTo(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (dst *Version) XXX_Merge(src proto.Message) { + xxx_messageInfo_Version.Merge(dst, src) +} +func (m *Version) XXX_Size() int { + return m.Size() +} +func (m *Version) XXX_DiscardUnknown() { + xxx_messageInfo_Version.DiscardUnknown(m) +} + +var xxx_messageInfo_Version proto.InternalMessageInfo + +func (m *Version) GetBlock() uint64 { + if m != nil { + return m.Block + } + return 0 +} + +func (m *Version) GetApp() uint64 { + if m != nil { + return m.App + } + return 0 +} + type BlockID struct { Hash []byte `protobuf:"bytes,1,opt,name=hash,proto3" json:"hash,omitempty"` PartsHeader PartSetHeader `protobuf:"bytes,2,opt,name=parts_header,json=partsHeader" json:"parts_header"` @@ -2724,7 +2866,7 @@ func (m *BlockID) Reset() { *m = BlockID{} } func (m *BlockID) String() string { return proto.CompactTextString(m) } func (*BlockID) ProtoMessage() {} func (*BlockID) Descriptor() ([]byte, []int) { - return fileDescriptor_types_4a7ab597ee120b05, []int{30} + return fileDescriptor_types_5b877df1938afe10, []int{32} } func (m *BlockID) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2779,7 +2921,7 @@ func (m *PartSetHeader) Reset() { *m = PartSetHeader{} } func (m *PartSetHeader) String() string { return proto.CompactTextString(m) } func (*PartSetHeader) ProtoMessage() {} func (*PartSetHeader) Descriptor() ([]byte, []int) { - return fileDescriptor_types_4a7ab597ee120b05, []int{31} + return fileDescriptor_types_5b877df1938afe10, []int{33} } func (m *PartSetHeader) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2836,7 +2978,7 @@ func (m *Validator) Reset() { *m = Validator{} } func (m *Validator) String() string { return proto.CompactTextString(m) } func (*Validator) ProtoMessage() {} func (*Validator) Descriptor() ([]byte, []int) { - return fileDescriptor_types_4a7ab597ee120b05, []int{32} + return fileDescriptor_types_5b877df1938afe10, []int{34} } func (m *Validator) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2892,7 +3034,7 @@ func (m *ValidatorUpdate) Reset() { *m = ValidatorUpdate{} } func (m *ValidatorUpdate) String() string { return proto.CompactTextString(m) } func (*ValidatorUpdate) ProtoMessage() {} func (*ValidatorUpdate) Descriptor() ([]byte, []int) { - return fileDescriptor_types_4a7ab597ee120b05, []int{33} + return fileDescriptor_types_5b877df1938afe10, []int{35} } func (m *ValidatorUpdate) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2948,7 +3090,7 @@ func (m *VoteInfo) Reset() { *m = VoteInfo{} } func (m *VoteInfo) String() string { return proto.CompactTextString(m) } func (*VoteInfo) ProtoMessage() {} func (*VoteInfo) Descriptor() ([]byte, []int) { - return fileDescriptor_types_4a7ab597ee120b05, []int{34} + return fileDescriptor_types_5b877df1938afe10, []int{36} } func (m *VoteInfo) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3003,7 +3145,7 @@ func (m *PubKey) Reset() { *m = PubKey{} } func (m *PubKey) String() string { return proto.CompactTextString(m) } func (*PubKey) ProtoMessage() {} func (*PubKey) Descriptor() ([]byte, []int) { - return fileDescriptor_types_4a7ab597ee120b05, []int{35} + return fileDescriptor_types_5b877df1938afe10, []int{37} } func (m *PubKey) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3061,7 +3203,7 @@ func (m *Evidence) Reset() { *m = Evidence{} } func (m *Evidence) String() string { return proto.CompactTextString(m) } func (*Evidence) ProtoMessage() {} func (*Evidence) Descriptor() ([]byte, []int) { - return fileDescriptor_types_4a7ab597ee120b05, []int{36} + return fileDescriptor_types_5b877df1938afe10, []int{38} } func (m *Evidence) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3178,14 +3320,18 @@ func init() { golang_proto.RegisterType((*ResponseCommit)(nil), "types.ResponseCommit") proto.RegisterType((*ConsensusParams)(nil), "types.ConsensusParams") golang_proto.RegisterType((*ConsensusParams)(nil), "types.ConsensusParams") - proto.RegisterType((*BlockSize)(nil), "types.BlockSize") - golang_proto.RegisterType((*BlockSize)(nil), "types.BlockSize") + proto.RegisterType((*BlockSizeParams)(nil), "types.BlockSizeParams") + golang_proto.RegisterType((*BlockSizeParams)(nil), "types.BlockSizeParams") proto.RegisterType((*EvidenceParams)(nil), "types.EvidenceParams") golang_proto.RegisterType((*EvidenceParams)(nil), "types.EvidenceParams") + proto.RegisterType((*ValidatorParams)(nil), "types.ValidatorParams") + golang_proto.RegisterType((*ValidatorParams)(nil), "types.ValidatorParams") proto.RegisterType((*LastCommitInfo)(nil), "types.LastCommitInfo") golang_proto.RegisterType((*LastCommitInfo)(nil), "types.LastCommitInfo") proto.RegisterType((*Header)(nil), "types.Header") golang_proto.RegisterType((*Header)(nil), "types.Header") + proto.RegisterType((*Version)(nil), "types.Version") + golang_proto.RegisterType((*Version)(nil), "types.Version") proto.RegisterType((*BlockID)(nil), "types.BlockID") golang_proto.RegisterType((*BlockID)(nil), "types.BlockID") proto.RegisterType((*PartSetHeader)(nil), "types.PartSetHeader") @@ -3571,6 +3717,12 @@ func (this *RequestInfo) Equal(that interface{}) bool { if this.Version != that1.Version { return false } + if this.BlockVersion != that1.BlockVersion { + return false + } + if this.P2PVersion != that1.P2PVersion { + return false + } if !bytes.Equal(this.XXX_unrecognized, that1.XXX_unrecognized) { return false } @@ -4256,6 +4408,9 @@ func (this *ResponseInfo) Equal(that interface{}) bool { if this.Version != that1.Version { return false } + if this.AppVersion != that1.AppVersion { + return false + } if this.LastBlockHeight != that1.LastBlockHeight { return false } @@ -4616,7 +4771,10 @@ func (this *ConsensusParams) Equal(that interface{}) bool { if !this.BlockSize.Equal(that1.BlockSize) { return false } - if !this.EvidenceParams.Equal(that1.EvidenceParams) { + if !this.Evidence.Equal(that1.Evidence) { + return false + } + if !this.Validator.Equal(that1.Validator) { return false } if !bytes.Equal(this.XXX_unrecognized, that1.XXX_unrecognized) { @@ -4624,14 +4782,14 @@ func (this *ConsensusParams) Equal(that interface{}) bool { } return true } -func (this *BlockSize) Equal(that interface{}) bool { +func (this *BlockSizeParams) Equal(that interface{}) bool { if that == nil { return this == nil } - that1, ok := that.(*BlockSize) + that1, ok := that.(*BlockSizeParams) if !ok { - that2, ok := that.(BlockSize) + that2, ok := that.(BlockSizeParams) if ok { that1 = &that2 } else { @@ -4681,6 +4839,38 @@ func (this *EvidenceParams) Equal(that interface{}) bool { } return true } +func (this *ValidatorParams) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + that1, ok := that.(*ValidatorParams) + if !ok { + that2, ok := that.(ValidatorParams) + if ok { + that1 = &that2 + } else { + return false + } + } + if that1 == nil { + return this == nil + } else if this == nil { + return false + } + if len(this.PubKeyTypes) != len(that1.PubKeyTypes) { + return false + } + for i := range this.PubKeyTypes { + if this.PubKeyTypes[i] != that1.PubKeyTypes[i] { + return false + } + } + if !bytes.Equal(this.XXX_unrecognized, that1.XXX_unrecognized) { + return false + } + return true +} func (this *LastCommitInfo) Equal(that interface{}) bool { if that == nil { return this == nil @@ -4735,6 +4925,9 @@ func (this *Header) Equal(that interface{}) bool { } else if this == nil { return false } + if !this.Version.Equal(&that1.Version) { + return false + } if this.ChainID != that1.ChainID { return false } @@ -4785,6 +4978,36 @@ func (this *Header) Equal(that interface{}) bool { } return true } +func (this *Version) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + that1, ok := that.(*Version) + if !ok { + that2, ok := that.(Version) + if ok { + that1 = &that2 + } else { + return false + } + } + if that1 == nil { + return this == nil + } else if this == nil { + return false + } + if this.Block != that1.Block { + return false + } + if this.App != that1.App { + return false + } + if !bytes.Equal(this.XXX_unrecognized, that1.XXX_unrecognized) { + return false + } + return true +} func (this *BlockID) Equal(that interface{}) bool { if that == nil { return this == nil @@ -5660,6 +5883,16 @@ func (m *RequestInfo) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintTypes(dAtA, i, uint64(len(m.Version))) i += copy(dAtA[i:], m.Version) } + if m.BlockVersion != 0 { + dAtA[i] = 0x10 + i++ + i = encodeVarintTypes(dAtA, i, uint64(m.BlockVersion)) + } + if m.P2PVersion != 0 { + dAtA[i] = 0x18 + i++ + i = encodeVarintTypes(dAtA, i, uint64(m.P2PVersion)) + } if m.XXX_unrecognized != nil { i += copy(dAtA[i:], m.XXX_unrecognized) } @@ -6264,13 +6497,18 @@ func (m *ResponseInfo) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintTypes(dAtA, i, uint64(len(m.Version))) i += copy(dAtA[i:], m.Version) } - if m.LastBlockHeight != 0 { + if m.AppVersion != 0 { dAtA[i] = 0x18 i++ + i = encodeVarintTypes(dAtA, i, uint64(m.AppVersion)) + } + if m.LastBlockHeight != 0 { + dAtA[i] = 0x20 + i++ i = encodeVarintTypes(dAtA, i, uint64(m.LastBlockHeight)) } if len(m.LastBlockAppHash) > 0 { - dAtA[i] = 0x22 + dAtA[i] = 0x2a i++ i = encodeVarintTypes(dAtA, i, uint64(len(m.LastBlockAppHash))) i += copy(dAtA[i:], m.LastBlockAppHash) @@ -6722,23 +6960,33 @@ func (m *ConsensusParams) MarshalTo(dAtA []byte) (int, error) { } i += n33 } - if m.EvidenceParams != nil { + if m.Evidence != nil { dAtA[i] = 0x12 i++ - i = encodeVarintTypes(dAtA, i, uint64(m.EvidenceParams.Size())) - n34, err := m.EvidenceParams.MarshalTo(dAtA[i:]) + i = encodeVarintTypes(dAtA, i, uint64(m.Evidence.Size())) + n34, err := m.Evidence.MarshalTo(dAtA[i:]) if err != nil { return 0, err } i += n34 } + if m.Validator != nil { + dAtA[i] = 0x1a + i++ + i = encodeVarintTypes(dAtA, i, uint64(m.Validator.Size())) + n35, err := m.Validator.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n35 + } if m.XXX_unrecognized != nil { i += copy(dAtA[i:], m.XXX_unrecognized) } return i, nil } -func (m *BlockSize) Marshal() (dAtA []byte, err error) { +func (m *BlockSizeParams) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalTo(dAtA) @@ -6748,7 +6996,7 @@ func (m *BlockSize) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *BlockSize) MarshalTo(dAtA []byte) (int, error) { +func (m *BlockSizeParams) MarshalTo(dAtA []byte) (int, error) { var i int _ = i var l int @@ -6795,6 +7043,42 @@ func (m *EvidenceParams) MarshalTo(dAtA []byte) (int, error) { return i, nil } +func (m *ValidatorParams) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ValidatorParams) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.PubKeyTypes) > 0 { + for _, s := range m.PubKeyTypes { + dAtA[i] = 0xa + i++ + l = len(s) + for l >= 1<<7 { + dAtA[i] = uint8(uint64(l)&0x7f | 0x80) + l >>= 7 + i++ + } + dAtA[i] = uint8(l) + i++ + i += copy(dAtA[i:], s) + } + } + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } + return i, nil +} + func (m *LastCommitInfo) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -6848,93 +7132,103 @@ func (m *Header) MarshalTo(dAtA []byte) (int, error) { _ = i var l int _ = l + dAtA[i] = 0xa + i++ + i = encodeVarintTypes(dAtA, i, uint64(m.Version.Size())) + n36, err := m.Version.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n36 if len(m.ChainID) > 0 { - dAtA[i] = 0xa + dAtA[i] = 0x12 i++ i = encodeVarintTypes(dAtA, i, uint64(len(m.ChainID))) i += copy(dAtA[i:], m.ChainID) } if m.Height != 0 { - dAtA[i] = 0x10 + dAtA[i] = 0x18 i++ i = encodeVarintTypes(dAtA, i, uint64(m.Height)) } - dAtA[i] = 0x1a + dAtA[i] = 0x22 i++ i = encodeVarintTypes(dAtA, i, uint64(github_com_gogo_protobuf_types.SizeOfStdTime(m.Time))) - n35, err := github_com_gogo_protobuf_types.StdTimeMarshalTo(m.Time, dAtA[i:]) + n37, err := github_com_gogo_protobuf_types.StdTimeMarshalTo(m.Time, dAtA[i:]) if err != nil { return 0, err } - i += n35 + i += n37 if m.NumTxs != 0 { - dAtA[i] = 0x20 + dAtA[i] = 0x28 i++ i = encodeVarintTypes(dAtA, i, uint64(m.NumTxs)) } if m.TotalTxs != 0 { - dAtA[i] = 0x28 + dAtA[i] = 0x30 i++ i = encodeVarintTypes(dAtA, i, uint64(m.TotalTxs)) } - dAtA[i] = 0x32 + dAtA[i] = 0x3a i++ i = encodeVarintTypes(dAtA, i, uint64(m.LastBlockId.Size())) - n36, err := m.LastBlockId.MarshalTo(dAtA[i:]) + n38, err := m.LastBlockId.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n36 + i += n38 if len(m.LastCommitHash) > 0 { - dAtA[i] = 0x3a + dAtA[i] = 0x42 i++ i = encodeVarintTypes(dAtA, i, uint64(len(m.LastCommitHash))) i += copy(dAtA[i:], m.LastCommitHash) } if len(m.DataHash) > 0 { - dAtA[i] = 0x42 + dAtA[i] = 0x4a i++ i = encodeVarintTypes(dAtA, i, uint64(len(m.DataHash))) i += copy(dAtA[i:], m.DataHash) } if len(m.ValidatorsHash) > 0 { - dAtA[i] = 0x4a + dAtA[i] = 0x52 i++ i = encodeVarintTypes(dAtA, i, uint64(len(m.ValidatorsHash))) i += copy(dAtA[i:], m.ValidatorsHash) } if len(m.NextValidatorsHash) > 0 { - dAtA[i] = 0x52 + dAtA[i] = 0x5a i++ i = encodeVarintTypes(dAtA, i, uint64(len(m.NextValidatorsHash))) i += copy(dAtA[i:], m.NextValidatorsHash) } if len(m.ConsensusHash) > 0 { - dAtA[i] = 0x5a + dAtA[i] = 0x62 i++ i = encodeVarintTypes(dAtA, i, uint64(len(m.ConsensusHash))) i += copy(dAtA[i:], m.ConsensusHash) } if len(m.AppHash) > 0 { - dAtA[i] = 0x62 + dAtA[i] = 0x6a i++ i = encodeVarintTypes(dAtA, i, uint64(len(m.AppHash))) i += copy(dAtA[i:], m.AppHash) } if len(m.LastResultsHash) > 0 { - dAtA[i] = 0x6a + dAtA[i] = 0x72 i++ i = encodeVarintTypes(dAtA, i, uint64(len(m.LastResultsHash))) i += copy(dAtA[i:], m.LastResultsHash) } if len(m.EvidenceHash) > 0 { - dAtA[i] = 0x72 + dAtA[i] = 0x7a i++ i = encodeVarintTypes(dAtA, i, uint64(len(m.EvidenceHash))) i += copy(dAtA[i:], m.EvidenceHash) } if len(m.ProposerAddress) > 0 { - dAtA[i] = 0x7a + dAtA[i] = 0x82 + i++ + dAtA[i] = 0x1 i++ i = encodeVarintTypes(dAtA, i, uint64(len(m.ProposerAddress))) i += copy(dAtA[i:], m.ProposerAddress) @@ -6945,6 +7239,37 @@ func (m *Header) MarshalTo(dAtA []byte) (int, error) { return i, nil } +func (m *Version) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Version) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.Block != 0 { + dAtA[i] = 0x8 + i++ + i = encodeVarintTypes(dAtA, i, uint64(m.Block)) + } + if m.App != 0 { + dAtA[i] = 0x10 + i++ + i = encodeVarintTypes(dAtA, i, uint64(m.App)) + } + if m.XXX_unrecognized != nil { + i += copy(dAtA[i:], m.XXX_unrecognized) + } + return i, nil +} + func (m *BlockID) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -6969,11 +7294,11 @@ func (m *BlockID) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintTypes(dAtA, i, uint64(m.PartsHeader.Size())) - n37, err := m.PartsHeader.MarshalTo(dAtA[i:]) + n39, err := m.PartsHeader.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n37 + i += n39 if m.XXX_unrecognized != nil { i += copy(dAtA[i:], m.XXX_unrecognized) } @@ -7062,11 +7387,11 @@ func (m *ValidatorUpdate) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintTypes(dAtA, i, uint64(m.PubKey.Size())) - n38, err := m.PubKey.MarshalTo(dAtA[i:]) + n40, err := m.PubKey.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n38 + i += n40 if m.Power != 0 { dAtA[i] = 0x10 i++ @@ -7096,11 +7421,11 @@ func (m *VoteInfo) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0xa i++ i = encodeVarintTypes(dAtA, i, uint64(m.Validator.Size())) - n39, err := m.Validator.MarshalTo(dAtA[i:]) + n41, err := m.Validator.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n39 + i += n41 if m.SignedLastBlock { dAtA[i] = 0x10 i++ @@ -7174,11 +7499,11 @@ func (m *Evidence) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x12 i++ i = encodeVarintTypes(dAtA, i, uint64(m.Validator.Size())) - n40, err := m.Validator.MarshalTo(dAtA[i:]) + n42, err := m.Validator.MarshalTo(dAtA[i:]) if err != nil { return 0, err } - i += n40 + i += n42 if m.Height != 0 { dAtA[i] = 0x18 i++ @@ -7187,11 +7512,11 @@ func (m *Evidence) MarshalTo(dAtA []byte) (int, error) { dAtA[i] = 0x22 i++ i = encodeVarintTypes(dAtA, i, uint64(github_com_gogo_protobuf_types.SizeOfStdTime(m.Time))) - n41, err := github_com_gogo_protobuf_types.StdTimeMarshalTo(m.Time, dAtA[i:]) + n43, err := github_com_gogo_protobuf_types.StdTimeMarshalTo(m.Time, dAtA[i:]) if err != nil { return 0, err } - i += n41 + i += n43 if m.TotalVotingPower != 0 { dAtA[i] = 0x28 i++ @@ -7320,8 +7645,10 @@ func NewPopulatedRequestFlush(r randyTypes, easy bool) *RequestFlush { func NewPopulatedRequestInfo(r randyTypes, easy bool) *RequestInfo { this := &RequestInfo{} this.Version = string(randStringTypes(r)) + this.BlockVersion = uint64(uint64(r.Uint32())) + this.P2PVersion = uint64(uint64(r.Uint32())) if !easy && r.Intn(10) != 0 { - this.XXX_unrecognized = randUnrecognizedTypes(r, 2) + this.XXX_unrecognized = randUnrecognizedTypes(r, 4) } return this } @@ -7578,6 +7905,7 @@ func NewPopulatedResponseInfo(r randyTypes, easy bool) *ResponseInfo { this := &ResponseInfo{} this.Data = string(randStringTypes(r)) this.Version = string(randStringTypes(r)) + this.AppVersion = uint64(uint64(r.Uint32())) this.LastBlockHeight = int64(r.Int63()) if r.Intn(2) == 0 { this.LastBlockHeight *= -1 @@ -7588,7 +7916,7 @@ func NewPopulatedResponseInfo(r randyTypes, easy bool) *ResponseInfo { this.LastBlockAppHash[i] = byte(r.Intn(256)) } if !easy && r.Intn(10) != 0 { - this.XXX_unrecognized = randUnrecognizedTypes(r, 5) + this.XXX_unrecognized = randUnrecognizedTypes(r, 6) } return this } @@ -7781,19 +8109,22 @@ func NewPopulatedResponseCommit(r randyTypes, easy bool) *ResponseCommit { func NewPopulatedConsensusParams(r randyTypes, easy bool) *ConsensusParams { this := &ConsensusParams{} if r.Intn(10) != 0 { - this.BlockSize = NewPopulatedBlockSize(r, easy) + this.BlockSize = NewPopulatedBlockSizeParams(r, easy) } if r.Intn(10) != 0 { - this.EvidenceParams = NewPopulatedEvidenceParams(r, easy) + this.Evidence = NewPopulatedEvidenceParams(r, easy) + } + if r.Intn(10) != 0 { + this.Validator = NewPopulatedValidatorParams(r, easy) } if !easy && r.Intn(10) != 0 { - this.XXX_unrecognized = randUnrecognizedTypes(r, 3) + this.XXX_unrecognized = randUnrecognizedTypes(r, 4) } return this } -func NewPopulatedBlockSize(r randyTypes, easy bool) *BlockSize { - this := &BlockSize{} +func NewPopulatedBlockSizeParams(r randyTypes, easy bool) *BlockSizeParams { + this := &BlockSizeParams{} this.MaxBytes = int64(r.Int63()) if r.Intn(2) == 0 { this.MaxBytes *= -1 @@ -7820,6 +8151,19 @@ func NewPopulatedEvidenceParams(r randyTypes, easy bool) *EvidenceParams { return this } +func NewPopulatedValidatorParams(r randyTypes, easy bool) *ValidatorParams { + this := &ValidatorParams{} + v31 := r.Intn(10) + this.PubKeyTypes = make([]string, v31) + for i := 0; i < v31; i++ { + this.PubKeyTypes[i] = string(randStringTypes(r)) + } + if !easy && r.Intn(10) != 0 { + this.XXX_unrecognized = randUnrecognizedTypes(r, 2) + } + return this +} + func NewPopulatedLastCommitInfo(r randyTypes, easy bool) *LastCommitInfo { this := &LastCommitInfo{} this.Round = int32(r.Int31()) @@ -7827,11 +8171,11 @@ func NewPopulatedLastCommitInfo(r randyTypes, easy bool) *LastCommitInfo { this.Round *= -1 } if r.Intn(10) != 0 { - v31 := r.Intn(5) - this.Votes = make([]VoteInfo, v31) - for i := 0; i < v31; i++ { - v32 := NewPopulatedVoteInfo(r, easy) - this.Votes[i] = *v32 + v32 := r.Intn(5) + this.Votes = make([]VoteInfo, v32) + for i := 0; i < v32; i++ { + v33 := NewPopulatedVoteInfo(r, easy) + this.Votes[i] = *v33 } } if !easy && r.Intn(10) != 0 { @@ -7842,13 +8186,15 @@ func NewPopulatedLastCommitInfo(r randyTypes, easy bool) *LastCommitInfo { func NewPopulatedHeader(r randyTypes, easy bool) *Header { this := &Header{} + v34 := NewPopulatedVersion(r, easy) + this.Version = *v34 this.ChainID = string(randStringTypes(r)) this.Height = int64(r.Int63()) if r.Intn(2) == 0 { this.Height *= -1 } - v33 := github_com_gogo_protobuf_types.NewPopulatedStdTime(r, easy) - this.Time = *v33 + v35 := github_com_gogo_protobuf_types.NewPopulatedStdTime(r, easy) + this.Time = *v35 this.NumTxs = int64(r.Int63()) if r.Intn(2) == 0 { this.NumTxs *= -1 @@ -7857,68 +8203,78 @@ func NewPopulatedHeader(r randyTypes, easy bool) *Header { if r.Intn(2) == 0 { this.TotalTxs *= -1 } - v34 := NewPopulatedBlockID(r, easy) - this.LastBlockId = *v34 - v35 := r.Intn(100) - this.LastCommitHash = make([]byte, v35) - for i := 0; i < v35; i++ { + v36 := NewPopulatedBlockID(r, easy) + this.LastBlockId = *v36 + v37 := r.Intn(100) + this.LastCommitHash = make([]byte, v37) + for i := 0; i < v37; i++ { this.LastCommitHash[i] = byte(r.Intn(256)) } - v36 := r.Intn(100) - this.DataHash = make([]byte, v36) - for i := 0; i < v36; i++ { + v38 := r.Intn(100) + this.DataHash = make([]byte, v38) + for i := 0; i < v38; i++ { this.DataHash[i] = byte(r.Intn(256)) } - v37 := r.Intn(100) - this.ValidatorsHash = make([]byte, v37) - for i := 0; i < v37; i++ { + v39 := r.Intn(100) + this.ValidatorsHash = make([]byte, v39) + for i := 0; i < v39; i++ { this.ValidatorsHash[i] = byte(r.Intn(256)) } - v38 := r.Intn(100) - this.NextValidatorsHash = make([]byte, v38) - for i := 0; i < v38; i++ { + v40 := r.Intn(100) + this.NextValidatorsHash = make([]byte, v40) + for i := 0; i < v40; i++ { this.NextValidatorsHash[i] = byte(r.Intn(256)) } - v39 := r.Intn(100) - this.ConsensusHash = make([]byte, v39) - for i := 0; i < v39; i++ { + v41 := r.Intn(100) + this.ConsensusHash = make([]byte, v41) + for i := 0; i < v41; i++ { this.ConsensusHash[i] = byte(r.Intn(256)) } - v40 := r.Intn(100) - this.AppHash = make([]byte, v40) - for i := 0; i < v40; i++ { + v42 := r.Intn(100) + this.AppHash = make([]byte, v42) + for i := 0; i < v42; i++ { this.AppHash[i] = byte(r.Intn(256)) } - v41 := r.Intn(100) - this.LastResultsHash = make([]byte, v41) - for i := 0; i < v41; i++ { + v43 := r.Intn(100) + this.LastResultsHash = make([]byte, v43) + for i := 0; i < v43; i++ { this.LastResultsHash[i] = byte(r.Intn(256)) } - v42 := r.Intn(100) - this.EvidenceHash = make([]byte, v42) - for i := 0; i < v42; i++ { + v44 := r.Intn(100) + this.EvidenceHash = make([]byte, v44) + for i := 0; i < v44; i++ { this.EvidenceHash[i] = byte(r.Intn(256)) } - v43 := r.Intn(100) - this.ProposerAddress = make([]byte, v43) - for i := 0; i < v43; i++ { + v45 := r.Intn(100) + this.ProposerAddress = make([]byte, v45) + for i := 0; i < v45; i++ { this.ProposerAddress[i] = byte(r.Intn(256)) } if !easy && r.Intn(10) != 0 { - this.XXX_unrecognized = randUnrecognizedTypes(r, 16) + this.XXX_unrecognized = randUnrecognizedTypes(r, 17) + } + return this +} + +func NewPopulatedVersion(r randyTypes, easy bool) *Version { + this := &Version{} + this.Block = uint64(uint64(r.Uint32())) + this.App = uint64(uint64(r.Uint32())) + if !easy && r.Intn(10) != 0 { + this.XXX_unrecognized = randUnrecognizedTypes(r, 3) } return this } func NewPopulatedBlockID(r randyTypes, easy bool) *BlockID { this := &BlockID{} - v44 := r.Intn(100) - this.Hash = make([]byte, v44) - for i := 0; i < v44; i++ { + v46 := r.Intn(100) + this.Hash = make([]byte, v46) + for i := 0; i < v46; i++ { this.Hash[i] = byte(r.Intn(256)) } - v45 := NewPopulatedPartSetHeader(r, easy) - this.PartsHeader = *v45 + v47 := NewPopulatedPartSetHeader(r, easy) + this.PartsHeader = *v47 if !easy && r.Intn(10) != 0 { this.XXX_unrecognized = randUnrecognizedTypes(r, 3) } @@ -7931,9 +8287,9 @@ func NewPopulatedPartSetHeader(r randyTypes, easy bool) *PartSetHeader { if r.Intn(2) == 0 { this.Total *= -1 } - v46 := r.Intn(100) - this.Hash = make([]byte, v46) - for i := 0; i < v46; i++ { + v48 := r.Intn(100) + this.Hash = make([]byte, v48) + for i := 0; i < v48; i++ { this.Hash[i] = byte(r.Intn(256)) } if !easy && r.Intn(10) != 0 { @@ -7944,9 +8300,9 @@ func NewPopulatedPartSetHeader(r randyTypes, easy bool) *PartSetHeader { func NewPopulatedValidator(r randyTypes, easy bool) *Validator { this := &Validator{} - v47 := r.Intn(100) - this.Address = make([]byte, v47) - for i := 0; i < v47; i++ { + v49 := r.Intn(100) + this.Address = make([]byte, v49) + for i := 0; i < v49; i++ { this.Address[i] = byte(r.Intn(256)) } this.Power = int64(r.Int63()) @@ -7961,8 +8317,8 @@ func NewPopulatedValidator(r randyTypes, easy bool) *Validator { func NewPopulatedValidatorUpdate(r randyTypes, easy bool) *ValidatorUpdate { this := &ValidatorUpdate{} - v48 := NewPopulatedPubKey(r, easy) - this.PubKey = *v48 + v50 := NewPopulatedPubKey(r, easy) + this.PubKey = *v50 this.Power = int64(r.Int63()) if r.Intn(2) == 0 { this.Power *= -1 @@ -7975,8 +8331,8 @@ func NewPopulatedValidatorUpdate(r randyTypes, easy bool) *ValidatorUpdate { func NewPopulatedVoteInfo(r randyTypes, easy bool) *VoteInfo { this := &VoteInfo{} - v49 := NewPopulatedValidator(r, easy) - this.Validator = *v49 + v51 := NewPopulatedValidator(r, easy) + this.Validator = *v51 this.SignedLastBlock = bool(bool(r.Intn(2) == 0)) if !easy && r.Intn(10) != 0 { this.XXX_unrecognized = randUnrecognizedTypes(r, 3) @@ -7987,9 +8343,9 @@ func NewPopulatedVoteInfo(r randyTypes, easy bool) *VoteInfo { func NewPopulatedPubKey(r randyTypes, easy bool) *PubKey { this := &PubKey{} this.Type = string(randStringTypes(r)) - v50 := r.Intn(100) - this.Data = make([]byte, v50) - for i := 0; i < v50; i++ { + v52 := r.Intn(100) + this.Data = make([]byte, v52) + for i := 0; i < v52; i++ { this.Data[i] = byte(r.Intn(256)) } if !easy && r.Intn(10) != 0 { @@ -8001,14 +8357,14 @@ func NewPopulatedPubKey(r randyTypes, easy bool) *PubKey { func NewPopulatedEvidence(r randyTypes, easy bool) *Evidence { this := &Evidence{} this.Type = string(randStringTypes(r)) - v51 := NewPopulatedValidator(r, easy) - this.Validator = *v51 + v53 := NewPopulatedValidator(r, easy) + this.Validator = *v53 this.Height = int64(r.Int63()) if r.Intn(2) == 0 { this.Height *= -1 } - v52 := github_com_gogo_protobuf_types.NewPopulatedStdTime(r, easy) - this.Time = *v52 + v54 := github_com_gogo_protobuf_types.NewPopulatedStdTime(r, easy) + this.Time = *v54 this.TotalVotingPower = int64(r.Int63()) if r.Intn(2) == 0 { this.TotalVotingPower *= -1 @@ -8038,9 +8394,9 @@ func randUTF8RuneTypes(r randyTypes) rune { return rune(ru + 61) } func randStringTypes(r randyTypes) string { - v53 := r.Intn(100) - tmps := make([]rune, v53) - for i := 0; i < v53; i++ { + v55 := r.Intn(100) + tmps := make([]rune, v55) + for i := 0; i < v55; i++ { tmps[i] = randUTF8RuneTypes(r) } return string(tmps) @@ -8062,11 +8418,11 @@ func randFieldTypes(dAtA []byte, r randyTypes, fieldNumber int, wire int) []byte switch wire { case 0: dAtA = encodeVarintPopulateTypes(dAtA, uint64(key)) - v54 := r.Int63() + v56 := r.Int63() if r.Intn(2) == 0 { - v54 *= -1 + v56 *= -1 } - dAtA = encodeVarintPopulateTypes(dAtA, uint64(v54)) + dAtA = encodeVarintPopulateTypes(dAtA, uint64(v56)) case 1: dAtA = encodeVarintPopulateTypes(dAtA, uint64(key)) dAtA = append(dAtA, byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256))) @@ -8231,6 +8587,12 @@ func (m *RequestInfo) Size() (n int) { if l > 0 { n += 1 + l + sovTypes(uint64(l)) } + if m.BlockVersion != 0 { + n += 1 + sovTypes(uint64(m.BlockVersion)) + } + if m.P2PVersion != 0 { + n += 1 + sovTypes(uint64(m.P2PVersion)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -8542,6 +8904,9 @@ func (m *ResponseInfo) Size() (n int) { if l > 0 { n += 1 + l + sovTypes(uint64(l)) } + if m.AppVersion != 0 { + n += 1 + sovTypes(uint64(m.AppVersion)) + } if m.LastBlockHeight != 0 { n += 1 + sovTypes(uint64(m.LastBlockHeight)) } @@ -8776,8 +9141,12 @@ func (m *ConsensusParams) Size() (n int) { l = m.BlockSize.Size() n += 1 + l + sovTypes(uint64(l)) } - if m.EvidenceParams != nil { - l = m.EvidenceParams.Size() + if m.Evidence != nil { + l = m.Evidence.Size() + n += 1 + l + sovTypes(uint64(l)) + } + if m.Validator != nil { + l = m.Validator.Size() n += 1 + l + sovTypes(uint64(l)) } if m.XXX_unrecognized != nil { @@ -8786,7 +9155,7 @@ func (m *ConsensusParams) Size() (n int) { return n } -func (m *BlockSize) Size() (n int) { +func (m *BlockSizeParams) Size() (n int) { var l int _ = l if m.MaxBytes != 0 { @@ -8813,6 +9182,21 @@ func (m *EvidenceParams) Size() (n int) { return n } +func (m *ValidatorParams) Size() (n int) { + var l int + _ = l + if len(m.PubKeyTypes) > 0 { + for _, s := range m.PubKeyTypes { + l = len(s) + n += 1 + l + sovTypes(uint64(l)) + } + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + func (m *LastCommitInfo) Size() (n int) { var l int _ = l @@ -8834,6 +9218,8 @@ func (m *LastCommitInfo) Size() (n int) { func (m *Header) Size() (n int) { var l int _ = l + l = m.Version.Size() + n += 1 + l + sovTypes(uint64(l)) l = len(m.ChainID) if l > 0 { n += 1 + l + sovTypes(uint64(l)) @@ -8885,7 +9271,22 @@ func (m *Header) Size() (n int) { } l = len(m.ProposerAddress) if l > 0 { - n += 1 + l + sovTypes(uint64(l)) + n += 2 + l + sovTypes(uint64(l)) + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *Version) Size() (n int) { + var l int + _ = l + if m.Block != 0 { + n += 1 + sovTypes(uint64(m.Block)) + } + if m.App != 0 { + n += 1 + sovTypes(uint64(m.App)) } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) @@ -9613,6 +10014,44 @@ func (m *RequestInfo) Unmarshal(dAtA []byte) error { } m.Version = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field BlockVersion", wireType) + } + m.BlockVersion = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.BlockVersion |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field P2PVersion", wireType) + } + m.P2PVersion = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.P2PVersion |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipTypes(dAtA[iNdEx:]) @@ -11291,6 +11730,25 @@ func (m *ResponseInfo) Unmarshal(dAtA []byte) error { m.Version = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field AppVersion", wireType) + } + m.AppVersion = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.AppVersion |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 4: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field LastBlockHeight", wireType) } @@ -11309,7 +11767,7 @@ func (m *ResponseInfo) Unmarshal(dAtA []byte) error { break } } - case 4: + case 5: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field LastBlockAppHash", wireType) } @@ -12775,7 +13233,7 @@ func (m *ConsensusParams) Unmarshal(dAtA []byte) error { return io.ErrUnexpectedEOF } if m.BlockSize == nil { - m.BlockSize = &BlockSize{} + m.BlockSize = &BlockSizeParams{} } if err := m.BlockSize.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err @@ -12783,7 +13241,7 @@ func (m *ConsensusParams) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 2: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field EvidenceParams", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Evidence", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -12807,10 +13265,43 @@ func (m *ConsensusParams) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - if m.EvidenceParams == nil { - m.EvidenceParams = &EvidenceParams{} + if m.Evidence == nil { + m.Evidence = &EvidenceParams{} } - if err := m.EvidenceParams.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + if err := m.Evidence.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Validator", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthTypes + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Validator == nil { + m.Validator = &ValidatorParams{} + } + if err := m.Validator.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex @@ -12836,7 +13327,7 @@ func (m *ConsensusParams) Unmarshal(dAtA []byte) error { } return nil } -func (m *BlockSize) Unmarshal(dAtA []byte) error { +func (m *BlockSizeParams) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -12859,10 +13350,10 @@ func (m *BlockSize) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: BlockSize: wiretype end group for non-group") + return fmt.Errorf("proto: BlockSizeParams: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: BlockSize: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: BlockSizeParams: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: @@ -12995,6 +13486,86 @@ func (m *EvidenceParams) Unmarshal(dAtA []byte) error { } return nil } +func (m *ValidatorParams) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ValidatorParams: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ValidatorParams: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field PubKeyTypes", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthTypes + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.PubKeyTypes = append(m.PubKeyTypes, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipTypes(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthTypes + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *LastCommitInfo) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -13126,6 +13697,36 @@ func (m *Header) Unmarshal(dAtA []byte) error { } switch fieldNum { case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Version", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthTypes + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if err := m.Version.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 2: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field ChainID", wireType) } @@ -13154,7 +13755,7 @@ func (m *Header) Unmarshal(dAtA []byte) error { } m.ChainID = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex - case 2: + case 3: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field Height", wireType) } @@ -13173,7 +13774,7 @@ func (m *Header) Unmarshal(dAtA []byte) error { break } } - case 3: + case 4: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Time", wireType) } @@ -13203,7 +13804,7 @@ func (m *Header) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex - case 4: + case 5: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field NumTxs", wireType) } @@ -13222,7 +13823,7 @@ func (m *Header) Unmarshal(dAtA []byte) error { break } } - case 5: + case 6: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field TotalTxs", wireType) } @@ -13241,7 +13842,7 @@ func (m *Header) Unmarshal(dAtA []byte) error { break } } - case 6: + case 7: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field LastBlockId", wireType) } @@ -13271,7 +13872,7 @@ func (m *Header) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex - case 7: + case 8: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field LastCommitHash", wireType) } @@ -13302,7 +13903,7 @@ func (m *Header) Unmarshal(dAtA []byte) error { m.LastCommitHash = []byte{} } iNdEx = postIndex - case 8: + case 9: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field DataHash", wireType) } @@ -13333,7 +13934,7 @@ func (m *Header) Unmarshal(dAtA []byte) error { m.DataHash = []byte{} } iNdEx = postIndex - case 9: + case 10: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field ValidatorsHash", wireType) } @@ -13364,7 +13965,7 @@ func (m *Header) Unmarshal(dAtA []byte) error { m.ValidatorsHash = []byte{} } iNdEx = postIndex - case 10: + case 11: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field NextValidatorsHash", wireType) } @@ -13395,7 +13996,7 @@ func (m *Header) Unmarshal(dAtA []byte) error { m.NextValidatorsHash = []byte{} } iNdEx = postIndex - case 11: + case 12: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field ConsensusHash", wireType) } @@ -13426,7 +14027,7 @@ func (m *Header) Unmarshal(dAtA []byte) error { m.ConsensusHash = []byte{} } iNdEx = postIndex - case 12: + case 13: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field AppHash", wireType) } @@ -13457,7 +14058,7 @@ func (m *Header) Unmarshal(dAtA []byte) error { m.AppHash = []byte{} } iNdEx = postIndex - case 13: + case 14: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field LastResultsHash", wireType) } @@ -13488,7 +14089,7 @@ func (m *Header) Unmarshal(dAtA []byte) error { m.LastResultsHash = []byte{} } iNdEx = postIndex - case 14: + case 15: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field EvidenceHash", wireType) } @@ -13519,7 +14120,7 @@ func (m *Header) Unmarshal(dAtA []byte) error { m.EvidenceHash = []byte{} } iNdEx = postIndex - case 15: + case 16: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field ProposerAddress", wireType) } @@ -13572,6 +14173,95 @@ func (m *Header) Unmarshal(dAtA []byte) error { } return nil } +func (m *Version) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Version: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Version: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Block", wireType) + } + m.Block = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Block |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field App", wireType) + } + m.App = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.App |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipTypes(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthTypes + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *BlockID) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -14481,143 +15171,150 @@ var ( ErrIntOverflowTypes = fmt.Errorf("proto: integer overflow") ) -func init() { proto.RegisterFile("abci/types/types.proto", fileDescriptor_types_4a7ab597ee120b05) } +func init() { proto.RegisterFile("abci/types/types.proto", fileDescriptor_types_5b877df1938afe10) } func init() { - golang_proto.RegisterFile("abci/types/types.proto", fileDescriptor_types_4a7ab597ee120b05) + golang_proto.RegisterFile("abci/types/types.proto", fileDescriptor_types_5b877df1938afe10) } -var fileDescriptor_types_4a7ab597ee120b05 = []byte{ - // 2107 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xdc, 0x58, 0xcd, 0x73, 0x23, 0x47, - 0x15, 0xf7, 0x48, 0xb2, 0xa4, 0x79, 0xb6, 0x25, 0xa7, 0xd7, 0x6b, 0x6b, 0x45, 0xb0, 0xb7, 0x26, - 0x90, 0xd8, 0xc4, 0x91, 0x53, 0x0e, 0xa1, 0xbc, 0xd9, 0x90, 0x2a, 0x6b, 0x77, 0xc1, 0xae, 0x04, - 0x30, 0xb3, 0xbb, 0xe6, 0x42, 0xd5, 0x54, 0x4b, 0xd3, 0x96, 0xa6, 0x56, 0x9a, 0x99, 0xcc, 0xb4, - 0x1c, 0x79, 0x8f, 0x9c, 0x73, 0xc8, 0x81, 0x2a, 0xfe, 0x05, 0xfe, 0x04, 0x8e, 0x9c, 0xa8, 0x1c, - 0x29, 0x8a, 0xf3, 0x02, 0xa6, 0x38, 0xc0, 0x95, 0xa2, 0x8a, 0x23, 0xd5, 0xaf, 0xbb, 0xe7, 0xcb, - 0xa3, 0x25, 0x1b, 0x6e, 0x5c, 0xa4, 0xee, 0xf7, 0xd1, 0x1f, 0x6f, 0xde, 0x7b, 0xbf, 0xf7, 0x1a, - 0x36, 0xe9, 0x60, 0xe8, 0x1d, 0xf0, 0xab, 0x90, 0xc5, 0xf2, 0xb7, 0x17, 0x46, 0x01, 0x0f, 0xc8, - 0x32, 0x4e, 0xba, 0xef, 0x8c, 0x3c, 0x3e, 0x9e, 0x0d, 0x7a, 0xc3, 0x60, 0x7a, 0x30, 0x0a, 0x46, - 0xc1, 0x01, 0x72, 0x07, 0xb3, 0x0b, 0x9c, 0xe1, 0x04, 0x47, 0x52, 0xab, 0xbb, 0x33, 0x0a, 0x82, - 0xd1, 0x84, 0xa5, 0x52, 0xdc, 0x9b, 0xb2, 0x98, 0xd3, 0x69, 0xa8, 0x04, 0x8e, 0x32, 0xeb, 0x71, - 0xe6, 0xbb, 0x2c, 0x9a, 0x7a, 0x3e, 0xcf, 0x0e, 0x27, 0xde, 0x20, 0x3e, 0x18, 0x06, 0xd3, 0x69, - 0xe0, 0x67, 0x0f, 0xd4, 0xbd, 0xff, 0x5f, 0x35, 0x87, 0xd1, 0x55, 0xc8, 0x83, 0x83, 0x29, 0x8b, - 0x9e, 0x4d, 0x98, 0xfa, 0x93, 0xca, 0xd6, 0xef, 0x6a, 0xd0, 0xb0, 0xd9, 0xa7, 0x33, 0x16, 0x73, - 0xb2, 0x0b, 0x35, 0x36, 0x1c, 0x07, 0x9d, 0xca, 0x5d, 0x63, 0x77, 0xe5, 0x90, 0xf4, 0xe4, 0x26, - 0x8a, 0xfb, 0x68, 0x38, 0x0e, 0x4e, 0x96, 0x6c, 0x94, 0x20, 0x6f, 0xc3, 0xf2, 0xc5, 0x64, 0x16, - 0x8f, 0x3b, 0x55, 0x14, 0xbd, 0x95, 0x17, 0xfd, 0x81, 0x60, 0x9d, 0x2c, 0xd9, 0x52, 0x46, 0x2c, - 0xeb, 0xf9, 0x17, 0x41, 0xa7, 0x56, 0xb6, 0xec, 0xa9, 0x7f, 0x81, 0xcb, 0x0a, 0x09, 0x72, 0x04, - 0x10, 0x33, 0xee, 0x04, 0x21, 0xf7, 0x02, 0xbf, 0xb3, 0x8c, 0xf2, 0x5b, 0x79, 0xf9, 0xc7, 0x8c, - 0xff, 0x04, 0xd9, 0x27, 0x4b, 0xb6, 0x19, 0xeb, 0x89, 0xd0, 0xf4, 0x7c, 0x8f, 0x3b, 0xc3, 0x31, - 0xf5, 0xfc, 0x4e, 0xbd, 0x4c, 0xf3, 0xd4, 0xf7, 0xf8, 0x03, 0xc1, 0x16, 0x9a, 0x9e, 0x9e, 0x88, - 0xab, 0x7c, 0x3a, 0x63, 0xd1, 0x55, 0xa7, 0x51, 0x76, 0x95, 0x9f, 0x0a, 0x96, 0xb8, 0x0a, 0xca, - 0x90, 0xfb, 0xb0, 0x32, 0x60, 0x23, 0xcf, 0x77, 0x06, 0x93, 0x60, 0xf8, 0xac, 0xd3, 0x44, 0x95, - 0x4e, 0x5e, 0xa5, 0x2f, 0x04, 0xfa, 0x82, 0x7f, 0xb2, 0x64, 0xc3, 0x20, 0x99, 0x91, 0x43, 0x68, - 0x0e, 0xc7, 0x6c, 0xf8, 0xcc, 0xe1, 0xf3, 0x8e, 0x89, 0x9a, 0xb7, 0xf3, 0x9a, 0x0f, 0x04, 0xf7, - 0xc9, 0xfc, 0x64, 0xc9, 0x6e, 0x0c, 0xe5, 0x90, 0xbc, 0x0f, 0x26, 0xf3, 0x5d, 0xb5, 0xdd, 0x0a, - 0x2a, 0x6d, 0x16, 0xbe, 0x8b, 0xef, 0xea, 0xcd, 0x9a, 0x4c, 0x8d, 0x49, 0x0f, 0xea, 0xc2, 0x51, - 0x3c, 0xde, 0x59, 0x45, 0x9d, 0x8d, 0xc2, 0x46, 0xc8, 0x3b, 0x59, 0xb2, 0x95, 0x94, 0x30, 0x9f, - 0xcb, 0x26, 0xde, 0x25, 0x8b, 0xc4, 0xe1, 0x6e, 0x95, 0x99, 0xef, 0xa1, 0xe4, 0xe3, 0xf1, 0x4c, - 0x57, 0x4f, 0xfa, 0x0d, 0x58, 0xbe, 0xa4, 0x93, 0x19, 0xb3, 0xde, 0x82, 0x95, 0x8c, 0xa7, 0x90, - 0x0e, 0x34, 0xa6, 0x2c, 0x8e, 0xe9, 0x88, 0x75, 0x8c, 0xbb, 0xc6, 0xae, 0x69, 0xeb, 0xa9, 0xd5, - 0x82, 0xd5, 0xac, 0x9f, 0x64, 0x14, 0x85, 0x2f, 0x08, 0xc5, 0x4b, 0x16, 0xc5, 0xc2, 0x01, 0x94, - 0xa2, 0x9a, 0x5a, 0x1f, 0xc0, 0x7a, 0xd1, 0x09, 0xc8, 0x3a, 0x54, 0x9f, 0xb1, 0x2b, 0x25, 0x29, - 0x86, 0x64, 0x43, 0x1d, 0x08, 0xbd, 0xd8, 0xb4, 0xd5, 0xe9, 0xbe, 0xa8, 0x24, 0xca, 0x89, 0x1f, - 0x90, 0x23, 0xa8, 0x89, 0x28, 0x44, 0xed, 0x95, 0xc3, 0x6e, 0x4f, 0x86, 0x68, 0x4f, 0x87, 0x68, - 0xef, 0x89, 0x0e, 0xd1, 0x7e, 0xf3, 0xcb, 0x17, 0x3b, 0x4b, 0x5f, 0xfc, 0x69, 0xc7, 0xb0, 0x51, - 0x83, 0xdc, 0x11, 0x9f, 0x92, 0x7a, 0xbe, 0xe3, 0xb9, 0x6a, 0x9f, 0x06, 0xce, 0x4f, 0x5d, 0x72, - 0x0c, 0xeb, 0xc3, 0xc0, 0x8f, 0x99, 0x1f, 0xcf, 0x62, 0x27, 0xa4, 0x11, 0x9d, 0xc6, 0x2a, 0x4a, - 0xf4, 0x87, 0x7b, 0xa0, 0xd9, 0x67, 0xc8, 0xb5, 0xdb, 0xc3, 0x3c, 0x81, 0x7c, 0x08, 0x70, 0x49, - 0x27, 0x9e, 0x4b, 0x79, 0x10, 0xc5, 0x9d, 0xda, 0xdd, 0x6a, 0x46, 0xf9, 0x5c, 0x33, 0x9e, 0x86, - 0x2e, 0xe5, 0xac, 0x5f, 0x13, 0x27, 0xb3, 0x33, 0xf2, 0xe4, 0x4d, 0x68, 0xd3, 0x30, 0x74, 0x62, - 0x4e, 0x39, 0x73, 0x06, 0x57, 0x9c, 0xc5, 0x18, 0x49, 0xab, 0xf6, 0x1a, 0x0d, 0xc3, 0xc7, 0x82, - 0xda, 0x17, 0x44, 0xcb, 0x4d, 0xbe, 0x03, 0x3a, 0x39, 0x21, 0x50, 0x73, 0x29, 0xa7, 0x68, 0x8d, - 0x55, 0x1b, 0xc7, 0x82, 0x16, 0x52, 0x3e, 0x56, 0x77, 0xc4, 0x31, 0xd9, 0x84, 0xfa, 0x98, 0x79, - 0xa3, 0x31, 0xc7, 0x6b, 0x55, 0x6d, 0x35, 0x13, 0x86, 0x0f, 0xa3, 0xe0, 0x92, 0x61, 0x9c, 0x37, - 0x6d, 0x39, 0xb1, 0xfe, 0x66, 0xc0, 0x6b, 0x37, 0x02, 0x43, 0xac, 0x3b, 0xa6, 0xf1, 0x58, 0xef, - 0x25, 0xc6, 0xe4, 0x6d, 0xb1, 0x2e, 0x75, 0x59, 0xa4, 0xf2, 0xcf, 0x9a, 0xba, 0xf1, 0x09, 0x12, - 0xd5, 0x45, 0x95, 0x08, 0x79, 0x04, 0xeb, 0x13, 0x1a, 0x73, 0x47, 0xfa, 0xaf, 0x83, 0xf9, 0xa5, - 0x9a, 0x8b, 0xa9, 0x4f, 0xa8, 0xf6, 0x73, 0xe1, 0x56, 0x4a, 0xbd, 0x35, 0xc9, 0x51, 0xc9, 0x09, - 0x6c, 0x0c, 0xae, 0x9e, 0x53, 0x9f, 0x7b, 0x3e, 0x73, 0x6e, 0xd8, 0xbc, 0xad, 0x96, 0x7a, 0x74, - 0xe9, 0xb9, 0xcc, 0x1f, 0x6a, 0x63, 0xdf, 0x4a, 0x54, 0x92, 0x8f, 0x11, 0x5b, 0x77, 0xa1, 0x95, - 0x8f, 0x62, 0xd2, 0x82, 0x0a, 0x9f, 0xab, 0x1b, 0x56, 0xf8, 0xdc, 0xb2, 0x12, 0x0f, 0x4c, 0x42, - 0xe9, 0x86, 0xcc, 0x1e, 0xb4, 0x0b, 0x61, 0x9d, 0x31, 0xb7, 0x91, 0x35, 0xb7, 0xd5, 0x86, 0xb5, - 0x5c, 0x34, 0x5b, 0x9f, 0x2f, 0x43, 0xd3, 0x66, 0x71, 0x28, 0x9c, 0x89, 0x1c, 0x81, 0xc9, 0xe6, - 0x43, 0x26, 0x13, 0xa9, 0x51, 0x48, 0x53, 0x52, 0xe6, 0x91, 0xe6, 0x8b, 0x80, 0x4e, 0x84, 0xc9, - 0x5e, 0x0e, 0x04, 0x6e, 0x15, 0x95, 0xb2, 0x28, 0xb0, 0x9f, 0x47, 0x81, 0x8d, 0x82, 0x6c, 0x01, - 0x06, 0xf6, 0x72, 0x30, 0x50, 0x5c, 0x38, 0x87, 0x03, 0xf7, 0x4a, 0x70, 0xa0, 0x78, 0xfc, 0x05, - 0x40, 0x70, 0xaf, 0x04, 0x08, 0x3a, 0x37, 0xf6, 0x2a, 0x45, 0x82, 0xfd, 0x3c, 0x12, 0x14, 0xaf, - 0x53, 0x80, 0x82, 0x0f, 0xcb, 0xa0, 0xe0, 0x4e, 0x41, 0x67, 0x21, 0x16, 0xbc, 0x77, 0x03, 0x0b, - 0x36, 0x0b, 0xaa, 0x25, 0x60, 0x70, 0x2f, 0x97, 0xa5, 0xa1, 0xf4, 0x6e, 0xe5, 0x69, 0x9a, 0x7c, - 0xef, 0x26, 0x8e, 0x6c, 0x15, 0x3f, 0x6d, 0x19, 0x90, 0x1c, 0x14, 0x80, 0xe4, 0x76, 0xf1, 0x94, - 0x05, 0x24, 0x49, 0xf1, 0x60, 0x4f, 0xc4, 0x7d, 0xc1, 0xd3, 0x44, 0x8e, 0x60, 0x51, 0x14, 0x44, - 0x2a, 0x61, 0xcb, 0x89, 0xb5, 0x2b, 0x32, 0x51, 0xea, 0x5f, 0x2f, 0xc1, 0x0e, 0x74, 0xfa, 0x8c, - 0x77, 0x59, 0xbf, 0x32, 0x52, 0x5d, 0x8c, 0xe8, 0x6c, 0x16, 0x33, 0x55, 0x16, 0xcb, 0x40, 0x4a, - 0x25, 0x07, 0x29, 0xe4, 0x3b, 0xf0, 0x1a, 0xa6, 0x11, 0xb4, 0x8b, 0x93, 0x4b, 0x6b, 0x6d, 0xc1, - 0x90, 0x06, 0x91, 0xf9, 0xed, 0x1d, 0xb8, 0x95, 0x91, 0x15, 0x29, 0x16, 0x53, 0x58, 0x0d, 0x83, - 0x77, 0x3d, 0x91, 0x3e, 0x0e, 0xc3, 0x13, 0x1a, 0x8f, 0xad, 0x1f, 0xa5, 0xf7, 0x4f, 0xe1, 0x8a, - 0x40, 0x6d, 0x18, 0xb8, 0xf2, 0x5a, 0x6b, 0x36, 0x8e, 0x05, 0x84, 0x4d, 0x82, 0x11, 0xee, 0x6a, - 0xda, 0x62, 0x28, 0xa4, 0x92, 0x48, 0x31, 0x65, 0x48, 0x58, 0xbf, 0x34, 0xd2, 0xf5, 0x52, 0x04, - 0x2b, 0x03, 0x1b, 0xe3, 0x7f, 0x01, 0x9b, 0xca, 0xab, 0x81, 0x8d, 0x75, 0x6d, 0xa4, 0x5f, 0x24, - 0x81, 0x91, 0xaf, 0x77, 0x45, 0xe1, 0x1c, 0x9e, 0xef, 0xb2, 0x39, 0x06, 0x7c, 0xd5, 0x96, 0x13, - 0x8d, 0xf0, 0x75, 0x34, 0x73, 0x1e, 0xe1, 0x1b, 0x48, 0x93, 0x13, 0xf2, 0x06, 0xc2, 0x4f, 0x70, - 0xa1, 0x22, 0x71, 0xad, 0xa7, 0xca, 0xdc, 0x33, 0x41, 0xb4, 0x25, 0x2f, 0x93, 0x4c, 0xcd, 0x1c, - 0x76, 0xbd, 0x0e, 0xa6, 0x38, 0x68, 0x1c, 0xd2, 0x21, 0xc3, 0xc0, 0x32, 0xed, 0x94, 0x60, 0x9d, - 0x01, 0xb9, 0x19, 0xd0, 0xe4, 0x03, 0xa8, 0x71, 0x3a, 0x12, 0xf6, 0x16, 0x26, 0x6b, 0xf5, 0x64, - 0x65, 0xde, 0xfb, 0xf8, 0xfc, 0x8c, 0x7a, 0x51, 0x7f, 0x53, 0x98, 0xea, 0x1f, 0x2f, 0x76, 0x5a, - 0x42, 0x66, 0x3f, 0x98, 0x7a, 0x9c, 0x4d, 0x43, 0x7e, 0x65, 0xa3, 0x8e, 0xf5, 0x4f, 0x43, 0x24, - 0xfa, 0x5c, 0xa0, 0x97, 0x1a, 0x4e, 0x7b, 0x73, 0x25, 0x83, 0xc9, 0x5f, 0xcd, 0x98, 0xdf, 0x04, - 0x18, 0xd1, 0xd8, 0xf9, 0x8c, 0xfa, 0x9c, 0xb9, 0xca, 0xa2, 0xe6, 0x88, 0xc6, 0x3f, 0x43, 0x82, - 0x28, 0x60, 0x04, 0x7b, 0x16, 0x33, 0x17, 0x4d, 0x5b, 0xb5, 0x1b, 0x23, 0x1a, 0x3f, 0x8d, 0x99, - 0x9b, 0xdc, 0xab, 0xf1, 0xea, 0xf7, 0xca, 0xdb, 0xb1, 0x59, 0xb4, 0xe3, 0xbf, 0x32, 0x3e, 0x9c, - 0x62, 0xe0, 0xff, 0xff, 0xbd, 0xff, 0x6e, 0x08, 0xe8, 0xcf, 0x67, 0x59, 0x72, 0x0a, 0xaf, 0x25, - 0x71, 0xe4, 0xcc, 0x30, 0xbe, 0xb4, 0x2f, 0xbd, 0x3c, 0xfc, 0xd6, 0x2f, 0xf3, 0xe4, 0x98, 0xfc, - 0x18, 0xb6, 0x0a, 0x59, 0x20, 0x59, 0xb0, 0xf2, 0xd2, 0x64, 0x70, 0x3b, 0x9f, 0x0c, 0xf4, 0x7a, - 0xda, 0x12, 0xd5, 0xaf, 0xe1, 0xd9, 0xdf, 0x12, 0x75, 0x50, 0x16, 0x1b, 0xca, 0xbe, 0xa5, 0xf5, - 0x0b, 0x03, 0xda, 0x85, 0xc3, 0x90, 0x03, 0x00, 0x99, 0x5a, 0x63, 0xef, 0xb9, 0xae, 0xc9, 0xd7, - 0xd5, 0xc1, 0xd1, 0x64, 0x8f, 0xbd, 0xe7, 0xcc, 0x36, 0x07, 0x7a, 0x48, 0x3e, 0x82, 0x36, 0x53, - 0x95, 0x99, 0xce, 0x7d, 0x95, 0x1c, 0x48, 0xe9, 0xba, 0x4d, 0xdd, 0xb6, 0xc5, 0x72, 0x73, 0xeb, - 0x18, 0xcc, 0x64, 0x5d, 0xf2, 0x0d, 0x30, 0xa7, 0x74, 0xae, 0xea, 0x65, 0x59, 0x69, 0x35, 0xa7, - 0x74, 0x8e, 0xa5, 0x32, 0xd9, 0x82, 0x86, 0x60, 0x8e, 0xa8, 0xdc, 0xa1, 0x6a, 0xd7, 0xa7, 0x74, - 0xfe, 0x43, 0x1a, 0x5b, 0x7b, 0xd0, 0xca, 0x6f, 0xa2, 0x45, 0x35, 0x76, 0x49, 0xd1, 0xe3, 0x11, - 0xb3, 0x1e, 0x43, 0x2b, 0x5f, 0x92, 0x8a, 0x3c, 0x16, 0x05, 0x33, 0xdf, 0x45, 0xc1, 0x65, 0x5b, - 0x4e, 0x44, 0x3f, 0x7a, 0x19, 0xc8, 0x4f, 0x97, 0xad, 0x41, 0xcf, 0x03, 0xce, 0x32, 0x85, 0xac, - 0x94, 0xb1, 0xfe, 0x50, 0x83, 0xba, 0xac, 0x8f, 0xc9, 0x9b, 0x99, 0x96, 0x04, 0xc1, 0xaf, 0xbf, - 0x72, 0xfd, 0x62, 0xa7, 0x81, 0x38, 0x71, 0xfa, 0x30, 0xed, 0x4f, 0xd2, 0x14, 0x58, 0xc9, 0xa5, - 0x40, 0xdd, 0x0c, 0x55, 0x5f, 0xb9, 0x19, 0xda, 0x82, 0x86, 0x3f, 0x9b, 0x3a, 0x7c, 0x1e, 0x63, - 0x24, 0x56, 0xed, 0xba, 0x3f, 0x9b, 0x3e, 0x99, 0xc7, 0xc2, 0xa6, 0x3c, 0xe0, 0x74, 0x82, 0x2c, - 0x19, 0x8a, 0x4d, 0x24, 0x08, 0xe6, 0x11, 0xac, 0x65, 0xe0, 0xd4, 0x73, 0x55, 0xad, 0xd6, 0xca, - 0x7e, 0xf1, 0xd3, 0x87, 0xea, 0xba, 0x2b, 0x09, 0xbc, 0x9e, 0xba, 0x64, 0x37, 0x5f, 0xfb, 0x23, - 0x0a, 0x4b, 0x28, 0xc8, 0x94, 0xf7, 0x02, 0x83, 0xc5, 0x01, 0x84, 0xbb, 0x49, 0x91, 0x26, 0x8a, - 0x34, 0x05, 0x01, 0x99, 0x6f, 0x41, 0x3b, 0x05, 0x32, 0x29, 0x62, 0xca, 0x55, 0x52, 0x32, 0x0a, - 0xbe, 0x0b, 0x1b, 0x3e, 0x9b, 0x73, 0xa7, 0x28, 0x0d, 0x28, 0x4d, 0x04, 0xef, 0x3c, 0xaf, 0xf1, - 0x6d, 0x68, 0xa5, 0x01, 0x89, 0xb2, 0x2b, 0xb2, 0x03, 0x4b, 0xa8, 0x28, 0x76, 0x07, 0x9a, 0x49, - 0x19, 0xb1, 0x8a, 0x02, 0x0d, 0x2a, 0xab, 0x87, 0xa4, 0x30, 0x89, 0x58, 0x3c, 0x9b, 0x70, 0xb5, - 0xc8, 0x1a, 0xca, 0x60, 0x61, 0x62, 0x4b, 0x3a, 0xca, 0xbe, 0x01, 0x6b, 0x49, 0x1c, 0xa0, 0x5c, - 0x0b, 0xe5, 0x56, 0x35, 0x11, 0x85, 0xf6, 0x60, 0x3d, 0x8c, 0x82, 0x30, 0x88, 0x59, 0xe4, 0x50, - 0xd7, 0x8d, 0x58, 0x1c, 0x77, 0xda, 0x72, 0x3d, 0x4d, 0x3f, 0x96, 0x64, 0xeb, 0xe7, 0xd0, 0x50, - 0xd6, 0x2f, 0xed, 0xd3, 0xbe, 0x0f, 0xab, 0x21, 0x8d, 0xc4, 0x99, 0xb2, 0xdd, 0x9a, 0xae, 0x96, - 0xcf, 0x68, 0x24, 0xda, 0xf3, 0x5c, 0xd3, 0xb6, 0x82, 0xf2, 0x92, 0x64, 0xdd, 0x83, 0xb5, 0x9c, - 0x8c, 0x08, 0x03, 0x74, 0x0a, 0x1d, 0x06, 0x38, 0x49, 0x76, 0xae, 0xa4, 0x3b, 0x5b, 0xf7, 0xc1, - 0x4c, 0x0c, 0x2d, 0x8a, 0x3a, 0x7d, 0x0f, 0x43, 0xd9, 0x4e, 0x4e, 0xb1, 0x11, 0x0d, 0x3e, 0x63, - 0x91, 0x2a, 0xe4, 0xe4, 0xc4, 0x7a, 0x0a, 0xed, 0x42, 0x3e, 0x25, 0xfb, 0xd0, 0x08, 0x67, 0x03, - 0x47, 0x3f, 0x20, 0xa4, 0x2d, 0xe7, 0xd9, 0x6c, 0xf0, 0x31, 0xbb, 0xd2, 0x2d, 0x67, 0x88, 0xb3, - 0x74, 0xd9, 0x4a, 0x76, 0xd9, 0x09, 0x34, 0x75, 0x68, 0x92, 0xef, 0x82, 0x99, 0xf8, 0x48, 0x21, - 0x81, 0x25, 0x5b, 0xab, 0x45, 0x53, 0x41, 0xf1, 0xa9, 0x63, 0x6f, 0xe4, 0x33, 0xd7, 0x49, 0xe3, - 0x01, 0xf7, 0x68, 0xda, 0x6d, 0xc9, 0xf8, 0x44, 0x3b, 0xbf, 0xf5, 0x2e, 0xd4, 0xe5, 0xd9, 0x84, - 0x7d, 0xc4, 0xca, 0xba, 0xce, 0x15, 0xe3, 0xd2, 0x4c, 0xfb, 0x47, 0x03, 0x9a, 0x3a, 0x45, 0x95, - 0x2a, 0xe5, 0x0e, 0x5d, 0xf9, 0xaa, 0x87, 0x5e, 0xf4, 0x08, 0xa0, 0xb3, 0x48, 0xed, 0x95, 0xb3, - 0xc8, 0x3e, 0x10, 0x99, 0x2c, 0x2e, 0x03, 0xee, 0xf9, 0x23, 0x47, 0xda, 0x5a, 0x66, 0x8d, 0x75, - 0xe4, 0x9c, 0x23, 0xe3, 0x4c, 0xd0, 0x0f, 0x3f, 0x5f, 0x86, 0xf6, 0x71, 0xff, 0xc1, 0xe9, 0x71, - 0x18, 0x4e, 0xbc, 0x21, 0xc5, 0xe2, 0xfa, 0x00, 0x6a, 0xd8, 0x3e, 0x94, 0x3c, 0x5c, 0x76, 0xcb, - 0xfa, 0x58, 0x72, 0x08, 0xcb, 0xd8, 0x45, 0x90, 0xb2, 0xf7, 0xcb, 0x6e, 0x69, 0x3b, 0x2b, 0x36, - 0x91, 0x7d, 0xc6, 0xcd, 0x67, 0xcc, 0x6e, 0x59, 0x4f, 0x4b, 0x3e, 0x02, 0x33, 0xad, 0xff, 0x17, - 0x3d, 0x66, 0x76, 0x17, 0x76, 0xb7, 0x42, 0x3f, 0xad, 0x95, 0x16, 0xbd, 0xc9, 0x75, 0x17, 0xb6, - 0x81, 0xe4, 0x08, 0x1a, 0xba, 0xc2, 0x2c, 0x7f, 0x6e, 0xec, 0x2e, 0xe8, 0x3c, 0x85, 0x79, 0x64, - 0x49, 0x5f, 0xf6, 0x26, 0xda, 0x2d, 0x6d, 0x8f, 0xc9, 0xfb, 0x50, 0x57, 0xb0, 0x5f, 0xfa, 0xe4, - 0xd8, 0x2d, 0xef, 0x1f, 0xc5, 0x25, 0xd3, 0xa6, 0x66, 0xd1, 0xbb, 0x6d, 0x77, 0x61, 0x1f, 0x4f, - 0x8e, 0x01, 0x32, 0x95, 0xf9, 0xc2, 0x07, 0xd9, 0xee, 0xe2, 0xfe, 0x9c, 0xdc, 0x87, 0x66, 0xfa, - 0xe6, 0x52, 0xfe, 0xc4, 0xda, 0x5d, 0xd4, 0x32, 0xf7, 0x5f, 0xff, 0xf7, 0x5f, 0xb6, 0x8d, 0x5f, - 0x5f, 0x6f, 0x1b, 0xbf, 0xb9, 0xde, 0x36, 0xbe, 0xbc, 0xde, 0x36, 0x7e, 0x7f, 0xbd, 0x6d, 0xfc, - 0xf9, 0x7a, 0xdb, 0xf8, 0xed, 0x5f, 0xb7, 0x8d, 0x41, 0x1d, 0xdd, 0xff, 0xbd, 0xff, 0x04, 0x00, - 0x00, 0xff, 0xff, 0x2c, 0x0a, 0x65, 0x88, 0x52, 0x18, 0x00, 0x00, +var fileDescriptor_types_5b877df1938afe10 = []byte{ + // 2214 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xdc, 0x58, 0xcb, 0x73, 0x1b, 0xc7, + 0xd1, 0xe7, 0x82, 0x20, 0x81, 0x6d, 0x10, 0x0f, 0x8d, 0x28, 0x09, 0xc2, 0xe7, 0x8f, 0x54, 0xad, + 0x12, 0x5b, 0x8c, 0x65, 0xd0, 0xa6, 0xa3, 0x14, 0x65, 0x39, 0xa9, 0x22, 0x24, 0xc5, 0x64, 0xd9, + 0x49, 0x98, 0x95, 0xc4, 0x5c, 0x52, 0xb5, 0x35, 0xc0, 0x8e, 0x80, 0x2d, 0x02, 0xbb, 0xeb, 0xdd, + 0x01, 0x0d, 0xea, 0x98, 0xb3, 0x0f, 0x3e, 0xe4, 0x8f, 0xc8, 0x35, 0x37, 0x1f, 0x73, 0x4a, 0xf9, + 0x98, 0x43, 0xce, 0x4a, 0xc2, 0x54, 0x0e, 0xc9, 0x35, 0x95, 0xaa, 0x1c, 0x53, 0xd3, 0x33, 0xb3, + 0x2f, 0x2e, 0x14, 0xcb, 0xc9, 0x29, 0x17, 0x60, 0xa6, 0x1f, 0xf3, 0xe8, 0xed, 0xee, 0x5f, 0xf7, + 0xc0, 0x75, 0x3a, 0x1c, 0x79, 0xbb, 0xfc, 0x3c, 0x64, 0xb1, 0xfc, 0xed, 0x87, 0x51, 0xc0, 0x03, + 0xb2, 0x86, 0x93, 0xde, 0x3b, 0x63, 0x8f, 0x4f, 0xe6, 0xc3, 0xfe, 0x28, 0x98, 0xed, 0x8e, 0x83, + 0x71, 0xb0, 0x8b, 0xdc, 0xe1, 0xfc, 0x39, 0xce, 0x70, 0x82, 0x23, 0xa9, 0xd5, 0xdb, 0x1e, 0x07, + 0xc1, 0x78, 0xca, 0x52, 0x29, 0xee, 0xcd, 0x58, 0xcc, 0xe9, 0x2c, 0x54, 0x02, 0xfb, 0x99, 0xf5, + 0x38, 0xf3, 0x5d, 0x16, 0xcd, 0x3c, 0x9f, 0x67, 0x87, 0x53, 0x6f, 0x18, 0xef, 0x8e, 0x82, 0xd9, + 0x2c, 0xf0, 0xb3, 0x07, 0xea, 0x3d, 0xf8, 0xb7, 0x9a, 0xa3, 0xe8, 0x3c, 0xe4, 0xc1, 0xee, 0x8c, + 0x45, 0xa7, 0x53, 0xa6, 0xfe, 0xa4, 0xb2, 0xf5, 0xdb, 0x2a, 0xd4, 0x6c, 0xf6, 0xe9, 0x9c, 0xc5, + 0x9c, 0xdc, 0x81, 0x2a, 0x1b, 0x4d, 0x82, 0x6e, 0xe5, 0x96, 0x71, 0xa7, 0xb1, 0x47, 0xfa, 0x72, + 0x13, 0xc5, 0x7d, 0x3c, 0x9a, 0x04, 0x87, 0x2b, 0x36, 0x4a, 0x90, 0xb7, 0x61, 0xed, 0xf9, 0x74, + 0x1e, 0x4f, 0xba, 0xab, 0x28, 0x7a, 0x35, 0x2f, 0xfa, 0x43, 0xc1, 0x3a, 0x5c, 0xb1, 0xa5, 0x8c, + 0x58, 0xd6, 0xf3, 0x9f, 0x07, 0xdd, 0x6a, 0xd9, 0xb2, 0x47, 0xfe, 0x73, 0x5c, 0x56, 0x48, 0x90, + 0x7d, 0x80, 0x98, 0x71, 0x27, 0x08, 0xb9, 0x17, 0xf8, 0xdd, 0x35, 0x94, 0xbf, 0x91, 0x97, 0x7f, + 0xc2, 0xf8, 0x4f, 0x90, 0x7d, 0xb8, 0x62, 0x9b, 0xb1, 0x9e, 0x08, 0x4d, 0xcf, 0xf7, 0xb8, 0x33, + 0x9a, 0x50, 0xcf, 0xef, 0xae, 0x97, 0x69, 0x1e, 0xf9, 0x1e, 0x7f, 0x28, 0xd8, 0x42, 0xd3, 0xd3, + 0x13, 0x71, 0x95, 0x4f, 0xe7, 0x2c, 0x3a, 0xef, 0xd6, 0xca, 0xae, 0xf2, 0x53, 0xc1, 0x12, 0x57, + 0x41, 0x19, 0xf2, 0x00, 0x1a, 0x43, 0x36, 0xf6, 0x7c, 0x67, 0x38, 0x0d, 0x46, 0xa7, 0xdd, 0x3a, + 0xaa, 0x74, 0xf3, 0x2a, 0x03, 0x21, 0x30, 0x10, 0xfc, 0xc3, 0x15, 0x1b, 0x86, 0xc9, 0x8c, 0xec, + 0x41, 0x7d, 0x34, 0x61, 0xa3, 0x53, 0x87, 0x2f, 0xba, 0x26, 0x6a, 0x5e, 0xcb, 0x6b, 0x3e, 0x14, + 0xdc, 0xa7, 0x8b, 0xc3, 0x15, 0xbb, 0x36, 0x92, 0x43, 0x72, 0x0f, 0x4c, 0xe6, 0xbb, 0x6a, 0xbb, + 0x06, 0x2a, 0x5d, 0x2f, 0x7c, 0x17, 0xdf, 0xd5, 0x9b, 0xd5, 0x99, 0x1a, 0x93, 0x3e, 0xac, 0x0b, + 0x47, 0xf1, 0x78, 0x77, 0x03, 0x75, 0x36, 0x0b, 0x1b, 0x21, 0xef, 0x70, 0xc5, 0x56, 0x52, 0xc2, + 0x7c, 0x2e, 0x9b, 0x7a, 0x67, 0x2c, 0x12, 0x87, 0xbb, 0x5a, 0x66, 0xbe, 0x47, 0x92, 0x8f, 0xc7, + 0x33, 0x5d, 0x3d, 0x19, 0xd4, 0x60, 0xed, 0x8c, 0x4e, 0xe7, 0xcc, 0x7a, 0x0b, 0x1a, 0x19, 0x4f, + 0x21, 0x5d, 0xa8, 0xcd, 0x58, 0x1c, 0xd3, 0x31, 0xeb, 0x1a, 0xb7, 0x8c, 0x3b, 0xa6, 0xad, 0xa7, + 0x56, 0x0b, 0x36, 0xb2, 0x7e, 0x62, 0xcd, 0x12, 0x45, 0xe1, 0x0b, 0x42, 0xf1, 0x8c, 0x45, 0xb1, + 0x70, 0x00, 0xa5, 0xa8, 0xa6, 0xe4, 0x36, 0x34, 0xd1, 0x0e, 0x8e, 0xe6, 0x0b, 0x3f, 0xad, 0xda, + 0x1b, 0x48, 0x3c, 0x51, 0x42, 0xdb, 0xd0, 0x08, 0xf7, 0xc2, 0x44, 0x64, 0x15, 0x45, 0x20, 0xdc, + 0x0b, 0x95, 0x80, 0xf5, 0x01, 0x74, 0x8a, 0xae, 0x44, 0x3a, 0xb0, 0x7a, 0xca, 0xce, 0xd5, 0x7e, + 0x62, 0x48, 0x36, 0xd5, 0xb5, 0x70, 0x0f, 0xd3, 0x56, 0x77, 0xfc, 0xa2, 0x92, 0x28, 0x27, 0xde, + 0x44, 0xf6, 0xa1, 0x2a, 0x62, 0x19, 0xb5, 0x1b, 0x7b, 0xbd, 0xbe, 0x0c, 0xf4, 0xbe, 0x0e, 0xf4, + 0xfe, 0x53, 0x1d, 0xe8, 0x83, 0xfa, 0x57, 0x2f, 0xb7, 0x57, 0xbe, 0xf8, 0xc3, 0xb6, 0x61, 0xa3, + 0x06, 0xb9, 0x29, 0x1c, 0x82, 0x7a, 0xbe, 0xe3, 0xb9, 0x6a, 0x9f, 0x1a, 0xce, 0x8f, 0x5c, 0x72, + 0x00, 0x9d, 0x51, 0xe0, 0xc7, 0xcc, 0x8f, 0xe7, 0xb1, 0x13, 0xd2, 0x88, 0xce, 0x62, 0x15, 0x6b, + 0xfa, 0xf3, 0x3f, 0xd4, 0xec, 0x63, 0xe4, 0xda, 0xed, 0x51, 0x9e, 0x40, 0x3e, 0x04, 0x38, 0xa3, + 0x53, 0xcf, 0xa5, 0x3c, 0x88, 0xe2, 0x6e, 0xf5, 0xd6, 0x6a, 0x46, 0xf9, 0x44, 0x33, 0x9e, 0x85, + 0x2e, 0xe5, 0x6c, 0x50, 0x15, 0x27, 0xb3, 0x33, 0xf2, 0xe4, 0x4d, 0x68, 0xd3, 0x30, 0x74, 0x62, + 0x4e, 0x39, 0x73, 0x86, 0xe7, 0x9c, 0xc5, 0x18, 0x8f, 0x1b, 0x76, 0x93, 0x86, 0xe1, 0x13, 0x41, + 0x1d, 0x08, 0xa2, 0xe5, 0x26, 0x5f, 0x13, 0x43, 0x85, 0x10, 0xa8, 0xba, 0x94, 0x53, 0xb4, 0xc6, + 0x86, 0x8d, 0x63, 0x41, 0x0b, 0x29, 0x9f, 0xa8, 0x3b, 0xe2, 0x98, 0x5c, 0x87, 0xf5, 0x09, 0xf3, + 0xc6, 0x13, 0x8e, 0xd7, 0x5a, 0xb5, 0xd5, 0x4c, 0x18, 0x3e, 0x8c, 0x82, 0x33, 0x86, 0xd9, 0xa2, + 0x6e, 0xcb, 0x89, 0xf5, 0x17, 0x03, 0xae, 0x5c, 0x0a, 0x2f, 0xb1, 0xee, 0x84, 0xc6, 0x13, 0xbd, + 0x97, 0x18, 0x93, 0xb7, 0xc5, 0xba, 0xd4, 0x65, 0x91, 0xca, 0x62, 0x4d, 0x75, 0xe3, 0x43, 0x24, + 0xaa, 0x8b, 0x2a, 0x11, 0xf2, 0x18, 0x3a, 0x53, 0x1a, 0x73, 0x47, 0x46, 0x81, 0x83, 0x59, 0x6a, + 0x35, 0x17, 0x99, 0x9f, 0x50, 0x1d, 0x2d, 0xc2, 0x39, 0x95, 0x7a, 0x6b, 0x9a, 0xa3, 0x92, 0x43, + 0xd8, 0x1c, 0x9e, 0xbf, 0xa0, 0x3e, 0xf7, 0x7c, 0xe6, 0x5c, 0xb2, 0x79, 0x5b, 0x2d, 0xf5, 0xf8, + 0xcc, 0x73, 0x99, 0x3f, 0xd2, 0xc6, 0xbe, 0x9a, 0xa8, 0x24, 0x1f, 0x23, 0xb6, 0x6e, 0x41, 0x2b, + 0x9f, 0x0b, 0x48, 0x0b, 0x2a, 0x7c, 0xa1, 0x6e, 0x58, 0xe1, 0x0b, 0xcb, 0x4a, 0x3c, 0x30, 0x09, + 0xc8, 0x4b, 0x32, 0x3b, 0xd0, 0x2e, 0x24, 0x87, 0x8c, 0xb9, 0x8d, 0xac, 0xb9, 0xad, 0x36, 0x34, + 0x73, 0x39, 0xc1, 0xfa, 0x7c, 0x0d, 0xea, 0x36, 0x8b, 0x43, 0xe1, 0x4c, 0x64, 0x1f, 0x4c, 0xb6, + 0x18, 0x31, 0x99, 0x8e, 0x8d, 0x42, 0xb2, 0x93, 0x32, 0x8f, 0x35, 0x5f, 0xa4, 0x85, 0x44, 0x98, + 0xec, 0xe4, 0xa0, 0xe4, 0x6a, 0x51, 0x29, 0x8b, 0x25, 0x77, 0xf3, 0x58, 0xb2, 0x59, 0x90, 0x2d, + 0x80, 0xc9, 0x4e, 0x0e, 0x4c, 0x8a, 0x0b, 0xe7, 0xd0, 0xe4, 0x7e, 0x09, 0x9a, 0x14, 0x8f, 0xbf, + 0x04, 0x4e, 0xee, 0x97, 0xc0, 0x49, 0xf7, 0xd2, 0x5e, 0xa5, 0x78, 0x72, 0x37, 0x8f, 0x27, 0xc5, + 0xeb, 0x14, 0x00, 0xe5, 0xc3, 0x32, 0x40, 0xb9, 0x59, 0xd0, 0x59, 0x8a, 0x28, 0xef, 0x5f, 0x42, + 0x94, 0xeb, 0x05, 0xd5, 0x12, 0x48, 0xb9, 0x9f, 0xcb, 0xf5, 0x50, 0x7a, 0xb7, 0xf2, 0x64, 0x4f, + 0xbe, 0x77, 0x19, 0x8d, 0x6e, 0x14, 0x3f, 0x6d, 0x19, 0x1c, 0xed, 0x16, 0xe0, 0xe8, 0x5a, 0xf1, + 0x94, 0x05, 0x3c, 0x4a, 0x51, 0x65, 0x47, 0xc4, 0x7d, 0xc1, 0xd3, 0x44, 0x8e, 0x60, 0x51, 0x14, + 0x44, 0x2a, 0x61, 0xcb, 0x89, 0x75, 0x47, 0x64, 0xa2, 0xd4, 0xbf, 0x5e, 0x81, 0x40, 0xe8, 0xf4, + 0x19, 0xef, 0xb2, 0xbe, 0x34, 0x52, 0x5d, 0x8c, 0xe8, 0x6c, 0x16, 0x33, 0x55, 0x16, 0xcb, 0x00, + 0x53, 0x25, 0x0f, 0x4c, 0xdb, 0xd0, 0x10, 0xb9, 0xb2, 0x80, 0x39, 0x34, 0xd4, 0x98, 0x43, 0xbe, + 0x03, 0x57, 0x30, 0xcf, 0x48, 0xf8, 0x52, 0x81, 0x58, 0xc5, 0x40, 0x6c, 0x0b, 0x86, 0xb4, 0x98, + 0x4c, 0x80, 0xef, 0xc0, 0xd5, 0x8c, 0xac, 0x58, 0x17, 0x73, 0x9c, 0x4c, 0xbe, 0x9d, 0x44, 0xfa, + 0x20, 0x0c, 0x0f, 0x69, 0x3c, 0xb1, 0x7e, 0x94, 0x1a, 0x28, 0xc5, 0x33, 0x02, 0xd5, 0x51, 0xe0, + 0xca, 0x7b, 0x37, 0x6d, 0x1c, 0x0b, 0x8c, 0x9b, 0x06, 0x63, 0x3c, 0x9c, 0x69, 0x8b, 0xa1, 0x90, + 0x4a, 0x42, 0xc9, 0x94, 0x31, 0x63, 0xfd, 0xd2, 0x48, 0xd7, 0x4b, 0x21, 0xae, 0x0c, 0x8d, 0x8c, + 0xff, 0x04, 0x8d, 0x2a, 0xaf, 0x87, 0x46, 0xd6, 0x85, 0x91, 0x7e, 0xb2, 0x04, 0x67, 0xbe, 0xd9, + 0x15, 0x85, 0xf7, 0x78, 0xbe, 0xcb, 0x16, 0x68, 0xd2, 0x55, 0x5b, 0x4e, 0x74, 0x09, 0xb0, 0x8e, + 0x66, 0xce, 0x97, 0x00, 0x35, 0xa4, 0xc9, 0x09, 0xb9, 0x8d, 0xf8, 0x14, 0x3c, 0x57, 0xa1, 0xda, + 0xec, 0xab, 0x6a, 0xfa, 0x58, 0x10, 0x6d, 0xc9, 0xcb, 0x64, 0x5b, 0x33, 0x07, 0x6e, 0x6f, 0x80, + 0x29, 0x0e, 0x1a, 0x87, 0x74, 0xc4, 0x30, 0xf2, 0x4c, 0x3b, 0x25, 0x58, 0xc7, 0x40, 0x2e, 0x47, + 0x3c, 0xf9, 0x00, 0xaa, 0x9c, 0x8e, 0x85, 0xbd, 0x85, 0xc9, 0x5a, 0x7d, 0xd9, 0x00, 0xf4, 0x3f, + 0x3e, 0x39, 0xa6, 0x5e, 0x34, 0xb8, 0x2e, 0x4c, 0xf5, 0xb7, 0x97, 0xdb, 0x2d, 0x21, 0x73, 0x37, + 0x98, 0x79, 0x9c, 0xcd, 0x42, 0x7e, 0x6e, 0xa3, 0x8e, 0xf5, 0x77, 0x43, 0x20, 0x41, 0x2e, 0x13, + 0x94, 0x1a, 0x4e, 0xbb, 0x7b, 0x25, 0x03, 0xda, 0x5f, 0xcf, 0x98, 0xff, 0x0f, 0x30, 0xa6, 0xb1, + 0xf3, 0x19, 0xf5, 0x39, 0x73, 0x95, 0x45, 0xcd, 0x31, 0x8d, 0x7f, 0x86, 0x04, 0x51, 0xe1, 0x08, + 0xf6, 0x3c, 0x66, 0x2e, 0x9a, 0x76, 0xd5, 0xae, 0x8d, 0x69, 0xfc, 0x2c, 0x66, 0x6e, 0x72, 0xaf, + 0xda, 0xeb, 0xdf, 0x2b, 0x6f, 0xc7, 0x7a, 0xd1, 0x8e, 0xff, 0xc8, 0xf8, 0x70, 0x0a, 0x92, 0xff, + 0xfb, 0xf7, 0xfe, 0xab, 0x21, 0x6a, 0x83, 0x7c, 0x1a, 0x26, 0x47, 0x70, 0x25, 0x89, 0x23, 0x67, + 0x8e, 0xf1, 0xa5, 0x7d, 0xe9, 0xd5, 0xe1, 0xd7, 0x39, 0xcb, 0x93, 0x63, 0xf2, 0x63, 0xb8, 0x51, + 0xc8, 0x02, 0xc9, 0x82, 0x95, 0x57, 0x26, 0x83, 0x6b, 0xf9, 0x64, 0xa0, 0xd7, 0xd3, 0x96, 0x58, + 0xfd, 0x06, 0x9e, 0xfd, 0x2d, 0x51, 0x28, 0x65, 0xc1, 0xa3, 0xec, 0x5b, 0x5a, 0xbf, 0x36, 0xa0, + 0x5d, 0x38, 0x0c, 0xb9, 0x07, 0x20, 0x53, 0x6b, 0xec, 0xbd, 0x60, 0x85, 0x2c, 0x86, 0x26, 0x7b, + 0xe2, 0xbd, 0x60, 0xea, 0xe0, 0xe6, 0x50, 0x13, 0xc8, 0x7b, 0x50, 0x67, 0xaa, 0x80, 0x53, 0xb7, + 0xbd, 0x56, 0xa8, 0xeb, 0x94, 0x4e, 0x22, 0x46, 0xbe, 0x0b, 0x66, 0x62, 0xc3, 0x42, 0xf1, 0x9e, + 0x98, 0x5c, 0x6f, 0x94, 0x08, 0x5a, 0x1f, 0x41, 0xbb, 0x70, 0x0c, 0xf2, 0x7f, 0x60, 0xce, 0xe8, + 0x42, 0x55, 0xe1, 0xb2, 0x7e, 0xab, 0xcf, 0xe8, 0x02, 0x0b, 0x70, 0x72, 0x03, 0x6a, 0x82, 0x39, + 0xa6, 0xf2, 0x2b, 0xac, 0xda, 0xeb, 0x33, 0xba, 0xf8, 0x88, 0xc6, 0xd6, 0x0e, 0xb4, 0xf2, 0x47, + 0xd3, 0xa2, 0x1a, 0x11, 0xa5, 0xe8, 0xc1, 0x98, 0x59, 0xf7, 0xa0, 0x5d, 0x38, 0x11, 0xb1, 0xa0, + 0x19, 0xce, 0x87, 0xce, 0x29, 0x3b, 0x77, 0xf0, 0xc8, 0xe8, 0x33, 0xa6, 0xdd, 0x08, 0xe7, 0xc3, + 0x8f, 0xd9, 0xf9, 0x53, 0x41, 0xb2, 0x9e, 0x40, 0x2b, 0x5f, 0x1f, 0x8b, 0x9c, 0x19, 0x05, 0x73, + 0xdf, 0xc5, 0xf5, 0xd7, 0x6c, 0x39, 0x11, 0x2d, 0xf6, 0x59, 0x20, 0xdd, 0x24, 0x5b, 0x10, 0x9f, + 0x04, 0x9c, 0x65, 0xaa, 0x6a, 0x29, 0x63, 0xfd, 0x62, 0x0d, 0xd6, 0x65, 0xb1, 0x4e, 0xfa, 0xf9, + 0x56, 0x50, 0xf8, 0x88, 0xd2, 0x94, 0x54, 0xa5, 0x98, 0xe0, 0xf0, 0x9b, 0xc5, 0x7e, 0x6a, 0xd0, + 0xb8, 0x78, 0xb9, 0x5d, 0x43, 0x0c, 0x3b, 0x7a, 0x94, 0x36, 0x57, 0xcb, 0x7a, 0x0f, 0xdd, 0xc9, + 0x55, 0x5f, 0xbb, 0x93, 0xbb, 0x01, 0x35, 0x7f, 0x3e, 0x73, 0xf8, 0x22, 0x56, 0xb9, 0x60, 0xdd, + 0x9f, 0xcf, 0x9e, 0x2e, 0xf0, 0xd3, 0xf1, 0x80, 0xd3, 0x29, 0xb2, 0x64, 0x26, 0xa8, 0x23, 0x41, + 0x30, 0xf7, 0xa1, 0x99, 0x81, 0x7a, 0xcf, 0x55, 0x25, 0x63, 0x2b, 0xeb, 0x8d, 0x47, 0x8f, 0xd4, + 0x2d, 0x1b, 0x09, 0xf4, 0x1f, 0xb9, 0xe4, 0x4e, 0xbe, 0x71, 0xc1, 0x0a, 0xa1, 0x8e, 0x8e, 0x9f, + 0xe9, 0x4d, 0x44, 0x7d, 0x20, 0x0e, 0x20, 0x42, 0x41, 0x8a, 0x98, 0x28, 0x52, 0x17, 0x04, 0x64, + 0xbe, 0x05, 0xed, 0x14, 0x64, 0xa5, 0x08, 0xc8, 0x55, 0x52, 0x32, 0x0a, 0xbe, 0x0b, 0x9b, 0x3e, + 0x5b, 0x70, 0xa7, 0x28, 0xdd, 0x40, 0x69, 0x22, 0x78, 0x27, 0x79, 0x8d, 0x6f, 0x43, 0x2b, 0x4d, + 0x16, 0x28, 0xbb, 0x21, 0xdb, 0xc7, 0x84, 0x8a, 0x62, 0x37, 0xa1, 0x9e, 0x94, 0x38, 0x4d, 0x14, + 0xa8, 0x51, 0x59, 0xd9, 0x24, 0x45, 0x53, 0xc4, 0xe2, 0xf9, 0x94, 0xab, 0x45, 0x5a, 0x28, 0x83, + 0x45, 0x93, 0x2d, 0xe9, 0x28, 0x7b, 0x1b, 0x9a, 0x3a, 0xec, 0xa4, 0x5c, 0x1b, 0xe5, 0x36, 0x34, + 0x11, 0x85, 0x76, 0xa0, 0x13, 0x46, 0x41, 0x18, 0xc4, 0x2c, 0x72, 0xa8, 0xeb, 0x46, 0x2c, 0x8e, + 0xbb, 0x1d, 0xb9, 0x9e, 0xa6, 0x1f, 0x48, 0xb2, 0xf5, 0x1e, 0xd4, 0x74, 0xed, 0xb6, 0x09, 0x6b, + 0x68, 0x75, 0x74, 0xc1, 0xaa, 0x2d, 0x27, 0x02, 0x25, 0x0e, 0xc2, 0x50, 0xbd, 0x40, 0x88, 0xa1, + 0xf5, 0x73, 0xa8, 0xa9, 0x0f, 0x56, 0xda, 0x97, 0x7e, 0x1f, 0x36, 0x42, 0x1a, 0x89, 0x6b, 0x64, + 0xbb, 0x53, 0xdd, 0x1d, 0x1c, 0xd3, 0x88, 0x3f, 0x61, 0x3c, 0xd7, 0xa4, 0x36, 0x50, 0x5e, 0x92, + 0xac, 0xfb, 0xd0, 0xcc, 0xc9, 0x88, 0x63, 0xa1, 0x1f, 0xe9, 0x48, 0xc3, 0x49, 0xb2, 0x73, 0x25, + 0xdd, 0xd9, 0x7a, 0x00, 0x66, 0xf2, 0x6d, 0x44, 0x11, 0xab, 0xaf, 0x6e, 0x28, 0x73, 0xcb, 0x29, + 0x36, 0xde, 0xc1, 0x67, 0x2c, 0x52, 0x31, 0x21, 0x27, 0xd6, 0xb3, 0x4c, 0x66, 0x90, 0x79, 0x9b, + 0xdc, 0x85, 0x9a, 0xca, 0x0c, 0x2a, 0x2a, 0x75, 0x8b, 0x7d, 0x8c, 0xa9, 0x41, 0xb7, 0xd8, 0x32, + 0x51, 0xa4, 0xcb, 0x56, 0xb2, 0xcb, 0x4e, 0xa1, 0xae, 0xa3, 0x3f, 0x9f, 0x26, 0xe5, 0x8a, 0x9d, + 0x62, 0x9a, 0x54, 0x8b, 0xa6, 0x82, 0xc2, 0x3b, 0x62, 0x6f, 0xec, 0x33, 0xd7, 0x49, 0x43, 0x08, + 0xf7, 0xa8, 0xdb, 0x6d, 0xc9, 0xf8, 0x44, 0xc7, 0x8b, 0xf5, 0x2e, 0xac, 0xcb, 0xb3, 0x09, 0xfb, + 0x88, 0x95, 0x75, 0x5d, 0x2f, 0xc6, 0xa5, 0xc0, 0xf1, 0x7b, 0x03, 0xea, 0x3a, 0x79, 0x96, 0x2a, + 0xe5, 0x0e, 0x5d, 0xf9, 0xba, 0x87, 0xfe, 0xef, 0x27, 0x9e, 0xbb, 0x40, 0x64, 0x7e, 0x39, 0x0b, + 0xb8, 0xe7, 0x8f, 0x1d, 0x69, 0x6b, 0x99, 0x83, 0x3a, 0xc8, 0x39, 0x41, 0xc6, 0xb1, 0xa0, 0xef, + 0x7d, 0xbe, 0x06, 0xed, 0x83, 0xc1, 0xc3, 0xa3, 0x83, 0x30, 0x9c, 0x7a, 0x23, 0x8a, 0xbd, 0xc2, + 0x2e, 0x54, 0xb1, 0x5d, 0x2a, 0x79, 0xee, 0xed, 0x95, 0xf5, 0xed, 0x64, 0x0f, 0xd6, 0xb0, 0x6b, + 0x22, 0x65, 0xaf, 0xbe, 0xbd, 0xd2, 0xf6, 0x5d, 0x6c, 0x22, 0xfb, 0xaa, 0xcb, 0x8f, 0xbf, 0xbd, + 0xb2, 0x1e, 0x9e, 0xfc, 0x00, 0xcc, 0xb4, 0x9d, 0x59, 0xf6, 0x04, 0xdc, 0x5b, 0xda, 0xcd, 0x0b, + 0xfd, 0xb4, 0xf4, 0x5b, 0xf6, 0x92, 0xd9, 0x5b, 0xda, 0xf6, 0x92, 0x7d, 0xa8, 0xe9, 0x82, 0xb9, + 0xfc, 0x91, 0xb6, 0xb7, 0xa4, 0xd3, 0x16, 0xe6, 0x91, 0x1d, 0x4a, 0xd9, 0x4b, 0x72, 0xaf, 0xf4, + 0x39, 0x80, 0xdc, 0x83, 0x75, 0x55, 0xc5, 0x94, 0x3e, 0xd4, 0xf6, 0xca, 0xfb, 0x65, 0x71, 0xc9, + 0xb4, 0x47, 0x5b, 0xf6, 0xda, 0xdd, 0x5b, 0xfa, 0x6e, 0x41, 0x0e, 0x00, 0x32, 0x8d, 0xc6, 0xd2, + 0x67, 0xec, 0xde, 0xf2, 0xf7, 0x08, 0xf2, 0x00, 0xea, 0xe9, 0x1b, 0x53, 0xf9, 0xc3, 0x74, 0x6f, + 0xd9, 0x13, 0xc1, 0xe0, 0x8d, 0x7f, 0xfe, 0x69, 0xcb, 0xf8, 0xd5, 0xc5, 0x96, 0xf1, 0xe5, 0xc5, + 0x96, 0xf1, 0xd5, 0xc5, 0x96, 0xf1, 0xbb, 0x8b, 0x2d, 0xe3, 0x8f, 0x17, 0x5b, 0xc6, 0x6f, 0xfe, + 0xbc, 0x65, 0x0c, 0xd7, 0xd1, 0xfd, 0xdf, 0xff, 0x57, 0x00, 0x00, 0x00, 0xff, 0xff, 0xfc, 0x8d, + 0xcb, 0x04, 0x88, 0x19, 0x00, 0x00, } diff --git a/abci/types/types.proto b/abci/types/types.proto index 39c96e0e..b48ff1e8 100644 --- a/abci/types/types.proto +++ b/abci/types/types.proto @@ -49,6 +49,8 @@ message RequestFlush { message RequestInfo { string version = 1; + uint64 block_version = 2; + uint64 p2p_version = 3; } // nondeterministic @@ -72,7 +74,6 @@ message RequestQuery { bool prove = 4; } -// NOTE: validators here have empty pubkeys. message RequestBeginBlock { bytes hash = 1; Header header = 2 [(gogoproto.nullable)=false]; @@ -129,9 +130,12 @@ message ResponseFlush { message ResponseInfo { string data = 1; + string version = 2; - int64 last_block_height = 3; - bytes last_block_app_hash = 4; + uint64 app_version = 3; + + int64 last_block_height = 4; + bytes last_block_app_hash = 5; } // nondeterministic @@ -203,12 +207,13 @@ message ResponseCommit { // ConsensusParams contains all consensus-relevant parameters // that can be adjusted by the abci app message ConsensusParams { - BlockSize block_size = 1; - EvidenceParams evidence_params = 2; + BlockSizeParams block_size = 1; + EvidenceParams evidence = 2; + ValidatorParams validator = 3; } // BlockSize contains limits on the block size. -message BlockSize { +message BlockSizeParams { // Note: must be greater than 0 int64 max_bytes = 1; // Note: must be greater or equal to -1 @@ -221,6 +226,11 @@ message EvidenceParams { int64 max_age = 1; } +// ValidatorParams contains limits on validators. +message ValidatorParams { + repeated string pub_key_types = 1; +} + message LastCommitInfo { int32 round = 1; repeated VoteInfo votes = 2 [(gogoproto.nullable)=false]; @@ -231,31 +241,38 @@ message LastCommitInfo { message Header { // basic block info - string chain_id = 1 [(gogoproto.customname)="ChainID"]; - int64 height = 2; - google.protobuf.Timestamp time = 3 [(gogoproto.nullable)=false, (gogoproto.stdtime)=true]; - int64 num_txs = 4; - int64 total_txs = 5; + Version version = 1 [(gogoproto.nullable)=false]; + string chain_id = 2 [(gogoproto.customname)="ChainID"]; + int64 height = 3; + google.protobuf.Timestamp time = 4 [(gogoproto.nullable)=false, (gogoproto.stdtime)=true]; + int64 num_txs = 5; + int64 total_txs = 6; // prev block info - BlockID last_block_id = 6 [(gogoproto.nullable)=false]; + BlockID last_block_id = 7 [(gogoproto.nullable)=false]; // hashes of block data - bytes last_commit_hash = 7; // commit from validators from the last block - bytes data_hash = 8; // transactions + bytes last_commit_hash = 8; // commit from validators from the last block + bytes data_hash = 9; // transactions // hashes from the app output from the prev block - bytes validators_hash = 9; // validators for the current block - bytes next_validators_hash = 10; // validators for the next block - bytes consensus_hash = 11; // consensus params for current block - bytes app_hash = 12; // state after txs from the previous block - bytes last_results_hash = 13;// root hash of all results from the txs from the previous block + bytes validators_hash = 10; // validators for the current block + bytes next_validators_hash = 11; // validators for the next block + bytes consensus_hash = 12; // consensus params for current block + bytes app_hash = 13; // state after txs from the previous block + bytes last_results_hash = 14;// root hash of all results from the txs from the previous block // consensus info - bytes evidence_hash = 14; // evidence included in the block - bytes proposer_address = 15; // original proposer of the block + bytes evidence_hash = 15; // evidence included in the block + bytes proposer_address = 16; // original proposer of the block } +message Version { + uint64 Block = 1; + uint64 App = 2; +} + + message BlockID { bytes hash = 1; PartSetHeader parts_header = 2 [(gogoproto.nullable)=false]; diff --git a/abci/types/typespb_test.go b/abci/types/typespb_test.go index 0ae0fea0..9375cc7f 100644 --- a/abci/types/typespb_test.go +++ b/abci/types/typespb_test.go @@ -1479,15 +1479,15 @@ func TestConsensusParamsMarshalTo(t *testing.T) { } } -func TestBlockSizeProto(t *testing.T) { +func TestBlockSizeParamsProto(t *testing.T) { seed := time.Now().UnixNano() popr := math_rand.New(math_rand.NewSource(seed)) - p := NewPopulatedBlockSize(popr, false) + p := NewPopulatedBlockSizeParams(popr, false) dAtA, err := github_com_gogo_protobuf_proto.Marshal(p) if err != nil { t.Fatalf("seed = %d, err = %v", seed, err) } - msg := &BlockSize{} + msg := &BlockSizeParams{} if err := github_com_gogo_protobuf_proto.Unmarshal(dAtA, msg); err != nil { t.Fatalf("seed = %d, err = %v", seed, err) } @@ -1510,10 +1510,10 @@ func TestBlockSizeProto(t *testing.T) { } } -func TestBlockSizeMarshalTo(t *testing.T) { +func TestBlockSizeParamsMarshalTo(t *testing.T) { seed := time.Now().UnixNano() popr := math_rand.New(math_rand.NewSource(seed)) - p := NewPopulatedBlockSize(popr, false) + p := NewPopulatedBlockSizeParams(popr, false) size := p.Size() dAtA := make([]byte, size) for i := range dAtA { @@ -1523,7 +1523,7 @@ func TestBlockSizeMarshalTo(t *testing.T) { if err != nil { t.Fatalf("seed = %d, err = %v", seed, err) } - msg := &BlockSize{} + msg := &BlockSizeParams{} if err := github_com_gogo_protobuf_proto.Unmarshal(dAtA, msg); err != nil { t.Fatalf("seed = %d, err = %v", seed, err) } @@ -1591,6 +1591,62 @@ func TestEvidenceParamsMarshalTo(t *testing.T) { } } +func TestValidatorParamsProto(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedValidatorParams(popr, false) + dAtA, err := github_com_gogo_protobuf_proto.Marshal(p) + if err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + msg := &ValidatorParams{} + if err := github_com_gogo_protobuf_proto.Unmarshal(dAtA, msg); err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + littlefuzz := make([]byte, len(dAtA)) + copy(littlefuzz, dAtA) + for i := range dAtA { + dAtA[i] = byte(popr.Intn(256)) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p) + } + if len(littlefuzz) > 0 { + fuzzamount := 100 + for i := 0; i < fuzzamount; i++ { + littlefuzz[popr.Intn(len(littlefuzz))] = byte(popr.Intn(256)) + littlefuzz = append(littlefuzz, byte(popr.Intn(256))) + } + // shouldn't panic + _ = github_com_gogo_protobuf_proto.Unmarshal(littlefuzz, msg) + } +} + +func TestValidatorParamsMarshalTo(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedValidatorParams(popr, false) + size := p.Size() + dAtA := make([]byte, size) + for i := range dAtA { + dAtA[i] = byte(popr.Intn(256)) + } + _, err := p.MarshalTo(dAtA) + if err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + msg := &ValidatorParams{} + if err := github_com_gogo_protobuf_proto.Unmarshal(dAtA, msg); err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + for i := range dAtA { + dAtA[i] = byte(popr.Intn(256)) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p) + } +} + func TestLastCommitInfoProto(t *testing.T) { seed := time.Now().UnixNano() popr := math_rand.New(math_rand.NewSource(seed)) @@ -1703,6 +1759,62 @@ func TestHeaderMarshalTo(t *testing.T) { } } +func TestVersionProto(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedVersion(popr, false) + dAtA, err := github_com_gogo_protobuf_proto.Marshal(p) + if err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + msg := &Version{} + if err := github_com_gogo_protobuf_proto.Unmarshal(dAtA, msg); err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + littlefuzz := make([]byte, len(dAtA)) + copy(littlefuzz, dAtA) + for i := range dAtA { + dAtA[i] = byte(popr.Intn(256)) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p) + } + if len(littlefuzz) > 0 { + fuzzamount := 100 + for i := 0; i < fuzzamount; i++ { + littlefuzz[popr.Intn(len(littlefuzz))] = byte(popr.Intn(256)) + littlefuzz = append(littlefuzz, byte(popr.Intn(256))) + } + // shouldn't panic + _ = github_com_gogo_protobuf_proto.Unmarshal(littlefuzz, msg) + } +} + +func TestVersionMarshalTo(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedVersion(popr, false) + size := p.Size() + dAtA := make([]byte, size) + for i := range dAtA { + dAtA[i] = byte(popr.Intn(256)) + } + _, err := p.MarshalTo(dAtA) + if err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + msg := &Version{} + if err := github_com_gogo_protobuf_proto.Unmarshal(dAtA, msg); err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + for i := range dAtA { + dAtA[i] = byte(popr.Intn(256)) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p) + } +} + func TestBlockIDProto(t *testing.T) { seed := time.Now().UnixNano() popr := math_rand.New(math_rand.NewSource(seed)) @@ -2563,16 +2675,16 @@ func TestConsensusParamsJSON(t *testing.T) { t.Fatalf("seed = %d, %#v !Json Equal %#v", seed, msg, p) } } -func TestBlockSizeJSON(t *testing.T) { +func TestBlockSizeParamsJSON(t *testing.T) { seed := time.Now().UnixNano() popr := math_rand.New(math_rand.NewSource(seed)) - p := NewPopulatedBlockSize(popr, true) + p := NewPopulatedBlockSizeParams(popr, true) marshaler := github_com_gogo_protobuf_jsonpb.Marshaler{} jsondata, err := marshaler.MarshalToString(p) if err != nil { t.Fatalf("seed = %d, err = %v", seed, err) } - msg := &BlockSize{} + msg := &BlockSizeParams{} err = github_com_gogo_protobuf_jsonpb.UnmarshalString(jsondata, msg) if err != nil { t.Fatalf("seed = %d, err = %v", seed, err) @@ -2599,6 +2711,24 @@ func TestEvidenceParamsJSON(t *testing.T) { t.Fatalf("seed = %d, %#v !Json Equal %#v", seed, msg, p) } } +func TestValidatorParamsJSON(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedValidatorParams(popr, true) + marshaler := github_com_gogo_protobuf_jsonpb.Marshaler{} + jsondata, err := marshaler.MarshalToString(p) + if err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + msg := &ValidatorParams{} + err = github_com_gogo_protobuf_jsonpb.UnmarshalString(jsondata, msg) + if err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Json Equal %#v", seed, msg, p) + } +} func TestLastCommitInfoJSON(t *testing.T) { seed := time.Now().UnixNano() popr := math_rand.New(math_rand.NewSource(seed)) @@ -2635,6 +2765,24 @@ func TestHeaderJSON(t *testing.T) { t.Fatalf("seed = %d, %#v !Json Equal %#v", seed, msg, p) } } +func TestVersionJSON(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedVersion(popr, true) + marshaler := github_com_gogo_protobuf_jsonpb.Marshaler{} + jsondata, err := marshaler.MarshalToString(p) + if err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + msg := &Version{} + err = github_com_gogo_protobuf_jsonpb.UnmarshalString(jsondata, msg) + if err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Json Equal %#v", seed, msg, p) + } +} func TestBlockIDJSON(t *testing.T) { seed := time.Now().UnixNano() popr := math_rand.New(math_rand.NewSource(seed)) @@ -3489,12 +3637,12 @@ func TestConsensusParamsProtoCompactText(t *testing.T) { } } -func TestBlockSizeProtoText(t *testing.T) { +func TestBlockSizeParamsProtoText(t *testing.T) { seed := time.Now().UnixNano() popr := math_rand.New(math_rand.NewSource(seed)) - p := NewPopulatedBlockSize(popr, true) + p := NewPopulatedBlockSizeParams(popr, true) dAtA := github_com_gogo_protobuf_proto.MarshalTextString(p) - msg := &BlockSize{} + msg := &BlockSizeParams{} if err := github_com_gogo_protobuf_proto.UnmarshalText(dAtA, msg); err != nil { t.Fatalf("seed = %d, err = %v", seed, err) } @@ -3503,12 +3651,12 @@ func TestBlockSizeProtoText(t *testing.T) { } } -func TestBlockSizeProtoCompactText(t *testing.T) { +func TestBlockSizeParamsProtoCompactText(t *testing.T) { seed := time.Now().UnixNano() popr := math_rand.New(math_rand.NewSource(seed)) - p := NewPopulatedBlockSize(popr, true) + p := NewPopulatedBlockSizeParams(popr, true) dAtA := github_com_gogo_protobuf_proto.CompactTextString(p) - msg := &BlockSize{} + msg := &BlockSizeParams{} if err := github_com_gogo_protobuf_proto.UnmarshalText(dAtA, msg); err != nil { t.Fatalf("seed = %d, err = %v", seed, err) } @@ -3545,6 +3693,34 @@ func TestEvidenceParamsProtoCompactText(t *testing.T) { } } +func TestValidatorParamsProtoText(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedValidatorParams(popr, true) + dAtA := github_com_gogo_protobuf_proto.MarshalTextString(p) + msg := &ValidatorParams{} + if err := github_com_gogo_protobuf_proto.UnmarshalText(dAtA, msg); err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p) + } +} + +func TestValidatorParamsProtoCompactText(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedValidatorParams(popr, true) + dAtA := github_com_gogo_protobuf_proto.CompactTextString(p) + msg := &ValidatorParams{} + if err := github_com_gogo_protobuf_proto.UnmarshalText(dAtA, msg); err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p) + } +} + func TestLastCommitInfoProtoText(t *testing.T) { seed := time.Now().UnixNano() popr := math_rand.New(math_rand.NewSource(seed)) @@ -3601,6 +3777,34 @@ func TestHeaderProtoCompactText(t *testing.T) { } } +func TestVersionProtoText(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedVersion(popr, true) + dAtA := github_com_gogo_protobuf_proto.MarshalTextString(p) + msg := &Version{} + if err := github_com_gogo_protobuf_proto.UnmarshalText(dAtA, msg); err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p) + } +} + +func TestVersionProtoCompactText(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedVersion(popr, true) + dAtA := github_com_gogo_protobuf_proto.CompactTextString(p) + msg := &Version{} + if err := github_com_gogo_protobuf_proto.UnmarshalText(dAtA, msg); err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + if !p.Equal(msg) { + t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p) + } +} + func TestBlockIDProtoText(t *testing.T) { seed := time.Now().UnixNano() popr := math_rand.New(math_rand.NewSource(seed)) @@ -4369,10 +4573,10 @@ func TestConsensusParamsSize(t *testing.T) { } } -func TestBlockSizeSize(t *testing.T) { +func TestBlockSizeParamsSize(t *testing.T) { seed := time.Now().UnixNano() popr := math_rand.New(math_rand.NewSource(seed)) - p := NewPopulatedBlockSize(popr, true) + p := NewPopulatedBlockSizeParams(popr, true) size2 := github_com_gogo_protobuf_proto.Size(p) dAtA, err := github_com_gogo_protobuf_proto.Marshal(p) if err != nil { @@ -4413,6 +4617,28 @@ func TestEvidenceParamsSize(t *testing.T) { } } +func TestValidatorParamsSize(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedValidatorParams(popr, true) + size2 := github_com_gogo_protobuf_proto.Size(p) + dAtA, err := github_com_gogo_protobuf_proto.Marshal(p) + if err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + size := p.Size() + if len(dAtA) != size { + t.Errorf("seed = %d, size %v != marshalled size %v", seed, size, len(dAtA)) + } + if size2 != size { + t.Errorf("seed = %d, size %v != before marshal proto.Size %v", seed, size, size2) + } + size3 := github_com_gogo_protobuf_proto.Size(p) + if size3 != size { + t.Errorf("seed = %d, size %v != after marshal proto.Size %v", seed, size, size3) + } +} + func TestLastCommitInfoSize(t *testing.T) { seed := time.Now().UnixNano() popr := math_rand.New(math_rand.NewSource(seed)) @@ -4457,6 +4683,28 @@ func TestHeaderSize(t *testing.T) { } } +func TestVersionSize(t *testing.T) { + seed := time.Now().UnixNano() + popr := math_rand.New(math_rand.NewSource(seed)) + p := NewPopulatedVersion(popr, true) + size2 := github_com_gogo_protobuf_proto.Size(p) + dAtA, err := github_com_gogo_protobuf_proto.Marshal(p) + if err != nil { + t.Fatalf("seed = %d, err = %v", seed, err) + } + size := p.Size() + if len(dAtA) != size { + t.Errorf("seed = %d, size %v != marshalled size %v", seed, size, len(dAtA)) + } + if size2 != size { + t.Errorf("seed = %d, size %v != before marshal proto.Size %v", seed, size, size2) + } + size3 := github_com_gogo_protobuf_proto.Size(p) + if size3 != size { + t.Errorf("seed = %d, size %v != after marshal proto.Size %v", seed, size, size3) + } +} + func TestBlockIDSize(t *testing.T) { seed := time.Now().UnixNano() popr := math_rand.New(math_rand.NewSource(seed)) diff --git a/benchmarks/codec_test.go b/benchmarks/codec_test.go index c0e13d16..eff5c734 100644 --- a/benchmarks/codec_test.go +++ b/benchmarks/codec_test.go @@ -4,7 +4,7 @@ import ( "testing" "time" - "github.com/tendermint/go-amino" + amino "github.com/tendermint/go-amino" proto "github.com/tendermint/tendermint/benchmarks/proto" "github.com/tendermint/tendermint/crypto/ed25519" @@ -12,23 +12,28 @@ import ( ctypes "github.com/tendermint/tendermint/rpc/core/types" ) +func testNodeInfo(id p2p.ID) p2p.DefaultNodeInfo { + return p2p.DefaultNodeInfo{ + ProtocolVersion: p2p.ProtocolVersion{1, 2, 3}, + ID_: id, + Moniker: "SOMENAME", + Network: "SOMENAME", + ListenAddr: "SOMEADDR", + Version: "SOMEVER", + Other: p2p.DefaultNodeInfoOther{ + TxIndex: "on", + RPCAddress: "0.0.0.0:26657", + }, + } +} + func BenchmarkEncodeStatusWire(b *testing.B) { b.StopTimer() cdc := amino.NewCodec() ctypes.RegisterAmino(cdc) nodeKey := p2p.NodeKey{PrivKey: ed25519.GenPrivKey()} status := &ctypes.ResultStatus{ - NodeInfo: p2p.NodeInfo{ - ID: nodeKey.ID(), - Moniker: "SOMENAME", - Network: "SOMENAME", - ListenAddr: "SOMEADDR", - Version: "SOMEVER", - Other: p2p.NodeInfoOther{ - AminoVersion: "SOMESTRING", - P2PVersion: "OTHERSTRING", - }, - }, + NodeInfo: testNodeInfo(nodeKey.ID()), SyncInfo: ctypes.SyncInfo{ LatestBlockHash: []byte("SOMEBYTES"), LatestBlockHeight: 123, @@ -56,17 +61,7 @@ func BenchmarkEncodeNodeInfoWire(b *testing.B) { cdc := amino.NewCodec() ctypes.RegisterAmino(cdc) nodeKey := p2p.NodeKey{PrivKey: ed25519.GenPrivKey()} - nodeInfo := p2p.NodeInfo{ - ID: nodeKey.ID(), - Moniker: "SOMENAME", - Network: "SOMENAME", - ListenAddr: "SOMEADDR", - Version: "SOMEVER", - Other: p2p.NodeInfoOther{ - AminoVersion: "SOMESTRING", - P2PVersion: "OTHERSTRING", - }, - } + nodeInfo := testNodeInfo(nodeKey.ID()) b.StartTimer() counter := 0 @@ -84,17 +79,7 @@ func BenchmarkEncodeNodeInfoBinary(b *testing.B) { cdc := amino.NewCodec() ctypes.RegisterAmino(cdc) nodeKey := p2p.NodeKey{PrivKey: ed25519.GenPrivKey()} - nodeInfo := p2p.NodeInfo{ - ID: nodeKey.ID(), - Moniker: "SOMENAME", - Network: "SOMENAME", - ListenAddr: "SOMEADDR", - Version: "SOMEVER", - Other: p2p.NodeInfoOther{ - AminoVersion: "SOMESTRING", - P2PVersion: "OTHERSTRING", - }, - } + nodeInfo := testNodeInfo(nodeKey.ID()) b.StartTimer() counter := 0 diff --git a/blockchain/pool.go b/blockchain/pool.go index c7864a64..804a4325 100644 --- a/blockchain/pool.go +++ b/blockchain/pool.go @@ -168,9 +168,12 @@ func (pool *BlockPool) IsCaughtUp() bool { return false } - // some conditions to determine if we're caught up - receivedBlockOrTimedOut := (pool.height > 0 || time.Since(pool.startTime) > 5*time.Second) - ourChainIsLongestAmongPeers := pool.maxPeerHeight == 0 || pool.height >= pool.maxPeerHeight + // Some conditions to determine if we're caught up. + // Ensures we've either received a block or waited some amount of time, + // and that we're synced to the highest known height. Note we use maxPeerHeight - 1 + // because to sync block H requires block H+1 to verify the LastCommit. + receivedBlockOrTimedOut := pool.height > 0 || time.Since(pool.startTime) > 5*time.Second + ourChainIsLongestAmongPeers := pool.maxPeerHeight == 0 || pool.height >= (pool.maxPeerHeight-1) isCaughtUp := receivedBlockOrTimedOut && ourChainIsLongestAmongPeers return isCaughtUp } @@ -252,7 +255,8 @@ func (pool *BlockPool) AddBlock(peerID p2p.ID, block *types.Block, blockSize int peer.decrPending(blockSize) } } else { - // Bad peer? + pool.Logger.Info("invalid peer", "peer", peerID, "blockHeight", block.Height) + pool.sendError(errors.New("invalid peer"), peerID) } } @@ -292,7 +296,7 @@ func (pool *BlockPool) RemovePeer(peerID p2p.ID) { func (pool *BlockPool) removePeer(peerID p2p.ID) { for _, requester := range pool.requesters { if requester.getPeerID() == peerID { - requester.redo() + requester.redo(peerID) } } delete(pool.peers, peerID) @@ -326,8 +330,11 @@ func (pool *BlockPool) makeNextRequester() { defer pool.mtx.Unlock() nextHeight := pool.height + pool.requestersLen() + if nextHeight > pool.maxPeerHeight { + return + } + request := newBPRequester(pool, nextHeight) - // request.SetLogger(pool.Logger.With("height", nextHeight)) pool.requesters[nextHeight] = request atomic.AddInt32(&pool.numPending, 1) @@ -356,7 +363,8 @@ func (pool *BlockPool) sendError(err error, peerID p2p.ID) { pool.errorsCh <- peerError{err, peerID} } -// unused by tendermint; left for debugging purposes +// for debugging purposes +//nolint:unused func (pool *BlockPool) debug() string { pool.mtx.Lock() defer pool.mtx.Unlock() @@ -453,7 +461,7 @@ type bpRequester struct { pool *BlockPool height int64 gotBlockCh chan struct{} - redoCh chan struct{} + redoCh chan p2p.ID //redo may send multitime, add peerId to identify repeat mtx sync.Mutex peerID p2p.ID @@ -465,7 +473,7 @@ func newBPRequester(pool *BlockPool, height int64) *bpRequester { pool: pool, height: height, gotBlockCh: make(chan struct{}, 1), - redoCh: make(chan struct{}, 1), + redoCh: make(chan p2p.ID, 1), peerID: "", block: nil, @@ -524,9 +532,9 @@ func (bpr *bpRequester) reset() { // Tells bpRequester to pick another peer and try again. // NOTE: Nonblocking, and does nothing if another redo // was already requested. -func (bpr *bpRequester) redo() { +func (bpr *bpRequester) redo(peerId p2p.ID) { select { - case bpr.redoCh <- struct{}{}: + case bpr.redoCh <- peerId: default: } } @@ -565,9 +573,13 @@ OUTER_LOOP: return case <-bpr.Quit(): return - case <-bpr.redoCh: - bpr.reset() - continue OUTER_LOOP + case peerID := <-bpr.redoCh: + if peerID == bpr.peerID { + bpr.reset() + continue OUTER_LOOP + } else { + continue WAIT_LOOP + } case <-bpr.gotBlockCh: // We got a block! // Continue the for-loop and wait til Quit. diff --git a/blockchain/pool_test.go b/blockchain/pool_test.go index 01187bcf..75a03f63 100644 --- a/blockchain/pool_test.go +++ b/blockchain/pool_test.go @@ -16,16 +16,52 @@ func init() { } type testPeer struct { - id p2p.ID - height int64 + id p2p.ID + height int64 + inputChan chan inputData //make sure each peer's data is sequential } -func makePeers(numPeers int, minHeight, maxHeight int64) map[p2p.ID]testPeer { - peers := make(map[p2p.ID]testPeer, numPeers) +type inputData struct { + t *testing.T + pool *BlockPool + request BlockRequest +} + +func (p testPeer) runInputRoutine() { + go func() { + for input := range p.inputChan { + p.simulateInput(input) + } + }() +} + +// Request desired, pretend like we got the block immediately. +func (p testPeer) simulateInput(input inputData) { + block := &types.Block{Header: types.Header{Height: input.request.Height}} + input.pool.AddBlock(input.request.PeerID, block, 123) + input.t.Logf("Added block from peer %v (height: %v)", input.request.PeerID, input.request.Height) +} + +type testPeers map[p2p.ID]testPeer + +func (ps testPeers) start() { + for _, v := range ps { + v.runInputRoutine() + } +} + +func (ps testPeers) stop() { + for _, v := range ps { + close(v.inputChan) + } +} + +func makePeers(numPeers int, minHeight, maxHeight int64) testPeers { + peers := make(testPeers, numPeers) for i := 0; i < numPeers; i++ { peerID := p2p.ID(cmn.RandStr(12)) height := minHeight + cmn.RandInt63n(maxHeight-minHeight) - peers[peerID] = testPeer{peerID, height} + peers[peerID] = testPeer{peerID, height, make(chan inputData, 10)} } return peers } @@ -45,6 +81,9 @@ func TestBasic(t *testing.T) { defer pool.Stop() + peers.start() + defer peers.stop() + // Introduce each peer. go func() { for _, peer := range peers { @@ -77,12 +116,8 @@ func TestBasic(t *testing.T) { if request.Height == 300 { return // Done! } - // Request desired, pretend like we got the block immediately. - go func() { - block := &types.Block{Header: types.Header{Height: request.Height}} - pool.AddBlock(request.PeerID, block, 123) - t.Logf("Added block from peer %v (height: %v)", request.PeerID, request.Height) - }() + + peers[request.PeerID].inputChan <- inputData{t, pool, request} } } } diff --git a/blockchain/reactor.go b/blockchain/reactor.go index fc1b1f4d..bed082cd 100644 --- a/blockchain/reactor.go +++ b/blockchain/reactor.go @@ -1,6 +1,7 @@ package blockchain import ( + "errors" "fmt" "reflect" "time" @@ -180,6 +181,12 @@ func (bcR *BlockchainReactor) Receive(chID byte, src p2p.Peer, msgBytes []byte) return } + if err = msg.ValidateBasic(); err != nil { + bcR.Logger.Error("Peer sent us invalid msg", "peer", src, "msg", msg, "err", err) + bcR.Switch.StopPeerForError(src, err) + return + } + bcR.Logger.Debug("Receive", "src", src, "chID", chID, "msg", msg) switch msg := msg.(type) { @@ -188,7 +195,6 @@ func (bcR *BlockchainReactor) Receive(chID byte, src p2p.Peer, msgBytes []byte) // Unfortunately not queued since the queue is full. } case *bcBlockResponseMessage: - // Got a block. bcR.pool.AddBlock(src.ID(), msg.Block, len(msgBytes)) case *bcStatusRequestMessage: // Send peer our state. @@ -258,8 +264,12 @@ FOR_LOOP: bcR.Logger.Info("Time to switch to consensus reactor!", "height", height) bcR.pool.Stop() - conR := bcR.Switch.Reactor("CONSENSUS").(consensusReactor) - conR.SwitchToConsensus(state, blocksSynced) + conR, ok := bcR.Switch.Reactor("CONSENSUS").(consensusReactor) + if ok { + conR.SwitchToConsensus(state, blocksSynced) + } else { + // should only happen during testing + } break FOR_LOOP } @@ -308,6 +318,13 @@ FOR_LOOP: // still need to clean up the rest. bcR.Switch.StopPeerForError(peer, fmt.Errorf("BlockchainReactor validation error: %v", err)) } + peerID2 := bcR.pool.RedoRequest(second.Height) + peer2 := bcR.Switch.Peers().Get(peerID2) + if peer2 != nil && peer2 != peer { + // NOTE: we've already removed the peer's request, but we + // still need to clean up the rest. + bcR.Switch.StopPeerForError(peer2, fmt.Errorf("BlockchainReactor validation error: %v", err)) + } continue FOR_LOOP } else { bcR.pool.PopRequest() @@ -352,7 +369,9 @@ func (bcR *BlockchainReactor) BroadcastStatusRequest() error { // Messages // BlockchainMessage is a generic message for this reactor. -type BlockchainMessage interface{} +type BlockchainMessage interface { + ValidateBasic() error +} func RegisterBlockchainMessages(cdc *amino.Codec) { cdc.RegisterInterface((*BlockchainMessage)(nil), nil) @@ -377,6 +396,14 @@ type bcBlockRequestMessage struct { Height int64 } +// ValidateBasic performs basic validation. +func (m *bcBlockRequestMessage) ValidateBasic() error { + if m.Height < 0 { + return errors.New("Negative Height") + } + return nil +} + func (m *bcBlockRequestMessage) String() string { return fmt.Sprintf("[bcBlockRequestMessage %v]", m.Height) } @@ -385,6 +412,14 @@ type bcNoBlockResponseMessage struct { Height int64 } +// ValidateBasic performs basic validation. +func (m *bcNoBlockResponseMessage) ValidateBasic() error { + if m.Height < 0 { + return errors.New("Negative Height") + } + return nil +} + func (brm *bcNoBlockResponseMessage) String() string { return fmt.Sprintf("[bcNoBlockResponseMessage %d]", brm.Height) } @@ -395,6 +430,11 @@ type bcBlockResponseMessage struct { Block *types.Block } +// ValidateBasic performs basic validation. +func (m *bcBlockResponseMessage) ValidateBasic() error { + return m.Block.ValidateBasic() +} + func (m *bcBlockResponseMessage) String() string { return fmt.Sprintf("[bcBlockResponseMessage %v]", m.Block.Height) } @@ -405,6 +445,14 @@ type bcStatusRequestMessage struct { Height int64 } +// ValidateBasic performs basic validation. +func (m *bcStatusRequestMessage) ValidateBasic() error { + if m.Height < 0 { + return errors.New("Negative Height") + } + return nil +} + func (m *bcStatusRequestMessage) String() string { return fmt.Sprintf("[bcStatusRequestMessage %v]", m.Height) } @@ -415,6 +463,14 @@ type bcStatusResponseMessage struct { Height int64 } +// ValidateBasic performs basic validation. +func (m *bcStatusResponseMessage) ValidateBasic() error { + if m.Height < 0 { + return errors.New("Negative Height") + } + return nil +} + func (m *bcStatusResponseMessage) String() string { return fmt.Sprintf("[bcStatusResponseMessage %v]", m.Height) } diff --git a/blockchain/reactor_test.go b/blockchain/reactor_test.go index b63a057e..138e1622 100644 --- a/blockchain/reactor_test.go +++ b/blockchain/reactor_test.go @@ -1,72 +1,151 @@ package blockchain import ( - "net" + "sort" "testing" + "time" + "github.com/stretchr/testify/assert" + + abci "github.com/tendermint/tendermint/abci/types" + cfg "github.com/tendermint/tendermint/config" cmn "github.com/tendermint/tendermint/libs/common" dbm "github.com/tendermint/tendermint/libs/db" "github.com/tendermint/tendermint/libs/log" - - cfg "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/p2p" "github.com/tendermint/tendermint/proxy" sm "github.com/tendermint/tendermint/state" "github.com/tendermint/tendermint/types" + tmtime "github.com/tendermint/tendermint/types/time" ) -func makeStateAndBlockStore(logger log.Logger) (sm.State, *BlockStore) { - config := cfg.ResetTestRoot("blockchain_reactor_test") - // blockDB := dbm.NewDebugDB("blockDB", dbm.NewMemDB()) - // stateDB := dbm.NewDebugDB("stateDB", dbm.NewMemDB()) +var config *cfg.Config + +func randGenesisDoc(numValidators int, randPower bool, minPower int64) (*types.GenesisDoc, []types.PrivValidator) { + validators := make([]types.GenesisValidator, numValidators) + privValidators := make([]types.PrivValidator, numValidators) + for i := 0; i < numValidators; i++ { + val, privVal := types.RandValidator(randPower, minPower) + validators[i] = types.GenesisValidator{ + PubKey: val.PubKey, + Power: val.VotingPower, + } + privValidators[i] = privVal + } + sort.Sort(types.PrivValidatorsByAddress(privValidators)) + + return &types.GenesisDoc{ + GenesisTime: tmtime.Now(), + ChainID: config.ChainID(), + Validators: validators, + }, privValidators +} + +func makeVote(header *types.Header, blockID types.BlockID, valset *types.ValidatorSet, privVal types.PrivValidator) *types.Vote { + addr := privVal.GetPubKey().Address() + idx, _ := valset.GetByAddress(addr) + vote := &types.Vote{ + ValidatorAddress: addr, + ValidatorIndex: idx, + Height: header.Height, + Round: 1, + Timestamp: tmtime.Now(), + Type: types.PrecommitType, + BlockID: blockID, + } + + privVal.SignVote(header.ChainID, vote) + + return vote +} + +type BlockchainReactorPair struct { + reactor *BlockchainReactor + app proxy.AppConns +} + +func newBlockchainReactor(logger log.Logger, genDoc *types.GenesisDoc, privVals []types.PrivValidator, maxBlockHeight int64) BlockchainReactorPair { + if len(privVals) != 1 { + panic("only support one validator") + } + + app := &testApp{} + cc := proxy.NewLocalClientCreator(app) + proxyApp := proxy.NewAppConns(cc) + err := proxyApp.Start() + if err != nil { + panic(cmn.ErrorWrap(err, "error start app")) + } + blockDB := dbm.NewMemDB() stateDB := dbm.NewMemDB() blockStore := NewBlockStore(blockDB) - state, err := sm.LoadStateFromDBOrGenesisFile(stateDB, config.GenesisFile()) + + state, err := sm.LoadStateFromDBOrGenesisDoc(stateDB, genDoc) if err != nil { panic(cmn.ErrorWrap(err, "error constructing state from genesis file")) } - return state, blockStore -} -func newBlockchainReactor(logger log.Logger, maxBlockHeight int64) *BlockchainReactor { - state, blockStore := makeStateAndBlockStore(logger) - - // Make the blockchainReactor itself + // Make the BlockchainReactor itself. + // NOTE we have to create and commit the blocks first because + // pool.height is determined from the store. fastSync := true - var nilApp proxy.AppConnConsensus - blockExec := sm.NewBlockExecutor(dbm.NewMemDB(), log.TestingLogger(), nilApp, + blockExec := sm.NewBlockExecutor(dbm.NewMemDB(), log.TestingLogger(), proxyApp.Consensus(), sm.MockMempool{}, sm.MockEvidencePool{}) + // let's add some blocks in + for blockHeight := int64(1); blockHeight <= maxBlockHeight; blockHeight++ { + lastCommit := &types.Commit{} + if blockHeight > 1 { + lastBlockMeta := blockStore.LoadBlockMeta(blockHeight - 1) + lastBlock := blockStore.LoadBlock(blockHeight - 1) + + vote := makeVote(&lastBlock.Header, lastBlockMeta.BlockID, state.Validators, privVals[0]).CommitSig() + lastCommit = &types.Commit{Precommits: []*types.CommitSig{vote}, BlockID: lastBlockMeta.BlockID} + } + + thisBlock := makeBlock(blockHeight, state, lastCommit) + + thisParts := thisBlock.MakePartSet(types.BlockPartSizeBytes) + blockID := types.BlockID{thisBlock.Hash(), thisParts.Header()} + + state, err = blockExec.ApplyBlock(state, blockID, thisBlock) + if err != nil { + panic(cmn.ErrorWrap(err, "error apply block")) + } + + blockStore.SaveBlock(thisBlock, thisParts, lastCommit) + } + bcReactor := NewBlockchainReactor(state.Copy(), blockExec, blockStore, fastSync) bcReactor.SetLogger(logger.With("module", "blockchain")) - // Next: we need to set a switch in order for peers to be added in - bcReactor.Switch = p2p.NewSwitch(cfg.DefaultP2PConfig(), nil) - - // Lastly: let's add some blocks in - for blockHeight := int64(1); blockHeight <= maxBlockHeight; blockHeight++ { - firstBlock := makeBlock(blockHeight, state) - secondBlock := makeBlock(blockHeight+1, state) - firstParts := firstBlock.MakePartSet(types.BlockPartSizeBytes) - blockStore.SaveBlock(firstBlock, firstParts, secondBlock.LastCommit) - } - - return bcReactor + return BlockchainReactorPair{bcReactor, proxyApp} } func TestNoBlockResponse(t *testing.T) { - maxBlockHeight := int64(20) + config = cfg.ResetTestRoot("blockchain_reactor_test") + genDoc, privVals := randGenesisDoc(1, false, 30) - bcr := newBlockchainReactor(log.TestingLogger(), maxBlockHeight) - bcr.Start() - defer bcr.Stop() + maxBlockHeight := int64(65) - // Add some peers in - peer := newbcrTestPeer(p2p.ID(cmn.RandStr(12))) - bcr.AddPeer(peer) + reactorPairs := make([]BlockchainReactorPair, 2) - chID := byte(0x01) + reactorPairs[0] = newBlockchainReactor(log.TestingLogger(), genDoc, privVals, maxBlockHeight) + reactorPairs[1] = newBlockchainReactor(log.TestingLogger(), genDoc, privVals, 0) + + p2p.MakeConnectedSwitches(config.P2P, 2, func(i int, s *p2p.Switch) *p2p.Switch { + s.AddReactor("BLOCKCHAIN", reactorPairs[i].reactor) + return s + + }, p2p.Connect2Switches) + + defer func() { + for _, r := range reactorPairs { + r.reactor.Stop() + r.app.Stop() + } + }() tests := []struct { height int64 @@ -78,72 +157,100 @@ func TestNoBlockResponse(t *testing.T) { {100, false}, } - // receive a request message from peer, - // wait for our response to be received on the peer - for _, tt := range tests { - reqBlockMsg := &bcBlockRequestMessage{tt.height} - reqBlockBytes := cdc.MustMarshalBinaryBare(reqBlockMsg) - bcr.Receive(chID, peer, reqBlockBytes) - msg := peer.lastBlockchainMessage() + for { + if reactorPairs[1].reactor.pool.IsCaughtUp() { + break + } + time.Sleep(10 * time.Millisecond) + } + + assert.Equal(t, maxBlockHeight, reactorPairs[0].reactor.store.Height()) + + for _, tt := range tests { + block := reactorPairs[1].reactor.store.LoadBlock(tt.height) if tt.existent { - if blockMsg, ok := msg.(*bcBlockResponseMessage); !ok { - t.Fatalf("Expected to receive a block response for height %d", tt.height) - } else if blockMsg.Block.Height != tt.height { - t.Fatalf("Expected response to be for height %d, got %d", tt.height, blockMsg.Block.Height) - } + assert.True(t, block != nil) } else { - if noBlockMsg, ok := msg.(*bcNoBlockResponseMessage); !ok { - t.Fatalf("Expected to receive a no block response for height %d", tt.height) - } else if noBlockMsg.Height != tt.height { - t.Fatalf("Expected response to be for height %d, got %d", tt.height, noBlockMsg.Height) - } + assert.True(t, block == nil) } } } -/* // NOTE: This is too hard to test without // an easy way to add test peer to switch // or without significant refactoring of the module. // Alternatively we could actually dial a TCP conn but // that seems extreme. func TestBadBlockStopsPeer(t *testing.T) { - maxBlockHeight := int64(20) + config = cfg.ResetTestRoot("blockchain_reactor_test") + genDoc, privVals := randGenesisDoc(1, false, 30) - bcr := newBlockchainReactor(log.TestingLogger(), maxBlockHeight) - bcr.Start() - defer bcr.Stop() + maxBlockHeight := int64(148) - // Add some peers in - peer := newbcrTestPeer(p2p.ID(cmn.RandStr(12))) + otherChain := newBlockchainReactor(log.TestingLogger(), genDoc, privVals, maxBlockHeight) + defer func() { + otherChain.reactor.Stop() + otherChain.app.Stop() + }() - // XXX: This doesn't add the peer to anything, - // so it's hard to check that it's later removed - bcr.AddPeer(peer) - assert.True(t, bcr.Switch.Peers().Size() > 0) + reactorPairs := make([]BlockchainReactorPair, 4) - // send a bad block from the peer - // default blocks already dont have commits, so should fail - block := bcr.store.LoadBlock(3) - msg := &bcBlockResponseMessage{Block: block} - peer.Send(BlockchainChannel, struct{ BlockchainMessage }{msg}) + reactorPairs[0] = newBlockchainReactor(log.TestingLogger(), genDoc, privVals, maxBlockHeight) + reactorPairs[1] = newBlockchainReactor(log.TestingLogger(), genDoc, privVals, 0) + reactorPairs[2] = newBlockchainReactor(log.TestingLogger(), genDoc, privVals, 0) + reactorPairs[3] = newBlockchainReactor(log.TestingLogger(), genDoc, privVals, 0) - ticker := time.NewTicker(time.Millisecond * 10) - timer := time.NewTimer(time.Second * 2) -LOOP: - for { - select { - case <-ticker.C: - if bcr.Switch.Peers().Size() == 0 { - break LOOP - } - case <-timer.C: - t.Fatal("Timed out waiting to disconnect peer") + switches := p2p.MakeConnectedSwitches(config.P2P, 4, func(i int, s *p2p.Switch) *p2p.Switch { + s.AddReactor("BLOCKCHAIN", reactorPairs[i].reactor) + return s + + }, p2p.Connect2Switches) + + defer func() { + for _, r := range reactorPairs { + r.reactor.Stop() + r.app.Stop() } + }() + + for { + if reactorPairs[3].reactor.pool.IsCaughtUp() { + break + } + + time.Sleep(1 * time.Second) } + + //at this time, reactors[0-3] is the newest + assert.Equal(t, 3, reactorPairs[1].reactor.Switch.Peers().Size()) + + //mark reactorPairs[3] is an invalid peer + reactorPairs[3].reactor.store = otherChain.reactor.store + + lastReactorPair := newBlockchainReactor(log.TestingLogger(), genDoc, privVals, 0) + reactorPairs = append(reactorPairs, lastReactorPair) + + switches = append(switches, p2p.MakeConnectedSwitches(config.P2P, 1, func(i int, s *p2p.Switch) *p2p.Switch { + s.AddReactor("BLOCKCHAIN", reactorPairs[len(reactorPairs)-1].reactor) + return s + + }, p2p.Connect2Switches)...) + + for i := 0; i < len(reactorPairs)-1; i++ { + p2p.Connect2Switches(switches, i, len(reactorPairs)-1) + } + + for { + if lastReactorPair.reactor.pool.IsCaughtUp() || lastReactorPair.reactor.Switch.Peers().Size() == 0 { + break + } + + time.Sleep(1 * time.Second) + } + + assert.True(t, lastReactorPair.reactor.Switch.Peers().Size() < len(reactorPairs)-1) } -*/ //---------------------------------------------- // utility funcs @@ -155,55 +262,41 @@ func makeTxs(height int64) (txs []types.Tx) { return txs } -func makeBlock(height int64, state sm.State) *types.Block { - block, _ := state.MakeBlock(height, makeTxs(height), new(types.Commit), nil, state.Validators.GetProposer().Address) +func makeBlock(height int64, state sm.State, lastCommit *types.Commit) *types.Block { + block, _ := state.MakeBlock(height, makeTxs(height), lastCommit, nil, state.Validators.GetProposer().Address) return block } -// The Test peer -type bcrTestPeer struct { - cmn.BaseService - id p2p.ID - ch chan interface{} +type testApp struct { + abci.BaseApplication } -var _ p2p.Peer = (*bcrTestPeer)(nil) +var _ abci.Application = (*testApp)(nil) -func newbcrTestPeer(id p2p.ID) *bcrTestPeer { - bcr := &bcrTestPeer{ - id: id, - ch: make(chan interface{}, 2), - } - bcr.BaseService = *cmn.NewBaseService(nil, "bcrTestPeer", bcr) - return bcr +func (app *testApp) Info(req abci.RequestInfo) (resInfo abci.ResponseInfo) { + return abci.ResponseInfo{} } -func (tp *bcrTestPeer) lastBlockchainMessage() interface{} { return <-tp.ch } - -func (tp *bcrTestPeer) TrySend(chID byte, msgBytes []byte) bool { - var msg BlockchainMessage - err := cdc.UnmarshalBinaryBare(msgBytes, &msg) - if err != nil { - panic(cmn.ErrorWrap(err, "Error while trying to parse a BlockchainMessage")) - } - if _, ok := msg.(*bcStatusResponseMessage); ok { - // Discard status response messages since they skew our results - // We only want to deal with: - // + bcBlockResponseMessage - // + bcNoBlockResponseMessage - } else { - tp.ch <- msg - } - return true +func (app *testApp) BeginBlock(req abci.RequestBeginBlock) abci.ResponseBeginBlock { + return abci.ResponseBeginBlock{} } -func (tp *bcrTestPeer) Send(chID byte, msgBytes []byte) bool { return tp.TrySend(chID, msgBytes) } -func (tp *bcrTestPeer) NodeInfo() p2p.NodeInfo { return p2p.NodeInfo{} } -func (tp *bcrTestPeer) Status() p2p.ConnectionStatus { return p2p.ConnectionStatus{} } -func (tp *bcrTestPeer) ID() p2p.ID { return tp.id } -func (tp *bcrTestPeer) IsOutbound() bool { return false } -func (tp *bcrTestPeer) IsPersistent() bool { return true } -func (tp *bcrTestPeer) Get(s string) interface{} { return s } -func (tp *bcrTestPeer) Set(string, interface{}) {} -func (tp *bcrTestPeer) RemoteIP() net.IP { return []byte{127, 0, 0, 1} } -func (tp *bcrTestPeer) OriginalAddr() *p2p.NetAddress { return nil } +func (app *testApp) EndBlock(req abci.RequestEndBlock) abci.ResponseEndBlock { + return abci.ResponseEndBlock{} +} + +func (app *testApp) DeliverTx(tx []byte) abci.ResponseDeliverTx { + return abci.ResponseDeliverTx{Tags: []cmn.KVPair{}} +} + +func (app *testApp) CheckTx(tx []byte) abci.ResponseCheckTx { + return abci.ResponseCheckTx{} +} + +func (app *testApp) Commit() abci.ResponseCommit { + return abci.ResponseCommit{} +} + +func (app *testApp) Query(reqQuery abci.RequestQuery) (resQuery abci.ResponseQuery) { + return +} diff --git a/blockchain/store.go b/blockchain/store.go index fa9ee518..498cca68 100644 --- a/blockchain/store.go +++ b/blockchain/store.go @@ -63,7 +63,7 @@ func (bs *BlockStore) LoadBlock(height int64) *types.Block { part := bs.LoadBlockPart(height, i) buf = append(buf, part.Bytes...) } - err := cdc.UnmarshalBinary(buf, block) + err := cdc.UnmarshalBinaryLengthPrefixed(buf, block) if err != nil { // NOTE: The existence of meta should imply the existence of the // block. So, make sure meta is only saved after blocks are saved. diff --git a/blockchain/store_test.go b/blockchain/store_test.go index 9c8fdb23..9abc210b 100644 --- a/blockchain/store_test.go +++ b/blockchain/store_test.go @@ -6,16 +6,43 @@ import ( "runtime/debug" "strings" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + cfg "github.com/tendermint/tendermint/config" + cmn "github.com/tendermint/tendermint/libs/common" "github.com/tendermint/tendermint/libs/db" + dbm "github.com/tendermint/tendermint/libs/db" "github.com/tendermint/tendermint/libs/log" + sm "github.com/tendermint/tendermint/state" "github.com/tendermint/tendermint/types" tmtime "github.com/tendermint/tendermint/types/time" ) +// make a Commit with a single vote containing just the height and a timestamp +func makeTestCommit(height int64, timestamp time.Time) *types.Commit { + return &types.Commit{ + Precommits: []*types.CommitSig{ + {Height: height, Timestamp: timestamp}, + }, + } +} + +func makeStateAndBlockStore(logger log.Logger) (sm.State, *BlockStore) { + config := cfg.ResetTestRoot("blockchain_reactor_test") + // blockDB := dbm.NewDebugDB("blockDB", dbm.NewMemDB()) + // stateDB := dbm.NewDebugDB("stateDB", dbm.NewMemDB()) + blockDB := dbm.NewMemDB() + stateDB := dbm.NewMemDB() + state, err := sm.LoadStateFromDBOrGenesisFile(stateDB, config.GenesisFile()) + if err != nil { + panic(cmn.ErrorWrap(err, "error constructing state from genesis file")) + } + return state, NewBlockStore(blockDB) +} + func TestLoadBlockStoreStateJSON(t *testing.T) { db := db.NewMemDB() @@ -65,12 +92,11 @@ func freshBlockStore() (*BlockStore, db.DB) { var ( state, _ = makeStateAndBlockStore(log.NewTMLogger(new(bytes.Buffer))) - block = makeBlock(1, state) + block = makeBlock(1, state, new(types.Commit)) partSet = block.MakePartSet(2) part1 = partSet.GetPart(0) part2 = partSet.GetPart(1) - seenCommit1 = &types.Commit{Precommits: []*types.Vote{{Height: 10, - Timestamp: tmtime.Now()}}} + seenCommit1 = makeTestCommit(10, tmtime.Now()) ) // TODO: This test should be simplified ... @@ -88,10 +114,9 @@ func TestBlockStoreSaveLoadBlock(t *testing.T) { } // save a block - block := makeBlock(bs.Height()+1, state) + block := makeBlock(bs.Height()+1, state, new(types.Commit)) validPartSet := block.MakePartSet(2) - seenCommit := &types.Commit{Precommits: []*types.Vote{{Height: 10, - Timestamp: tmtime.Now()}}} + seenCommit := makeTestCommit(10, tmtime.Now()) bs.SaveBlock(block, partSet, seenCommit) require.Equal(t, bs.Height(), block.Header.Height, "expecting the new height to be changed") @@ -110,8 +135,7 @@ func TestBlockStoreSaveLoadBlock(t *testing.T) { // End of setup, test data - commitAtH10 := &types.Commit{Precommits: []*types.Vote{{Height: 10, - Timestamp: tmtime.Now()}}} + commitAtH10 := makeTestCommit(10, tmtime.Now()) tuples := []struct { block *types.Block parts *types.PartSet @@ -294,7 +318,7 @@ func TestLoadBlockPart(t *testing.T) { gotPart, _, panicErr := doFn(loadPart) require.Nil(t, panicErr, "an existent and proper block should not panic") require.Nil(t, res, "a properly saved block should return a proper block") - require.Equal(t, gotPart.(*types.Part).Hash(), part1.Hash(), + require.Equal(t, gotPart.(*types.Part), part1, "expecting successful retrieval of previously saved block") } @@ -331,12 +355,10 @@ func TestLoadBlockMeta(t *testing.T) { func TestBlockFetchAtHeight(t *testing.T) { state, bs := makeStateAndBlockStore(log.NewTMLogger(new(bytes.Buffer))) require.Equal(t, bs.Height(), int64(0), "initially the height should be zero") - block := makeBlock(bs.Height()+1, state) + block := makeBlock(bs.Height()+1, state, new(types.Commit)) partSet := block.MakePartSet(2) - seenCommit := &types.Commit{Precommits: []*types.Vote{{Height: 10, - Timestamp: tmtime.Now()}}} - + seenCommit := makeTestCommit(10, tmtime.Now()) bs.SaveBlock(block, partSet, seenCommit) require.Equal(t, bs.Height(), block.Header.Height, "expecting the new height to be changed") diff --git a/blockchain/wire.go b/blockchain/wire.go index 91156fa8..487fbe2b 100644 --- a/blockchain/wire.go +++ b/blockchain/wire.go @@ -1,7 +1,7 @@ package blockchain import ( - "github.com/tendermint/go-amino" + amino "github.com/tendermint/go-amino" "github.com/tendermint/tendermint/types" ) diff --git a/cmd/priv_val_server/main.go b/cmd/priv_val_server/main.go index 03aa57f4..768b9cf6 100644 --- a/cmd/priv_val_server/main.go +++ b/cmd/priv_val_server/main.go @@ -3,6 +3,7 @@ package main import ( "flag" "os" + "time" "github.com/tendermint/tendermint/crypto/ed25519" cmn "github.com/tendermint/tendermint/libs/common" @@ -13,9 +14,10 @@ import ( func main() { var ( - addr = flag.String("addr", ":26659", "Address of client to connect to") - chainID = flag.String("chain-id", "mychain", "chain id") - privValPath = flag.String("priv", "", "priv val file path") + addr = flag.String("addr", ":26659", "Address of client to connect to") + chainID = flag.String("chain-id", "mychain", "chain id") + privValKeyPath = flag.String("priv-key", "", "priv val key file path") + privValStatePath = flag.String("priv-state", "", "priv val state file path") logger = log.NewTMLogger( log.NewSyncWriter(os.Stdout), @@ -27,18 +29,26 @@ func main() { "Starting private validator", "addr", *addr, "chainID", *chainID, - "privPath", *privValPath, + "privKeyPath", *privValKeyPath, + "privStatePath", *privValStatePath, ) - pv := privval.LoadFilePV(*privValPath) + pv := privval.LoadFilePV(*privValKeyPath, *privValStatePath) - rs := privval.NewRemoteSigner( - logger, - *chainID, - *addr, - pv, - ed25519.GenPrivKey(), - ) + var dialer privval.Dialer + protocol, address := cmn.ProtocolAndAddress(*addr) + switch protocol { + case "unix": + dialer = privval.DialUnixFn(address) + case "tcp": + connTimeout := 3 * time.Second // TODO + dialer = privval.DialTCPFn(address, connTimeout, ed25519.GenPrivKey()) + default: + logger.Error("Unknown protocol", "protocol", protocol) + os.Exit(1) + } + + rs := privval.NewRemoteSigner(logger, *chainID, pv, dialer) err := rs.Start() if err != nil { panic(err) diff --git a/cmd/tendermint/commands/gen_validator.go b/cmd/tendermint/commands/gen_validator.go index 20d43d4d..572bc974 100644 --- a/cmd/tendermint/commands/gen_validator.go +++ b/cmd/tendermint/commands/gen_validator.go @@ -17,7 +17,7 @@ var GenValidatorCmd = &cobra.Command{ } func genValidator(cmd *cobra.Command, args []string) { - pv := privval.GenFilePV("") + pv := privval.GenFilePV("", "") jsbz, err := cdc.MarshalJSON(pv) if err != nil { panic(err) diff --git a/cmd/tendermint/commands/init.go b/cmd/tendermint/commands/init.go index 85ee4491..1d6e24d7 100644 --- a/cmd/tendermint/commands/init.go +++ b/cmd/tendermint/commands/init.go @@ -4,7 +4,6 @@ import ( "fmt" "github.com/spf13/cobra" - cfg "github.com/tendermint/tendermint/config" cmn "github.com/tendermint/tendermint/libs/common" "github.com/tendermint/tendermint/p2p" @@ -26,15 +25,18 @@ func initFiles(cmd *cobra.Command, args []string) error { func initFilesWithConfig(config *cfg.Config) error { // private validator - privValFile := config.PrivValidatorFile() + privValKeyFile := config.PrivValidatorKeyFile() + privValStateFile := config.PrivValidatorStateFile() var pv *privval.FilePV - if cmn.FileExists(privValFile) { - pv = privval.LoadFilePV(privValFile) - logger.Info("Found private validator", "path", privValFile) + if cmn.FileExists(privValKeyFile) { + pv = privval.LoadFilePV(privValKeyFile, privValStateFile) + logger.Info("Found private validator", "keyFile", privValKeyFile, + "stateFile", privValStateFile) } else { - pv = privval.GenFilePV(privValFile) + pv = privval.GenFilePV(privValKeyFile, privValStateFile) pv.Save() - logger.Info("Generated private validator", "path", privValFile) + logger.Info("Generated private validator", "keyFile", privValKeyFile, + "stateFile", privValStateFile) } nodeKeyFile := config.NodeKeyFile() @@ -57,9 +59,10 @@ func initFilesWithConfig(config *cfg.Config) error { GenesisTime: tmtime.Now(), ConsensusParams: types.DefaultConsensusParams(), } + key := pv.GetPubKey() genDoc.Validators = []types.GenesisValidator{{ - Address: pv.GetPubKey().Address(), - PubKey: pv.GetPubKey(), + Address: key.Address(), + PubKey: key, Power: 10, }} diff --git a/cmd/tendermint/commands/lite.go b/cmd/tendermint/commands/lite.go index bc51d7de..eb2817b6 100644 --- a/cmd/tendermint/commands/lite.go +++ b/cmd/tendermint/commands/lite.go @@ -26,12 +26,12 @@ just with added trust and running locally.`, } var ( - listenAddr string - nodeAddr string - chainID string - home string - maxOpenConnections int - cacheSize int + listenAddr string + nodeAddr string + chainID string + home string + maxOpenConnections int + cacheSize int ) func init() { @@ -39,7 +39,7 @@ func init() { LiteCmd.Flags().StringVar(&nodeAddr, "node", "tcp://localhost:26657", "Connect to a Tendermint node at this address") LiteCmd.Flags().StringVar(&chainID, "chain-id", "tendermint", "Specify the Tendermint chain ID") LiteCmd.Flags().StringVar(&home, "home-dir", ".tendermint-lite", "Specify the home directory") - LiteCmd.Flags().IntVar(&maxOpenConnections,"max-open-connections",900,"Maximum number of simultaneous connections (including WebSocket).") + LiteCmd.Flags().IntVar(&maxOpenConnections, "max-open-connections", 900, "Maximum number of simultaneous connections (including WebSocket).") LiteCmd.Flags().IntVar(&cacheSize, "cache-size", 10, "Specify the memory trust store cache size") } diff --git a/cmd/tendermint/commands/reset_priv_validator.go b/cmd/tendermint/commands/reset_priv_validator.go index 53d34712..122c2a72 100644 --- a/cmd/tendermint/commands/reset_priv_validator.go +++ b/cmd/tendermint/commands/reset_priv_validator.go @@ -5,6 +5,7 @@ import ( "github.com/spf13/cobra" + cmn "github.com/tendermint/tendermint/libs/common" "github.com/tendermint/tendermint/libs/log" "github.com/tendermint/tendermint/privval" ) @@ -27,36 +28,41 @@ var ResetPrivValidatorCmd = &cobra.Command{ // XXX: this is totally unsafe. // it's only suitable for testnets. func resetAll(cmd *cobra.Command, args []string) { - ResetAll(config.DBDir(), config.P2P.AddrBookFile(), config.PrivValidatorFile(), logger) + ResetAll(config.DBDir(), config.P2P.AddrBookFile(), config.PrivValidatorKeyFile(), + config.PrivValidatorStateFile(), logger) } // XXX: this is totally unsafe. // it's only suitable for testnets. func resetPrivValidator(cmd *cobra.Command, args []string) { - resetFilePV(config.PrivValidatorFile(), logger) + resetFilePV(config.PrivValidatorKeyFile(), config.PrivValidatorStateFile(), logger) } -// ResetAll removes the privValidator and address book files plus all data. +// ResetAll removes address book files plus all data, and resets the privValdiator data. // Exported so other CLI tools can use it. -func ResetAll(dbDir, addrBookFile, privValFile string, logger log.Logger) { - resetFilePV(privValFile, logger) +func ResetAll(dbDir, addrBookFile, privValKeyFile, privValStateFile string, logger log.Logger) { removeAddrBook(addrBookFile, logger) if err := os.RemoveAll(dbDir); err == nil { logger.Info("Removed all blockchain history", "dir", dbDir) } else { logger.Error("Error removing all blockchain history", "dir", dbDir, "err", err) } + // recreate the dbDir since the privVal state needs to live there + cmn.EnsureDir(dbDir, 0700) + resetFilePV(privValKeyFile, privValStateFile, logger) } -func resetFilePV(privValFile string, logger log.Logger) { - if _, err := os.Stat(privValFile); err == nil { - pv := privval.LoadFilePV(privValFile) +func resetFilePV(privValKeyFile, privValStateFile string, logger log.Logger) { + if _, err := os.Stat(privValKeyFile); err == nil { + pv := privval.LoadFilePVEmptyState(privValKeyFile, privValStateFile) pv.Reset() - logger.Info("Reset private validator file to genesis state", "file", privValFile) + logger.Info("Reset private validator file to genesis state", "keyFile", privValKeyFile, + "stateFile", privValStateFile) } else { - pv := privval.GenFilePV(privValFile) + pv := privval.GenFilePV(privValKeyFile, privValStateFile) pv.Save() - logger.Info("Generated private validator file", "file", privValFile) + logger.Info("Generated private validator file", "file", "keyFile", privValKeyFile, + "stateFile", privValStateFile) } } diff --git a/cmd/tendermint/commands/root.go b/cmd/tendermint/commands/root.go index 89ffbe74..6d79f75c 100644 --- a/cmd/tendermint/commands/root.go +++ b/cmd/tendermint/commands/root.go @@ -54,6 +54,9 @@ var RootCmd = &cobra.Command{ if err != nil { return err } + if config.LogFormat == cfg.LogFormatJSON { + logger = log.NewTMJSONLogger(log.NewSyncWriter(os.Stdout)) + } logger, err = tmflags.ParseLogLevel(config.LogLevel, logger, cfg.DefaultLogLevel()) if err != nil { return err diff --git a/cmd/tendermint/commands/root_test.go b/cmd/tendermint/commands/root_test.go index e8095b38..892a49b7 100644 --- a/cmd/tendermint/commands/root_test.go +++ b/cmd/tendermint/commands/root_test.go @@ -22,10 +22,6 @@ var ( defaultRoot = os.ExpandEnv("$HOME/.some/test/dir") ) -const ( - rootName = "root" -) - // clearConfig clears env vars, the given root dir, and resets viper. func clearConfig(dir string) { if err := os.Unsetenv("TMHOME"); err != nil { diff --git a/cmd/tendermint/commands/run_node.go b/cmd/tendermint/commands/run_node.go index 6dabacb1..ef205aa6 100644 --- a/cmd/tendermint/commands/run_node.go +++ b/cmd/tendermint/commands/run_node.go @@ -24,7 +24,7 @@ func AddNodeFlags(cmd *cobra.Command) { cmd.Flags().Bool("fast_sync", config.FastSync, "Fast blockchain syncing") // abci flags - cmd.Flags().String("proxy_app", config.ProxyApp, "Proxy app address, or 'nilapp' or 'kvstore' for local testing.") + cmd.Flags().String("proxy_app", config.ProxyApp, "Proxy app address, or one of: 'kvstore', 'persistent_kvstore', 'counter', 'counter_serial' or 'noop' for local testing.") cmd.Flags().String("abci", config.ABCI, "Specify abci transport (socket | grpc)") // rpc flags diff --git a/cmd/tendermint/commands/show_validator.go b/cmd/tendermint/commands/show_validator.go index 54765164..78bc0603 100644 --- a/cmd/tendermint/commands/show_validator.go +++ b/cmd/tendermint/commands/show_validator.go @@ -16,7 +16,7 @@ var ShowValidatorCmd = &cobra.Command{ } func showValidator(cmd *cobra.Command, args []string) { - privValidator := privval.LoadOrGenFilePV(config.PrivValidatorFile()) + privValidator := privval.LoadOrGenFilePV(config.PrivValidatorKeyFile(), config.PrivValidatorStateFile()) pubKeyJSONBytes, _ := cdc.MarshalJSON(privValidator.GetPubKey()) fmt.Println(string(pubKeyJSONBytes)) } diff --git a/cmd/tendermint/commands/testnet.go b/cmd/tendermint/commands/testnet.go index 0f7dd79a..c3ef8619 100644 --- a/cmd/tendermint/commands/testnet.go +++ b/cmd/tendermint/commands/testnet.go @@ -85,11 +85,18 @@ func testnetFiles(cmd *cobra.Command, args []string) error { _ = os.RemoveAll(outputDir) return err } + err = os.MkdirAll(filepath.Join(nodeDir, "data"), nodeDirPerm) + if err != nil { + _ = os.RemoveAll(outputDir) + return err + } initFilesWithConfig(config) - pvFile := filepath.Join(nodeDir, config.BaseConfig.PrivValidator) - pv := privval.LoadFilePV(pvFile) + pvKeyFile := filepath.Join(nodeDir, config.BaseConfig.PrivValidatorKey) + pvStateFile := filepath.Join(nodeDir, config.BaseConfig.PrivValidatorState) + + pv := privval.LoadFilePV(pvKeyFile, pvStateFile) genVals[i] = types.GenesisValidator{ Address: pv.GetPubKey().Address(), PubKey: pv.GetPubKey(), @@ -127,14 +134,32 @@ func testnetFiles(cmd *cobra.Command, args []string) error { } } + // Gather persistent peer addresses. + var ( + persistentPeers string + err error + ) if populatePersistentPeers { - err := populatePersistentPeersInConfigAndWriteIt(config) + persistentPeers, err = persistentPeersString(config) if err != nil { _ = os.RemoveAll(outputDir) return err } } + // Overwrite default config. + for i := 0; i < nValidators+nNonValidators; i++ { + nodeDir := filepath.Join(outputDir, fmt.Sprintf("%s%d", nodeDirPrefix, i)) + config.SetRoot(nodeDir) + config.P2P.AddrBookStrict = false + config.P2P.AllowDuplicateIP = true + if populatePersistentPeers { + config.P2P.PersistentPeers = persistentPeers + } + + cfg.WriteConfigFile(filepath.Join(nodeDir, "config", "config.toml"), config) + } + fmt.Printf("Successfully initialized %v node directories\n", nValidators+nNonValidators) return nil } @@ -157,28 +182,16 @@ func hostnameOrIP(i int) string { return fmt.Sprintf("%s%d", hostnamePrefix, i) } -func populatePersistentPeersInConfigAndWriteIt(config *cfg.Config) error { +func persistentPeersString(config *cfg.Config) (string, error) { persistentPeers := make([]string, nValidators+nNonValidators) for i := 0; i < nValidators+nNonValidators; i++ { nodeDir := filepath.Join(outputDir, fmt.Sprintf("%s%d", nodeDirPrefix, i)) config.SetRoot(nodeDir) nodeKey, err := p2p.LoadNodeKey(config.NodeKeyFile()) if err != nil { - return err + return "", err } persistentPeers[i] = p2p.IDAddressString(nodeKey.ID(), fmt.Sprintf("%s:%d", hostnameOrIP(i), p2pPort)) } - persistentPeersList := strings.Join(persistentPeers, ",") - - for i := 0; i < nValidators+nNonValidators; i++ { - nodeDir := filepath.Join(outputDir, fmt.Sprintf("%s%d", nodeDirPrefix, i)) - config.SetRoot(nodeDir) - config.P2P.PersistentPeers = persistentPeersList - config.P2P.AddrBookStrict = false - - // overwrite default config - cfg.WriteConfigFile(filepath.Join(nodeDir, "config", "config.toml"), config) - } - - return nil + return strings.Join(persistentPeers, ","), nil } diff --git a/cmd/tendermint/commands/wire.go b/cmd/tendermint/commands/wire.go index 0f0b536d..322f92b3 100644 --- a/cmd/tendermint/commands/wire.go +++ b/cmd/tendermint/commands/wire.go @@ -1,7 +1,7 @@ package commands import ( - "github.com/tendermint/go-amino" + amino "github.com/tendermint/go-amino" cryptoAmino "github.com/tendermint/tendermint/crypto/encoding/amino" ) diff --git a/config/config.go b/config/config.go index f2bac5c6..fc50d7c6 100644 --- a/config/config.go +++ b/config/config.go @@ -14,6 +14,11 @@ const ( FuzzModeDrop = iota // FuzzModeDelay is a mode in which we randomly sleep FuzzModeDelay + + // LogFormatPlain is a format for colored text + LogFormatPlain = "plain" + // LogFormatJSON is a format for json output + LogFormatJSON = "json" ) // NOTE: Most of the structs & relevant comments + the @@ -30,15 +35,24 @@ var ( defaultConfigFileName = "config.toml" defaultGenesisJSONName = "genesis.json" - defaultPrivValName = "priv_validator.json" + defaultPrivValKeyName = "priv_validator_key.json" + defaultPrivValStateName = "priv_validator_state.json" + defaultNodeKeyName = "node_key.json" defaultAddrBookName = "addrbook.json" - defaultConfigFilePath = filepath.Join(defaultConfigDir, defaultConfigFileName) - defaultGenesisJSONPath = filepath.Join(defaultConfigDir, defaultGenesisJSONName) - defaultPrivValPath = filepath.Join(defaultConfigDir, defaultPrivValName) - defaultNodeKeyPath = filepath.Join(defaultConfigDir, defaultNodeKeyName) - defaultAddrBookPath = filepath.Join(defaultConfigDir, defaultAddrBookName) + defaultConfigFilePath = filepath.Join(defaultConfigDir, defaultConfigFileName) + defaultGenesisJSONPath = filepath.Join(defaultConfigDir, defaultGenesisJSONName) + defaultPrivValKeyPath = filepath.Join(defaultConfigDir, defaultPrivValKeyName) + defaultPrivValStatePath = filepath.Join(defaultDataDir, defaultPrivValStateName) + + defaultNodeKeyPath = filepath.Join(defaultConfigDir, defaultNodeKeyName) + defaultAddrBookPath = filepath.Join(defaultConfigDir, defaultAddrBookName) +) + +var ( + oldPrivVal = "priv_validator.json" + oldPrivValPath = filepath.Join(defaultConfigDir, oldPrivVal) ) // Config defines the top level configuration for a Tendermint node @@ -94,6 +108,9 @@ func (cfg *Config) SetRoot(root string) *Config { // ValidateBasic performs basic validation (checking param bounds, etc.) and // returns an error if any check fails. func (cfg *Config) ValidateBasic() error { + if err := cfg.BaseConfig.ValidateBasic(); err != nil { + return err + } if err := cfg.RPC.ValidateBasic(); err != nil { return errors.Wrap(err, "Error in [rpc] section") } @@ -145,11 +162,17 @@ type BaseConfig struct { // Output level for logging LogLevel string `mapstructure:"log_level"` + // Output format: 'plain' (colored text) or 'json' + LogFormat string `mapstructure:"log_format"` + // Path to the JSON file containing the initial validator set and other meta data Genesis string `mapstructure:"genesis_file"` // Path to the JSON file containing the private key to use as a validator in the consensus protocol - PrivValidator string `mapstructure:"priv_validator_file"` + PrivValidatorKey string `mapstructure:"priv_validator_key_file"` + + // Path to the JSON file containing the last sign state of a validator + PrivValidatorState string `mapstructure:"priv_validator_state_file"` // TCP or UNIX socket address for Tendermint to listen on for // connections from an external PrivValidator process @@ -172,18 +195,20 @@ type BaseConfig struct { // DefaultBaseConfig returns a default base configuration for a Tendermint node func DefaultBaseConfig() BaseConfig { return BaseConfig{ - Genesis: defaultGenesisJSONPath, - PrivValidator: defaultPrivValPath, - NodeKey: defaultNodeKeyPath, - Moniker: defaultMoniker, - ProxyApp: "tcp://127.0.0.1:26658", - ABCI: "socket", - LogLevel: DefaultPackageLogLevels(), - ProfListenAddress: "", - FastSync: true, - FilterPeers: false, - DBBackend: "leveldb", - DBPath: "data", + Genesis: defaultGenesisJSONPath, + PrivValidatorKey: defaultPrivValKeyPath, + PrivValidatorState: defaultPrivValStatePath, + NodeKey: defaultNodeKeyPath, + Moniker: defaultMoniker, + ProxyApp: "tcp://127.0.0.1:26658", + ABCI: "socket", + LogLevel: DefaultPackageLogLevels(), + LogFormat: LogFormatPlain, + ProfListenAddress: "", + FastSync: true, + FilterPeers: false, + DBBackend: "leveldb", + DBPath: "data", } } @@ -206,9 +231,20 @@ func (cfg BaseConfig) GenesisFile() string { return rootify(cfg.Genesis, cfg.RootDir) } -// PrivValidatorFile returns the full path to the priv_validator.json file -func (cfg BaseConfig) PrivValidatorFile() string { - return rootify(cfg.PrivValidator, cfg.RootDir) +// PrivValidatorKeyFile returns the full path to the priv_validator_key.json file +func (cfg BaseConfig) PrivValidatorKeyFile() string { + return rootify(cfg.PrivValidatorKey, cfg.RootDir) +} + +// PrivValidatorFile returns the full path to the priv_validator_state.json file +func (cfg BaseConfig) PrivValidatorStateFile() string { + return rootify(cfg.PrivValidatorState, cfg.RootDir) +} + +// OldPrivValidatorFile returns the full path of the priv_validator.json from pre v0.28.0. +// TODO: eventually remove. +func (cfg BaseConfig) OldPrivValidatorFile() string { + return rootify(oldPrivValPath, cfg.RootDir) } // NodeKeyFile returns the full path to the node_key.json file @@ -221,6 +257,17 @@ func (cfg BaseConfig) DBDir() string { return rootify(cfg.DBPath, cfg.RootDir) } +// ValidateBasic performs basic validation (checking param bounds, etc.) and +// returns an error if any check fails. +func (cfg BaseConfig) ValidateBasic() error { + switch cfg.LogFormat { + case LogFormatPlain, LogFormatJSON: + default: + return errors.New("unknown log_format (must be 'plain' or 'json')") + } + return nil +} + // DefaultLogLevel returns a default log level of "error" func DefaultLogLevel() string { return "error" @@ -242,13 +289,25 @@ type RPCConfig struct { // TCP or UNIX socket address for the RPC server to listen on ListenAddress string `mapstructure:"laddr"` + // A list of origins a cross-domain request can be executed from. + // If the special '*' value is present in the list, all origins will be allowed. + // An origin may contain a wildcard (*) to replace 0 or more characters (i.e.: http://*.domain.com). + // Only one wildcard can be used per origin. + CORSAllowedOrigins []string `mapstructure:"cors_allowed_origins"` + + // A list of methods the client is allowed to use with cross-domain requests. + CORSAllowedMethods []string `mapstructure:"cors_allowed_methods"` + + // A list of non simple headers the client is allowed to use with cross-domain requests. + CORSAllowedHeaders []string `mapstructure:"cors_allowed_headers"` + // TCP or UNIX socket address for the gRPC server to listen on // NOTE: This server only supports /broadcast_tx_commit GRPCListenAddress string `mapstructure:"grpc_laddr"` // Maximum number of simultaneous connections. // Does not include RPC (HTTP&WebSocket) connections. See max_open_connections - // If you want to accept more significant number than the default, make sure + // If you want to accept a larger number than the default, make sure // you increase your OS limits. // 0 - unlimited. GRPCMaxOpenConnections int `mapstructure:"grpc_max_open_connections"` @@ -258,7 +317,7 @@ type RPCConfig struct { // Maximum number of simultaneous connections (including WebSocket). // Does not include gRPC connections. See grpc_max_open_connections - // If you want to accept more significant number than the default, make sure + // If you want to accept a larger number than the default, make sure // you increase your OS limits. // 0 - unlimited. // Should be < {ulimit -Sn} - {MaxNumInboundPeers} - {MaxNumOutboundPeers} - {N of wal, db and other open files} @@ -269,8 +328,10 @@ type RPCConfig struct { // DefaultRPCConfig returns a default configuration for the RPC server func DefaultRPCConfig() *RPCConfig { return &RPCConfig{ - ListenAddress: "tcp://0.0.0.0:26657", - + ListenAddress: "tcp://0.0.0.0:26657", + CORSAllowedOrigins: []string{}, + CORSAllowedMethods: []string{"HEAD", "GET", "POST"}, + CORSAllowedHeaders: []string{"Origin", "Accept", "Content-Type", "X-Requested-With", "X-Server-Time"}, GRPCListenAddress: "", GRPCMaxOpenConnections: 900, @@ -300,6 +361,11 @@ func (cfg *RPCConfig) ValidateBasic() error { return nil } +// IsCorsEnabled returns true if cross-origin resource sharing is enabled. +func (cfg *RPCConfig) IsCorsEnabled() bool { + return len(cfg.CORSAllowedOrigins) != 0 +} + //----------------------------------------------------------------------------- // P2PConfig @@ -392,7 +458,7 @@ func DefaultP2PConfig() *P2PConfig { RecvRate: 5120000, // 5 mB/s PexReactor: true, SeedMode: false, - AllowDuplicateIP: true, // so non-breaking yet + AllowDuplicateIP: false, HandshakeTimeout: 20 * time.Second, DialTimeout: 3 * time.Second, TestDialFail: false, @@ -497,6 +563,11 @@ func (cfg *MempoolConfig) WalDir() string { return rootify(cfg.WalPath, cfg.RootDir) } +// WalEnabled returns true if the WAL is enabled. +func (cfg *MempoolConfig) WalEnabled() bool { + return cfg.WalPath != "" +} + // ValidateBasic performs basic validation (checking param bounds, etc.) and // returns an error if any check fails. func (cfg *MempoolConfig) ValidateBasic() error { @@ -565,7 +636,7 @@ func DefaultConsensusConfig() *ConsensusConfig { // TestConsensusConfig returns a configuration for testing the consensus service func TestConsensusConfig() *ConsensusConfig { cfg := DefaultConsensusConfig() - cfg.TimeoutPropose = 100 * time.Millisecond + cfg.TimeoutPropose = 40 * time.Millisecond cfg.TimeoutProposeDelta = 1 * time.Millisecond cfg.TimeoutPrevote = 10 * time.Millisecond cfg.TimeoutPrevoteDelta = 1 * time.Millisecond @@ -727,12 +798,12 @@ type InstrumentationConfig struct { PrometheusListenAddr string `mapstructure:"prometheus_listen_addr"` // Maximum number of simultaneous connections. - // If you want to accept more significant number than the default, make sure + // If you want to accept a larger number than the default, make sure // you increase your OS limits. // 0 - unlimited. MaxOpenConnections int `mapstructure:"max_open_connections"` - // Tendermint instrumentation namespace. + // Instrumentation namespace. Namespace string `mapstructure:"namespace"` } diff --git a/config/toml.go b/config/toml.go index ddfe5f05..e842e9e3 100644 --- a/config/toml.go +++ b/config/toml.go @@ -2,6 +2,7 @@ package config import ( "bytes" + "fmt" "os" "path/filepath" "text/template" @@ -86,13 +87,19 @@ db_dir = "{{ js .BaseConfig.DBPath }}" # Output level for logging, including package level options log_level = "{{ .BaseConfig.LogLevel }}" +# Output format: 'plain' (colored text) or 'json' +log_format = "{{ .BaseConfig.LogFormat }}" + ##### additional base config options ##### # Path to the JSON file containing the initial validator set and other meta data genesis_file = "{{ js .BaseConfig.Genesis }}" # Path to the JSON file containing the private key to use as a validator in the consensus protocol -priv_validator_file = "{{ js .BaseConfig.PrivValidator }}" +priv_validator_key_file = "{{ js .BaseConfig.PrivValidatorKey }}" + +# Path to the JSON file containing the last sign state of a validator +priv_validator_state_file = "{{ js .BaseConfig.PrivValidatorState }}" # TCP or UNIX socket address for Tendermint to listen on for # connections from an external PrivValidator process @@ -119,13 +126,24 @@ filter_peers = {{ .BaseConfig.FilterPeers }} # TCP or UNIX socket address for the RPC server to listen on laddr = "{{ .RPC.ListenAddress }}" +# A list of origins a cross-domain request can be executed from +# Default value '[]' disables cors support +# Use '["*"]' to allow any origin +cors_allowed_origins = [{{ range .RPC.CORSAllowedOrigins }}{{ printf "%q, " . }}{{end}}] + +# A list of methods the client is allowed to use with cross-domain requests +cors_allowed_methods = [{{ range .RPC.CORSAllowedMethods }}{{ printf "%q, " . }}{{end}}] + +# A list of non simple headers the client is allowed to use with cross-domain requests +cors_allowed_headers = [{{ range .RPC.CORSAllowedHeaders }}{{ printf "%q, " . }}{{end}}] + # TCP or UNIX socket address for the gRPC server to listen on # NOTE: This server only supports /broadcast_tx_commit grpc_laddr = "{{ .RPC.GRPCListenAddress }}" # Maximum number of simultaneous connections. # Does not include RPC (HTTP&WebSocket) connections. See max_open_connections -# If you want to accept more significant number than the default, make sure +# If you want to accept a larger number than the default, make sure # you increase your OS limits. # 0 - unlimited. # Should be < {ulimit -Sn} - {MaxNumInboundPeers} - {MaxNumOutboundPeers} - {N of wal, db and other open files} @@ -137,7 +155,7 @@ unsafe = {{ .RPC.Unsafe }} # Maximum number of simultaneous connections (including WebSocket). # Does not include gRPC connections. See grpc_max_open_connections -# If you want to accept more significant number than the default, make sure +# If you want to accept a larger number than the default, make sure # you increase your OS limits. # 0 - unlimited. # Should be < {ulimit -Sn} - {MaxNumInboundPeers} - {MaxNumOutboundPeers} - {N of wal, db and other open files} @@ -246,14 +264,17 @@ create_empty_blocks_interval = "{{ .Consensus.CreateEmptyBlocksInterval }}" peer_gossip_sleep_duration = "{{ .Consensus.PeerGossipSleepDuration }}" peer_query_maj23_sleep_duration = "{{ .Consensus.PeerQueryMaj23SleepDuration }}" +# Block time parameters. Corresponds to the minimum time increment between consecutive blocks. +blocktime_iota = "{{ .Consensus.BlockTimeIota }}" + ##### transactions indexer configuration options ##### [tx_index] # What indexer to use for transactions # # Options: -# 1) "null" (default) -# 2) "kv" - the simplest possible indexer, backed by key-value storage (defaults to levelDB; see DBBackend). +# 1) "null" +# 2) "kv" (default) - the simplest possible indexer, backed by key-value storage (defaults to levelDB; see DBBackend). indexer = "{{ .TxIndex.Indexer }}" # Comma-separated list of tags to index (by default the only tag is "tx.hash") @@ -285,7 +306,7 @@ prometheus = {{ .Instrumentation.Prometheus }} prometheus_listen_addr = "{{ .Instrumentation.PrometheusListenAddr }}" # Maximum number of simultaneous connections. -# If you want to accept more significant number than the default, make sure +# If you want to accept a larger number than the default, make sure # you increase your OS limits. # 0 - unlimited. max_open_connections = {{ .Instrumentation.MaxOpenConnections }} @@ -297,6 +318,10 @@ namespace = "{{ .Instrumentation.Namespace }}" /****** these are for test settings ***********/ func ResetTestRoot(testName string) *Config { + return ResetTestRootWithChainID(testName, "") +} + +func ResetTestRootWithChainID(testName string, chainID string) *Config { rootDir := os.ExpandEnv("$HOME/.tendermint_test") rootDir = filepath.Join(rootDir, testName) // Remove ~/.tendermint_test_bak @@ -325,25 +350,31 @@ func ResetTestRoot(testName string) *Config { baseConfig := DefaultBaseConfig() configFilePath := filepath.Join(rootDir, defaultConfigFilePath) genesisFilePath := filepath.Join(rootDir, baseConfig.Genesis) - privFilePath := filepath.Join(rootDir, baseConfig.PrivValidator) + privKeyFilePath := filepath.Join(rootDir, baseConfig.PrivValidatorKey) + privStateFilePath := filepath.Join(rootDir, baseConfig.PrivValidatorState) // Write default config file if missing. if !cmn.FileExists(configFilePath) { writeDefaultConfigFile(configFilePath) } if !cmn.FileExists(genesisFilePath) { + if chainID == "" { + chainID = "tendermint_test" + } + testGenesis := fmt.Sprintf(testGenesisFmt, chainID) cmn.MustWriteFile(genesisFilePath, []byte(testGenesis), 0644) } // we always overwrite the priv val - cmn.MustWriteFile(privFilePath, []byte(testPrivValidator), 0644) + cmn.MustWriteFile(privKeyFilePath, []byte(testPrivValidatorKey), 0644) + cmn.MustWriteFile(privStateFilePath, []byte(testPrivValidatorState), 0644) config := TestConfig().SetRoot(rootDir) return config } -var testGenesis = `{ - "genesis_time": "0001-01-01T00:00:00.000Z", - "chain_id": "tendermint_test", +var testGenesisFmt = `{ + "genesis_time": "2018-10-10T08:20:13.695936996Z", + "chain_id": "%s", "validators": [ { "pub_key": { @@ -357,7 +388,7 @@ var testGenesis = `{ "app_hash": "" }` -var testPrivValidator = `{ +var testPrivValidatorKey = `{ "address": "A3258DCBF45DCA0DF052981870F2D1441A36D145", "pub_key": { "type": "tendermint/PubKeyEd25519", @@ -366,8 +397,11 @@ var testPrivValidator = `{ "priv_key": { "type": "tendermint/PrivKeyEd25519", "value": "EVkqJO/jIXp3rkASXfh9YnyToYXRXhBr6g9cQVxPFnQBP/5povV4HTjvsy530kybxKHwEi85iU8YL0qQhSYVoQ==" - }, - "last_height": "0", - "last_round": "0", - "last_step": 0 + } +}` + +var testPrivValidatorState = `{ + "height": "0", + "round": "0", + "step": 0 }` diff --git a/config/toml_test.go b/config/toml_test.go index a1637f67..59528db1 100644 --- a/config/toml_test.go +++ b/config/toml_test.go @@ -60,7 +60,7 @@ func TestEnsureTestRoot(t *testing.T) { // TODO: make sure the cfg returned and testconfig are the same! baseConfig := DefaultBaseConfig() - ensureFiles(t, rootDir, defaultDataDir, baseConfig.Genesis, baseConfig.PrivValidator) + ensureFiles(t, rootDir, defaultDataDir, baseConfig.Genesis, baseConfig.PrivValidatorKey, baseConfig.PrivValidatorState) } func checkConfig(configFile string) bool { diff --git a/consensus/byzantine_test.go b/consensus/byzantine_test.go index 3903e6b9..ba69d0cc 100644 --- a/consensus/byzantine_test.go +++ b/consensus/byzantine_test.go @@ -76,8 +76,7 @@ func TestByzantine(t *testing.T) { conR.SetLogger(logger.With("validator", i)) conR.SetEventBus(eventBus) - var conRI p2p.Reactor // nolint: gotype, gosimple - conRI = conR + var conRI p2p.Reactor = conR // make first val byzantine if i == 0 { @@ -179,16 +178,16 @@ func byzantineDecideProposalFunc(t *testing.T, height int64, round int, cs *Cons // Create a new proposal block from state/txs from the mempool. block1, blockParts1 := cs.createProposalBlock() - polRound, polBlockID := cs.Votes.POLInfo() - proposal1 := types.NewProposal(height, round, blockParts1.Header(), polRound, polBlockID) + polRound, propBlockID := cs.ValidRound, types.BlockID{block1.Hash(), blockParts1.Header()} + proposal1 := types.NewProposal(height, round, polRound, propBlockID) if err := cs.privValidator.SignProposal(cs.state.ChainID, proposal1); err != nil { t.Error(err) } // Create a new proposal block from state/txs from the mempool. block2, blockParts2 := cs.createProposalBlock() - polRound, polBlockID = cs.Votes.POLInfo() - proposal2 := types.NewProposal(height, round, blockParts2.Header(), polRound, polBlockID) + polRound, propBlockID = cs.ValidRound, types.BlockID{block2.Hash(), blockParts2.Header()} + proposal2 := types.NewProposal(height, round, polRound, propBlockID) if err := cs.privValidator.SignProposal(cs.state.ChainID, proposal2); err != nil { t.Error(err) } @@ -226,8 +225,8 @@ func sendProposalAndParts(height int64, round int, cs *ConsensusState, peer p2p. // votes cs.mtx.Lock() - prevote, _ := cs.signVote(types.VoteTypePrevote, blockHash, parts.Header()) - precommit, _ := cs.signVote(types.VoteTypePrecommit, blockHash, parts.Header()) + prevote, _ := cs.signVote(types.PrevoteType, blockHash, parts.Header()) + precommit, _ := cs.signVote(types.PrecommitType, blockHash, parts.Header()) cs.mtx.Unlock() peer.Send(VoteChannel, cdc.MustMarshalBinaryBare(&VoteMessage{prevote})) @@ -263,7 +262,7 @@ func (br *ByzantineReactor) AddPeer(peer p2p.Peer) { // Send our state to peer. // If we're fast_syncing, broadcast a RoundStepMessage later upon SwitchToConsensus(). if !br.reactor.fastSync { - br.reactor.sendNewRoundStepMessages(peer) + br.reactor.sendNewRoundStepMessage(peer) } } func (br *ByzantineReactor) RemovePeer(peer p2p.Peer, reason interface{}) { diff --git a/consensus/common_test.go b/consensus/common_test.go index 2a5cc8e7..e6e64c25 100644 --- a/consensus/common_test.go +++ b/consensus/common_test.go @@ -6,14 +6,18 @@ import ( "fmt" "io/ioutil" "os" - "path" + "path/filepath" "reflect" "sort" "sync" "testing" "time" + "github.com/go-kit/kit/log/term" + abcicli "github.com/tendermint/tendermint/abci/client" + "github.com/tendermint/tendermint/abci/example/counter" + "github.com/tendermint/tendermint/abci/example/kvstore" abci "github.com/tendermint/tendermint/abci/types" bc "github.com/tendermint/tendermint/blockchain" cfg "github.com/tendermint/tendermint/config" @@ -27,11 +31,6 @@ import ( sm "github.com/tendermint/tendermint/state" "github.com/tendermint/tendermint/types" tmtime "github.com/tendermint/tendermint/types/time" - - "github.com/tendermint/tendermint/abci/example/counter" - "github.com/tendermint/tendermint/abci/example/kvstore" - - "github.com/go-kit/kit/log/term" ) const ( @@ -39,8 +38,8 @@ const ( ) // genesis, chain_id, priv_val -var config *cfg.Config // NOTE: must be reset for each _test.go file -var ensureTimeout = time.Second * 1 // must be in seconds because CreateEmptyBlocksInterval is +var config *cfg.Config // NOTE: must be reset for each _test.go file +var ensureTimeout = time.Millisecond * 100 func ensureDir(dir string, mode os.FileMode) { if err := cmn.EnsureDir(dir, mode); err != nil { @@ -71,10 +70,11 @@ func NewValidatorStub(privValidator types.PrivValidator, valIndex int) *validato } } -func (vs *validatorStub) signVote(voteType byte, hash []byte, header types.PartSetHeader) (*types.Vote, error) { +func (vs *validatorStub) signVote(voteType types.SignedMsgType, hash []byte, header types.PartSetHeader) (*types.Vote, error) { + addr := vs.PrivValidator.GetPubKey().Address() vote := &types.Vote{ ValidatorIndex: vs.Index, - ValidatorAddress: vs.PrivValidator.GetAddress(), + ValidatorAddress: addr, Height: vs.Height, Round: vs.Round, Timestamp: tmtime.Now(), @@ -86,7 +86,7 @@ func (vs *validatorStub) signVote(voteType byte, hash []byte, header types.PartS } // Sign vote for type/hash/header -func signVote(vs *validatorStub, voteType byte, hash []byte, header types.PartSetHeader) *types.Vote { +func signVote(vs *validatorStub, voteType types.SignedMsgType, hash []byte, header types.PartSetHeader) *types.Vote { v, err := vs.signVote(voteType, hash, header) if err != nil { panic(fmt.Errorf("failed to sign vote: %v", err)) @@ -94,7 +94,7 @@ func signVote(vs *validatorStub, voteType byte, hash []byte, header types.PartSe return v } -func signVotes(voteType byte, hash []byte, header types.PartSetHeader, vss ...*validatorStub) []*types.Vote { +func signVotes(voteType types.SignedMsgType, hash []byte, header types.PartSetHeader, vss ...*validatorStub) []*types.Vote { votes := make([]*types.Vote, len(vss)) for i, vs := range vss { votes[i] = signVote(vs, voteType, hash, header) @@ -130,8 +130,8 @@ func decideProposal(cs1 *ConsensusState, vs *validatorStub, height int64, round } // Make proposal - polRound, polBlockID := cs1.Votes.POLInfo() - proposal = types.NewProposal(height, round, blockParts.Header(), polRound, polBlockID) + polRound, propBlockID := cs1.ValidRound, types.BlockID{block.Hash(), blockParts.Header()} + proposal = types.NewProposal(height, round, polRound, propBlockID) if err := vs.SignProposal(cs1.state.ChainID, proposal); err != nil { panic(err) } @@ -144,15 +144,16 @@ func addVotes(to *ConsensusState, votes ...*types.Vote) { } } -func signAddVotes(to *ConsensusState, voteType byte, hash []byte, header types.PartSetHeader, vss ...*validatorStub) { +func signAddVotes(to *ConsensusState, voteType types.SignedMsgType, hash []byte, header types.PartSetHeader, vss ...*validatorStub) { votes := signVotes(voteType, hash, header, vss...) addVotes(to, votes...) } func validatePrevote(t *testing.T, cs *ConsensusState, round int, privVal *validatorStub, blockHash []byte) { prevotes := cs.Votes.Prevotes(round) + address := privVal.GetPubKey().Address() var vote *types.Vote - if vote = prevotes.GetByAddress(privVal.GetAddress()); vote == nil { + if vote = prevotes.GetByAddress(address); vote == nil { panic("Failed to find prevote from validator") } if blockHash == nil { @@ -168,8 +169,9 @@ func validatePrevote(t *testing.T, cs *ConsensusState, round int, privVal *valid func validateLastPrecommit(t *testing.T, cs *ConsensusState, privVal *validatorStub, blockHash []byte) { votes := cs.LastCommit + address := privVal.GetPubKey().Address() var vote *types.Vote - if vote = votes.GetByAddress(privVal.GetAddress()); vote == nil { + if vote = votes.GetByAddress(address); vote == nil { panic("Failed to find precommit from validator") } if !bytes.Equal(vote.BlockID.Hash, blockHash) { @@ -179,8 +181,9 @@ func validateLastPrecommit(t *testing.T, cs *ConsensusState, privVal *validatorS func validatePrecommit(t *testing.T, cs *ConsensusState, thisRound, lockRound int, privVal *validatorStub, votedBlockHash, lockedBlockHash []byte) { precommits := cs.Votes.Precommits(thisRound) + address := privVal.GetPubKey().Address() var vote *types.Vote - if vote = precommits.GetByAddress(privVal.GetAddress()); vote == nil { + if vote = precommits.GetByAddress(address); vote == nil { panic("Failed to find precommit from validator") } @@ -281,9 +284,10 @@ func newConsensusStateWithConfigAndBlockStore(thisConfig *cfg.Config, state sm.S } func loadPrivValidator(config *cfg.Config) *privval.FilePV { - privValidatorFile := config.PrivValidatorFile() - ensureDir(path.Dir(privValidatorFile), 0700) - privValidator := privval.LoadOrGenFilePV(privValidatorFile) + privValidatorKeyFile := config.PrivValidatorKeyFile() + ensureDir(filepath.Dir(privValidatorKeyFile), 0700) + privValidatorStateFile := config.PrivValidatorStateFile() + privValidator := privval.LoadOrGenFilePV(privValidatorKeyFile, privValidatorStateFile) privValidator.Reset() return privValidator } @@ -317,67 +321,163 @@ func ensureNoNewEvent(ch <-chan interface{}, timeout time.Duration, } } -func ensureNoNewStep(stepCh <-chan interface{}) { - ensureNoNewEvent(stepCh, ensureTimeout, "We should be stuck waiting, "+ - "not moving to the next step") +func ensureNoNewEventOnChannel(ch <-chan interface{}) { + ensureNoNewEvent( + ch, + ensureTimeout, + "We should be stuck waiting, not receiving new event on the channel") +} + +func ensureNoNewRoundStep(stepCh <-chan interface{}) { + ensureNoNewEvent( + stepCh, + ensureTimeout, + "We should be stuck waiting, not receiving NewRoundStep event") +} + +func ensureNoNewUnlock(unlockCh <-chan interface{}) { + ensureNoNewEvent( + unlockCh, + ensureTimeout, + "We should be stuck waiting, not receiving Unlock event") } func ensureNoNewTimeout(stepCh <-chan interface{}, timeout int64) { timeoutDuration := time.Duration(timeout*5) * time.Nanosecond - ensureNoNewEvent(stepCh, timeoutDuration, "We should be stuck waiting, "+ - "not moving to the next step") + ensureNoNewEvent( + stepCh, + timeoutDuration, + "We should be stuck waiting, not receiving NewTimeout event") } -func ensureNewEvent(ch <-chan interface{}, timeout time.Duration, errorMessage string) { +func ensureNewEvent( + ch <-chan interface{}, + height int64, + round int, + timeout time.Duration, + errorMessage string) { + select { case <-time.After(timeout): panic(errorMessage) - case <-ch: - break + case ev := <-ch: + rs, ok := ev.(types.EventDataRoundState) + if !ok { + panic( + fmt.Sprintf( + "expected a EventDataRoundState, got %v.Wrong subscription channel?", + reflect.TypeOf(rs))) + } + if rs.Height != height { + panic(fmt.Sprintf("expected height %v, got %v", height, rs.Height)) + } + if rs.Round != round { + panic(fmt.Sprintf("expected round %v, got %v", round, rs.Round)) + } + // TODO: We could check also for a step at this point! } } -func ensureNewStep(stepCh <-chan interface{}) { - ensureNewEvent(stepCh, ensureTimeout, - "Timeout expired while waiting for NewStep event") +func ensureNewRound(roundCh <-chan interface{}, height int64, round int) { + select { + case <-time.After(ensureTimeout): + panic("Timeout expired while waiting for NewRound event") + case ev := <-roundCh: + rs, ok := ev.(types.EventDataNewRound) + if !ok { + panic( + fmt.Sprintf( + "expected a EventDataNewRound, got %v.Wrong subscription channel?", + reflect.TypeOf(rs))) + } + if rs.Height != height { + panic(fmt.Sprintf("expected height %v, got %v", height, rs.Height)) + } + if rs.Round != round { + panic(fmt.Sprintf("expected round %v, got %v", round, rs.Round)) + } + } } -func ensureNewRound(roundCh <-chan interface{}) { - ensureNewEvent(roundCh, ensureTimeout, - "Timeout expired while waiting for NewRound event") -} - -func ensureNewTimeout(timeoutCh <-chan interface{}, timeout int64) { - timeoutDuration := time.Duration(timeout*5) * time.Nanosecond - ensureNewEvent(timeoutCh, timeoutDuration, +func ensureNewTimeout(timeoutCh <-chan interface{}, height int64, round int, timeout int64) { + timeoutDuration := time.Duration(timeout*3) * time.Nanosecond + ensureNewEvent(timeoutCh, height, round, timeoutDuration, "Timeout expired while waiting for NewTimeout event") } -func ensureNewProposal(proposalCh <-chan interface{}) { - ensureNewEvent(proposalCh, ensureTimeout, - "Timeout expired while waiting for NewProposal event") +func ensureNewProposal(proposalCh <-chan interface{}, height int64, round int) { + select { + case <-time.After(ensureTimeout): + panic("Timeout expired while waiting for NewProposal event") + case ev := <-proposalCh: + rs, ok := ev.(types.EventDataCompleteProposal) + if !ok { + panic( + fmt.Sprintf( + "expected a EventDataCompleteProposal, got %v.Wrong subscription channel?", + reflect.TypeOf(rs))) + } + if rs.Height != height { + panic(fmt.Sprintf("expected height %v, got %v", height, rs.Height)) + } + if rs.Round != round { + panic(fmt.Sprintf("expected round %v, got %v", round, rs.Round)) + } + } } -func ensureNewBlock(blockCh <-chan interface{}) { - ensureNewEvent(blockCh, ensureTimeout, - "Timeout expired while waiting for NewBlock event") +func ensureNewValidBlock(validBlockCh <-chan interface{}, height int64, round int) { + ensureNewEvent(validBlockCh, height, round, ensureTimeout, + "Timeout expired while waiting for NewValidBlock event") } -func ensureNewVote(voteCh <-chan interface{}) { - ensureNewEvent(voteCh, ensureTimeout, - "Timeout expired while waiting for NewVote event") +func ensureNewBlock(blockCh <-chan interface{}, height int64) { + select { + case <-time.After(ensureTimeout): + panic("Timeout expired while waiting for NewBlock event") + case ev := <-blockCh: + block, ok := ev.(types.EventDataNewBlock) + if !ok { + panic(fmt.Sprintf("expected a *types.EventDataNewBlock, "+ + "got %v. wrong subscription channel?", + reflect.TypeOf(block))) + } + if block.Block.Height != height { + panic(fmt.Sprintf("expected height %v, got %v", height, block.Block.Height)) + } + } } -func ensureNewUnlock(unlockCh <-chan interface{}) { - ensureNewEvent(unlockCh, ensureTimeout, +func ensureNewBlockHeader(blockCh <-chan interface{}, height int64, blockHash cmn.HexBytes) { + select { + case <-time.After(ensureTimeout): + panic("Timeout expired while waiting for NewBlockHeader event") + case ev := <-blockCh: + blockHeader, ok := ev.(types.EventDataNewBlockHeader) + if !ok { + panic(fmt.Sprintf("expected a *types.EventDataNewBlockHeader, "+ + "got %v. wrong subscription channel?", + reflect.TypeOf(blockHeader))) + } + if blockHeader.Header.Height != height { + panic(fmt.Sprintf("expected height %v, got %v", height, blockHeader.Header.Height)) + } + if !bytes.Equal(blockHeader.Header.Hash(), blockHash) { + panic(fmt.Sprintf("expected header %X, got %X", blockHash, blockHeader.Header.Hash())) + } + } +} + +func ensureNewUnlock(unlockCh <-chan interface{}, height int64, round int) { + ensureNewEvent(unlockCh, height, round, ensureTimeout, "Timeout expired while waiting for NewUnlock event") } -func ensureVote(voteCh chan interface{}, height int64, round int, - voteType byte) { +func ensureVote(voteCh <-chan interface{}, height int64, round int, + voteType types.SignedMsgType) { select { case <-time.After(ensureTimeout): - break + panic("Timeout expired while waiting for NewVote event") case v := <-voteCh: edv, ok := v.(types.EventDataVote) if !ok { @@ -398,6 +498,46 @@ func ensureVote(voteCh chan interface{}, height int64, round int, } } +func ensureProposal(proposalCh <-chan interface{}, height int64, round int, propId types.BlockID) { + select { + case <-time.After(ensureTimeout): + panic("Timeout expired while waiting for NewProposal event") + case ev := <-proposalCh: + rs, ok := ev.(types.EventDataCompleteProposal) + if !ok { + panic( + fmt.Sprintf( + "expected a EventDataCompleteProposal, got %v.Wrong subscription channel?", + reflect.TypeOf(rs))) + } + if rs.Height != height { + panic(fmt.Sprintf("expected height %v, got %v", height, rs.Height)) + } + if rs.Round != round { + panic(fmt.Sprintf("expected round %v, got %v", round, rs.Round)) + } + if !rs.BlockID.Equals(propId) { + panic("Proposed block does not match expected block") + } + } +} + +func ensurePrecommit(voteCh <-chan interface{}, height int64, round int) { + ensureVote(voteCh, height, round, types.PrecommitType) +} + +func ensurePrevote(voteCh <-chan interface{}, height int64, round int) { + ensureVote(voteCh, height, round, types.PrevoteType) +} + +func ensureNewEventOnChannel(ch <-chan interface{}) { + select { + case <-time.After(ensureTimeout): + panic("Timeout expired while waiting for new activity on the channel") + case <-ch: + } +} + //------------------------------------------------------------------------------- // consensus nets @@ -425,7 +565,7 @@ func randConsensusNet(nValidators int, testName string, tickerFunc func() Timeou for _, opt := range configOpts { opt(thisConfig) } - ensureDir(path.Dir(thisConfig.Consensus.WalFile()), 0700) // dir for wal + ensureDir(filepath.Dir(thisConfig.Consensus.WalFile()), 0700) // dir for wal app := appFunc() vals := types.TM2PB.ValidatorUpdates(state.Validators) app.InitChain(abci.RequestInitChain{Validators: vals}) @@ -446,16 +586,21 @@ func randConsensusNetWithPeers(nValidators, nPeers int, testName string, tickerF stateDB := dbm.NewMemDB() // each state needs its own db state, _ := sm.LoadStateFromDBOrGenesisDoc(stateDB, genDoc) thisConfig := ResetConfig(fmt.Sprintf("%s_%d", testName, i)) - ensureDir(path.Dir(thisConfig.Consensus.WalFile()), 0700) // dir for wal + ensureDir(filepath.Dir(thisConfig.Consensus.WalFile()), 0700) // dir for wal var privVal types.PrivValidator if i < nValidators { privVal = privVals[i] } else { - tempFile, err := ioutil.TempFile("", "priv_validator_") + tempKeyFile, err := ioutil.TempFile("", "priv_validator_key_") if err != nil { panic(err) } - privVal = privval.GenFilePV(tempFile.Name()) + tempStateFile, err := ioutil.TempFile("", "priv_validator_state_") + if err != nil { + panic(err) + } + + privVal = privval.GenFilePV(tempKeyFile.Name(), tempStateFile.Name()) } app := appFunc() @@ -471,7 +616,7 @@ func randConsensusNetWithPeers(nValidators, nPeers int, testName string, tickerF func getSwitchIndex(switches []*p2p.Switch, peer p2p.Peer) int { for i, s := range switches { - if peer.NodeInfo().ID == s.NodeInfo().ID { + if peer.NodeInfo().ID() == s.NodeInfo().ID() { return i } } @@ -505,8 +650,6 @@ func randGenesisDoc(numValidators int, randPower bool, minPower int64) (*types.G func randGenesisState(numValidators int, randPower bool, minPower int64) (sm.State, []types.PrivValidator) { genDoc, privValidators := randGenesisDoc(numValidators, randPower, minPower) s0, _ := sm.MakeGenesisState(genDoc) - db := dbm.NewMemDB() // remove this ? - sm.SaveState(db, s0) return s0, privValidators } diff --git a/consensus/mempool_test.go b/consensus/mempool_test.go index 179766fd..bb4bf6eb 100644 --- a/consensus/mempool_test.go +++ b/consensus/mempool_test.go @@ -10,7 +10,7 @@ import ( "github.com/tendermint/tendermint/abci/example/code" abci "github.com/tendermint/tendermint/abci/types" - + sm "github.com/tendermint/tendermint/state" "github.com/tendermint/tendermint/types" ) @@ -18,22 +18,27 @@ func init() { config = ResetConfig("consensus_mempool_test") } +// for testing +func assertMempool(txn txNotifier) sm.Mempool { + return txn.(sm.Mempool) +} + func TestMempoolNoProgressUntilTxsAvailable(t *testing.T) { config := ResetConfig("consensus_mempool_txs_available_test") config.Consensus.CreateEmptyBlocks = false state, privVals := randGenesisState(1, false, 10) cs := newConsensusStateWithConfig(config, state, privVals[0], NewCounterApplication()) - cs.mempool.EnableTxsAvailable() + assertMempool(cs.txNotifier).EnableTxsAvailable() height, round := cs.Height, cs.Round newBlockCh := subscribe(cs.eventBus, types.EventQueryNewBlock) startTestRound(cs, height, round) - ensureNewStep(newBlockCh) // first block gets committed - ensureNoNewStep(newBlockCh) + ensureNewEventOnChannel(newBlockCh) // first block gets committed + ensureNoNewEventOnChannel(newBlockCh) deliverTxsRange(cs, 0, 1) - ensureNewStep(newBlockCh) // commit txs - ensureNewStep(newBlockCh) // commit updated app hash - ensureNoNewStep(newBlockCh) + ensureNewEventOnChannel(newBlockCh) // commit txs + ensureNewEventOnChannel(newBlockCh) // commit updated app hash + ensureNoNewEventOnChannel(newBlockCh) } func TestMempoolProgressAfterCreateEmptyBlocksInterval(t *testing.T) { @@ -41,14 +46,14 @@ func TestMempoolProgressAfterCreateEmptyBlocksInterval(t *testing.T) { config.Consensus.CreateEmptyBlocksInterval = ensureTimeout state, privVals := randGenesisState(1, false, 10) cs := newConsensusStateWithConfig(config, state, privVals[0], NewCounterApplication()) - cs.mempool.EnableTxsAvailable() + assertMempool(cs.txNotifier).EnableTxsAvailable() height, round := cs.Height, cs.Round newBlockCh := subscribe(cs.eventBus, types.EventQueryNewBlock) startTestRound(cs, height, round) - ensureNewStep(newBlockCh) // first block gets committed - ensureNoNewStep(newBlockCh) // then we dont make a block ... - ensureNewStep(newBlockCh) // until the CreateEmptyBlocksInterval has passed + ensureNewEventOnChannel(newBlockCh) // first block gets committed + ensureNoNewEventOnChannel(newBlockCh) // then we dont make a block ... + ensureNewEventOnChannel(newBlockCh) // until the CreateEmptyBlocksInterval has passed } func TestMempoolProgressInHigherRound(t *testing.T) { @@ -56,7 +61,7 @@ func TestMempoolProgressInHigherRound(t *testing.T) { config.Consensus.CreateEmptyBlocks = false state, privVals := randGenesisState(1, false, 10) cs := newConsensusStateWithConfig(config, state, privVals[0], NewCounterApplication()) - cs.mempool.EnableTxsAvailable() + assertMempool(cs.txNotifier).EnableTxsAvailable() height, round := cs.Height, cs.Round newBlockCh := subscribe(cs.eventBus, types.EventQueryNewBlock) newRoundCh := subscribe(cs.eventBus, types.EventQueryNewRound) @@ -72,13 +77,19 @@ func TestMempoolProgressInHigherRound(t *testing.T) { } startTestRound(cs, height, round) - ensureNewStep(newRoundCh) // first round at first height - ensureNewStep(newBlockCh) // first block gets committed - ensureNewStep(newRoundCh) // first round at next height - deliverTxsRange(cs, 0, 1) // we deliver txs, but dont set a proposal so we get the next round - <-timeoutCh - ensureNewStep(newRoundCh) // wait for the next round - ensureNewStep(newBlockCh) // now we can commit the block + ensureNewRound(newRoundCh, height, round) // first round at first height + ensureNewEventOnChannel(newBlockCh) // first block gets committed + + height = height + 1 // moving to the next height + round = 0 + + ensureNewRound(newRoundCh, height, round) // first round at next height + deliverTxsRange(cs, 0, 1) // we deliver txs, but dont set a proposal so we get the next round + ensureNewTimeout(timeoutCh, height, round, cs.config.TimeoutPropose.Nanoseconds()) + + round = round + 1 // moving to the next round + ensureNewRound(newRoundCh, height, round) // wait for the next round + ensureNewEventOnChannel(newBlockCh) // now we can commit the block } func deliverTxsRange(cs *ConsensusState, start, end int) { @@ -86,7 +97,7 @@ func deliverTxsRange(cs *ConsensusState, start, end int) { for i := start; i < end; i++ { txBytes := make([]byte, 8) binary.BigEndian.PutUint64(txBytes, uint64(i)) - err := cs.mempool.CheckTx(txBytes, nil) + err := assertMempool(cs.txNotifier).CheckTx(txBytes, nil) if err != nil { panic(fmt.Sprintf("Error after CheckTx: %v", err)) } @@ -136,7 +147,7 @@ func TestMempoolRmBadTx(t *testing.T) { // Try to send the tx through the mempool. // CheckTx should not err, but the app should return a bad abci code // and the tx should get removed from the pool - err := cs.mempool.CheckTx(txBytes, func(r *abci.Response) { + err := assertMempool(cs.txNotifier).CheckTx(txBytes, func(r *abci.Response) { if r.GetCheckTx().Code != code.CodeTypeBadNonce { t.Fatalf("expected checktx to return bad nonce, got %v", r) } @@ -148,7 +159,7 @@ func TestMempoolRmBadTx(t *testing.T) { // check for the tx for { - txs := cs.mempool.ReapMaxBytesMaxGas(int64(len(txBytes)), -1) + txs := assertMempool(cs.txNotifier).ReapMaxBytesMaxGas(int64(len(txBytes)), -1) if len(txs) == 0 { emptyMempoolCh <- struct{}{} return diff --git a/consensus/metrics.go b/consensus/metrics.go index 7b4a3fbc..b5207742 100644 --- a/consensus/metrics.go +++ b/consensus/metrics.go @@ -8,7 +8,11 @@ import ( stdprometheus "github.com/prometheus/client_golang/prometheus" ) -const MetricsSubsystem = "consensus" +const ( + // MetricsSubsystem is a subsystem shared by all metrics exposed by this + // package. + MetricsSubsystem = "consensus" +) // Metrics contains metrics exposed by this package. type Metrics struct { @@ -50,101 +54,107 @@ type Metrics struct { } // PrometheusMetrics returns Metrics build using Prometheus client library. -func PrometheusMetrics(namespace string) *Metrics { +// Optionally, labels can be provided along with their values ("foo", +// "fooValue"). +func PrometheusMetrics(namespace string, labelsAndValues ...string) *Metrics { + labels := []string{} + for i := 0; i < len(labelsAndValues); i += 2 { + labels = append(labels, labelsAndValues[i]) + } return &Metrics{ Height: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ Namespace: namespace, Subsystem: MetricsSubsystem, Name: "height", Help: "Height of the chain.", - }, []string{}), + }, labels).With(labelsAndValues...), Rounds: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ Namespace: namespace, Subsystem: MetricsSubsystem, Name: "rounds", Help: "Number of rounds.", - }, []string{}), + }, labels).With(labelsAndValues...), Validators: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ Namespace: namespace, Subsystem: MetricsSubsystem, Name: "validators", Help: "Number of validators.", - }, []string{}), + }, labels).With(labelsAndValues...), ValidatorsPower: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ Namespace: namespace, Subsystem: MetricsSubsystem, Name: "validators_power", Help: "Total power of all validators.", - }, []string{}), + }, labels).With(labelsAndValues...), MissingValidators: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ Namespace: namespace, Subsystem: MetricsSubsystem, Name: "missing_validators", Help: "Number of validators who did not sign.", - }, []string{}), + }, labels).With(labelsAndValues...), MissingValidatorsPower: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ Namespace: namespace, Subsystem: MetricsSubsystem, Name: "missing_validators_power", Help: "Total power of the missing validators.", - }, []string{}), + }, labels).With(labelsAndValues...), ByzantineValidators: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ Namespace: namespace, Subsystem: MetricsSubsystem, Name: "byzantine_validators", Help: "Number of validators who tried to double sign.", - }, []string{}), + }, labels).With(labelsAndValues...), ByzantineValidatorsPower: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ Namespace: namespace, Subsystem: MetricsSubsystem, Name: "byzantine_validators_power", Help: "Total power of the byzantine validators.", - }, []string{}), + }, labels).With(labelsAndValues...), BlockIntervalSeconds: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ Namespace: namespace, Subsystem: MetricsSubsystem, Name: "block_interval_seconds", Help: "Time between this and the last block.", - }, []string{}), + }, labels).With(labelsAndValues...), NumTxs: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ Namespace: namespace, Subsystem: MetricsSubsystem, Name: "num_txs", Help: "Number of transactions.", - }, []string{}), + }, labels).With(labelsAndValues...), BlockSizeBytes: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ Namespace: namespace, Subsystem: MetricsSubsystem, Name: "block_size_bytes", Help: "Size of the block.", - }, []string{}), + }, labels).With(labelsAndValues...), TotalTxs: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ Namespace: namespace, Subsystem: MetricsSubsystem, Name: "total_txs", Help: "Total number of transactions.", - }, []string{}), + }, labels).With(labelsAndValues...), CommittedHeight: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ Namespace: namespace, Subsystem: MetricsSubsystem, Name: "latest_block_height", Help: "The latest block height.", - }, []string{}), + }, labels).With(labelsAndValues...), FastSyncing: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ Namespace: namespace, Subsystem: MetricsSubsystem, Name: "fast_syncing", Help: "Whether or not a node is fast syncing. 1 if yes, 0 if no.", - }, []string{}), + }, labels).With(labelsAndValues...), BlockParts: prometheus.NewCounterFrom(stdprometheus.CounterOpts{ Namespace: namespace, Subsystem: MetricsSubsystem, Name: "block_parts", Help: "Number of blockparts transmitted by peer.", - }, []string{"peer_id"}), + }, append(labels, "peer_id")).With(labelsAndValues...), } } diff --git a/consensus/reactor.go b/consensus/reactor.go index ca63e899..b92ae1f7 100644 --- a/consensus/reactor.go +++ b/consensus/reactor.go @@ -8,8 +8,7 @@ import ( "github.com/pkg/errors" - "github.com/tendermint/go-amino" - + amino "github.com/tendermint/go-amino" cstypes "github.com/tendermint/tendermint/consensus/types" cmn "github.com/tendermint/tendermint/libs/common" tmevents "github.com/tendermint/tendermint/libs/events" @@ -174,7 +173,7 @@ func (conR *ConsensusReactor) AddPeer(peer p2p.Peer) { // Send our state to peer. // If we're fast_syncing, broadcast a RoundStepMessage later upon SwitchToConsensus(). if !conR.FastSync() { - conR.sendNewRoundStepMessages(peer) + conR.sendNewRoundStepMessage(peer) } } @@ -184,7 +183,11 @@ func (conR *ConsensusReactor) RemovePeer(peer p2p.Peer, reason interface{}) { return } // TODO - //peer.Get(PeerStateKey).(*PeerState).Disconnect() + // ps, ok := peer.Get(PeerStateKey).(*PeerState) + // if !ok { + // panic(fmt.Sprintf("Peer %v has no state", peer)) + // } + // ps.Disconnect() } // Receive implements Reactor @@ -205,18 +208,28 @@ func (conR *ConsensusReactor) Receive(chID byte, src p2p.Peer, msgBytes []byte) conR.Switch.StopPeerForError(src, err) return } + + if err = msg.ValidateBasic(); err != nil { + conR.Logger.Error("Peer sent us invalid msg", "peer", src, "msg", msg, "err", err) + conR.Switch.StopPeerForError(src, err) + return + } + conR.Logger.Debug("Receive", "src", src, "chId", chID, "msg", msg) // Get peer states - ps := src.Get(types.PeerStateKey).(*PeerState) + ps, ok := src.Get(types.PeerStateKey).(*PeerState) + if !ok { + panic(fmt.Sprintf("Peer %v has no state", src)) + } switch chID { case StateChannel: switch msg := msg.(type) { case *NewRoundStepMessage: ps.ApplyNewRoundStepMessage(msg) - case *CommitStepMessage: - ps.ApplyCommitStepMessage(msg) + case *NewValidBlockMessage: + ps.ApplyNewValidBlockMessage(msg) case *HasVoteMessage: ps.ApplyHasVoteMessage(msg) case *VoteSetMaj23Message: @@ -237,13 +250,12 @@ func (conR *ConsensusReactor) Receive(chID byte, src p2p.Peer, msgBytes []byte) // (and consequently shows which we don't have) var ourVotes *cmn.BitArray switch msg.Type { - case types.VoteTypePrevote: + case types.PrevoteType: ourVotes = votes.Prevotes(msg.Round).BitArrayByBlockID(msg.BlockID) - case types.VoteTypePrecommit: + case types.PrecommitType: ourVotes = votes.Precommits(msg.Round).BitArrayByBlockID(msg.BlockID) default: - conR.Logger.Error("Bad VoteSetBitsMessage field Type") - return + panic("Bad VoteSetBitsMessage field Type. Forgot to add a check in ValidateBasic?") } src.TrySend(VoteSetBitsChannel, cdc.MustMarshalBinaryBare(&VoteSetBitsMessage{ Height: msg.Height, @@ -252,11 +264,6 @@ func (conR *ConsensusReactor) Receive(chID byte, src p2p.Peer, msgBytes []byte) BlockID: msg.BlockID, Votes: ourVotes, })) - case *ProposalHeartbeatMessage: - hb := msg.Heartbeat - conR.Logger.Debug("Received proposal heartbeat message", - "height", hb.Height, "round", hb.Round, "sequence", hb.Sequence, - "valIdx", hb.ValidatorIndex, "valAddr", hb.ValidatorAddress) default: conR.Logger.Error(fmt.Sprintf("Unknown message type %v", reflect.TypeOf(msg))) } @@ -288,9 +295,9 @@ func (conR *ConsensusReactor) Receive(chID byte, src p2p.Peer, msgBytes []byte) switch msg := msg.(type) { case *VoteMessage: cs := conR.conS - cs.mtx.Lock() + cs.mtx.RLock() height, valSize, lastCommitSize := cs.Height, cs.Validators.Size(), cs.LastCommit.Size() - cs.mtx.Unlock() + cs.mtx.RUnlock() ps.EnsureVoteBitArrays(height, valSize) ps.EnsureVoteBitArrays(height-1, lastCommitSize) ps.SetHasVote(msg.Vote) @@ -317,13 +324,12 @@ func (conR *ConsensusReactor) Receive(chID byte, src p2p.Peer, msgBytes []byte) if height == msg.Height { var ourVotes *cmn.BitArray switch msg.Type { - case types.VoteTypePrevote: + case types.PrevoteType: ourVotes = votes.Prevotes(msg.Round).BitArrayByBlockID(msg.BlockID) - case types.VoteTypePrecommit: + case types.PrecommitType: ourVotes = votes.Precommits(msg.Round).BitArrayByBlockID(msg.BlockID) default: - conR.Logger.Error("Bad VoteSetBitsMessage field Type") - return + panic("Bad VoteSetBitsMessage field Type. Forgot to add a check in ValidateBasic?") } ps.ApplyVoteSetBitsMessage(msg, ourVotes) } else { @@ -358,14 +364,19 @@ func (conR *ConsensusReactor) FastSync() bool { //-------------------------------------- -// subscribeToBroadcastEvents subscribes for new round steps, votes and -// proposal heartbeats using internal pubsub defined on state to broadcast +// subscribeToBroadcastEvents subscribes for new round steps and votes +// using internal pubsub defined on state to broadcast // them to peers upon receiving. func (conR *ConsensusReactor) subscribeToBroadcastEvents() { const subscriber = "consensus-reactor" conR.conS.evsw.AddListenerForEvent(subscriber, types.EventNewRoundStep, func(data tmevents.EventData) { - conR.broadcastNewRoundStepMessages(data.(*cstypes.RoundState)) + conR.broadcastNewRoundStepMessage(data.(*cstypes.RoundState)) + }) + + conR.conS.evsw.AddListenerForEvent(subscriber, types.EventValidBlock, + func(data tmevents.EventData) { + conR.broadcastNewValidBlockMessage(data.(*cstypes.RoundState)) }) conR.conS.evsw.AddListenerForEvent(subscriber, types.EventVote, @@ -373,10 +384,6 @@ func (conR *ConsensusReactor) subscribeToBroadcastEvents() { conR.broadcastHasVoteMessage(data.(*types.Vote)) }) - conR.conS.evsw.AddListenerForEvent(subscriber, types.EventProposalHeartbeat, - func(data tmevents.EventData) { - conR.broadcastProposalHeartbeatMessage(data.(*types.Heartbeat)) - }) } func (conR *ConsensusReactor) unsubscribeFromBroadcastEvents() { @@ -384,21 +391,20 @@ func (conR *ConsensusReactor) unsubscribeFromBroadcastEvents() { conR.conS.evsw.RemoveListener(subscriber) } -func (conR *ConsensusReactor) broadcastProposalHeartbeatMessage(hb *types.Heartbeat) { - conR.Logger.Debug("Broadcasting proposal heartbeat message", - "height", hb.Height, "round", hb.Round, "sequence", hb.Sequence) - msg := &ProposalHeartbeatMessage{hb} - conR.Switch.Broadcast(StateChannel, cdc.MustMarshalBinaryBare(msg)) +func (conR *ConsensusReactor) broadcastNewRoundStepMessage(rs *cstypes.RoundState) { + nrsMsg := makeRoundStepMessage(rs) + conR.Switch.Broadcast(StateChannel, cdc.MustMarshalBinaryBare(nrsMsg)) } -func (conR *ConsensusReactor) broadcastNewRoundStepMessages(rs *cstypes.RoundState) { - nrsMsg, csMsg := makeRoundStepMessages(rs) - if nrsMsg != nil { - conR.Switch.Broadcast(StateChannel, cdc.MustMarshalBinaryBare(nrsMsg)) - } - if csMsg != nil { - conR.Switch.Broadcast(StateChannel, cdc.MustMarshalBinaryBare(csMsg)) +func (conR *ConsensusReactor) broadcastNewValidBlockMessage(rs *cstypes.RoundState) { + csMsg := &NewValidBlockMessage{ + Height: rs.Height, + Round: rs.Round, + BlockPartsHeader: rs.ProposalBlockParts.Header(), + BlockParts: rs.ProposalBlockParts.BitArray(), + IsCommit: rs.Step == cstypes.RoundStepCommit, } + conR.Switch.Broadcast(StateChannel, cdc.MustMarshalBinaryBare(csMsg)) } // Broadcasts HasVoteMessage to peers that care. @@ -413,7 +419,10 @@ func (conR *ConsensusReactor) broadcastHasVoteMessage(vote *types.Vote) { /* // TODO: Make this broadcast more selective. for _, peer := range conR.Switch.Peers().List() { - ps := peer.Get(PeerStateKey).(*PeerState) + ps, ok := peer.Get(PeerStateKey).(*PeerState) + if !ok { + panic(fmt.Sprintf("Peer %v has no state", peer)) + } prs := ps.GetRoundState() if prs.Height == vote.Height { // TODO: Also filter on round? @@ -427,33 +436,21 @@ func (conR *ConsensusReactor) broadcastHasVoteMessage(vote *types.Vote) { */ } -func makeRoundStepMessages(rs *cstypes.RoundState) (nrsMsg *NewRoundStepMessage, csMsg *CommitStepMessage) { +func makeRoundStepMessage(rs *cstypes.RoundState) (nrsMsg *NewRoundStepMessage) { nrsMsg = &NewRoundStepMessage{ - Height: rs.Height, - Round: rs.Round, - Step: rs.Step, + Height: rs.Height, + Round: rs.Round, + Step: rs.Step, SecondsSinceStartTime: int(time.Since(rs.StartTime).Seconds()), LastCommitRound: rs.LastCommit.Round(), } - if rs.Step == cstypes.RoundStepCommit { - csMsg = &CommitStepMessage{ - Height: rs.Height, - BlockPartsHeader: rs.ProposalBlockParts.Header(), - BlockParts: rs.ProposalBlockParts.BitArray(), - } - } return } -func (conR *ConsensusReactor) sendNewRoundStepMessages(peer p2p.Peer) { +func (conR *ConsensusReactor) sendNewRoundStepMessage(peer p2p.Peer) { rs := conR.conS.GetRoundState() - nrsMsg, csMsg := makeRoundStepMessages(rs) - if nrsMsg != nil { - peer.Send(StateChannel, cdc.MustMarshalBinaryBare(nrsMsg)) - } - if csMsg != nil { - peer.Send(StateChannel, cdc.MustMarshalBinaryBare(csMsg)) - } + nrsMsg := makeRoundStepMessage(rs) + peer.Send(StateChannel, cdc.MustMarshalBinaryBare(nrsMsg)) } func (conR *ConsensusReactor) gossipDataRoutine(peer p2p.Peer, ps *PeerState) { @@ -524,6 +521,7 @@ OUTER_LOOP: msg := &ProposalMessage{Proposal: rs.Proposal} logger.Debug("Sending proposal", "height", prs.Height, "round", prs.Round) if peer.Send(DataChannel, cdc.MustMarshalBinaryBare(msg)) { + // NOTE[ZM]: A peer might have received different proposal msg so this Proposal msg will be rejected! ps.SetHasProposal(rs.Proposal) } } @@ -739,7 +737,7 @@ OUTER_LOOP: peer.TrySend(StateChannel, cdc.MustMarshalBinaryBare(&VoteSetMaj23Message{ Height: prs.Height, Round: prs.Round, - Type: types.VoteTypePrevote, + Type: types.PrevoteType, BlockID: maj23, })) time.Sleep(conR.conS.config.PeerQueryMaj23SleepDuration) @@ -756,7 +754,7 @@ OUTER_LOOP: peer.TrySend(StateChannel, cdc.MustMarshalBinaryBare(&VoteSetMaj23Message{ Height: prs.Height, Round: prs.Round, - Type: types.VoteTypePrecommit, + Type: types.PrecommitType, BlockID: maj23, })) time.Sleep(conR.conS.config.PeerQueryMaj23SleepDuration) @@ -773,7 +771,7 @@ OUTER_LOOP: peer.TrySend(StateChannel, cdc.MustMarshalBinaryBare(&VoteSetMaj23Message{ Height: prs.Height, Round: prs.ProposalPOLRound, - Type: types.VoteTypePrevote, + Type: types.PrevoteType, BlockID: maj23, })) time.Sleep(conR.conS.config.PeerQueryMaj23SleepDuration) @@ -792,7 +790,7 @@ OUTER_LOOP: peer.TrySend(StateChannel, cdc.MustMarshalBinaryBare(&VoteSetMaj23Message{ Height: prs.Height, Round: commit.Round(), - Type: types.VoteTypePrecommit, + Type: types.PrecommitType, BlockID: commit.BlockID, })) time.Sleep(conR.conS.config.PeerQueryMaj23SleepDuration) @@ -822,7 +820,10 @@ func (conR *ConsensusReactor) peerStatsRoutine() { continue } // Get peer state - ps := peer.Get(types.PeerStateKey).(*PeerState) + ps, ok := peer.Get(types.PeerStateKey).(*PeerState) + if !ok { + panic(fmt.Sprintf("Peer %v has no state", peer)) + } switch msg.Msg.(type) { case *VoteMessage: if numVotes := ps.RecordVote(); numVotes%votesToContributeToBecomeGoodPeer == 0 { @@ -855,7 +856,10 @@ func (conR *ConsensusReactor) StringIndented(indent string) string { s := "ConsensusReactor{\n" s += indent + " " + conR.conS.StringIndented(indent+" ") + "\n" for _, peer := range conR.Switch.Peers().List() { - ps := peer.Get(types.PeerStateKey).(*PeerState) + ps, ok := peer.Get(types.PeerStateKey).(*PeerState) + if !ok { + panic(fmt.Sprintf("Peer %v has no state", peer)) + } s += indent + " " + ps.StringIndented(indent+" ") + "\n" } s += indent + "}" @@ -964,13 +968,20 @@ func (ps *PeerState) SetHasProposal(proposal *types.Proposal) { if ps.PRS.Height != proposal.Height || ps.PRS.Round != proposal.Round { return } + if ps.PRS.Proposal { return } ps.PRS.Proposal = true - ps.PRS.ProposalBlockPartsHeader = proposal.BlockPartsHeader - ps.PRS.ProposalBlockParts = cmn.NewBitArray(proposal.BlockPartsHeader.Total) + + // ps.PRS.ProposalBlockParts is set due to NewValidBlockMessage + if ps.PRS.ProposalBlockParts != nil { + return + } + + ps.PRS.ProposalBlockPartsHeader = proposal.BlockID.PartsHeader + ps.PRS.ProposalBlockParts = cmn.NewBitArray(proposal.BlockID.PartsHeader.Total) ps.PRS.ProposalPOLRound = proposal.POLRound ps.PRS.ProposalPOL = nil // Nil until ProposalPOLMessage received. } @@ -1006,7 +1017,11 @@ func (ps *PeerState) PickSendVote(votes types.VoteSetReader) bool { if vote, ok := ps.PickVoteToSend(votes); ok { msg := &VoteMessage{vote} ps.logger.Debug("Sending vote message", "ps", ps, "vote", vote) - return ps.peer.Send(VoteChannel, cdc.MustMarshalBinaryBare(msg)) + if ps.peer.Send(VoteChannel, cdc.MustMarshalBinaryBare(msg)) { + ps.SetHasVote(vote) + return true + } + return false } return false } @@ -1022,7 +1037,7 @@ func (ps *PeerState) PickVoteToSend(votes types.VoteSetReader) (vote *types.Vote return nil, false } - height, round, type_, size := votes.Height(), votes.Round(), votes.Type(), votes.Size() + height, round, type_, size := votes.Height(), votes.Round(), types.SignedMsgType(votes.Type()), votes.Size() // Lazily set data using 'votes'. if votes.IsCommit() { @@ -1035,13 +1050,12 @@ func (ps *PeerState) PickVoteToSend(votes types.VoteSetReader) (vote *types.Vote return nil, false // Not something worth sending } if index, ok := votes.BitArray().Sub(psVotes).PickRandom(); ok { - ps.setHasVote(height, round, type_, index) return votes.GetByIndex(index), true } return nil, false } -func (ps *PeerState) getVoteBitArray(height int64, round int, type_ byte) *cmn.BitArray { +func (ps *PeerState) getVoteBitArray(height int64, round int, type_ types.SignedMsgType) *cmn.BitArray { if !types.IsVoteTypeValid(type_) { return nil } @@ -1049,25 +1063,25 @@ func (ps *PeerState) getVoteBitArray(height int64, round int, type_ byte) *cmn.B if ps.PRS.Height == height { if ps.PRS.Round == round { switch type_ { - case types.VoteTypePrevote: + case types.PrevoteType: return ps.PRS.Prevotes - case types.VoteTypePrecommit: + case types.PrecommitType: return ps.PRS.Precommits } } if ps.PRS.CatchupCommitRound == round { switch type_ { - case types.VoteTypePrevote: + case types.PrevoteType: return nil - case types.VoteTypePrecommit: + case types.PrecommitType: return ps.PRS.CatchupCommit } } if ps.PRS.ProposalPOLRound == round { switch type_ { - case types.VoteTypePrevote: + case types.PrevoteType: return ps.PRS.ProposalPOL - case types.VoteTypePrecommit: + case types.PrecommitType: return nil } } @@ -1076,9 +1090,9 @@ func (ps *PeerState) getVoteBitArray(height int64, round int, type_ byte) *cmn.B if ps.PRS.Height == height+1 { if ps.PRS.LastCommitRound == round { switch type_ { - case types.VoteTypePrevote: + case types.PrevoteType: return nil - case types.VoteTypePrecommit: + case types.PrecommitType: return ps.PRS.LastCommit } } @@ -1187,7 +1201,7 @@ func (ps *PeerState) SetHasVote(vote *types.Vote) { ps.setHasVote(vote.Height, vote.Round, vote.Type, vote.ValidatorIndex) } -func (ps *PeerState) setHasVote(height int64, round int, type_ byte, index int) { +func (ps *PeerState) setHasVote(height int64, round int, type_ types.SignedMsgType, index int) { logger := ps.logger.With("peerH/R", fmt.Sprintf("%d/%d", ps.PRS.Height, ps.PRS.Round), "H/R", fmt.Sprintf("%d/%d", height, round)) logger.Debug("setHasVote", "type", type_, "index", index) @@ -1211,7 +1225,6 @@ func (ps *PeerState) ApplyNewRoundStepMessage(msg *NewRoundStepMessage) { // Just remember these values. psHeight := ps.PRS.Height psRound := ps.PRS.Round - //psStep := ps.PRS.Step psCatchupCommitRound := ps.PRS.CatchupCommitRound psCatchupCommit := ps.PRS.CatchupCommit @@ -1252,8 +1265,8 @@ func (ps *PeerState) ApplyNewRoundStepMessage(msg *NewRoundStepMessage) { } } -// ApplyCommitStepMessage updates the peer state for the new commit. -func (ps *PeerState) ApplyCommitStepMessage(msg *CommitStepMessage) { +// ApplyNewValidBlockMessage updates the peer state for the new valid block. +func (ps *PeerState) ApplyNewValidBlockMessage(msg *NewValidBlockMessage) { ps.mtx.Lock() defer ps.mtx.Unlock() @@ -1261,6 +1274,10 @@ func (ps *PeerState) ApplyCommitStepMessage(msg *CommitStepMessage) { return } + if ps.PRS.Round != msg.Round && !msg.IsCommit { + return + } + ps.PRS.ProposalBlockPartsHeader = msg.BlockPartsHeader ps.PRS.ProposalBlockParts = msg.BlockParts } @@ -1339,12 +1356,14 @@ func (ps *PeerState) StringIndented(indent string) string { // Messages // ConsensusMessage is a message that can be sent and received on the ConsensusReactor -type ConsensusMessage interface{} +type ConsensusMessage interface { + ValidateBasic() error +} func RegisterConsensusMessages(cdc *amino.Codec) { cdc.RegisterInterface((*ConsensusMessage)(nil), nil) cdc.RegisterConcrete(&NewRoundStepMessage{}, "tendermint/NewRoundStepMessage", nil) - cdc.RegisterConcrete(&CommitStepMessage{}, "tendermint/CommitStep", nil) + cdc.RegisterConcrete(&NewValidBlockMessage{}, "tendermint/NewValidBlockMessage", nil) cdc.RegisterConcrete(&ProposalMessage{}, "tendermint/Proposal", nil) cdc.RegisterConcrete(&ProposalPOLMessage{}, "tendermint/ProposalPOL", nil) cdc.RegisterConcrete(&BlockPartMessage{}, "tendermint/BlockPart", nil) @@ -1352,7 +1371,6 @@ func RegisterConsensusMessages(cdc *amino.Codec) { cdc.RegisterConcrete(&HasVoteMessage{}, "tendermint/HasVote", nil) cdc.RegisterConcrete(&VoteSetMaj23Message{}, "tendermint/VoteSetMaj23", nil) cdc.RegisterConcrete(&VoteSetBitsMessage{}, "tendermint/VoteSetBits", nil) - cdc.RegisterConcrete(&ProposalHeartbeatMessage{}, "tendermint/ProposalHeartbeat", nil) } func decodeMsg(bz []byte) (msg ConsensusMessage, err error) { @@ -1375,6 +1393,27 @@ type NewRoundStepMessage struct { LastCommitRound int } +// ValidateBasic performs basic validation. +func (m *NewRoundStepMessage) ValidateBasic() error { + if m.Height < 0 { + return errors.New("Negative Height") + } + if m.Round < 0 { + return errors.New("Negative Round") + } + if !m.Step.IsValid() { + return errors.New("Invalid Step") + } + + // NOTE: SecondsSinceStartTime may be negative + + if (m.Height == 1 && m.LastCommitRound != -1) || + (m.Height > 1 && m.LastCommitRound < -1) { // TODO: #2737 LastCommitRound should always be >= 0 for heights > 1 + return errors.New("Invalid LastCommitRound (for 1st block: -1, for others: >= 0)") + } + return nil +} + // String returns a string representation. func (m *NewRoundStepMessage) String() string { return fmt.Sprintf("[NewRoundStep H:%v R:%v S:%v LCR:%v]", @@ -1383,16 +1422,40 @@ func (m *NewRoundStepMessage) String() string { //------------------------------------- -// CommitStepMessage is sent when a block is committed. -type CommitStepMessage struct { +// NewValidBlockMessage is sent when a validator observes a valid block B in some round r, +//i.e., there is a Proposal for block B and 2/3+ prevotes for the block B in the round r. +// In case the block is also committed, then IsCommit flag is set to true. +type NewValidBlockMessage struct { Height int64 + Round int BlockPartsHeader types.PartSetHeader BlockParts *cmn.BitArray + IsCommit bool +} + +// ValidateBasic performs basic validation. +func (m *NewValidBlockMessage) ValidateBasic() error { + if m.Height < 0 { + return errors.New("Negative Height") + } + if m.Round < 0 { + return errors.New("Negative Round") + } + if err := m.BlockPartsHeader.ValidateBasic(); err != nil { + return fmt.Errorf("Wrong BlockPartsHeader: %v", err) + } + if m.BlockParts.Size() != m.BlockPartsHeader.Total { + return fmt.Errorf("BlockParts bit array size %d not equal to BlockPartsHeader.Total %d", + m.BlockParts.Size(), + m.BlockPartsHeader.Total) + } + return nil } // String returns a string representation. -func (m *CommitStepMessage) String() string { - return fmt.Sprintf("[CommitStep H:%v BP:%v BA:%v]", m.Height, m.BlockPartsHeader, m.BlockParts) +func (m *NewValidBlockMessage) String() string { + return fmt.Sprintf("[ValidBlockMessage H:%v R:%v BP:%v BA:%v IsCommit:%v]", + m.Height, m.Round, m.BlockPartsHeader, m.BlockParts, m.IsCommit) } //------------------------------------- @@ -1402,6 +1465,11 @@ type ProposalMessage struct { Proposal *types.Proposal } +// ValidateBasic performs basic validation. +func (m *ProposalMessage) ValidateBasic() error { + return m.Proposal.ValidateBasic() +} + // String returns a string representation. func (m *ProposalMessage) String() string { return fmt.Sprintf("[Proposal %v]", m.Proposal) @@ -1416,6 +1484,20 @@ type ProposalPOLMessage struct { ProposalPOL *cmn.BitArray } +// ValidateBasic performs basic validation. +func (m *ProposalPOLMessage) ValidateBasic() error { + if m.Height < 0 { + return errors.New("Negative Height") + } + if m.ProposalPOLRound < 0 { + return errors.New("Negative ProposalPOLRound") + } + if m.ProposalPOL.Size() == 0 { + return errors.New("Empty ProposalPOL bit array") + } + return nil +} + // String returns a string representation. func (m *ProposalPOLMessage) String() string { return fmt.Sprintf("[ProposalPOL H:%v POLR:%v POL:%v]", m.Height, m.ProposalPOLRound, m.ProposalPOL) @@ -1430,6 +1512,20 @@ type BlockPartMessage struct { Part *types.Part } +// ValidateBasic performs basic validation. +func (m *BlockPartMessage) ValidateBasic() error { + if m.Height < 0 { + return errors.New("Negative Height") + } + if m.Round < 0 { + return errors.New("Negative Round") + } + if err := m.Part.ValidateBasic(); err != nil { + return fmt.Errorf("Wrong Part: %v", err) + } + return nil +} + // String returns a string representation. func (m *BlockPartMessage) String() string { return fmt.Sprintf("[BlockPart H:%v R:%v P:%v]", m.Height, m.Round, m.Part) @@ -1442,6 +1538,11 @@ type VoteMessage struct { Vote *types.Vote } +// ValidateBasic performs basic validation. +func (m *VoteMessage) ValidateBasic() error { + return m.Vote.ValidateBasic() +} + // String returns a string representation. func (m *VoteMessage) String() string { return fmt.Sprintf("[Vote %v]", m.Vote) @@ -1453,10 +1554,27 @@ func (m *VoteMessage) String() string { type HasVoteMessage struct { Height int64 Round int - Type byte + Type types.SignedMsgType Index int } +// ValidateBasic performs basic validation. +func (m *HasVoteMessage) ValidateBasic() error { + if m.Height < 0 { + return errors.New("Negative Height") + } + if m.Round < 0 { + return errors.New("Negative Round") + } + if !types.IsVoteTypeValid(m.Type) { + return errors.New("Invalid Type") + } + if m.Index < 0 { + return errors.New("Negative Index") + } + return nil +} + // String returns a string representation. func (m *HasVoteMessage) String() string { return fmt.Sprintf("[HasVote VI:%v V:{%v/%02d/%v}]", m.Index, m.Height, m.Round, m.Type) @@ -1468,10 +1586,27 @@ func (m *HasVoteMessage) String() string { type VoteSetMaj23Message struct { Height int64 Round int - Type byte + Type types.SignedMsgType BlockID types.BlockID } +// ValidateBasic performs basic validation. +func (m *VoteSetMaj23Message) ValidateBasic() error { + if m.Height < 0 { + return errors.New("Negative Height") + } + if m.Round < 0 { + return errors.New("Negative Round") + } + if !types.IsVoteTypeValid(m.Type) { + return errors.New("Invalid Type") + } + if err := m.BlockID.ValidateBasic(); err != nil { + return fmt.Errorf("Wrong BlockID: %v", err) + } + return nil +} + // String returns a string representation. func (m *VoteSetMaj23Message) String() string { return fmt.Sprintf("[VSM23 %v/%02d/%v %v]", m.Height, m.Round, m.Type, m.BlockID) @@ -1483,24 +1618,32 @@ func (m *VoteSetMaj23Message) String() string { type VoteSetBitsMessage struct { Height int64 Round int - Type byte + Type types.SignedMsgType BlockID types.BlockID Votes *cmn.BitArray } +// ValidateBasic performs basic validation. +func (m *VoteSetBitsMessage) ValidateBasic() error { + if m.Height < 0 { + return errors.New("Negative Height") + } + if m.Round < 0 { + return errors.New("Negative Round") + } + if !types.IsVoteTypeValid(m.Type) { + return errors.New("Invalid Type") + } + if err := m.BlockID.ValidateBasic(); err != nil { + return fmt.Errorf("Wrong BlockID: %v", err) + } + // NOTE: Votes.Size() can be zero if the node does not have any + return nil +} + // String returns a string representation. func (m *VoteSetBitsMessage) String() string { return fmt.Sprintf("[VSB %v/%02d/%v %v %v]", m.Height, m.Round, m.Type, m.BlockID, m.Votes) } //------------------------------------- - -// ProposalHeartbeatMessage is sent to signal that a node is alive and waiting for transactions for a proposal. -type ProposalHeartbeatMessage struct { - Heartbeat *types.Heartbeat -} - -// String returns a string representation. -func (m *ProposalHeartbeatMessage) String() string { - return fmt.Sprintf("[HEARTBEAT %v]", m.Heartbeat) -} diff --git a/consensus/reactor_test.go b/consensus/reactor_test.go index 2758f3fa..28e245ae 100644 --- a/consensus/reactor_test.go +++ b/consensus/reactor_test.go @@ -14,7 +14,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/tendermint/tendermint/abci/client" + abcicli "github.com/tendermint/tendermint/abci/client" "github.com/tendermint/tendermint/abci/example/kvstore" abci "github.com/tendermint/tendermint/abci/types" bc "github.com/tendermint/tendermint/blockchain" @@ -143,7 +143,8 @@ func TestReactorWithEvidence(t *testing.T) { // mock the evidence pool // everyone includes evidence of another double signing vIdx := (i + 1) % nValidators - evpool := newMockEvidencePool(privVals[vIdx].GetAddress()) + addr := privVals[vIdx].GetPubKey().Address() + evpool := newMockEvidencePool(addr) // Make ConsensusState blockExec := sm.NewBlockExecutor(stateDB, log.TestingLogger(), proxyAppConnCon, mempool, evpool) @@ -213,8 +214,8 @@ func (m *mockEvidencePool) Update(block *types.Block, state sm.State) { //------------------------------------ -// Ensure a testnet sends proposal heartbeats and makes blocks when there are txs -func TestReactorProposalHeartbeats(t *testing.T) { +// Ensure a testnet makes blocks when there are txs +func TestReactorCreatesBlockWhenEmptyBlocksFalse(t *testing.T) { N := 4 css := randConsensusNet(N, "consensus_reactor_test", newMockTickerFunc(true), newCounter, func(c *cfg.Config) { @@ -222,20 +223,9 @@ func TestReactorProposalHeartbeats(t *testing.T) { }) reactors, eventChans, eventBuses := startConsensusNet(t, css, N) defer stopConsensusNet(log.TestingLogger(), reactors, eventBuses) - heartbeatChans := make([]chan interface{}, N) - var err error - for i := 0; i < N; i++ { - heartbeatChans[i] = make(chan interface{}, 1) - err = eventBuses[i].Subscribe(context.Background(), testSubscriber, types.EventQueryProposalHeartbeat, heartbeatChans[i]) - require.NoError(t, err) - } - // wait till everyone sends a proposal heartbeat - timeoutWaitGroup(t, N, func(j int) { - <-heartbeatChans[j] - }, css) // send a tx - if err := css[3].mempool.CheckTx([]byte{1, 2, 3}, nil); err != nil { + if err := assertMempool(css[3].txNotifier).CheckTx([]byte{1, 2, 3}, nil); err != nil { //t.Fatal(err) } @@ -279,7 +269,8 @@ func TestReactorVotingPowerChange(t *testing.T) { // map of active validators activeVals := make(map[string]struct{}) for i := 0; i < nVals; i++ { - activeVals[string(css[i].privValidator.GetAddress())] = struct{}{} + addr := css[i].privValidator.GetPubKey().Address() + activeVals[string(addr)] = struct{}{} } // wait till everyone makes block 1 @@ -342,7 +333,8 @@ func TestReactorValidatorSetChanges(t *testing.T) { // map of active validators activeVals := make(map[string]struct{}) for i := 0; i < nVals; i++ { - activeVals[string(css[i].privValidator.GetAddress())] = struct{}{} + addr := css[i].privValidator.GetPubKey().Address() + activeVals[string(addr)] = struct{}{} } // wait till everyone makes block 1 @@ -456,7 +448,7 @@ func waitForAndValidateBlock(t *testing.T, n int, activeVals map[string]struct{} err := validateBlock(newBlock, activeVals) assert.Nil(t, err) for _, tx := range txs { - err := css[j].mempool.CheckTx(tx, nil) + err := assertMempool(css[j].txNotifier).CheckTx(tx, nil) assert.Nil(t, err) } }, css) diff --git a/consensus/replay.go b/consensus/replay.go index c92654f2..21fef6b2 100644 --- a/consensus/replay.go +++ b/consensus/replay.go @@ -6,6 +6,7 @@ import ( "hash/crc32" "io" "reflect" + //"strconv" //"strings" "time" @@ -73,7 +74,7 @@ func (cs *ConsensusState) readReplayMessage(msg *TimedWALMessage, newStepCh chan case *ProposalMessage: p := msg.Proposal cs.Logger.Info("Replay: Proposal", "height", p.Height, "round", p.Round, "header", - p.BlockPartsHeader, "pol", p.POLRound, "peer", peerID) + p.BlockID.PartsHeader, "pol", p.POLRound, "peer", peerID) case *BlockPartMessage: cs.Logger.Info("Replay: BlockPart", "height", msg.Height, "round", msg.Round, "peer", peerID) case *VoteMessage: @@ -143,8 +144,8 @@ func (cs *ConsensusState) catchupReplay(csHeight int64) error { if err == io.EOF { break } else if IsDataCorruptionError(err) { - cs.Logger.Debug("data has been corrupted in last height of consensus WAL", "err", err, "height", csHeight) - panic(fmt.Sprintf("data has been corrupted (%v) in last height %d of consensus WAL", err, csHeight)) + cs.Logger.Error("data has been corrupted in last height of consensus WAL", "err", err, "height", csHeight) + return err } else if err != nil { return err } @@ -196,6 +197,7 @@ type Handshaker struct { stateDB dbm.DB initialState sm.State store sm.BlockStore + eventBus types.BlockEventPublisher genDoc *types.GenesisDoc logger log.Logger @@ -209,6 +211,7 @@ func NewHandshaker(stateDB dbm.DB, state sm.State, stateDB: stateDB, initialState: state, store: store, + eventBus: types.NopEventBus{}, genDoc: genDoc, logger: log.NewNopLogger(), nBlocks: 0, @@ -219,6 +222,12 @@ func (h *Handshaker) SetLogger(l log.Logger) { h.logger = l } +// SetEventBus - sets the event bus for publishing block related events. +// If not called, it defaults to types.NopEventBus. +func (h *Handshaker) SetEventBus(eventBus types.BlockEventPublisher) { + h.eventBus = eventBus +} + func (h *Handshaker) NBlocks() int { return h.nBlocks } @@ -227,7 +236,7 @@ func (h *Handshaker) NBlocks() int { func (h *Handshaker) Handshake(proxyApp proxy.AppConns) error { // Handshake is done via ABCI Info on the query conn. - res, err := proxyApp.Query().InfoSync(abci.RequestInfo{Version: version.Version}) + res, err := proxyApp.Query().InfoSync(proxy.RequestInfo) if err != nil { return fmt.Errorf("Error calling Info: %v", err) } @@ -238,9 +247,16 @@ func (h *Handshaker) Handshake(proxyApp proxy.AppConns) error { } appHash := res.LastBlockAppHash - h.logger.Info("ABCI Handshake", "appHeight", blockHeight, "appHash", fmt.Sprintf("%X", appHash)) + h.logger.Info("ABCI Handshake App Info", + "height", blockHeight, + "hash", fmt.Sprintf("%X", appHash), + "software-version", res.Version, + "protocol-version", res.AppVersion, + ) - // TODO: check app version. + // Set AppVersion on the state. + h.initialState.Version.Consensus.App = version.Protocol(res.AppVersion) + sm.SaveState(h.stateDB, h.initialState) // Replay blocks up to the latest in the blockstore. _, err = h.ReplayBlocks(h.initialState, appHash, blockHeight, proxyApp) @@ -258,15 +274,24 @@ func (h *Handshaker) Handshake(proxyApp proxy.AppConns) error { // Replay all blocks since appBlockHeight and ensure the result matches the current state. // Returns the final AppHash or an error. -func (h *Handshaker) ReplayBlocks(state sm.State, appHash []byte, appBlockHeight int64, proxyApp proxy.AppConns) ([]byte, error) { - +func (h *Handshaker) ReplayBlocks( + state sm.State, + appHash []byte, + appBlockHeight int64, + proxyApp proxy.AppConns, +) ([]byte, error) { storeBlockHeight := h.store.Height() stateBlockHeight := state.LastBlockHeight h.logger.Info("ABCI Replay Blocks", "appHeight", appBlockHeight, "storeHeight", storeBlockHeight, "stateHeight", stateBlockHeight) // If appBlockHeight == 0 it means that we are at genesis and hence should send InitChain. if appBlockHeight == 0 { - nextVals := types.TM2PB.ValidatorUpdates(state.NextValidators) // state.Validators would work too. + validators := make([]*types.Validator, len(h.genDoc.Validators)) + for i, val := range h.genDoc.Validators { + validators[i] = types.NewValidator(val.PubKey, val.Power) + } + validatorSet := types.NewValidatorSet(validators) + nextVals := types.TM2PB.ValidatorUpdates(validatorSet) csParams := types.TM2PB.ConsensusParams(h.genDoc.ConsensusParams) req := abci.RequestInitChain{ Time: h.genDoc.GenesisTime, @@ -280,18 +305,27 @@ func (h *Handshaker) ReplayBlocks(state sm.State, appHash []byte, appBlockHeight return nil, err } - // If the app returned validators or consensus params, update the state. - if len(res.Validators) > 0 { - vals, err := types.PB2TM.ValidatorUpdates(res.Validators) - if err != nil { - return nil, err + if stateBlockHeight == 0 { //we only update state when we are in initial state + // If the app returned validators or consensus params, update the state. + if len(res.Validators) > 0 { + vals, err := types.PB2TM.ValidatorUpdates(res.Validators) + if err != nil { + return nil, err + } + state.Validators = types.NewValidatorSet(vals) + state.NextValidators = types.NewValidatorSet(vals) + } else { + // If validator set is not set in genesis and still empty after InitChain, exit. + if len(h.genDoc.Validators) == 0 { + return nil, fmt.Errorf("Validator set is nil in genesis and still empty after InitChain") + } } - state.Validators = types.NewValidatorSet(vals) + + if res.ConsensusParams != nil { + state.ConsensusParams = types.PB2TM.ConsensusParams(res.ConsensusParams) + } + sm.SaveState(h.stateDB, state) } - if res.ConsensusParams != nil { - state.ConsensusParams = types.PB2TM.ConsensusParams(res.ConsensusParams) - } - sm.SaveState(h.stateDB, state) } // First handle edge cases and constraints on the storeBlockHeight. @@ -407,6 +441,7 @@ func (h *Handshaker) replayBlock(state sm.State, height int64, proxyApp proxy.Ap meta := h.store.LoadBlockMeta(height) blockExec := sm.NewBlockExecutor(h.stateDB, h.logger, proxyApp, sm.MockMempool{}, sm.MockEvidencePool{}) + blockExec.SetEventBus(h.eventBus) var err error state, err = blockExec.ApplyBlock(state, meta.BlockID, block) diff --git a/consensus/replay_file.go b/consensus/replay_file.go index 685eb71f..3e92bad6 100644 --- a/consensus/replay_file.go +++ b/consensus/replay_file.go @@ -58,7 +58,18 @@ func (cs *ConsensusState) ReplayFile(file string, console bool) error { if err != nil { return errors.Errorf("failed to subscribe %s to %v", subscriber, types.EventQueryNewRoundStep) } - defer cs.eventBus.Unsubscribe(ctx, subscriber, types.EventQueryNewRoundStep) + defer func() { + // drain newStepCh to make sure we don't block + LOOP: + for { + select { + case <-newStepCh: + default: + break LOOP + } + } + cs.eventBus.Unsubscribe(ctx, subscriber, types.EventQueryNewRoundStep) + }() // just open the file for reading, no need to use wal fp, err := os.OpenFile(file, os.O_RDONLY, 0600) @@ -126,7 +137,7 @@ func (pb *playback) replayReset(count int, newStepCh chan interface{}) error { pb.cs.Wait() newCS := NewConsensusState(pb.cs.config, pb.genesisState.Copy(), pb.cs.blockExec, - pb.cs.blockStore, pb.cs.mempool, pb.cs.evpool) + pb.cs.blockStore, pb.cs.txNotifier, pb.cs.evpool) newCS.SetEventBus(pb.cs.eventBus) newCS.startForReplay() @@ -221,7 +232,18 @@ func (pb *playback) replayConsoleLoop() int { if err != nil { cmn.Exit(fmt.Sprintf("failed to subscribe %s to %v", subscriber, types.EventQueryNewRoundStep)) } - defer pb.cs.eventBus.Unsubscribe(ctx, subscriber, types.EventQueryNewRoundStep) + defer func() { + // drain newStepCh to make sure we don't block + LOOP: + for { + select { + case <-newStepCh: + default: + break LOOP + } + } + pb.cs.eventBus.Unsubscribe(ctx, subscriber, types.EventQueryNewRoundStep) + }() if len(tokens) == 1 { if err := pb.replayReset(1, newStepCh); err != nil { @@ -304,17 +326,18 @@ func newConsensusStateForReplay(config cfg.BaseConfig, csConfig *cfg.ConsensusCo cmn.Exit(fmt.Sprintf("Error starting proxy app conns: %v", err)) } - handshaker := NewHandshaker(stateDB, state, blockStore, gdoc) - err = handshaker.Handshake(proxyApp) - if err != nil { - cmn.Exit(fmt.Sprintf("Error on handshake: %v", err)) - } - eventBus := types.NewEventBus() if err := eventBus.Start(); err != nil { cmn.Exit(fmt.Sprintf("Failed to start event bus: %v", err)) } + handshaker := NewHandshaker(stateDB, state, blockStore, gdoc) + handshaker.SetEventBus(eventBus) + err = handshaker.Handshake(proxyApp) + if err != nil { + cmn.Exit(fmt.Sprintf("Error on handshake: %v", err)) + } + mempool, evpool := sm.MockMempool{}, sm.MockEvidencePool{} blockExec := sm.NewBlockExecutor(stateDB, log.TestingLogger(), proxyApp.Consensus(), mempool, evpool) diff --git a/consensus/replay_test.go b/consensus/replay_test.go index 7a828da6..e7269254 100644 --- a/consensus/replay_test.go +++ b/consensus/replay_test.go @@ -17,9 +17,10 @@ import ( "github.com/tendermint/tendermint/abci/example/kvstore" abci "github.com/tendermint/tendermint/abci/types" - crypto "github.com/tendermint/tendermint/crypto" + "github.com/tendermint/tendermint/crypto" auto "github.com/tendermint/tendermint/libs/autofile" dbm "github.com/tendermint/tendermint/libs/db" + "github.com/tendermint/tendermint/version" cfg "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/libs/log" @@ -86,7 +87,7 @@ func sendTxs(cs *ConsensusState, ctx context.Context) { return default: tx := []byte{byte(i)} - cs.mempool.CheckTx(tx, nil) + assertMempool(cs.txNotifier).CheckTx(tx, nil) i++ } } @@ -314,30 +315,23 @@ func testHandshakeReplay(t *testing.T, nBlocks int, mode uint) { config := ResetConfig("proxy_test_") walBody, err := WALWithNBlocks(NUM_BLOCKS) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) walFile := tempWALWithData(walBody) config.Consensus.SetWalFile(walFile) - privVal := privval.LoadFilePV(config.PrivValidatorFile()) + privVal := privval.LoadFilePV(config.PrivValidatorKeyFile(), config.PrivValidatorStateFile()) wal, err := NewWAL(walFile) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) wal.SetLogger(log.TestingLogger()) - if err := wal.Start(); err != nil { - t.Fatal(err) - } + err = wal.Start() + require.NoError(t, err) defer wal.Stop() chain, commits, err := makeBlockchainFromWAL(wal) - if err != nil { - t.Fatalf(err.Error()) - } + require.NoError(t, err) - stateDB, state, store := stateAndStore(config, privVal.GetPubKey()) + stateDB, state, store := stateAndStore(config, privVal.GetPubKey(), kvstore.ProtocolVersion) store.chain = chain store.commits = commits @@ -352,7 +346,7 @@ func testHandshakeReplay(t *testing.T, nBlocks int, mode uint) { // run nBlocks against a new client to build up the app state. // use a throwaway tendermint state proxyApp := proxy.NewAppConns(clientCreator2) - stateDB, state, _ := stateAndStore(config, privVal.GetPubKey()) + stateDB, state, _ := stateAndStore(config, privVal.GetPubKey(), kvstore.ProtocolVersion) buildAppStateFromChain(proxyApp, stateDB, state, chain, nBlocks, mode) } @@ -442,7 +436,7 @@ func buildAppStateFromChain(proxyApp proxy.AppConns, stateDB dbm.DB, func buildTMStateFromChain(config *cfg.Config, stateDB dbm.DB, state sm.State, chain []*types.Block, mode uint) sm.State { // run the whole chain against this client to build up the tendermint state clientCreator := proxy.NewLocalClientCreator(kvstore.NewPersistentKVStoreApplication(path.Join(config.DBDir(), "1"))) - proxyApp := proxy.NewAppConns(clientCreator) // sm.NewHandshaker(config, state, store, ReplayLastBlock)) + proxyApp := proxy.NewAppConns(clientCreator) if err := proxyApp.Start(); err != nil { panic(err) } @@ -519,7 +513,7 @@ func makeBlockchainFromWAL(wal WAL) ([]*types.Block, []*types.Commit, error) { // if its not the first one, we have a full block if thisBlockParts != nil { var block = new(types.Block) - _, err = cdc.UnmarshalBinaryReader(thisBlockParts.GetReader(), block, 0) + _, err = cdc.UnmarshalBinaryLengthPrefixedReader(thisBlockParts.GetReader(), block, 0) if err != nil { panic(err) } @@ -542,17 +536,17 @@ func makeBlockchainFromWAL(wal WAL) ([]*types.Block, []*types.Commit, error) { return nil, nil, err } case *types.Vote: - if p.Type == types.VoteTypePrecommit { + if p.Type == types.PrecommitType { thisBlockCommit = &types.Commit{ BlockID: p.BlockID, - Precommits: []*types.Vote{p}, + Precommits: []*types.CommitSig{p.CommitSig()}, } } } } // grab the last block too var block = new(types.Block) - _, err = cdc.UnmarshalBinaryReader(thisBlockParts.GetReader(), block, 0) + _, err = cdc.UnmarshalBinaryLengthPrefixedReader(thisBlockParts.GetReader(), block, 0) if err != nil { panic(err) } @@ -574,7 +568,7 @@ func readPieceFromWAL(msg *TimedWALMessage) interface{} { case msgInfo: switch msg := m.Msg.(type) { case *ProposalMessage: - return &msg.Proposal.BlockPartsHeader + return &msg.Proposal.BlockID.PartsHeader case *BlockPartMessage: return msg.Part case *VoteMessage: @@ -588,9 +582,10 @@ func readPieceFromWAL(msg *TimedWALMessage) interface{} { } // fresh state and mock store -func stateAndStore(config *cfg.Config, pubKey crypto.PubKey) (dbm.DB, sm.State, *mockBlockStore) { +func stateAndStore(config *cfg.Config, pubKey crypto.PubKey, appVersion version.Protocol) (dbm.DB, sm.State, *mockBlockStore) { stateDB := dbm.NewMemDB() state, _ := sm.MakeGenesisStateFromFile(config.GenesisFile()) + state.Version.Consensus.App = appVersion store := NewMockBlockStore(config, state.ConsensusParams) return stateDB, state, store } @@ -638,8 +633,8 @@ func TestInitChainUpdateValidators(t *testing.T) { clientCreator := proxy.NewLocalClientCreator(app) config := ResetConfig("proxy_test_") - privVal := privval.LoadFilePV(config.PrivValidatorFile()) - stateDB, state, store := stateAndStore(config, privVal.GetPubKey()) + privVal := privval.LoadFilePV(config.PrivValidatorKeyFile(), config.PrivValidatorStateFile()) + stateDB, state, store := stateAndStore(config, privVal.GetPubKey(), 0x0) oldValAddr := state.Validators.Validators[0].Address @@ -664,12 +659,6 @@ func TestInitChainUpdateValidators(t *testing.T) { assert.Equal(t, newValAddr, expectValAddr) } -func newInitChainApp(vals []abci.ValidatorUpdate) *initChainApp { - return &initChainApp{ - vals: vals, - } -} - // returns the vals on InitChain type initChainApp struct { abci.BaseApplication diff --git a/consensus/state.go b/consensus/state.go index 0100a150..74165801 100644 --- a/consensus/state.go +++ b/consensus/state.go @@ -2,15 +2,16 @@ package consensus import ( "bytes" - "errors" "fmt" "reflect" "runtime/debug" "sync" "time" - fail "github.com/ebuchman/fail-test" + "github.com/pkg/errors" + cmn "github.com/tendermint/tendermint/libs/common" + "github.com/tendermint/tendermint/libs/fail" "github.com/tendermint/tendermint/libs/log" tmtime "github.com/tendermint/tendermint/types/time" @@ -22,13 +23,6 @@ import ( "github.com/tendermint/tendermint/types" ) -//----------------------------------------------------------------------------- -// Config - -const ( - proposalHeartbeatIntervalSeconds = 2 -) - //----------------------------------------------------------------------------- // Errors @@ -63,6 +57,16 @@ func (ti *timeoutInfo) String() string { return fmt.Sprintf("%v ; %d/%d %v", ti.Duration, ti.Height, ti.Round, ti.Step) } +// interface to the mempool +type txNotifier interface { + TxsAvailable() <-chan struct{} +} + +// interface to the evidence pool +type evidencePool interface { + AddEvidence(types.Evidence) error +} + // ConsensusState handles execution of the consensus algorithm. // It processes votes and proposals, and upon reaching agreement, // commits blocks to the chain and executes them against the application. @@ -74,11 +78,18 @@ type ConsensusState struct { config *cfg.ConsensusConfig privValidator types.PrivValidator // for signing votes - // services for creating and executing blocks - blockExec *sm.BlockExecutor + // store blocks and commits blockStore sm.BlockStore - mempool sm.Mempool - evpool sm.EvidencePool + + // create and execute blocks + blockExec *sm.BlockExecutor + + // notify us if txs are available + txNotifier txNotifier + + // add evidence to the pool + // when it's detected + evpool evidencePool // internal state mtx sync.RWMutex @@ -117,7 +128,7 @@ type ConsensusState struct { done chan struct{} // synchronous pubsub between consensus state and reactor. - // state only emits EventNewRoundStep, EventVote and EventProposalHeartbeat + // state only emits EventNewRoundStep and EventVote evsw tmevents.EventSwitch // for reporting metrics @@ -133,15 +144,15 @@ func NewConsensusState( state sm.State, blockExec *sm.BlockExecutor, blockStore sm.BlockStore, - mempool sm.Mempool, - evpool sm.EvidencePool, + txNotifier txNotifier, + evpool evidencePool, options ...StateOption, ) *ConsensusState { cs := &ConsensusState{ config: config, blockExec: blockExec, blockStore: blockStore, - mempool: mempool, + txNotifier: txNotifier, peerMsgQueue: make(chan msgInfo, msgQueueSize), internalMsgQueue: make(chan msgInfo, msgQueueSize), timeoutTicker: NewTimeoutTicker(), @@ -206,18 +217,16 @@ func (cs *ConsensusState) GetState() sm.State { // GetLastHeight returns the last height committed. // If there were no blocks, returns 0. func (cs *ConsensusState) GetLastHeight() int64 { - cs.mtx.Lock() - defer cs.mtx.Unlock() - + cs.mtx.RLock() + defer cs.mtx.RUnlock() return cs.RoundState.Height - 1 } // GetRoundState returns a shallow copy of the internal consensus state. func (cs *ConsensusState) GetRoundState() *cstypes.RoundState { cs.mtx.RLock() - defer cs.mtx.RUnlock() - rs := cs.RoundState // copy + cs.mtx.RUnlock() return &rs } @@ -225,7 +234,6 @@ func (cs *ConsensusState) GetRoundState() *cstypes.RoundState { func (cs *ConsensusState) GetRoundStateJSON() ([]byte, error) { cs.mtx.RLock() defer cs.mtx.RUnlock() - return cdc.MarshalJSON(cs.RoundState) } @@ -233,7 +241,6 @@ func (cs *ConsensusState) GetRoundStateJSON() ([]byte, error) { func (cs *ConsensusState) GetRoundStateSimpleJSON() ([]byte, error) { cs.mtx.RLock() defer cs.mtx.RUnlock() - return cdc.MarshalJSON(cs.RoundState.RoundStateSimple()) } @@ -247,15 +254,15 @@ func (cs *ConsensusState) GetValidators() (int64, []*types.Validator) { // SetPrivValidator sets the private validator account for signing votes. func (cs *ConsensusState) SetPrivValidator(priv types.PrivValidator) { cs.mtx.Lock() - defer cs.mtx.Unlock() cs.privValidator = priv + cs.mtx.Unlock() } // SetTimeoutTicker sets the local timer. It may be useful to overwrite for testing. func (cs *ConsensusState) SetTimeoutTicker(timeoutTicker TimeoutTicker) { cs.mtx.Lock() - defer cs.mtx.Unlock() cs.timeoutTicker = timeoutTicker + cs.mtx.Unlock() } // LoadCommit loads the commit for a given height. @@ -300,6 +307,23 @@ func (cs *ConsensusState) OnStart() error { // reload from consensus log to catchup if cs.doWALCatchup { if err := cs.catchupReplay(cs.Height); err != nil { + // don't try to recover from data corruption error + if IsDataCorruptionError(err) { + cs.Logger.Error("Encountered corrupt WAL file", "err", err.Error()) + cs.Logger.Error("Please repair the WAL file before restarting") + fmt.Println(`You can attempt to repair the WAL as follows: + +---- +WALFILE=~/.tendermint/data/cs.wal/wal +cp $WALFILE ${WALFILE}.bak # backup the file +go run scripts/wal2json/main.go $WALFILE > wal.json # this will panic, but can be ignored +rm $WALFILE # remove the corrupt file +go run scripts/json2wal/main.go wal.json $WALFILE # rebuild the file without corruption +----`) + + return err + } + cs.Logger.Error("Error on catchup replay. Proceeding to start ConsensusState anyway", "err", err.Error()) // NOTE: if we ever do return an error here, // make sure to stop the timeoutTicker @@ -327,10 +351,11 @@ func (cs *ConsensusState) startRoutines(maxSteps int) { go cs.receiveRoutine(maxSteps) } -// OnStop implements cmn.Service. It stops all routines and waits for the WAL to finish. +// OnStop implements cmn.Service. func (cs *ConsensusState) OnStop() { cs.evsw.Stop() cs.timeoutTicker.Stop() + // WAL is stopped in receiveRoutine. } // Wait waits for the the main routine to return. @@ -429,7 +454,7 @@ func (cs *ConsensusState) updateRoundStep(round int, step cstypes.RoundStepType) // enterNewRound(height, 0) at cs.StartTime. func (cs *ConsensusState) scheduleRound0(rs *cstypes.RoundState) { //cs.Logger.Info("scheduleRound0", "now", tmtime.Now(), "startTime", cs.StartTime) - sleepDuration := rs.StartTime.Sub(tmtime.Now()) // nolint: gotype, gosimple + sleepDuration := rs.StartTime.Sub(tmtime.Now()) cs.scheduleTimeout(sleepDuration, rs.Height, 0, cstypes.RoundStepNewHeight) } @@ -459,12 +484,12 @@ func (cs *ConsensusState) reconstructLastCommit(state sm.State) { return } seenCommit := cs.blockStore.LoadSeenCommit(state.LastBlockHeight) - lastPrecommits := types.NewVoteSet(state.ChainID, state.LastBlockHeight, seenCommit.Round(), types.VoteTypePrecommit, state.LastValidators) + lastPrecommits := types.NewVoteSet(state.ChainID, state.LastBlockHeight, seenCommit.Round(), types.PrecommitType, state.LastValidators) for _, precommit := range seenCommit.Precommits { if precommit == nil { continue } - added, err := lastPrecommits.AddVote(precommit) + added, err := lastPrecommits.AddVote(seenCommit.ToVote(precommit)) if !added || err != nil { cmn.PanicCrisis(fmt.Sprintf("Failed to reconstruct LastCommit: %v", err)) } @@ -492,7 +517,7 @@ func (cs *ConsensusState) updateToState(state sm.State) { // If state isn't further out than cs.state, just ignore. // This happens when SwitchToConsensus() is called in the reactor. // We don't want to reset e.g. the Votes, but we still want to - // signal the new round step, because other services (eg. mempool) + // signal the new round step, because other services (eg. txNotifier) // depend on having an up-to-date peer state! if !cs.state.IsEmpty() && (state.LastBlockHeight <= cs.state.LastBlockHeight) { cs.Logger.Info("Ignoring updateToState()", "newHeight", state.LastBlockHeight+1, "oldHeight", cs.state.LastBlockHeight+1) @@ -531,10 +556,10 @@ func (cs *ConsensusState) updateToState(state sm.State) { cs.Proposal = nil cs.ProposalBlock = nil cs.ProposalBlockParts = nil - cs.LockedRound = 0 + cs.LockedRound = -1 cs.LockedBlock = nil cs.LockedBlockParts = nil - cs.ValidRound = 0 + cs.ValidRound = -1 cs.ValidBlock = nil cs.ValidBlockParts = nil cs.Votes = cstypes.NewHeightVoteSet(state.ChainID, height, validators) @@ -607,7 +632,7 @@ func (cs *ConsensusState) receiveRoutine(maxSteps int) { var mi msgInfo select { - case <-cs.mempool.TxsAvailable(): + case <-cs.txNotifier.TxsAvailable(): cs.handleTxsAvailable() case mi = <-cs.peerMsgQueue: cs.wal.Write(mi) @@ -616,6 +641,15 @@ func (cs *ConsensusState) receiveRoutine(maxSteps int) { cs.handleMsg(mi) case mi = <-cs.internalMsgQueue: cs.wal.WriteSync(mi) // NOTE: fsync + + if _, ok := mi.Msg.(*VoteMessage); ok { + // we actually want to simulate failing during + // the previous WriteSync, but this isn't easy to do. + // Equivalent would be to fail here and manually remove + // some bytes from the end of the wal. + fail.Fail() // XXX + } + // handles proposals, block parts, votes cs.handleMsg(mi) case ti := <-cs.timeoutTicker.Chan(): // tockChan: @@ -711,6 +745,7 @@ func (cs *ConsensusState) handleTimeout(ti timeoutInfo, rs cstypes.RoundState) { cs.enterPrecommit(ti.Height, ti.Round) case cstypes.RoundStepPrecommitWait: cs.eventBus.PublishEventTimeoutWait(cs.RoundStateEvent()) + cs.enterPrecommit(ti.Height, ti.Round) cs.enterNewRound(ti.Height, ti.Round+1) default: panic(fmt.Sprintf("Invalid timeout step: %v", ti.Step)) @@ -722,6 +757,7 @@ func (cs *ConsensusState) handleTxsAvailable() { cs.mtx.Lock() defer cs.mtx.Unlock() // we only need to do this for round 0 + cs.enterNewRound(cs.Height, 0) cs.enterPropose(cs.Height, 0) } @@ -753,7 +789,7 @@ func (cs *ConsensusState) enterNewRound(height int64, round int) { validators := cs.Validators if cs.Round < round { validators = validators.Copy() - validators.IncrementAccum(round - cs.Round) + validators.IncrementProposerPriority(round - cs.Round) } // Setup new round @@ -772,8 +808,9 @@ func (cs *ConsensusState) enterNewRound(height int64, round int) { cs.ProposalBlockParts = nil } cs.Votes.SetRound(round + 1) // also track next round (round+1) to allow round-skipping + cs.TriggeredTimeoutPrecommit = false - cs.eventBus.PublishEventNewRound(cs.RoundStateEvent()) + cs.eventBus.PublishEventNewRound(cs.NewRoundEvent()) cs.metrics.Rounds.Set(float64(round)) // Wait for txs to be available in the mempool @@ -782,9 +819,9 @@ func (cs *ConsensusState) enterNewRound(height int64, round int) { waitForTxs := cs.config.WaitForTxs() && round == 0 && !cs.needProofBlock(height) if waitForTxs { if cs.config.CreateEmptyBlocksInterval > 0 { - cs.scheduleTimeout(cs.config.CreateEmptyBlocksInterval, height, round, cstypes.RoundStepNewRound) + cs.scheduleTimeout(cs.config.CreateEmptyBlocksInterval, height, round, + cstypes.RoundStepNewRound) } - go cs.proposalHeartbeat(height, round) } else { cs.enterPropose(height, round) } @@ -801,32 +838,6 @@ func (cs *ConsensusState) needProofBlock(height int64) bool { return !bytes.Equal(cs.state.AppHash, lastBlockMeta.Header.AppHash) } -func (cs *ConsensusState) proposalHeartbeat(height int64, round int) { - counter := 0 - addr := cs.privValidator.GetAddress() - valIndex, _ := cs.Validators.GetByAddress(addr) - chainID := cs.state.ChainID - for { - rs := cs.GetRoundState() - // if we've already moved on, no need to send more heartbeats - if rs.Step > cstypes.RoundStepNewRound || rs.Round > round || rs.Height > height { - return - } - heartbeat := &types.Heartbeat{ - Height: rs.Height, - Round: rs.Round, - Sequence: counter, - ValidatorAddress: addr, - ValidatorIndex: valIndex, - } - cs.privValidator.SignHeartbeat(chainID, heartbeat) - cs.eventBus.PublishEventProposalHeartbeat(types.EventDataProposalHeartbeat{heartbeat}) - cs.evsw.FireEvent(types.EventProposalHeartbeat, heartbeat) - counter++ - time.Sleep(proposalHeartbeatIntervalSeconds * time.Second) - } -} - // Enter (CreateEmptyBlocks): from enterNewRound(height,round) // Enter (CreateEmptyBlocks, CreateEmptyBlocksInterval > 0 ): after enterNewRound(height,round), after timeout of CreateEmptyBlocksInterval // Enter (!CreateEmptyBlocks) : after enterNewRound(height,round), once txs are in the mempool @@ -862,13 +873,14 @@ func (cs *ConsensusState) enterPropose(height int64, round int) { } // if not a validator, we're done - if !cs.Validators.HasAddress(cs.privValidator.GetAddress()) { - logger.Debug("This node is not a validator", "addr", cs.privValidator.GetAddress(), "vals", cs.Validators) + address := cs.privValidator.GetPubKey().Address() + if !cs.Validators.HasAddress(address) { + logger.Debug("This node is not a validator", "addr", address, "vals", cs.Validators) return } logger.Debug("This node is a validator") - if cs.isProposer() { + if cs.isProposer(address) { logger.Info("enterPropose: Our turn to propose", "proposer", cs.Validators.GetProposer().Address, "privValidator", cs.privValidator) cs.decideProposal(height, round) } else { @@ -876,8 +888,8 @@ func (cs *ConsensusState) enterPropose(height int64, round int) { } } -func (cs *ConsensusState) isProposer() bool { - return bytes.Equal(cs.Validators.GetProposer().Address, cs.privValidator.GetAddress()) +func (cs *ConsensusState) isProposer(address []byte) bool { + return bytes.Equal(cs.Validators.GetProposer().Address, address) } func (cs *ConsensusState) defaultDecideProposal(height int64, round int) { @@ -885,10 +897,7 @@ func (cs *ConsensusState) defaultDecideProposal(height int64, round int) { var blockParts *types.PartSet // Decide on block - if cs.LockedBlock != nil { - // If we're locked onto a block, just choose that. - block, blockParts = cs.LockedBlock, cs.LockedBlockParts - } else if cs.ValidBlock != nil { + if cs.ValidBlock != nil { // If there is valid block, choose that. block, blockParts = cs.ValidBlock, cs.ValidBlockParts } else { @@ -900,15 +909,9 @@ func (cs *ConsensusState) defaultDecideProposal(height int64, round int) { } // Make proposal - polRound, polBlockID := cs.Votes.POLInfo() - proposal := types.NewProposal(height, round, blockParts.Header(), polRound, polBlockID) + propBlockId := types.BlockID{block.Hash(), blockParts.Header()} + proposal := types.NewProposal(height, round, cs.ValidRound, propBlockId) if err := cs.privValidator.SignProposal(cs.state.ChainID, proposal); err == nil { - // Set fields - /* fields set by setProposal and addBlockPart - cs.Proposal = proposal - cs.ProposalBlock = block - cs.ProposalBlockParts = blockParts - */ // send proposal and block parts on internal msg queue cs.sendInternalMessage(msgInfo{&ProposalMessage{proposal}, ""}) @@ -961,25 +964,12 @@ func (cs *ConsensusState) createProposalBlock() (block *types.Block, blockParts return } - maxBytes := cs.state.ConsensusParams.BlockSize.MaxBytes - maxGas := cs.state.ConsensusParams.BlockSize.MaxGas - // bound evidence to 1/10th of the block - evidence := cs.evpool.PendingEvidence(types.MaxEvidenceBytesPerBlock(maxBytes)) - // Mempool validated transactions - txs := cs.mempool.ReapMaxBytesMaxGas(types.MaxDataBytes( - maxBytes, - cs.state.Validators.Size(), - len(evidence), - ), maxGas) - proposerAddr := cs.privValidator.GetAddress() - block, parts := cs.state.MakeBlock(cs.Height, txs, commit, evidence, proposerAddr) - - return block, parts + proposerAddr := cs.privValidator.GetPubKey().Address() + return cs.blockExec.CreateProposalBlock(cs.Height, cs.state, commit, proposerAddr) } // Enter: `timeoutPropose` after entering Propose. // Enter: proposal block and POL is ready. -// Enter: any +2/3 prevotes for future round. // Prevote for LockedBlock if we're locked, or ProposalBlock if valid. // Otherwise vote nil. func (cs *ConsensusState) enterPrevote(height int64, round int) { @@ -994,14 +984,6 @@ func (cs *ConsensusState) enterPrevote(height int64, round int) { cs.newStep() }() - // fire event for how we got here - if cs.isProposalComplete() { - cs.eventBus.PublishEventCompleteProposal(cs.RoundStateEvent()) - } else { - // we received +2/3 prevotes for a future round - // TODO: catchup event? - } - cs.Logger.Info(fmt.Sprintf("enterPrevote(%v/%v). Current: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step)) // Sign and broadcast vote as necessary @@ -1013,17 +995,18 @@ func (cs *ConsensusState) enterPrevote(height int64, round int) { func (cs *ConsensusState) defaultDoPrevote(height int64, round int) { logger := cs.Logger.With("height", height, "round", round) + // If a block is locked, prevote that. if cs.LockedBlock != nil { logger.Info("enterPrevote: Block was locked") - cs.signAddVote(types.VoteTypePrevote, cs.LockedBlock.Hash(), cs.LockedBlockParts.Header()) + cs.signAddVote(types.PrevoteType, cs.LockedBlock.Hash(), cs.LockedBlockParts.Header()) return } // If ProposalBlock is nil, prevote nil. if cs.ProposalBlock == nil { logger.Info("enterPrevote: ProposalBlock is nil") - cs.signAddVote(types.VoteTypePrevote, nil, types.PartSetHeader{}) + cs.signAddVote(types.PrevoteType, nil, types.PartSetHeader{}) return } @@ -1032,7 +1015,7 @@ func (cs *ConsensusState) defaultDoPrevote(height int64, round int) { if err != nil { // ProposalBlock is invalid, prevote nil. logger.Error("enterPrevote: ProposalBlock is invalid", "err", err) - cs.signAddVote(types.VoteTypePrevote, nil, types.PartSetHeader{}) + cs.signAddVote(types.PrevoteType, nil, types.PartSetHeader{}) return } @@ -1040,7 +1023,7 @@ func (cs *ConsensusState) defaultDoPrevote(height int64, round int) { // NOTE: the proposal signature is validated when it is received, // and the proposal block parts are validated as they are received (against the merkle hash in the proposal) logger.Info("enterPrevote: ProposalBlock is valid") - cs.signAddVote(types.VoteTypePrevote, cs.ProposalBlock.Hash(), cs.ProposalBlockParts.Header()) + cs.signAddVote(types.PrevoteType, cs.ProposalBlock.Hash(), cs.ProposalBlockParts.Header()) } // Enter: any +2/3 prevotes at next round. @@ -1067,8 +1050,8 @@ func (cs *ConsensusState) enterPrevoteWait(height int64, round int) { } // Enter: `timeoutPrevote` after any +2/3 prevotes. +// Enter: `timeoutPrecommit` after any +2/3 precommits. // Enter: +2/3 precomits for block or nil. -// Enter: any +2/3 precommits for next round. // Lock & precommit the ProposalBlock if we have enough prevotes for it (a POL in this round) // else, unlock an existing lock and precommit nil if +2/3 of prevotes were nil, // else, precommit nil otherwise. @@ -1098,7 +1081,7 @@ func (cs *ConsensusState) enterPrecommit(height int64, round int) { } else { logger.Info("enterPrecommit: No +2/3 prevotes during enterPrecommit. Precommitting nil.") } - cs.signAddVote(types.VoteTypePrecommit, nil, types.PartSetHeader{}) + cs.signAddVote(types.PrecommitType, nil, types.PartSetHeader{}) return } @@ -1117,12 +1100,12 @@ func (cs *ConsensusState) enterPrecommit(height int64, round int) { logger.Info("enterPrecommit: +2/3 prevoted for nil.") } else { logger.Info("enterPrecommit: +2/3 prevoted for nil. Unlocking") - cs.LockedRound = 0 + cs.LockedRound = -1 cs.LockedBlock = nil cs.LockedBlockParts = nil cs.eventBus.PublishEventUnlock(cs.RoundStateEvent()) } - cs.signAddVote(types.VoteTypePrecommit, nil, types.PartSetHeader{}) + cs.signAddVote(types.PrecommitType, nil, types.PartSetHeader{}) return } @@ -1133,7 +1116,7 @@ func (cs *ConsensusState) enterPrecommit(height int64, round int) { logger.Info("enterPrecommit: +2/3 prevoted locked block. Relocking") cs.LockedRound = round cs.eventBus.PublishEventRelock(cs.RoundStateEvent()) - cs.signAddVote(types.VoteTypePrecommit, blockID.Hash, blockID.PartsHeader) + cs.signAddVote(types.PrecommitType, blockID.Hash, blockID.PartsHeader) return } @@ -1148,7 +1131,7 @@ func (cs *ConsensusState) enterPrecommit(height int64, round int) { cs.LockedBlock = cs.ProposalBlock cs.LockedBlockParts = cs.ProposalBlockParts cs.eventBus.PublishEventLock(cs.RoundStateEvent()) - cs.signAddVote(types.VoteTypePrecommit, blockID.Hash, blockID.PartsHeader) + cs.signAddVote(types.PrecommitType, blockID.Hash, blockID.PartsHeader) return } @@ -1156,7 +1139,7 @@ func (cs *ConsensusState) enterPrecommit(height int64, round int) { // Fetch that block, unlock, and precommit nil. // The +2/3 prevotes for this round is the POL for our unlock. // TODO: In the future save the POL prevotes for justification. - cs.LockedRound = 0 + cs.LockedRound = -1 cs.LockedBlock = nil cs.LockedBlockParts = nil if !cs.ProposalBlockParts.HasHeader(blockID.PartsHeader) { @@ -1164,15 +1147,19 @@ func (cs *ConsensusState) enterPrecommit(height int64, round int) { cs.ProposalBlockParts = types.NewPartSetFromHeader(blockID.PartsHeader) } cs.eventBus.PublishEventUnlock(cs.RoundStateEvent()) - cs.signAddVote(types.VoteTypePrecommit, nil, types.PartSetHeader{}) + cs.signAddVote(types.PrecommitType, nil, types.PartSetHeader{}) } // Enter: any +2/3 precommits for next round. func (cs *ConsensusState) enterPrecommitWait(height int64, round int) { logger := cs.Logger.With("height", height, "round", round) - if cs.Height != height || round < cs.Round || (cs.Round == round && cstypes.RoundStepPrecommitWait <= cs.Step) { - logger.Debug(fmt.Sprintf("enterPrecommitWait(%v/%v): Invalid args. Current step: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step)) + if cs.Height != height || round < cs.Round || (cs.Round == round && cs.TriggeredTimeoutPrecommit) { + logger.Debug( + fmt.Sprintf( + "enterPrecommitWait(%v/%v): Invalid args. "+ + "Current state is Height/Round: %v/%v/, TriggeredTimeoutPrecommit:%v", + height, round, cs.Height, cs.Round, cs.TriggeredTimeoutPrecommit)) return } if !cs.Votes.Precommits(round).HasTwoThirdsAny() { @@ -1182,7 +1169,7 @@ func (cs *ConsensusState) enterPrecommitWait(height int64, round int) { defer func() { // Done enterPrecommitWait: - cs.updateRoundStep(round, cstypes.RoundStepPrecommitWait) + cs.TriggeredTimeoutPrecommit = true cs.newStep() }() @@ -1235,6 +1222,8 @@ func (cs *ConsensusState) enterCommit(height int64, commitRound int) { // Set up ProposalBlockParts and keep waiting. cs.ProposalBlock = nil cs.ProposalBlockParts = types.NewPartSetFromHeader(blockID.PartsHeader) + cs.eventBus.PublishEventValidBlock(cs.RoundStateEvent()) + cs.evsw.FireEvent(types.EventValidBlock, &cs.RoundState) } else { // We just need to keep waiting. } @@ -1367,7 +1356,7 @@ func (cs *ConsensusState) recordMetrics(height int64, block *types.Block) { missingValidators := 0 missingValidatorsPower := int64(0) for i, val := range cs.Validators.Validators { - var vote *types.Vote + var vote *types.CommitSig if i < len(block.LastCommit.Precommits) { vote = block.LastCommit.Precommits[i] } @@ -1415,14 +1404,9 @@ func (cs *ConsensusState) defaultSetProposal(proposal *types.Proposal) error { return nil } - // We don't care about the proposal if we're already in cstypes.RoundStepCommit. - if cstypes.RoundStepCommit <= cs.Step { - return nil - } - - // Verify POLRound, which must be -1 or between 0 and proposal.Round exclusive. - if proposal.POLRound != -1 && - (proposal.POLRound < 0 || proposal.Round <= proposal.POLRound) { + // Verify POLRound, which must be -1 or in range [0, proposal.Round). + if proposal.POLRound < -1 || + (proposal.POLRound >= 0 && proposal.POLRound >= proposal.Round) { return ErrInvalidProposalPOLRound } @@ -1432,7 +1416,12 @@ func (cs *ConsensusState) defaultSetProposal(proposal *types.Proposal) error { } cs.Proposal = proposal - cs.ProposalBlockParts = types.NewPartSetFromHeader(proposal.BlockPartsHeader) + // We don't update cs.ProposalBlockParts if it is already set. + // This happens if we're already in cstypes.RoundStepCommit or if there is a valid block in the current round. + // TODO: We can check if Proposal is for a different block as this is a sign of misbehavior! + if cs.ProposalBlockParts == nil { + cs.ProposalBlockParts = types.NewPartSetFromHeader(proposal.BlockID.PartsHeader) + } cs.Logger.Info("Received proposal", "proposal", proposal) return nil } @@ -1463,7 +1452,7 @@ func (cs *ConsensusState) addProposalBlockPart(msg *BlockPartMessage, peerID p2p } if added && cs.ProposalBlockParts.IsComplete() { // Added and completed! - _, err = cdc.UnmarshalBinaryReader( + _, err = cdc.UnmarshalBinaryLengthPrefixedReader( cs.ProposalBlockParts.GetReader(), &cs.ProposalBlock, int64(cs.state.ConsensusParams.BlockSize.MaxBytes), @@ -1473,6 +1462,7 @@ func (cs *ConsensusState) addProposalBlockPart(msg *BlockPartMessage, peerID p2p } // NOTE: it's possible to receive complete proposal blocks for future rounds without having the proposal cs.Logger.Info("Received complete proposal block", "height", cs.ProposalBlock.Height, "hash", cs.ProposalBlock.Hash()) + cs.eventBus.PublishEventCompleteProposal(cs.CompleteProposalEvent()) // Update Valid* if we can. prevotes := cs.Votes.Prevotes(cs.Round) @@ -1495,6 +1485,9 @@ func (cs *ConsensusState) addProposalBlockPart(msg *BlockPartMessage, peerID p2p if cs.Step <= cstypes.RoundStepPropose && cs.isProposalComplete() { // Move onto the next step cs.enterPrevote(height, cs.Round) + if hasTwoThirds { // this is optimisation as this will be triggered when prevote is added + cs.enterPrecommit(height, cs.Round) + } } else if cs.Step == cstypes.RoundStepCommit { // If we're waiting on the proposal block... cs.tryFinalizeCommit(height) @@ -1514,7 +1507,8 @@ func (cs *ConsensusState) tryAddVote(vote *types.Vote, peerID p2p.ID) (bool, err if err == ErrVoteHeightMismatch { return added, err } else if voteErr, ok := err.(*types.ErrVoteConflictingVotes); ok { - if bytes.Equal(vote.ValidatorAddress, cs.privValidator.GetAddress()) { + addr := cs.privValidator.GetPubKey().Address() + if bytes.Equal(vote.ValidatorAddress, addr) { cs.Logger.Error("Found conflicting vote from ourselves. Did you unsafe_reset a validator?", "height", vote.Height, "round", vote.Round, "type", vote.Type) return added, err } @@ -1538,7 +1532,7 @@ func (cs *ConsensusState) addVote(vote *types.Vote, peerID p2p.ID) (added bool, // A precommit for the previous height? // These come in while we wait timeoutCommit if vote.Height+1 == cs.Height { - if !(cs.Step == cstypes.RoundStepNewHeight && vote.Type == types.VoteTypePrecommit) { + if !(cs.Step == cstypes.RoundStepNewHeight && vote.Type == types.PrecommitType) { // TODO: give the reason .. // fmt.Errorf("tryAddVote: Wrong height, not a LastCommit straggler commit.") return added, ErrVoteHeightMismatch @@ -1566,7 +1560,7 @@ func (cs *ConsensusState) addVote(vote *types.Vote, peerID p2p.ID) (added bool, // Not necessarily a bad peer, but not favourable behaviour. if vote.Height != cs.Height { err = ErrVoteHeightMismatch - cs.Logger.Info("Vote ignored and not added", "voteHeight", vote.Height, "csHeight", cs.Height, "err", err) + cs.Logger.Info("Vote ignored and not added", "voteHeight", vote.Height, "csHeight", cs.Height, "peerID", peerID) return } @@ -1581,7 +1575,7 @@ func (cs *ConsensusState) addVote(vote *types.Vote, peerID p2p.ID) (added bool, cs.evsw.FireEvent(types.EventVote, vote) switch vote.Type { - case types.VoteTypePrevote: + case types.PrevoteType: prevotes := cs.Votes.Prevotes(vote.Round) cs.Logger.Info("Added to prevote", "vote", vote, "prevotes", prevotes.StringShort()) @@ -1600,7 +1594,7 @@ func (cs *ConsensusState) addVote(vote *types.Vote, peerID p2p.ID) (added bool, !cs.LockedBlock.HashesTo(blockID.Hash) { cs.Logger.Info("Unlocking because of POL.", "lockedRound", cs.LockedRound, "POLRound", vote.Round) - cs.LockedRound = 0 + cs.LockedRound = -1 cs.LockedBlock = nil cs.LockedBlockParts = nil cs.eventBus.PublishEventUnlock(cs.RoundStateEvent()) @@ -1608,27 +1602,38 @@ func (cs *ConsensusState) addVote(vote *types.Vote, peerID p2p.ID) (added bool, // Update Valid* if we can. // NOTE: our proposal block may be nil or not what received a polka.. - // TODO: we may want to still update the ValidBlock and obtain it via gossipping - if !blockID.IsZero() && - (cs.ValidRound < vote.Round) && - (vote.Round <= cs.Round) && - cs.ProposalBlock.HashesTo(blockID.Hash) { + if len(blockID.Hash) != 0 && (cs.ValidRound < vote.Round) && (vote.Round == cs.Round) { - cs.Logger.Info("Updating ValidBlock because of POL.", "validRound", cs.ValidRound, "POLRound", vote.Round) - cs.ValidRound = vote.Round - cs.ValidBlock = cs.ProposalBlock - cs.ValidBlockParts = cs.ProposalBlockParts + if cs.ProposalBlock.HashesTo(blockID.Hash) { + cs.Logger.Info( + "Updating ValidBlock because of POL.", "validRound", cs.ValidRound, "POLRound", vote.Round) + cs.ValidRound = vote.Round + cs.ValidBlock = cs.ProposalBlock + cs.ValidBlockParts = cs.ProposalBlockParts + } else { + cs.Logger.Info( + "Valid block we don't know about. Set ProposalBlock=nil", + "proposal", cs.ProposalBlock.Hash(), "blockId", blockID.Hash) + // We're getting the wrong block. + cs.ProposalBlock = nil + } + if !cs.ProposalBlockParts.HasHeader(blockID.PartsHeader) { + cs.ProposalBlockParts = types.NewPartSetFromHeader(blockID.PartsHeader) + } + cs.evsw.FireEvent(types.EventValidBlock, &cs.RoundState) + cs.eventBus.PublishEventValidBlock(cs.RoundStateEvent()) } } - // If +2/3 prevotes for *anything* for this or future round: - if cs.Round <= vote.Round && prevotes.HasTwoThirdsAny() { - // Round-skip over to PrevoteWait or goto Precommit. - cs.enterNewRound(height, vote.Round) // if the vote is ahead of us - if prevotes.HasTwoThirdsMajority() { + // If +2/3 prevotes for *anything* for future round: + if cs.Round < vote.Round && prevotes.HasTwoThirdsAny() { + // Round-skip if there is any 2/3+ of votes ahead of us + cs.enterNewRound(height, vote.Round) + } else if cs.Round == vote.Round && cstypes.RoundStepPrevote <= cs.Step { // current round + blockID, ok := prevotes.TwoThirdsMajority() + if ok && (cs.isProposalComplete() || len(blockID.Hash) == 0) { cs.enterPrecommit(height, vote.Round) - } else { - cs.enterPrevote(height, vote.Round) // if the vote is ahead of us + } else if prevotes.HasTwoThirdsAny() { cs.enterPrevoteWait(height, vote.Round) } } else if cs.Proposal != nil && 0 <= cs.Proposal.POLRound && cs.Proposal.POLRound == vote.Round { @@ -1638,24 +1643,28 @@ func (cs *ConsensusState) addVote(vote *types.Vote, peerID p2p.ID) (added bool, } } - case types.VoteTypePrecommit: + case types.PrecommitType: precommits := cs.Votes.Precommits(vote.Round) cs.Logger.Info("Added to precommit", "vote", vote, "precommits", precommits.StringShort()) + blockID, ok := precommits.TwoThirdsMajority() - if ok && len(blockID.Hash) != 0 { + if ok { // Executed as TwoThirdsMajority could be from a higher round cs.enterNewRound(height, vote.Round) cs.enterPrecommit(height, vote.Round) - cs.enterCommit(height, vote.Round) - - if cs.config.SkipTimeoutCommit && precommits.HasAll() { - cs.enterNewRound(cs.Height, 0) + if len(blockID.Hash) != 0 { + cs.enterCommit(height, vote.Round) + if cs.config.SkipTimeoutCommit && precommits.HasAll() { + cs.enterNewRound(cs.Height, 0) + } + } else { + cs.enterPrecommitWait(height, vote.Round) } } else if cs.Round <= vote.Round && precommits.HasTwoThirdsAny() { cs.enterNewRound(height, vote.Round) - cs.enterPrecommit(height, vote.Round) cs.enterPrecommitWait(height, vote.Round) } + default: panic(fmt.Sprintf("Unexpected vote type %X", vote.Type)) // go-wire should prevent this. } @@ -1663,8 +1672,8 @@ func (cs *ConsensusState) addVote(vote *types.Vote, peerID p2p.ID) (added bool, return } -func (cs *ConsensusState) signVote(type_ byte, hash []byte, header types.PartSetHeader) (*types.Vote, error) { - addr := cs.privValidator.GetAddress() +func (cs *ConsensusState) signVote(type_ types.SignedMsgType, hash []byte, header types.PartSetHeader) (*types.Vote, error) { + addr := cs.privValidator.GetPubKey().Address() valIndex, _ := cs.Validators.GetByAddress(addr) vote := &types.Vote{ @@ -1698,9 +1707,9 @@ func (cs *ConsensusState) voteTime() time.Time { } // sign the vote and publish on internalMsgQueue -func (cs *ConsensusState) signAddVote(type_ byte, hash []byte, header types.PartSetHeader) *types.Vote { +func (cs *ConsensusState) signAddVote(type_ types.SignedMsgType, hash []byte, header types.PartSetHeader) *types.Vote { // if we don't have a key or we're not in the validator set, do nothing - if cs.privValidator == nil || !cs.Validators.HasAddress(cs.privValidator.GetAddress()) { + if cs.privValidator == nil || !cs.Validators.HasAddress(cs.privValidator.GetPubKey().Address()) { return nil } vote, err := cs.signVote(type_, hash, header) diff --git a/consensus/state_test.go b/consensus/state_test.go index 831f77f4..153f51e1 100644 --- a/consensus/state_test.go +++ b/consensus/state_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" cstypes "github.com/tendermint/tendermint/consensus/types" @@ -21,10 +22,6 @@ func init() { config = ResetConfig("consensus_state_test") } -func ensureProposeTimeout(timeoutPropose time.Duration) time.Duration { - return time.Duration(timeoutPropose.Nanoseconds()*2) * time.Nanosecond -} - /* ProposeSuite @@ -68,25 +65,27 @@ func TestStateProposerSelection0(t *testing.T) { startTestRound(cs1, height, round) // Wait for new round so proposer is set. - ensureNewRound(newRoundCh) + ensureNewRound(newRoundCh, height, round) // Commit a block and ensure proposer for the next height is correct. prop := cs1.GetRoundState().Validators.GetProposer() - if !bytes.Equal(prop.Address, cs1.privValidator.GetAddress()) { + address := cs1.privValidator.GetPubKey().Address() + if !bytes.Equal(prop.Address, address) { t.Fatalf("expected proposer to be validator %d. Got %X", 0, prop.Address) } // Wait for complete proposal. - ensureNewProposal(proposalCh) + ensureNewProposal(proposalCh, height, round) rs := cs1.GetRoundState() - signAddVotes(cs1, types.VoteTypePrecommit, rs.ProposalBlock.Hash(), rs.ProposalBlockParts.Header(), vss[1:]...) + signAddVotes(cs1, types.PrecommitType, rs.ProposalBlock.Hash(), rs.ProposalBlockParts.Header(), vss[1:]...) // Wait for new round so next validator is set. - ensureNewRound(newRoundCh) + ensureNewRound(newRoundCh, height+1, 0) prop = cs1.GetRoundState().Validators.GetProposer() - if !bytes.Equal(prop.Address, vss[1].GetAddress()) { + addr := vss[1].GetPubKey().Address() + if !bytes.Equal(prop.Address, addr) { panic(fmt.Sprintf("expected proposer to be validator %d. Got %X", 1, prop.Address)) } } @@ -94,27 +93,30 @@ func TestStateProposerSelection0(t *testing.T) { // Now let's do it all again, but starting from round 2 instead of 0 func TestStateProposerSelection2(t *testing.T) { cs1, vss := randConsensusState(4) // test needs more work for more than 3 validators - + height := cs1.Height newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound) // this time we jump in at round 2 incrementRound(vss[1:]...) incrementRound(vss[1:]...) - startTestRound(cs1, cs1.Height, 2) - ensureNewRound(newRoundCh) // wait for the new round + round := 2 + startTestRound(cs1, height, round) + + ensureNewRound(newRoundCh, height, round) // wait for the new round // everyone just votes nil. we get a new proposer each round for i := 0; i < len(vss); i++ { prop := cs1.GetRoundState().Validators.GetProposer() - correctProposer := vss[(i+2)%len(vss)].GetAddress() + addr := vss[(i+round)%len(vss)].GetPubKey().Address() + correctProposer := addr if !bytes.Equal(prop.Address, correctProposer) { panic(fmt.Sprintf("expected RoundState.Validators.GetProposer() to be validator %d. Got %X", (i+2)%len(vss), prop.Address)) } rs := cs1.GetRoundState() - signAddVotes(cs1, types.VoteTypePrecommit, nil, rs.ProposalBlockParts.Header(), vss[1:]...) - ensureNewRound(newRoundCh) // wait for the new round event each round + signAddVotes(cs1, types.PrecommitType, nil, rs.ProposalBlockParts.Header(), vss[1:]...) + ensureNewRound(newRoundCh, height, i+round+1) // wait for the new round event each round incrementRound(vss[1:]...) } @@ -132,7 +134,7 @@ func TestStateEnterProposeNoPrivValidator(t *testing.T) { startTestRound(cs, height, round) // if we're not a validator, EnterPropose should timeout - ensureNewTimeout(timeoutCh, cs.config.TimeoutPropose.Nanoseconds()) + ensureNewTimeout(timeoutCh, height, round, cs.config.TimeoutPropose.Nanoseconds()) if cs.GetRoundState().Proposal != nil { t.Error("Expected to make no proposal, since no privValidator") @@ -152,7 +154,7 @@ func TestStateEnterProposeYesPrivValidator(t *testing.T) { cs.enterNewRound(height, round) cs.startRoutines(3) - ensureNewProposal(proposalCh) + ensureNewProposal(proposalCh, height, round) // Check that Proposal, ProposalBlock, ProposalBlockParts are set. rs := cs.GetRoundState() @@ -194,7 +196,8 @@ func TestStateBadProposal(t *testing.T) { stateHash[0] = byte((stateHash[0] + 1) % 255) propBlock.AppHash = stateHash propBlockParts := propBlock.MakePartSet(partSize) - proposal := types.NewProposal(vs2.Height, round, propBlockParts.Header(), -1, types.BlockID{}) + blockID := types.BlockID{propBlock.Hash(), propBlockParts.Header()} + proposal := types.NewProposal(vs2.Height, round, -1, blockID) if err := vs2.SignProposal(config.ChainID(), proposal); err != nil { t.Fatal("failed to sign bad proposal", err) } @@ -208,22 +211,20 @@ func TestStateBadProposal(t *testing.T) { startTestRound(cs1, height, round) // wait for proposal - ensureNewProposal(proposalCh) + ensureProposal(proposalCh, height, round, blockID) // wait for prevote - ensureNewVote(voteCh) - + ensurePrevote(voteCh, height, round) validatePrevote(t, cs1, round, vss[0], nil) // add bad prevote from vs2 and wait for it - signAddVotes(cs1, types.VoteTypePrevote, propBlock.Hash(), propBlock.MakePartSet(partSize).Header(), vs2) - ensureNewVote(voteCh) + signAddVotes(cs1, types.PrevoteType, propBlock.Hash(), propBlock.MakePartSet(partSize).Header(), vs2) + ensurePrevote(voteCh, height, round) // wait for precommit - ensureNewVote(voteCh) - - validatePrecommit(t, cs1, round, 0, vss[0], nil, nil) - signAddVotes(cs1, types.VoteTypePrecommit, propBlock.Hash(), propBlock.MakePartSet(partSize).Header(), vs2) + ensurePrecommit(voteCh, height, round) + validatePrecommit(t, cs1, round, -1, vss[0], nil, nil) + signAddVotes(cs1, types.PrecommitType, propBlock.Hash(), propBlock.MakePartSet(partSize).Header(), vs2) } //---------------------------------------------------------------------------------------------------- @@ -246,21 +247,21 @@ func TestStateFullRound1(t *testing.T) { propCh := subscribe(cs.eventBus, types.EventQueryCompleteProposal) newRoundCh := subscribe(cs.eventBus, types.EventQueryNewRound) + // Maybe it would be better to call explicitly startRoutines(4) startTestRound(cs, height, round) - ensureNewRound(newRoundCh) + ensureNewRound(newRoundCh, height, round) - // grab proposal - re := <-propCh - propBlockHash := re.(types.EventDataRoundState).RoundState.(*cstypes.RoundState).ProposalBlock.Hash() + ensureNewProposal(propCh, height, round) + propBlockHash := cs.GetRoundState().ProposalBlock.Hash() - ensureNewVote(voteCh) // wait for prevote + ensurePrevote(voteCh, height, round) // wait for prevote validatePrevote(t, cs, round, vss[0], propBlockHash) - ensureNewVote(voteCh) // wait for precommit + ensurePrecommit(voteCh, height, round) // wait for precommit // we're going to roll right into new height - ensureNewRound(newRoundCh) + ensureNewRound(newRoundCh, height+1, 0) validateLastPrecommit(t, cs, vss[0], propBlockHash) } @@ -275,11 +276,11 @@ func TestStateFullRoundNil(t *testing.T) { cs.enterPrevote(height, round) cs.startRoutines(4) - ensureNewVote(voteCh) // prevote - ensureNewVote(voteCh) // precommit + ensurePrevote(voteCh, height, round) // prevote + ensurePrecommit(voteCh, height, round) // precommit // should prevote and precommit nil - validatePrevoteAndPrecommit(t, cs, round, 0, vss[0], nil, nil) + validatePrevoteAndPrecommit(t, cs, round, -1, vss[0], nil, nil) } // run through propose, prevote, precommit commit with two validators @@ -295,29 +296,28 @@ func TestStateFullRound2(t *testing.T) { // start round and wait for propose and prevote startTestRound(cs1, height, round) - ensureNewVote(voteCh) // prevote + ensurePrevote(voteCh, height, round) // prevote // we should be stuck in limbo waiting for more prevotes rs := cs1.GetRoundState() propBlockHash, propPartsHeader := rs.ProposalBlock.Hash(), rs.ProposalBlockParts.Header() // prevote arrives from vs2: - signAddVotes(cs1, types.VoteTypePrevote, propBlockHash, propPartsHeader, vs2) - ensureNewVote(voteCh) - - ensureNewVote(voteCh) //precommit + signAddVotes(cs1, types.PrevoteType, propBlockHash, propPartsHeader, vs2) + ensurePrevote(voteCh, height, round) // prevote + ensurePrecommit(voteCh, height, round) //precommit // the proposed block should now be locked and our precommit added validatePrecommit(t, cs1, 0, 0, vss[0], propBlockHash, propBlockHash) // we should be stuck in limbo waiting for more precommits // precommit arrives from vs2: - signAddVotes(cs1, types.VoteTypePrecommit, propBlockHash, propPartsHeader, vs2) - ensureNewVote(voteCh) + signAddVotes(cs1, types.PrecommitType, propBlockHash, propPartsHeader, vs2) + ensurePrecommit(voteCh, height, round) // wait to finish commit, propose in next height - ensureNewBlock(newBlockCh) + ensureNewBlock(newBlockCh, height) } //------------------------------------------------------------------------------------------ @@ -328,7 +328,7 @@ func TestStateFullRound2(t *testing.T) { func TestStateLockNoPOL(t *testing.T) { cs1, vss := randConsensusState(2) vs2 := vss[1] - height := cs1.Height + height, round := cs1.Height, cs1.Round partSize := types.BlockPartSizeBytes @@ -343,41 +343,43 @@ func TestStateLockNoPOL(t *testing.T) { */ // start round and wait for prevote - cs1.enterNewRound(height, 0) + cs1.enterNewRound(height, round) cs1.startRoutines(0) - re := <-proposalCh - rs := re.(types.EventDataRoundState).RoundState.(*cstypes.RoundState) - theBlockHash := rs.ProposalBlock.Hash() + ensureNewRound(newRoundCh, height, round) - ensureNewVote(voteCh) // prevote + ensureNewProposal(proposalCh, height, round) + roundState := cs1.GetRoundState() + theBlockHash := roundState.ProposalBlock.Hash() + thePartSetHeader := roundState.ProposalBlockParts.Header() + + ensurePrevote(voteCh, height, round) // prevote // we should now be stuck in limbo forever, waiting for more prevotes // prevote arrives from vs2: - signAddVotes(cs1, types.VoteTypePrevote, cs1.ProposalBlock.Hash(), cs1.ProposalBlockParts.Header(), vs2) - ensureNewVote(voteCh) // prevote - - ensureNewVote(voteCh) // precommit + signAddVotes(cs1, types.PrevoteType, theBlockHash, thePartSetHeader, vs2) + ensurePrevote(voteCh, height, round) // prevote + ensurePrecommit(voteCh, height, round) // precommit // the proposed block should now be locked and our precommit added - validatePrecommit(t, cs1, 0, 0, vss[0], theBlockHash, theBlockHash) + validatePrecommit(t, cs1, round, round, vss[0], theBlockHash, theBlockHash) // we should now be stuck in limbo forever, waiting for more precommits // lets add one for a different block - // NOTE: in practice we should never get to a point where there are precommits for different blocks at the same round hash := make([]byte, len(theBlockHash)) copy(hash, theBlockHash) hash[0] = byte((hash[0] + 1) % 255) - signAddVotes(cs1, types.VoteTypePrecommit, hash, rs.ProposalBlock.MakePartSet(partSize).Header(), vs2) - ensureNewVote(voteCh) // precommit + signAddVotes(cs1, types.PrecommitType, hash, thePartSetHeader, vs2) + ensurePrecommit(voteCh, height, round) // precommit // (note we're entering precommit for a second time this round) // but with invalid args. then we enterPrecommitWait, and the timeout to new round - ensureNewTimeout(timeoutWaitCh, cs1.config.TimeoutPrecommit.Nanoseconds()) + ensureNewTimeout(timeoutWaitCh, height, round, cs1.config.TimeoutPrecommit.Nanoseconds()) /// - ensureNewRound(newRoundCh) + round = round + 1 // moving to the next round + ensureNewRound(newRoundCh, height, round) t.Log("#### ONTO ROUND 1") /* Round2 (cs1, B) // B B2 @@ -386,43 +388,42 @@ func TestStateLockNoPOL(t *testing.T) { incrementRound(vs2) // now we're on a new round and not the proposer, so wait for timeout - re = <-timeoutProposeCh - rs = re.(types.EventDataRoundState).RoundState.(*cstypes.RoundState) + ensureNewTimeout(timeoutProposeCh, height, round, cs1.config.TimeoutPropose.Nanoseconds()) + + rs := cs1.GetRoundState() if rs.ProposalBlock != nil { panic("Expected proposal block to be nil") } // wait to finish prevote - ensureNewVote(voteCh) - + ensurePrevote(voteCh, height, round) // we should have prevoted our locked block - validatePrevote(t, cs1, 1, vss[0], rs.LockedBlock.Hash()) + validatePrevote(t, cs1, round, vss[0], rs.LockedBlock.Hash()) // add a conflicting prevote from the other validator - signAddVotes(cs1, types.VoteTypePrevote, hash, rs.LockedBlock.MakePartSet(partSize).Header(), vs2) - ensureNewVote(voteCh) + signAddVotes(cs1, types.PrevoteType, hash, rs.LockedBlock.MakePartSet(partSize).Header(), vs2) + ensurePrevote(voteCh, height, round) // now we're going to enter prevote again, but with invalid args // and then prevote wait, which should timeout. then wait for precommit - ensureNewTimeout(timeoutWaitCh, cs1.config.TimeoutPrevote.Nanoseconds()) - - ensureNewVote(voteCh) // precommit + ensureNewTimeout(timeoutWaitCh, height, round, cs1.config.TimeoutPrevote.Nanoseconds()) + ensurePrecommit(voteCh, height, round) // precommit // the proposed block should still be locked and our precommit added // we should precommit nil and be locked on the proposal - validatePrecommit(t, cs1, 1, 0, vss[0], nil, theBlockHash) + validatePrecommit(t, cs1, round, 0, vss[0], nil, theBlockHash) // add conflicting precommit from vs2 - // NOTE: in practice we should never get to a point where there are precommits for different blocks at the same round - signAddVotes(cs1, types.VoteTypePrecommit, hash, rs.LockedBlock.MakePartSet(partSize).Header(), vs2) - ensureNewVote(voteCh) + signAddVotes(cs1, types.PrecommitType, hash, rs.LockedBlock.MakePartSet(partSize).Header(), vs2) + ensurePrecommit(voteCh, height, round) // (note we're entering precommit for a second time this round, but with invalid args // then we enterPrecommitWait and timeout into NewRound - ensureNewTimeout(timeoutWaitCh, cs1.config.TimeoutPrecommit.Nanoseconds()) + ensureNewTimeout(timeoutWaitCh, height, round, cs1.config.TimeoutPrecommit.Nanoseconds()) - ensureNewRound(newRoundCh) + round = round + 1 // entering new round + ensureNewRound(newRoundCh, height, round) t.Log("#### ONTO ROUND 2") /* Round3 (vs2, _) // B, B2 @@ -430,40 +431,41 @@ func TestStateLockNoPOL(t *testing.T) { incrementRound(vs2) - re = <-proposalCh - rs = re.(types.EventDataRoundState).RoundState.(*cstypes.RoundState) + ensureNewProposal(proposalCh, height, round) + rs = cs1.GetRoundState() // now we're on a new round and are the proposer if !bytes.Equal(rs.ProposalBlock.Hash(), rs.LockedBlock.Hash()) { panic(fmt.Sprintf("Expected proposal block to be locked block. Got %v, Expected %v", rs.ProposalBlock, rs.LockedBlock)) } - ensureNewVote(voteCh) // prevote + ensurePrevote(voteCh, height, round) // prevote + validatePrevote(t, cs1, round, vss[0], rs.LockedBlock.Hash()) - validatePrevote(t, cs1, 2, vss[0], rs.LockedBlock.Hash()) + signAddVotes(cs1, types.PrevoteType, hash, rs.ProposalBlock.MakePartSet(partSize).Header(), vs2) + ensurePrevote(voteCh, height, round) - signAddVotes(cs1, types.VoteTypePrevote, hash, rs.ProposalBlock.MakePartSet(partSize).Header(), vs2) - ensureNewVote(voteCh) + ensureNewTimeout(timeoutWaitCh, height, round, cs1.config.TimeoutPrevote.Nanoseconds()) + ensurePrecommit(voteCh, height, round) // precommit - ensureNewTimeout(timeoutWaitCh, cs1.config.TimeoutPrevote.Nanoseconds()) - ensureNewVote(voteCh) // precommit + validatePrecommit(t, cs1, round, 0, vss[0], nil, theBlockHash) // precommit nil but be locked on proposal - validatePrecommit(t, cs1, 2, 0, vss[0], nil, theBlockHash) // precommit nil but be locked on proposal + signAddVotes(cs1, types.PrecommitType, hash, rs.ProposalBlock.MakePartSet(partSize).Header(), vs2) // NOTE: conflicting precommits at same height + ensurePrecommit(voteCh, height, round) - signAddVotes(cs1, types.VoteTypePrecommit, hash, rs.ProposalBlock.MakePartSet(partSize).Header(), vs2) // NOTE: conflicting precommits at same height - ensureNewVote(voteCh) - - ensureNewTimeout(timeoutWaitCh, cs1.config.TimeoutPrecommit.Nanoseconds()) + ensureNewTimeout(timeoutWaitCh, height, round, cs1.config.TimeoutPrecommit.Nanoseconds()) + cs2, _ := randConsensusState(2) // needed so generated block is different than locked block // before we time out into new round, set next proposal block - prop, propBlock := decideProposal(cs1, vs2, vs2.Height, vs2.Round+1) + prop, propBlock := decideProposal(cs2, vs2, vs2.Height, vs2.Round+1) if prop == nil || propBlock == nil { t.Fatal("Failed to create proposal block with vs2") } incrementRound(vs2) - ensureNewRound(newRoundCh) + round = round + 1 // entering new round + ensureNewRound(newRoundCh, height, round) t.Log("#### ONTO ROUND 3") /* Round4 (vs2, C) // B C // B C @@ -475,35 +477,35 @@ func TestStateLockNoPOL(t *testing.T) { t.Fatal(err) } - ensureNewProposal(proposalCh) - ensureNewVote(voteCh) // prevote - + ensureNewProposal(proposalCh, height, round) + ensurePrevote(voteCh, height, round) // prevote // prevote for locked block (not proposal) - validatePrevote(t, cs1, 0, vss[0], cs1.LockedBlock.Hash()) + validatePrevote(t, cs1, 3, vss[0], cs1.LockedBlock.Hash()) - signAddVotes(cs1, types.VoteTypePrevote, propBlock.Hash(), propBlock.MakePartSet(partSize).Header(), vs2) - ensureNewVote(voteCh) + // prevote for proposed block + signAddVotes(cs1, types.PrevoteType, propBlock.Hash(), propBlock.MakePartSet(partSize).Header(), vs2) + ensurePrevote(voteCh, height, round) - ensureNewTimeout(timeoutWaitCh, cs1.config.TimeoutPrevote.Nanoseconds()) - ensureNewVote(voteCh) + ensureNewTimeout(timeoutWaitCh, height, round, cs1.config.TimeoutPrevote.Nanoseconds()) + ensurePrecommit(voteCh, height, round) + validatePrecommit(t, cs1, round, 0, vss[0], nil, theBlockHash) // precommit nil but locked on proposal - validatePrecommit(t, cs1, 2, 0, vss[0], nil, theBlockHash) // precommit nil but locked on proposal - - signAddVotes(cs1, types.VoteTypePrecommit, propBlock.Hash(), propBlock.MakePartSet(partSize).Header(), vs2) // NOTE: conflicting precommits at same height - ensureNewVote(voteCh) + signAddVotes(cs1, types.PrecommitType, propBlock.Hash(), propBlock.MakePartSet(partSize).Header(), vs2) // NOTE: conflicting precommits at same height + ensurePrecommit(voteCh, height, round) } // 4 vals, one precommits, other 3 polka at next round, so we unlock and precomit the polka func TestStateLockPOLRelock(t *testing.T) { cs1, vss := randConsensusState(4) vs2, vs3, vs4 := vss[1], vss[2], vss[3] + height, round := cs1.Height, cs1.Round partSize := types.BlockPartSizeBytes - timeoutProposeCh := subscribe(cs1.eventBus, types.EventQueryTimeoutPropose) timeoutWaitCh := subscribe(cs1.eventBus, types.EventQueryTimeoutWait) proposalCh := subscribe(cs1.eventBus, types.EventQueryCompleteProposal) - voteCh := subscribe(cs1.eventBus, types.EventQueryVote) + addr := cs1.privValidator.GetPubKey().Address() + voteCh := subscribeToVoter(cs1, addr) newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound) newBlockCh := subscribe(cs1.eventBus, types.EventQueryNewBlockHeader) @@ -516,28 +518,25 @@ func TestStateLockPOLRelock(t *testing.T) { */ // start round and wait for propose and prevote - startTestRound(cs1, cs1.Height, 0) + startTestRound(cs1, height, round) - ensureNewRound(newRoundCh) - re := <-proposalCh - rs := re.(types.EventDataRoundState).RoundState.(*cstypes.RoundState) + ensureNewRound(newRoundCh, height, round) + ensureNewProposal(proposalCh, height, round) + rs := cs1.GetRoundState() theBlockHash := rs.ProposalBlock.Hash() + theBlockParts := rs.ProposalBlockParts.Header() - ensureNewVote(voteCh) // prevote + ensurePrevote(voteCh, height, round) // prevote - signAddVotes(cs1, types.VoteTypePrevote, cs1.ProposalBlock.Hash(), cs1.ProposalBlockParts.Header(), vs2, vs3, vs4) - // prevotes - discardFromChan(voteCh, 3) + signAddVotes(cs1, types.PrevoteType, theBlockHash, theBlockParts, vs2, vs3, vs4) - ensureNewVote(voteCh) // our precommit + ensurePrecommit(voteCh, height, round) // our precommit // the proposed block should now be locked and our precommit added - validatePrecommit(t, cs1, 0, 0, vss[0], theBlockHash, theBlockHash) + validatePrecommit(t, cs1, round, round, vss[0], theBlockHash, theBlockHash) // add precommits from the rest - signAddVotes(cs1, types.VoteTypePrecommit, nil, types.PartSetHeader{}, vs2, vs4) - signAddVotes(cs1, types.VoteTypePrecommit, cs1.ProposalBlock.Hash(), cs1.ProposalBlockParts.Header(), vs3) - // precommites - discardFromChan(voteCh, 3) + signAddVotes(cs1, types.PrecommitType, nil, types.PartSetHeader{}, vs2, vs4) + signAddVotes(cs1, types.PrecommitType, theBlockHash, theBlockParts, vs3) // before we timeout to the new round set the new proposal prop, propBlock := decideProposal(cs1, vs2, vs2.Height, vs2.Round+1) @@ -547,14 +546,15 @@ func TestStateLockPOLRelock(t *testing.T) { incrementRound(vs2, vs3, vs4) // timeout to new round - ensureNewTimeout(timeoutWaitCh, cs1.config.TimeoutPrecommit.Nanoseconds()) + ensureNewTimeout(timeoutWaitCh, height, round, cs1.config.TimeoutPrecommit.Nanoseconds()) + round = round + 1 // moving to the next round //XXX: this isnt guaranteed to get there before the timeoutPropose ... if err := cs1.SetProposalAndBlock(prop, propBlock, propBlockParts, "some peer"); err != nil { t.Fatal(err) } - ensureNewRound(newRoundCh) + ensureNewRound(newRoundCh, height, round) t.Log("### ONTO ROUND 1") /* @@ -565,64 +565,39 @@ func TestStateLockPOLRelock(t *testing.T) { // now we're on a new round and not the proposer // but we should receive the proposal - select { - case <-proposalCh: - case <-timeoutProposeCh: - <-proposalCh - } + ensureNewProposal(proposalCh, height, round) // go to prevote, prevote for locked block (not proposal), move on - ensureNewVote(voteCh) - validatePrevote(t, cs1, 0, vss[0], theBlockHash) + ensurePrevote(voteCh, height, round) + validatePrevote(t, cs1, round, vss[0], theBlockHash) // now lets add prevotes from everyone else for the new block - signAddVotes(cs1, types.VoteTypePrevote, propBlockHash, propBlockParts.Header(), vs2, vs3, vs4) - // prevotes - discardFromChan(voteCh, 3) - - // now either we go to PrevoteWait or Precommit - select { - case <-timeoutWaitCh: // we're in PrevoteWait, go to Precommit - // XXX: there's no guarantee we see the polka, this might be a precommit for nil, - // in which case the test fails! - <-voteCh - case <-voteCh: // we went straight to Precommit - } + signAddVotes(cs1, types.PrevoteType, propBlockHash, propBlockParts.Header(), vs2, vs3, vs4) + ensurePrecommit(voteCh, height, round) // we should have unlocked and locked on the new block - validatePrecommit(t, cs1, 1, 1, vss[0], propBlockHash, propBlockHash) + validatePrecommit(t, cs1, round, round, vss[0], propBlockHash, propBlockHash) - signAddVotes(cs1, types.VoteTypePrecommit, propBlockHash, propBlockParts.Header(), vs2, vs3) - discardFromChan(voteCh, 2) + signAddVotes(cs1, types.PrecommitType, propBlockHash, propBlockParts.Header(), vs2, vs3) + ensureNewBlockHeader(newBlockCh, height, propBlockHash) - be := <-newBlockCh - b := be.(types.EventDataNewBlockHeader) - re = <-newRoundCh - rs = re.(types.EventDataRoundState).RoundState.(*cstypes.RoundState) - if rs.Height != 2 { - panic("Expected height to increment") - } - - if !bytes.Equal(b.Header.Hash(), propBlockHash) { - panic("Expected new block to be proposal block") - } + ensureNewRound(newRoundCh, height+1, 0) } // 4 vals, one precommits, other 3 polka at next round, so we unlock and precomit the polka func TestStateLockPOLUnlock(t *testing.T) { cs1, vss := randConsensusState(4) vs2, vs3, vs4 := vss[1], vss[2], vss[3] - h := cs1.GetRoundState().Height - r := cs1.GetRoundState().Round + height, round := cs1.Height, cs1.Round partSize := types.BlockPartSizeBytes proposalCh := subscribe(cs1.eventBus, types.EventQueryCompleteProposal) - timeoutProposeCh := subscribe(cs1.eventBus, types.EventQueryTimeoutPropose) timeoutWaitCh := subscribe(cs1.eventBus, types.EventQueryTimeoutWait) newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound) unlockCh := subscribe(cs1.eventBus, types.EventQueryUnlock) - voteCh := subscribeToVoter(cs1, cs1.privValidator.GetAddress()) + addr := cs1.privValidator.GetPubKey().Address() + voteCh := subscribeToVoter(cs1, addr) // everything done from perspective of cs1 @@ -633,75 +608,71 @@ func TestStateLockPOLUnlock(t *testing.T) { */ // start round and wait for propose and prevote - startTestRound(cs1, h, r) - ensureNewRound(newRoundCh) - re := <-proposalCh - rs := re.(types.EventDataRoundState).RoundState.(*cstypes.RoundState) + startTestRound(cs1, height, round) + ensureNewRound(newRoundCh, height, round) + + ensureNewProposal(proposalCh, height, round) + rs := cs1.GetRoundState() theBlockHash := rs.ProposalBlock.Hash() + theBlockParts := rs.ProposalBlockParts.Header() - ensureVote(voteCh, h, r, types.VoteTypePrevote) + ensurePrevote(voteCh, height, round) + validatePrevote(t, cs1, round, vss[0], theBlockHash) - signAddVotes(cs1, types.VoteTypePrevote, cs1.ProposalBlock.Hash(), cs1.ProposalBlockParts.Header(), vs2, vs3, vs4) - - ensureVote(voteCh, h, r, types.VoteTypePrecommit) + signAddVotes(cs1, types.PrevoteType, theBlockHash, theBlockParts, vs2, vs3, vs4) + ensurePrecommit(voteCh, height, round) // the proposed block should now be locked and our precommit added - validatePrecommit(t, cs1, r, 0, vss[0], theBlockHash, theBlockHash) + validatePrecommit(t, cs1, round, round, vss[0], theBlockHash, theBlockHash) rs = cs1.GetRoundState() // add precommits from the rest - signAddVotes(cs1, types.VoteTypePrecommit, nil, types.PartSetHeader{}, vs2, vs4) - signAddVotes(cs1, types.VoteTypePrecommit, cs1.ProposalBlock.Hash(), cs1.ProposalBlockParts.Header(), vs3) + signAddVotes(cs1, types.PrecommitType, nil, types.PartSetHeader{}, vs2, vs4) + signAddVotes(cs1, types.PrecommitType, theBlockHash, theBlockParts, vs3) // before we time out into new round, set next proposal block prop, propBlock := decideProposal(cs1, vs2, vs2.Height, vs2.Round+1) propBlockParts := propBlock.MakePartSet(partSize) - incrementRound(vs2, vs3, vs4) - // timeout to new round - re = <-timeoutWaitCh - rs = re.(types.EventDataRoundState).RoundState.(*cstypes.RoundState) + ensureNewTimeout(timeoutWaitCh, height, round, cs1.config.TimeoutPrecommit.Nanoseconds()) + rs = cs1.GetRoundState() lockedBlockHash := rs.LockedBlock.Hash() - //XXX: this isnt guaranteed to get there before the timeoutPropose ... - if err := cs1.SetProposalAndBlock(prop, propBlock, propBlockParts, "some peer"); err != nil { - t.Fatal(err) - } + incrementRound(vs2, vs3, vs4) + round = round + 1 // moving to the next round - ensureNewRound(newRoundCh) + ensureNewRound(newRoundCh, height, round) t.Log("#### ONTO ROUND 1") /* Round2 (vs2, C) // B nil nil nil // nil nil nil _ cs1 unlocks! */ - - // now we're on a new round and not the proposer, - // but we should receive the proposal - select { - case <-proposalCh: - case <-timeoutProposeCh: - <-proposalCh + //XXX: this isnt guaranteed to get there before the timeoutPropose ... + if err := cs1.SetProposalAndBlock(prop, propBlock, propBlockParts, "some peer"); err != nil { + t.Fatal(err) } + ensureNewProposal(proposalCh, height, round) + // go to prevote, prevote for locked block (not proposal) - ensureVote(voteCh, h, r+1, types.VoteTypePrevote) - validatePrevote(t, cs1, 0, vss[0], lockedBlockHash) + ensurePrevote(voteCh, height, round) + validatePrevote(t, cs1, round, vss[0], lockedBlockHash) // now lets add prevotes from everyone else for nil (a polka!) - signAddVotes(cs1, types.VoteTypePrevote, nil, types.PartSetHeader{}, vs2, vs3, vs4) + signAddVotes(cs1, types.PrevoteType, nil, types.PartSetHeader{}, vs2, vs3, vs4) // the polka makes us unlock and precommit nil - ensureNewUnlock(unlockCh) - ensureVote(voteCh, h, r+1, types.VoteTypePrecommit) + ensureNewUnlock(unlockCh, height, round) + ensurePrecommit(voteCh, height, round) // we should have unlocked and committed nil - // NOTE: since we don't relock on nil, the lock round is 0 - validatePrecommit(t, cs1, r+1, 0, vss[0], nil, nil) + // NOTE: since we don't relock on nil, the lock round is -1 + validatePrecommit(t, cs1, round, -1, vss[0], nil, nil) - signAddVotes(cs1, types.VoteTypePrecommit, nil, types.PartSetHeader{}, vs2, vs3) - ensureNewRound(newRoundCh) + signAddVotes(cs1, types.PrecommitType, nil, types.PartSetHeader{}, vs2, vs3) + ensureNewRound(newRoundCh, height, round+1) } // 4 vals @@ -711,8 +682,7 @@ func TestStateLockPOLUnlock(t *testing.T) { func TestStateLockPOLSafety1(t *testing.T) { cs1, vss := randConsensusState(4) vs2, vs3, vs4 := vss[1], vss[2], vss[3] - h := cs1.GetRoundState().Height - r := cs1.GetRoundState().Round + height, round := cs1.Height, cs1.Round partSize := types.BlockPartSizeBytes @@ -720,41 +690,32 @@ func TestStateLockPOLSafety1(t *testing.T) { timeoutProposeCh := subscribe(cs1.eventBus, types.EventQueryTimeoutPropose) timeoutWaitCh := subscribe(cs1.eventBus, types.EventQueryTimeoutWait) newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound) - voteCh := subscribeToVoter(cs1, cs1.privValidator.GetAddress()) + addr := cs1.privValidator.GetPubKey().Address() + voteCh := subscribeToVoter(cs1, addr) // start round and wait for propose and prevote - startTestRound(cs1, cs1.Height, 0) - ensureNewRound(newRoundCh) - re := <-proposalCh - rs := re.(types.EventDataRoundState).RoundState.(*cstypes.RoundState) + startTestRound(cs1, cs1.Height, round) + ensureNewRound(newRoundCh, height, round) + + ensureNewProposal(proposalCh, height, round) + rs := cs1.GetRoundState() propBlock := rs.ProposalBlock - ensureVote(voteCh, h, r, types.VoteTypePrevote) - - validatePrevote(t, cs1, 0, vss[0], propBlock.Hash()) + ensurePrevote(voteCh, height, round) + validatePrevote(t, cs1, round, vss[0], propBlock.Hash()) // the others sign a polka but we don't see it - prevotes := signVotes(types.VoteTypePrevote, propBlock.Hash(), propBlock.MakePartSet(partSize).Header(), vs2, vs3, vs4) - - // before we time out into new round, set next proposer - // and next proposal block - - //TODO: Should we remove this? - /* - _, v1 := cs1.Validators.GetByAddress(vss[0].Address) - v1.VotingPower = 1 - if updated := cs1.Validators.Update(v1); !updated { - panic("failed to update validator") - }*/ + prevotes := signVotes(types.PrevoteType, propBlock.Hash(), propBlock.MakePartSet(partSize).Header(), vs2, vs3, vs4) t.Logf("old prop hash %v", fmt.Sprintf("%X", propBlock.Hash())) // we do see them precommit nil - signAddVotes(cs1, types.VoteTypePrecommit, nil, types.PartSetHeader{}, vs2, vs3, vs4) + signAddVotes(cs1, types.PrecommitType, nil, types.PartSetHeader{}, vs2, vs3, vs4) - ensureVote(voteCh, h, r, types.VoteTypePrecommit) + // cs1 precommit nil + ensurePrecommit(voteCh, height, round) + ensureNewTimeout(timeoutWaitCh, height, round, cs1.config.TimeoutPrecommit.Nanoseconds()) - ensureNewRound(newRoundCh) t.Log("### ONTO ROUND 1") prop, propBlock := decideProposal(cs1, vs2, vs2.Height, vs2.Round+1) @@ -763,6 +724,9 @@ func TestStateLockPOLSafety1(t *testing.T) { incrementRound(vs2, vs3, vs4) + round = round + 1 // moving to the next round + ensureNewRound(newRoundCh, height, round) + //XXX: this isnt guaranteed to get there before the timeoutPropose ... if err := cs1.SetProposalAndBlock(prop, propBlock, propBlockParts, "some peer"); err != nil { t.Fatal(err) @@ -772,39 +736,34 @@ func TestStateLockPOLSafety1(t *testing.T) { // a polka happened but we didn't see it! */ - // now we're on a new round and not the proposer, - // but we should receive the proposal - select { - case re = <-proposalCh: - case <-timeoutProposeCh: - re = <-proposalCh - } + ensureNewProposal(proposalCh, height, round) - rs = re.(types.EventDataRoundState).RoundState.(*cstypes.RoundState) + rs = cs1.GetRoundState() if rs.LockedBlock != nil { panic("we should not be locked!") } t.Logf("new prop hash %v", fmt.Sprintf("%X", propBlockHash)) + // go to prevote, prevote for proposal block - ensureVote(voteCh, h, r+1, types.VoteTypePrevote) - validatePrevote(t, cs1, 1, vss[0], propBlockHash) + ensurePrevote(voteCh, height, round) + validatePrevote(t, cs1, round, vss[0], propBlockHash) // now we see the others prevote for it, so we should lock on it - signAddVotes(cs1, types.VoteTypePrevote, propBlockHash, propBlockParts.Header(), vs2, vs3, vs4) - - ensureVote(voteCh, h, r+1, types.VoteTypePrecommit) + signAddVotes(cs1, types.PrevoteType, propBlockHash, propBlockParts.Header(), vs2, vs3, vs4) + ensurePrecommit(voteCh, height, round) // we should have precommitted - validatePrecommit(t, cs1, 1, 1, vss[0], propBlockHash, propBlockHash) + validatePrecommit(t, cs1, round, round, vss[0], propBlockHash, propBlockHash) - signAddVotes(cs1, types.VoteTypePrecommit, nil, types.PartSetHeader{}, vs2, vs3) + signAddVotes(cs1, types.PrecommitType, nil, types.PartSetHeader{}, vs2, vs3, vs4) - ensureNewTimeout(timeoutWaitCh, cs1.config.TimeoutPrecommit.Nanoseconds()) + ensureNewTimeout(timeoutWaitCh, height, round, cs1.config.TimeoutPrecommit.Nanoseconds()) incrementRound(vs2, vs3, vs4) + round = round + 1 // moving to the next round - ensureNewRound(newRoundCh) + ensureNewRound(newRoundCh, height, round) t.Log("### ONTO ROUND 2") /*Round3 @@ -812,22 +771,22 @@ func TestStateLockPOLSafety1(t *testing.T) { */ // timeout of propose - ensureNewTimeout(timeoutProposeCh, cs1.config.TimeoutPropose.Nanoseconds()) + ensureNewTimeout(timeoutProposeCh, height, round, cs1.config.TimeoutPropose.Nanoseconds()) // finish prevote - ensureVote(voteCh, h, r+2, types.VoteTypePrevote) - + ensurePrevote(voteCh, height, round) // we should prevote what we're locked on - validatePrevote(t, cs1, 2, vss[0], propBlockHash) + validatePrevote(t, cs1, round, vss[0], propBlockHash) newStepCh := subscribe(cs1.eventBus, types.EventQueryNewRoundStep) + // before prevotes from the previous round are added // add prevotes from the earlier round addVotes(cs1, prevotes...) t.Log("Done adding prevotes!") - ensureNoNewStep(newStepCh) + ensureNoNewRoundStep(newStepCh) } // 4 vals. @@ -840,66 +799,66 @@ func TestStateLockPOLSafety1(t *testing.T) { func TestStateLockPOLSafety2(t *testing.T) { cs1, vss := randConsensusState(4) vs2, vs3, vs4 := vss[1], vss[2], vss[3] - h := cs1.GetRoundState().Height - r := cs1.GetRoundState().Round + height, round := cs1.Height, cs1.Round partSize := types.BlockPartSizeBytes proposalCh := subscribe(cs1.eventBus, types.EventQueryCompleteProposal) - timeoutProposeCh := subscribe(cs1.eventBus, types.EventQueryTimeoutPropose) timeoutWaitCh := subscribe(cs1.eventBus, types.EventQueryTimeoutWait) newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound) unlockCh := subscribe(cs1.eventBus, types.EventQueryUnlock) - voteCh := subscribeToVoter(cs1, cs1.privValidator.GetAddress()) + addr := cs1.privValidator.GetPubKey().Address() + voteCh := subscribeToVoter(cs1, addr) // the block for R0: gets polkad but we miss it // (even though we signed it, shhh) - _, propBlock0 := decideProposal(cs1, vss[0], cs1.Height, cs1.Round) + _, propBlock0 := decideProposal(cs1, vss[0], height, round) propBlockHash0 := propBlock0.Hash() propBlockParts0 := propBlock0.MakePartSet(partSize) + propBlockID0 := types.BlockID{propBlockHash0, propBlockParts0.Header()} // the others sign a polka but we don't see it - prevotes := signVotes(types.VoteTypePrevote, propBlockHash0, propBlockParts0.Header(), vs2, vs3, vs4) + prevotes := signVotes(types.PrevoteType, propBlockHash0, propBlockParts0.Header(), vs2, vs3, vs4) // the block for round 1 prop1, propBlock1 := decideProposal(cs1, vs2, vs2.Height, vs2.Round+1) propBlockHash1 := propBlock1.Hash() propBlockParts1 := propBlock1.MakePartSet(partSize) - propBlockID1 := types.BlockID{propBlockHash1, propBlockParts1.Header()} incrementRound(vs2, vs3, vs4) - cs1.updateRoundStep(0, cstypes.RoundStepPrecommitWait) - + round = round + 1 // moving to the next round t.Log("### ONTO Round 1") // jump in at round 1 - startTestRound(cs1, h, r+1) - ensureNewRound(newRoundCh) + startTestRound(cs1, height, round) + ensureNewRound(newRoundCh, height, round) if err := cs1.SetProposalAndBlock(prop1, propBlock1, propBlockParts1, "some peer"); err != nil { t.Fatal(err) } - ensureNewProposal(proposalCh) + ensureNewProposal(proposalCh, height, round) - ensureVote(voteCh, h, r+1, types.VoteTypePrevote) + ensurePrevote(voteCh, height, round) + validatePrevote(t, cs1, round, vss[0], propBlockHash1) - signAddVotes(cs1, types.VoteTypePrevote, propBlockHash1, propBlockParts1.Header(), vs2, vs3, vs4) + signAddVotes(cs1, types.PrevoteType, propBlockHash1, propBlockParts1.Header(), vs2, vs3, vs4) - ensureVote(voteCh, h, r+1, types.VoteTypePrecommit) + ensurePrecommit(voteCh, height, round) // the proposed block should now be locked and our precommit added - validatePrecommit(t, cs1, 1, 1, vss[0], propBlockHash1, propBlockHash1) + validatePrecommit(t, cs1, round, round, vss[0], propBlockHash1, propBlockHash1) // add precommits from the rest - signAddVotes(cs1, types.VoteTypePrecommit, nil, types.PartSetHeader{}, vs2, vs4) - signAddVotes(cs1, types.VoteTypePrecommit, propBlockHash1, propBlockParts1.Header(), vs3) + signAddVotes(cs1, types.PrecommitType, nil, types.PartSetHeader{}, vs2, vs4) + signAddVotes(cs1, types.PrecommitType, propBlockHash1, propBlockParts1.Header(), vs3) incrementRound(vs2, vs3, vs4) // timeout of precommit wait to new round - ensureNewTimeout(timeoutWaitCh, cs1.config.TimeoutPrecommit.Nanoseconds()) + ensureNewTimeout(timeoutWaitCh, height, round, cs1.config.TimeoutPrecommit.Nanoseconds()) + round = round + 1 // moving to the next round // in round 2 we see the polkad block from round 0 - newProp := types.NewProposal(h, 2, propBlockParts0.Header(), 0, propBlockID1) + newProp := types.NewProposal(height, round, 0, propBlockID0) if err := vs3.SignProposal(config.ChainID(), newProp); err != nil { t.Fatal(err) } @@ -910,26 +869,221 @@ func TestStateLockPOLSafety2(t *testing.T) { // Add the pol votes addVotes(cs1, prevotes...) - ensureNewRound(newRoundCh) + ensureNewRound(newRoundCh, height, round) t.Log("### ONTO Round 2") /*Round2 // now we see the polka from round 1, but we shouldnt unlock */ + ensureNewProposal(proposalCh, height, round) - select { - case <-timeoutProposeCh: - <-proposalCh - case <-proposalCh: + ensureNoNewUnlock(unlockCh) + ensurePrevote(voteCh, height, round) + validatePrevote(t, cs1, round, vss[0], propBlockHash1) + +} + +// 4 vals. +// polka P0 at R0 for B0. We lock B0 on P0 at R0. P0 unlocks value at R1. + +// What we want: +// P0 proposes B0 at R3. +func TestProposeValidBlock(t *testing.T) { + cs1, vss := randConsensusState(4) + vs2, vs3, vs4 := vss[1], vss[2], vss[3] + height, round := cs1.Height, cs1.Round + + partSize := types.BlockPartSizeBytes + + proposalCh := subscribe(cs1.eventBus, types.EventQueryCompleteProposal) + timeoutWaitCh := subscribe(cs1.eventBus, types.EventQueryTimeoutWait) + timeoutProposeCh := subscribe(cs1.eventBus, types.EventQueryTimeoutPropose) + newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound) + unlockCh := subscribe(cs1.eventBus, types.EventQueryUnlock) + addr := cs1.privValidator.GetPubKey().Address() + voteCh := subscribeToVoter(cs1, addr) + + // start round and wait for propose and prevote + startTestRound(cs1, cs1.Height, round) + ensureNewRound(newRoundCh, height, round) + + ensureNewProposal(proposalCh, height, round) + rs := cs1.GetRoundState() + propBlock := rs.ProposalBlock + propBlockHash := propBlock.Hash() + + ensurePrevote(voteCh, height, round) + validatePrevote(t, cs1, round, vss[0], propBlockHash) + + // the others sign a polka + signAddVotes(cs1, types.PrevoteType, propBlockHash, propBlock.MakePartSet(partSize).Header(), vs2, vs3, vs4) + + ensurePrecommit(voteCh, height, round) + // we should have precommitted + validatePrecommit(t, cs1, round, round, vss[0], propBlockHash, propBlockHash) + + signAddVotes(cs1, types.PrecommitType, nil, types.PartSetHeader{}, vs2, vs3, vs4) + + ensureNewTimeout(timeoutWaitCh, height, round, cs1.config.TimeoutPrecommit.Nanoseconds()) + + incrementRound(vs2, vs3, vs4) + round = round + 1 // moving to the next round + + ensureNewRound(newRoundCh, height, round) + + t.Log("### ONTO ROUND 2") + + // timeout of propose + ensureNewTimeout(timeoutProposeCh, height, round, cs1.config.TimeoutPropose.Nanoseconds()) + + ensurePrevote(voteCh, height, round) + validatePrevote(t, cs1, round, vss[0], propBlockHash) + + signAddVotes(cs1, types.PrevoteType, nil, types.PartSetHeader{}, vs2, vs3, vs4) + + ensureNewUnlock(unlockCh, height, round) + + ensurePrecommit(voteCh, height, round) + // we should have precommitted + validatePrecommit(t, cs1, round, -1, vss[0], nil, nil) + + incrementRound(vs2, vs3, vs4) + incrementRound(vs2, vs3, vs4) + + signAddVotes(cs1, types.PrecommitType, nil, types.PartSetHeader{}, vs2, vs3, vs4) + + round = round + 2 // moving to the next round + + ensureNewRound(newRoundCh, height, round) + t.Log("### ONTO ROUND 3") + + ensureNewTimeout(timeoutWaitCh, height, round, cs1.config.TimeoutPrecommit.Nanoseconds()) + + round = round + 1 // moving to the next round + + ensureNewRound(newRoundCh, height, round) + + t.Log("### ONTO ROUND 4") + + ensureNewProposal(proposalCh, height, round) + + rs = cs1.GetRoundState() + assert.True(t, bytes.Equal(rs.ProposalBlock.Hash(), propBlockHash)) + assert.True(t, bytes.Equal(rs.ProposalBlock.Hash(), rs.ValidBlock.Hash())) + assert.True(t, rs.Proposal.POLRound == rs.ValidRound) + assert.True(t, bytes.Equal(rs.Proposal.BlockID.Hash, rs.ValidBlock.Hash())) +} + +// What we want: +// P0 miss to lock B but set valid block to B after receiving delayed prevote. +func TestSetValidBlockOnDelayedPrevote(t *testing.T) { + cs1, vss := randConsensusState(4) + vs2, vs3, vs4 := vss[1], vss[2], vss[3] + height, round := cs1.Height, cs1.Round + + partSize := types.BlockPartSizeBytes + + proposalCh := subscribe(cs1.eventBus, types.EventQueryCompleteProposal) + timeoutWaitCh := subscribe(cs1.eventBus, types.EventQueryTimeoutWait) + newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound) + validBlockCh := subscribe(cs1.eventBus, types.EventQueryValidBlock) + addr := cs1.privValidator.GetPubKey().Address() + voteCh := subscribeToVoter(cs1, addr) + + // start round and wait for propose and prevote + startTestRound(cs1, cs1.Height, round) + ensureNewRound(newRoundCh, height, round) + + ensureNewProposal(proposalCh, height, round) + rs := cs1.GetRoundState() + propBlock := rs.ProposalBlock + propBlockHash := propBlock.Hash() + propBlockParts := propBlock.MakePartSet(partSize) + + ensurePrevote(voteCh, height, round) + validatePrevote(t, cs1, round, vss[0], propBlockHash) + + // vs2 send prevote for propBlock + signAddVotes(cs1, types.PrevoteType, propBlockHash, propBlockParts.Header(), vs2) + + // vs3 send prevote nil + signAddVotes(cs1, types.PrevoteType, nil, types.PartSetHeader{}, vs3) + + ensureNewTimeout(timeoutWaitCh, height, round, cs1.config.TimeoutPrevote.Nanoseconds()) + + ensurePrecommit(voteCh, height, round) + // we should have precommitted + validatePrecommit(t, cs1, round, -1, vss[0], nil, nil) + + rs = cs1.GetRoundState() + + assert.True(t, rs.ValidBlock == nil) + assert.True(t, rs.ValidBlockParts == nil) + assert.True(t, rs.ValidRound == -1) + + // vs2 send (delayed) prevote for propBlock + signAddVotes(cs1, types.PrevoteType, propBlockHash, propBlockParts.Header(), vs4) + + ensureNewValidBlock(validBlockCh, height, round) + + rs = cs1.GetRoundState() + + assert.True(t, bytes.Equal(rs.ValidBlock.Hash(), propBlockHash)) + assert.True(t, rs.ValidBlockParts.Header().Equals(propBlockParts.Header())) + assert.True(t, rs.ValidRound == round) +} + +// What we want: +// P0 miss to lock B as Proposal Block is missing, but set valid block to B after +// receiving delayed Block Proposal. +func TestSetValidBlockOnDelayedProposal(t *testing.T) { + cs1, vss := randConsensusState(4) + vs2, vs3, vs4 := vss[1], vss[2], vss[3] + height, round := cs1.Height, cs1.Round + + partSize := types.BlockPartSizeBytes + + timeoutWaitCh := subscribe(cs1.eventBus, types.EventQueryTimeoutWait) + timeoutProposeCh := subscribe(cs1.eventBus, types.EventQueryTimeoutPropose) + newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound) + validBlockCh := subscribe(cs1.eventBus, types.EventQueryValidBlock) + addr := cs1.privValidator.GetPubKey().Address() + voteCh := subscribeToVoter(cs1, addr) + proposalCh := subscribe(cs1.eventBus, types.EventQueryCompleteProposal) + + round = round + 1 // move to round in which P0 is not proposer + incrementRound(vs2, vs3, vs4) + + startTestRound(cs1, cs1.Height, round) + ensureNewRound(newRoundCh, height, round) + + ensureNewTimeout(timeoutProposeCh, height, round, cs1.config.TimeoutPropose.Nanoseconds()) + + ensurePrevote(voteCh, height, round) + validatePrevote(t, cs1, round, vss[0], nil) + + prop, propBlock := decideProposal(cs1, vs2, vs2.Height, vs2.Round+1) + propBlockHash := propBlock.Hash() + propBlockParts := propBlock.MakePartSet(partSize) + + // vs2, vs3 and vs4 send prevote for propBlock + signAddVotes(cs1, types.PrevoteType, propBlockHash, propBlockParts.Header(), vs2, vs3, vs4) + ensureNewValidBlock(validBlockCh, height, round) + + ensureNewTimeout(timeoutWaitCh, height, round, cs1.config.TimeoutPrevote.Nanoseconds()) + + ensurePrecommit(voteCh, height, round) + validatePrecommit(t, cs1, round, -1, vss[0], nil, nil) + + if err := cs1.SetProposalAndBlock(prop, propBlock, propBlockParts, "some peer"); err != nil { + t.Fatal(err) } - select { - case <-unlockCh: - panic("validator unlocked using an old polka") - case <-voteCh: - // prevote our locked block - } - validatePrevote(t, cs1, 2, vss[0], propBlockHash1) + ensureNewProposal(proposalCh, height, round) + rs := cs1.GetRoundState() + assert.True(t, bytes.Equal(rs.ValidBlock.Hash(), propBlockHash)) + assert.True(t, rs.ValidBlockParts.Header().Equals(propBlockParts.Header())) + assert.True(t, rs.ValidRound == round) } // 4 vals, 3 Nil Precommits at P0 @@ -938,18 +1092,252 @@ func TestStateLockPOLSafety2(t *testing.T) { func TestWaitingTimeoutOnNilPolka(t *testing.T) { cs1, vss := randConsensusState(4) vs2, vs3, vs4 := vss[1], vss[2], vss[3] + height, round := cs1.Height, cs1.Round timeoutWaitCh := subscribe(cs1.eventBus, types.EventQueryTimeoutWait) newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound) // start round - startTestRound(cs1, cs1.Height, 0) - ensureNewRound(newRoundCh) + startTestRound(cs1, height, round) + ensureNewRound(newRoundCh, height, round) - signAddVotes(cs1, types.VoteTypePrecommit, nil, types.PartSetHeader{}, vs2, vs3, vs4) + signAddVotes(cs1, types.PrecommitType, nil, types.PartSetHeader{}, vs2, vs3, vs4) - ensureNewTimeout(timeoutWaitCh, cs1.config.TimeoutPrecommit.Nanoseconds()) - ensureNewRound(newRoundCh) + ensureNewTimeout(timeoutWaitCh, height, round, cs1.config.TimeoutPrecommit.Nanoseconds()) + ensureNewRound(newRoundCh, height, round+1) +} + +// 4 vals, 3 Prevotes for nil from the higher round. +// What we want: +// P0 waits for timeoutPropose in the next round before entering prevote +func TestWaitingTimeoutProposeOnNewRound(t *testing.T) { + cs1, vss := randConsensusState(4) + vs2, vs3, vs4 := vss[1], vss[2], vss[3] + height, round := cs1.Height, cs1.Round + + timeoutWaitCh := subscribe(cs1.eventBus, types.EventQueryTimeoutPropose) + newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound) + addr := cs1.privValidator.GetPubKey().Address() + voteCh := subscribeToVoter(cs1, addr) + + // start round + startTestRound(cs1, height, round) + ensureNewRound(newRoundCh, height, round) + + ensurePrevote(voteCh, height, round) + + incrementRound(vss[1:]...) + signAddVotes(cs1, types.PrevoteType, nil, types.PartSetHeader{}, vs2, vs3, vs4) + + round = round + 1 // moving to the next round + ensureNewRound(newRoundCh, height, round) + + rs := cs1.GetRoundState() + assert.True(t, rs.Step == cstypes.RoundStepPropose) // P0 does not prevote before timeoutPropose expires + + ensureNewTimeout(timeoutWaitCh, height, round, cs1.config.TimeoutPropose.Nanoseconds()) + + ensurePrevote(voteCh, height, round) + validatePrevote(t, cs1, round, vss[0], nil) +} + +// 4 vals, 3 Precommits for nil from the higher round. +// What we want: +// P0 jump to higher round, precommit and start precommit wait +func TestRoundSkipOnNilPolkaFromHigherRound(t *testing.T) { + cs1, vss := randConsensusState(4) + vs2, vs3, vs4 := vss[1], vss[2], vss[3] + height, round := cs1.Height, cs1.Round + + timeoutWaitCh := subscribe(cs1.eventBus, types.EventQueryTimeoutWait) + newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound) + addr := cs1.privValidator.GetPubKey().Address() + voteCh := subscribeToVoter(cs1, addr) + + // start round + startTestRound(cs1, height, round) + ensureNewRound(newRoundCh, height, round) + + ensurePrevote(voteCh, height, round) + + incrementRound(vss[1:]...) + signAddVotes(cs1, types.PrecommitType, nil, types.PartSetHeader{}, vs2, vs3, vs4) + + round = round + 1 // moving to the next round + ensureNewRound(newRoundCh, height, round) + + ensurePrecommit(voteCh, height, round) + validatePrecommit(t, cs1, round, -1, vss[0], nil, nil) + + ensureNewTimeout(timeoutWaitCh, height, round, cs1.config.TimeoutPrecommit.Nanoseconds()) + + round = round + 1 // moving to the next round + ensureNewRound(newRoundCh, height, round) +} + +// 4 vals, 3 Prevotes for nil in the current round. +// What we want: +// P0 wait for timeoutPropose to expire before sending prevote. +func TestWaitTimeoutProposeOnNilPolkaForTheCurrentRound(t *testing.T) { + cs1, vss := randConsensusState(4) + vs2, vs3, vs4 := vss[1], vss[2], vss[3] + height, round := cs1.Height, 1 + + timeoutProposeCh := subscribe(cs1.eventBus, types.EventQueryTimeoutPropose) + newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound) + addr := cs1.privValidator.GetPubKey().Address() + voteCh := subscribeToVoter(cs1, addr) + + // start round in which PO is not proposer + startTestRound(cs1, height, round) + ensureNewRound(newRoundCh, height, round) + + incrementRound(vss[1:]...) + signAddVotes(cs1, types.PrevoteType, nil, types.PartSetHeader{}, vs2, vs3, vs4) + + ensureNewTimeout(timeoutProposeCh, height, round, cs1.config.TimeoutPropose.Nanoseconds()) + + ensurePrevote(voteCh, height, round) + validatePrevote(t, cs1, round, vss[0], nil) +} + +// What we want: +// P0 emit NewValidBlock event upon receiving 2/3+ Precommit for B but hasn't received block B yet +func TestEmitNewValidBlockEventOnCommitWithoutBlock(t *testing.T) { + cs1, vss := randConsensusState(4) + vs2, vs3, vs4 := vss[1], vss[2], vss[3] + height, round := cs1.Height, 1 + + incrementRound(vs2, vs3, vs4) + + partSize := types.BlockPartSizeBytes + + newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound) + validBlockCh := subscribe(cs1.eventBus, types.EventQueryValidBlock) + + _, propBlock := decideProposal(cs1, vs2, vs2.Height, vs2.Round) + propBlockHash := propBlock.Hash() + propBlockParts := propBlock.MakePartSet(partSize) + + // start round in which PO is not proposer + startTestRound(cs1, height, round) + ensureNewRound(newRoundCh, height, round) + + // vs2, vs3 and vs4 send precommit for propBlock + signAddVotes(cs1, types.PrecommitType, propBlockHash, propBlockParts.Header(), vs2, vs3, vs4) + ensureNewValidBlock(validBlockCh, height, round) + + rs := cs1.GetRoundState() + assert.True(t, rs.Step == cstypes.RoundStepCommit) + assert.True(t, rs.ProposalBlock == nil) + assert.True(t, rs.ProposalBlockParts.Header().Equals(propBlockParts.Header())) + +} + +// What we want: +// P0 receives 2/3+ Precommit for B for round 0, while being in round 1. It emits NewValidBlock event. +// After receiving block, it executes block and moves to the next height. +func TestCommitFromPreviousRound(t *testing.T) { + cs1, vss := randConsensusState(4) + vs2, vs3, vs4 := vss[1], vss[2], vss[3] + height, round := cs1.Height, 1 + + partSize := types.BlockPartSizeBytes + + newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound) + validBlockCh := subscribe(cs1.eventBus, types.EventQueryValidBlock) + proposalCh := subscribe(cs1.eventBus, types.EventQueryCompleteProposal) + + prop, propBlock := decideProposal(cs1, vs2, vs2.Height, vs2.Round) + propBlockHash := propBlock.Hash() + propBlockParts := propBlock.MakePartSet(partSize) + + // start round in which PO is not proposer + startTestRound(cs1, height, round) + ensureNewRound(newRoundCh, height, round) + + // vs2, vs3 and vs4 send precommit for propBlock for the previous round + signAddVotes(cs1, types.PrecommitType, propBlockHash, propBlockParts.Header(), vs2, vs3, vs4) + + ensureNewValidBlock(validBlockCh, height, round) + + rs := cs1.GetRoundState() + assert.True(t, rs.Step == cstypes.RoundStepCommit) + assert.True(t, rs.CommitRound == vs2.Round) + assert.True(t, rs.ProposalBlock == nil) + assert.True(t, rs.ProposalBlockParts.Header().Equals(propBlockParts.Header())) + + if err := cs1.SetProposalAndBlock(prop, propBlock, propBlockParts, "some peer"); err != nil { + t.Fatal(err) + } + + ensureNewProposal(proposalCh, height, round) + ensureNewRound(newRoundCh, height+1, 0) +} + +type fakeTxNotifier struct { + ch chan struct{} +} + +func (n *fakeTxNotifier) TxsAvailable() <-chan struct{} { + return n.ch +} + +func (n *fakeTxNotifier) Notify() { + n.ch <- struct{}{} +} + +func TestStartNextHeightCorrectly(t *testing.T) { + cs1, vss := randConsensusState(4) + cs1.config.SkipTimeoutCommit = false + cs1.txNotifier = &fakeTxNotifier{ch: make(chan struct{})} + + vs2, vs3, vs4 := vss[1], vss[2], vss[3] + height, round := cs1.Height, cs1.Round + + proposalCh := subscribe(cs1.eventBus, types.EventQueryCompleteProposal) + timeoutProposeCh := subscribe(cs1.eventBus, types.EventQueryTimeoutPropose) + + newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound) + newBlockHeader := subscribe(cs1.eventBus, types.EventQueryNewBlockHeader) + addr := cs1.privValidator.GetPubKey().Address() + voteCh := subscribeToVoter(cs1, addr) + + // start round and wait for propose and prevote + startTestRound(cs1, height, round) + ensureNewRound(newRoundCh, height, round) + + ensureNewProposal(proposalCh, height, round) + rs := cs1.GetRoundState() + theBlockHash := rs.ProposalBlock.Hash() + theBlockParts := rs.ProposalBlockParts.Header() + + ensurePrevote(voteCh, height, round) + validatePrevote(t, cs1, round, vss[0], theBlockHash) + + signAddVotes(cs1, types.PrevoteType, theBlockHash, theBlockParts, vs2, vs3, vs4) + + ensurePrecommit(voteCh, height, round) + // the proposed block should now be locked and our precommit added + validatePrecommit(t, cs1, round, round, vss[0], theBlockHash, theBlockHash) + + rs = cs1.GetRoundState() + + // add precommits + signAddVotes(cs1, types.PrecommitType, nil, types.PartSetHeader{}, vs2) + signAddVotes(cs1, types.PrecommitType, theBlockHash, theBlockParts, vs3) + signAddVotes(cs1, types.PrecommitType, theBlockHash, theBlockParts, vs4) + + ensureNewBlockHeader(newBlockHeader, height, theBlockHash) + + rs = cs1.GetRoundState() + assert.True(t, rs.TriggeredTimeoutPrecommit) + + cs1.txNotifier.(*fakeTxNotifier).Notify() + + ensureNewTimeout(timeoutProposeCh, height+1, round, cs1.config.TimeoutPropose.Nanoseconds()) + rs = cs1.GetRoundState() + assert.False(t, rs.TriggeredTimeoutPrecommit, "triggeredTimeoutPrecommit should be false at the beginning of each round") } //------------------------------------------------------------------------------------------ @@ -979,7 +1367,7 @@ func TestStateSlashingPrevotes(t *testing.T) { // add one for a different block should cause us to go into prevote wait hash := rs.ProposalBlock.Hash() hash[0] = byte(hash[0]+1) % 255 - signAddVotes(cs1, types.VoteTypePrevote, hash, rs.ProposalBlockParts.Header(), vs2) + signAddVotes(cs1, types.PrevoteType, hash, rs.ProposalBlockParts.Header(), vs2) <-timeoutWaitCh @@ -987,7 +1375,7 @@ func TestStateSlashingPrevotes(t *testing.T) { // away and ignore more prevotes (and thus fail to slash!) // add the conflicting vote - signAddVotes(cs1, types.VoteTypePrevote, rs.ProposalBlock.Hash(), rs.ProposalBlockParts.Header(), vs2) + signAddVotes(cs1, types.PrevoteType, rs.ProposalBlock.Hash(), rs.ProposalBlockParts.Header(), vs2) // XXX: Check for existence of Dupeout info } @@ -1009,7 +1397,7 @@ func TestStateSlashingPrecommits(t *testing.T) { <-voteCh // prevote // add prevote from vs2 - signAddVotes(cs1, types.VoteTypePrevote, rs.ProposalBlock.Hash(), rs.ProposalBlockParts.Header(), vs2) + signAddVotes(cs1, types.PrevoteType, rs.ProposalBlock.Hash(), rs.ProposalBlockParts.Header(), vs2) <-voteCh // precommit @@ -1017,13 +1405,13 @@ func TestStateSlashingPrecommits(t *testing.T) { // add one for a different block should cause us to go into prevote wait hash := rs.ProposalBlock.Hash() hash[0] = byte(hash[0]+1) % 255 - signAddVotes(cs1, types.VoteTypePrecommit, hash, rs.ProposalBlockParts.Header(), vs2) + signAddVotes(cs1, types.PrecommitType, hash, rs.ProposalBlockParts.Header(), vs2) // NOTE: we have to send the vote for different block first so we don't just go into precommit round right // away and ignore more prevotes (and thus fail to slash!) // add precommit from vs2 - signAddVotes(cs1, types.VoteTypePrecommit, rs.ProposalBlock.Hash(), rs.ProposalBlockParts.Header(), vs2) + signAddVotes(cs1, types.PrecommitType, rs.ProposalBlock.Hash(), rs.ProposalBlockParts.Header(), vs2) // XXX: Check for existence of Dupeout info } @@ -1040,44 +1428,48 @@ func TestStateSlashingPrecommits(t *testing.T) { func TestStateHalt1(t *testing.T) { cs1, vss := randConsensusState(4) vs2, vs3, vs4 := vss[1], vss[2], vss[3] - h := cs1.GetRoundState().Height - r := cs1.GetRoundState().Round + height, round := cs1.Height, cs1.Round partSize := types.BlockPartSizeBytes proposalCh := subscribe(cs1.eventBus, types.EventQueryCompleteProposal) timeoutWaitCh := subscribe(cs1.eventBus, types.EventQueryTimeoutWait) newRoundCh := subscribe(cs1.eventBus, types.EventQueryNewRound) newBlockCh := subscribe(cs1.eventBus, types.EventQueryNewBlock) - voteCh := subscribeToVoter(cs1, cs1.privValidator.GetAddress()) + addr := cs1.privValidator.GetPubKey().Address() + voteCh := subscribeToVoter(cs1, addr) // start round and wait for propose and prevote - startTestRound(cs1, cs1.Height, 0) - ensureNewRound(newRoundCh) - re := <-proposalCh - rs := re.(types.EventDataRoundState).RoundState.(*cstypes.RoundState) + startTestRound(cs1, height, round) + ensureNewRound(newRoundCh, height, round) + + ensureNewProposal(proposalCh, height, round) + rs := cs1.GetRoundState() propBlock := rs.ProposalBlock propBlockParts := propBlock.MakePartSet(partSize) - ensureVote(voteCh, h, r, types.VoteTypePrevote) + ensurePrevote(voteCh, height, round) - signAddVotes(cs1, types.VoteTypePrevote, propBlock.Hash(), propBlockParts.Header(), vs3, vs4) - ensureVote(voteCh, h, r, types.VoteTypePrecommit) + signAddVotes(cs1, types.PrevoteType, propBlock.Hash(), propBlockParts.Header(), vs2, vs3, vs4) + ensurePrecommit(voteCh, height, round) // the proposed block should now be locked and our precommit added - validatePrecommit(t, cs1, 0, 0, vss[0], propBlock.Hash(), propBlock.Hash()) + validatePrecommit(t, cs1, round, round, vss[0], propBlock.Hash(), propBlock.Hash()) // add precommits from the rest - signAddVotes(cs1, types.VoteTypePrecommit, nil, types.PartSetHeader{}, vs2) // didnt receive proposal - signAddVotes(cs1, types.VoteTypePrecommit, propBlock.Hash(), propBlockParts.Header(), vs3) + signAddVotes(cs1, types.PrecommitType, nil, types.PartSetHeader{}, vs2) // didnt receive proposal + signAddVotes(cs1, types.PrecommitType, propBlock.Hash(), propBlockParts.Header(), vs3) // we receive this later, but vs3 might receive it earlier and with ours will go to commit! - precommit4 := signVote(vs4, types.VoteTypePrecommit, propBlock.Hash(), propBlockParts.Header()) + precommit4 := signVote(vs4, types.PrecommitType, propBlock.Hash(), propBlockParts.Header()) incrementRound(vs2, vs3, vs4) // timeout to new round - ensureNewTimeout(timeoutWaitCh, cs1.config.TimeoutPrecommit.Nanoseconds()) - re = <-newRoundCh - rs = re.(types.EventDataRoundState).RoundState.(*cstypes.RoundState) + ensureNewTimeout(timeoutWaitCh, height, round, cs1.config.TimeoutPrecommit.Nanoseconds()) + + round = round + 1 // moving to the next round + + ensureNewRound(newRoundCh, height, round) + rs = cs1.GetRoundState() t.Log("### ONTO ROUND 1") /*Round2 @@ -1086,20 +1478,16 @@ func TestStateHalt1(t *testing.T) { */ // go to prevote, prevote for locked block - ensureVote(voteCh, h, r+1, types.VoteTypePrevote) - validatePrevote(t, cs1, 0, vss[0], rs.LockedBlock.Hash()) + ensurePrevote(voteCh, height, round) + validatePrevote(t, cs1, round, vss[0], rs.LockedBlock.Hash()) // now we receive the precommit from the previous round addVotes(cs1, precommit4) // receiving that precommit should take us straight to commit - ensureNewBlock(newBlockCh) - re = <-newRoundCh - rs = re.(types.EventDataRoundState).RoundState.(*cstypes.RoundState) + ensureNewBlock(newBlockCh, height) - if rs.Height != 2 { - panic("expected height to increment") - } + ensureNewRound(newRoundCh, height+1, 0) } func TestStateOutputsBlockPartsStats(t *testing.T) { @@ -1150,7 +1538,7 @@ func TestStateOutputVoteStats(t *testing.T) { // create dummy peer peer := p2pdummy.NewPeer() - vote := signVote(vss[1], types.VoteTypePrecommit, []byte("test"), types.PartSetHeader{}) + vote := signVote(vss[1], types.PrecommitType, []byte("test"), types.PartSetHeader{}) voteMessage := &VoteMessage{vote} cs.handleMsg(msgInfo{voteMessage, peer.ID()}) @@ -1164,7 +1552,7 @@ func TestStateOutputVoteStats(t *testing.T) { // sending the vote for the bigger height incrementHeight(vss[1]) - vote = signVote(vss[1], types.VoteTypePrecommit, []byte("test"), types.PartSetHeader{}) + vote = signVote(vss[1], types.PrecommitType, []byte("test"), types.PartSetHeader{}) cs.handleMsg(msgInfo{&VoteMessage{vote}, peer.ID()}) @@ -1185,10 +1573,3 @@ func subscribe(eventBus *types.EventBus, q tmpubsub.Query) <-chan interface{} { } return out } - -// discardFromChan reads n values from the channel. -func discardFromChan(ch <-chan interface{}, n int) { - for i := 0; i < n; i++ { - <-ch - } -} diff --git a/consensus/types/height_vote_set.go b/consensus/types/height_vote_set.go index 1c8ac67c..eee013ee 100644 --- a/consensus/types/height_vote_set.go +++ b/consensus/types/height_vote_set.go @@ -99,8 +99,8 @@ func (hvs *HeightVoteSet) addRound(round int) { cmn.PanicSanity("addRound() for an existing round") } // log.Debug("addRound(round)", "round", round) - prevotes := types.NewVoteSet(hvs.chainID, hvs.height, round, types.VoteTypePrevote, hvs.valSet) - precommits := types.NewVoteSet(hvs.chainID, hvs.height, round, types.VoteTypePrecommit, hvs.valSet) + prevotes := types.NewVoteSet(hvs.chainID, hvs.height, round, types.PrevoteType, hvs.valSet) + precommits := types.NewVoteSet(hvs.chainID, hvs.height, round, types.PrecommitType, hvs.valSet) hvs.roundVoteSets[round] = RoundVoteSet{ Prevotes: prevotes, Precommits: precommits, @@ -134,13 +134,13 @@ func (hvs *HeightVoteSet) AddVote(vote *types.Vote, peerID p2p.ID) (added bool, func (hvs *HeightVoteSet) Prevotes(round int) *types.VoteSet { hvs.mtx.Lock() defer hvs.mtx.Unlock() - return hvs.getVoteSet(round, types.VoteTypePrevote) + return hvs.getVoteSet(round, types.PrevoteType) } func (hvs *HeightVoteSet) Precommits(round int) *types.VoteSet { hvs.mtx.Lock() defer hvs.mtx.Unlock() - return hvs.getVoteSet(round, types.VoteTypePrecommit) + return hvs.getVoteSet(round, types.PrecommitType) } // Last round and blockID that has +2/3 prevotes for a particular block or nil. @@ -149,7 +149,7 @@ func (hvs *HeightVoteSet) POLInfo() (polRound int, polBlockID types.BlockID) { hvs.mtx.Lock() defer hvs.mtx.Unlock() for r := hvs.round; r >= 0; r-- { - rvs := hvs.getVoteSet(r, types.VoteTypePrevote) + rvs := hvs.getVoteSet(r, types.PrevoteType) polBlockID, ok := rvs.TwoThirdsMajority() if ok { return r, polBlockID @@ -158,15 +158,15 @@ func (hvs *HeightVoteSet) POLInfo() (polRound int, polBlockID types.BlockID) { return -1, types.BlockID{} } -func (hvs *HeightVoteSet) getVoteSet(round int, type_ byte) *types.VoteSet { +func (hvs *HeightVoteSet) getVoteSet(round int, type_ types.SignedMsgType) *types.VoteSet { rvs, ok := hvs.roundVoteSets[round] if !ok { return nil } switch type_ { - case types.VoteTypePrevote: + case types.PrevoteType: return rvs.Prevotes - case types.VoteTypePrecommit: + case types.PrecommitType: return rvs.Precommits default: cmn.PanicSanity(fmt.Sprintf("Unexpected vote type %X", type_)) @@ -178,7 +178,7 @@ func (hvs *HeightVoteSet) getVoteSet(round int, type_ byte) *types.VoteSet { // NOTE: if there are too many peers, or too much peer churn, // this can cause memory issues. // TODO: implement ability to remove peers too -func (hvs *HeightVoteSet) SetPeerMaj23(round int, type_ byte, peerID p2p.ID, blockID types.BlockID) error { +func (hvs *HeightVoteSet) SetPeerMaj23(round int, type_ types.SignedMsgType, peerID p2p.ID, blockID types.BlockID) error { hvs.mtx.Lock() defer hvs.mtx.Unlock() if !types.IsVoteTypeValid(type_) { diff --git a/consensus/types/height_vote_set_test.go b/consensus/types/height_vote_set_test.go index 5f469221..4460cd3e 100644 --- a/consensus/types/height_vote_set_test.go +++ b/consensus/types/height_vote_set_test.go @@ -50,13 +50,14 @@ func TestPeerCatchupRounds(t *testing.T) { func makeVoteHR(t *testing.T, height int64, round int, privVals []types.PrivValidator, valIndex int) *types.Vote { privVal := privVals[valIndex] + addr := privVal.GetPubKey().Address() vote := &types.Vote{ - ValidatorAddress: privVal.GetAddress(), + ValidatorAddress: addr, ValidatorIndex: valIndex, Height: height, Round: round, Timestamp: tmtime.Now(), - Type: types.VoteTypePrecommit, + Type: types.PrecommitType, BlockID: types.BlockID{[]byte("fakehash"), types.PartSetHeader{}}, } chainID := config.ChainID() diff --git a/consensus/types/peer_round_state.go b/consensus/types/peer_round_state.go index e42395bc..16e29294 100644 --- a/consensus/types/peer_round_state.go +++ b/consensus/types/peer_round_state.go @@ -55,3 +55,31 @@ func (prs PeerRoundState) StringIndented(indent string) string { indent, prs.CatchupCommit, prs.CatchupCommitRound, indent) } + +//----------------------------------------------------------- +// These methods are for Protobuf Compatibility + +// Size returns the size of the amino encoding, in bytes. +func (ps *PeerRoundState) Size() int { + bs, _ := ps.Marshal() + return len(bs) +} + +// Marshal returns the amino encoding. +func (ps *PeerRoundState) Marshal() ([]byte, error) { + return cdc.MarshalBinaryBare(ps) +} + +// MarshalTo calls Marshal and copies to the given buffer. +func (ps *PeerRoundState) MarshalTo(data []byte) (int, error) { + bs, err := ps.Marshal() + if err != nil { + return -1, err + } + return copy(data, bs), nil +} + +// Unmarshal deserializes from amino encoded form. +func (ps *PeerRoundState) Unmarshal(bs []byte) error { + return cdc.UnmarshalBinaryBare(bs, ps) +} diff --git a/consensus/types/round_state.go b/consensus/types/round_state.go index d3f6468b..eab13b6c 100644 --- a/consensus/types/round_state.go +++ b/consensus/types/round_state.go @@ -26,8 +26,15 @@ const ( RoundStepPrecommitWait = RoundStepType(0x07) // Did receive any +2/3 precommits, start timeout RoundStepCommit = RoundStepType(0x08) // Entered commit state machine // NOTE: RoundStepNewHeight acts as RoundStepCommitWait. + + // NOTE: Update IsValid method if you change this! ) +// IsValid returns true if the step is valid, false if unknown/undefined. +func (rs RoundStepType) IsValid() bool { + return uint8(rs) >= 0x01 && uint8(rs) <= 0x08 +} + // String returns a string func (rs RoundStepType) String() string { switch rs { @@ -58,25 +65,26 @@ func (rs RoundStepType) String() string { // NOTE: Not thread safe. Should only be manipulated by functions downstream // of the cs.receiveRoutine type RoundState struct { - Height int64 `json:"height"` // Height we are working on - Round int `json:"round"` - Step RoundStepType `json:"step"` - StartTime time.Time `json:"start_time"` - CommitTime time.Time `json:"commit_time"` // Subjective time when +2/3 precommits for Block at Round were found - Validators *types.ValidatorSet `json:"validators"` - Proposal *types.Proposal `json:"proposal"` - ProposalBlock *types.Block `json:"proposal_block"` - ProposalBlockParts *types.PartSet `json:"proposal_block_parts"` - LockedRound int `json:"locked_round"` - LockedBlock *types.Block `json:"locked_block"` - LockedBlockParts *types.PartSet `json:"locked_block_parts"` - ValidRound int `json:"valid_round"` // Last known round with POL for non-nil valid block. - ValidBlock *types.Block `json:"valid_block"` // Last known block of POL mentioned above. - ValidBlockParts *types.PartSet `json:"valid_block_parts"` // Last known block parts of POL metnioned above. - Votes *HeightVoteSet `json:"votes"` - CommitRound int `json:"commit_round"` // - LastCommit *types.VoteSet `json:"last_commit"` // Last precommits at Height-1 - LastValidators *types.ValidatorSet `json:"last_validators"` + Height int64 `json:"height"` // Height we are working on + Round int `json:"round"` + Step RoundStepType `json:"step"` + StartTime time.Time `json:"start_time"` + CommitTime time.Time `json:"commit_time"` // Subjective time when +2/3 precommits for Block at Round were found + Validators *types.ValidatorSet `json:"validators"` + Proposal *types.Proposal `json:"proposal"` + ProposalBlock *types.Block `json:"proposal_block"` + ProposalBlockParts *types.PartSet `json:"proposal_block_parts"` + LockedRound int `json:"locked_round"` + LockedBlock *types.Block `json:"locked_block"` + LockedBlockParts *types.PartSet `json:"locked_block_parts"` + ValidRound int `json:"valid_round"` // Last known round with POL for non-nil valid block. + ValidBlock *types.Block `json:"valid_block"` // Last known block of POL mentioned above. + ValidBlockParts *types.PartSet `json:"valid_block_parts"` // Last known block parts of POL metnioned above. + Votes *HeightVoteSet `json:"votes"` + CommitRound int `json:"commit_round"` // + LastCommit *types.VoteSet `json:"last_commit"` // Last precommits at Height-1 + LastValidators *types.ValidatorSet `json:"last_validators"` + TriggeredTimeoutPrecommit bool `json:"triggered_timeout_precommit"` } // Compressed version of the RoundState for use in RPC @@ -105,18 +113,50 @@ func (rs *RoundState) RoundStateSimple() RoundStateSimple { } } +// NewRoundEvent returns the RoundState with proposer information as an event. +func (rs *RoundState) NewRoundEvent() types.EventDataNewRound { + addr := rs.Validators.GetProposer().Address + idx, _ := rs.Validators.GetByAddress(addr) + + return types.EventDataNewRound{ + Height: rs.Height, + Round: rs.Round, + Step: rs.Step.String(), + Proposer: types.ValidatorInfo{ + Address: addr, + Index: idx, + }, + } +} + +// CompleteProposalEvent returns information about a proposed block as an event. +func (rs *RoundState) CompleteProposalEvent() types.EventDataCompleteProposal { + // We must construct BlockID from ProposalBlock and ProposalBlockParts + // cs.Proposal is not guaranteed to be set when this function is called + blockId := types.BlockID{ + Hash: rs.ProposalBlock.Hash(), + PartsHeader: rs.ProposalBlockParts.Header(), + } + + return types.EventDataCompleteProposal{ + Height: rs.Height, + Round: rs.Round, + Step: rs.Step.String(), + BlockID: blockId, + } +} + // RoundStateEvent returns the H/R/S of the RoundState as an event. func (rs *RoundState) RoundStateEvent() types.EventDataRoundState { // copy the RoundState. // TODO: if we want to avoid this, we may need synchronous events after all rsCopy := *rs - edrs := types.EventDataRoundState{ + return types.EventDataRoundState{ Height: rs.Height, Round: rs.Round, Step: rs.Step.String(), RoundState: &rsCopy, } - return edrs } // String returns a string @@ -162,3 +202,31 @@ func (rs *RoundState) StringShort() string { return fmt.Sprintf(`RoundState{H:%v R:%v S:%v ST:%v}`, rs.Height, rs.Round, rs.Step, rs.StartTime) } + +//----------------------------------------------------------- +// These methods are for Protobuf Compatibility + +// Size returns the size of the amino encoding, in bytes. +func (rs *RoundStateSimple) Size() int { + bs, _ := rs.Marshal() + return len(bs) +} + +// Marshal returns the amino encoding. +func (rs *RoundStateSimple) Marshal() ([]byte, error) { + return cdc.MarshalBinaryBare(rs) +} + +// MarshalTo calls Marshal and copies to the given buffer. +func (rs *RoundStateSimple) MarshalTo(data []byte) (int, error) { + bs, err := rs.Marshal() + if err != nil { + return -1, err + } + return copy(data, bs), nil +} + +// Unmarshal deserializes from amino encoded form. +func (rs *RoundStateSimple) Unmarshal(bs []byte) error { + return cdc.UnmarshalBinaryBare(bs, rs) +} diff --git a/consensus/types/round_state_test.go b/consensus/types/round_state_test.go index a330981f..cb16f939 100644 --- a/consensus/types/round_state_test.go +++ b/consensus/types/round_state_test.go @@ -3,7 +3,7 @@ package types import ( "testing" - "github.com/tendermint/go-amino" + amino "github.com/tendermint/go-amino" "github.com/tendermint/tendermint/crypto/ed25519" cmn "github.com/tendermint/tendermint/libs/common" "github.com/tendermint/tendermint/types" @@ -16,7 +16,7 @@ func BenchmarkRoundStateDeepCopy(b *testing.B) { // Random validators nval, ntxs := 100, 100 vset, _ := types.RandValidatorSet(nval, 1) - precommits := make([]*types.Vote, nval) + precommits := make([]*types.CommitSig, nval) blockID := types.BlockID{ Hash: cmn.RandBytes(20), PartsHeader: types.PartSetHeader{ @@ -25,12 +25,12 @@ func BenchmarkRoundStateDeepCopy(b *testing.B) { } sig := make([]byte, ed25519.SignatureSize) for i := 0; i < nval; i++ { - precommits[i] = &types.Vote{ + precommits[i] = (&types.Vote{ ValidatorAddress: types.Address(cmn.RandBytes(20)), Timestamp: tmtime.Now(), BlockID: blockID, Signature: sig, - } + }).CommitSig() } txs := make([]types.Tx, ntxs) for i := 0; i < ntxs; i++ { @@ -63,11 +63,8 @@ func BenchmarkRoundStateDeepCopy(b *testing.B) { // Random Proposal proposal := &types.Proposal{ Timestamp: tmtime.Now(), - BlockPartsHeader: types.PartSetHeader{ - Hash: cmn.RandBytes(20), - }, - POLBlockID: blockID, - Signature: sig, + BlockID: blockID, + Signature: sig, } // Random HeightVoteSet // TODO: hvs := diff --git a/consensus/version.go b/consensus/version.go deleted file mode 100644 index c04d2ac7..00000000 --- a/consensus/version.go +++ /dev/null @@ -1,11 +0,0 @@ -package consensus - -import "fmt" - -// kind of arbitrary -var Spec = "1" // async -var Major = "0" // -var Minor = "2" // replay refactor -var Revision = "2" // validation -> commit - -var Version = fmt.Sprintf("v%s/%s.%s.%s", Spec, Major, Minor, Revision) diff --git a/consensus/wal.go b/consensus/wal.go index 6472c257..d56ede26 100644 --- a/consensus/wal.go +++ b/consensus/wal.go @@ -13,6 +13,7 @@ import ( amino "github.com/tendermint/go-amino" auto "github.com/tendermint/tendermint/libs/autofile" cmn "github.com/tendermint/tendermint/libs/common" + "github.com/tendermint/tendermint/libs/log" "github.com/tendermint/tendermint/types" tmtime "github.com/tendermint/tendermint/types/time" ) @@ -95,6 +96,11 @@ func (wal *baseWAL) Group() *auto.Group { return wal.group } +func (wal *baseWAL) SetLogger(l log.Logger) { + wal.BaseService.Logger = l + wal.group.SetLogger(l) +} + func (wal *baseWAL) OnStart() error { size, err := wal.group.Head.Size() if err != nil { @@ -106,11 +112,20 @@ func (wal *baseWAL) OnStart() error { return err } +// Stop the underlying autofile group. +// Use Wait() to ensure it's finished shutting down +// before cleaning up files. func (wal *baseWAL) OnStop() { wal.group.Stop() wal.group.Close() } +// Wait for the underlying autofile group to finish shutting down +// so it's safe to cleanup files. +func (wal *baseWAL) Wait() { + wal.group.Wait() +} + // Write is called in newStep and for each receive on the // peerMsgQueue and the timeoutTicker. // NOTE: does not call fsync() @@ -157,7 +172,7 @@ func (wal *baseWAL) SearchForEndHeight(height int64, options *WALSearchOptions) // NOTE: starting from the last file in the group because we're usually // searching for the last height. See replay.go min, max := wal.group.MinIndex(), wal.group.MaxIndex() - wal.Logger.Debug("Searching for height", "height", height, "min", min, "max", max) + wal.Logger.Info("Searching for height", "height", height, "min", min, "max", max) for index := max; index >= min; index-- { gr, err = wal.group.NewReader(index) if err != nil { @@ -177,7 +192,7 @@ func (wal *baseWAL) SearchForEndHeight(height int64, options *WALSearchOptions) break } if options.IgnoreDataCorruptionErrors && IsDataCorruptionError(err) { - wal.Logger.Debug("Corrupted entry. Skipping...", "err", err) + wal.Logger.Error("Corrupted entry. Skipping...", "err", err) // do nothing continue } else if err != nil { @@ -188,7 +203,7 @@ func (wal *baseWAL) SearchForEndHeight(height int64, options *WALSearchOptions) if m, ok := msg.Msg.(EndHeightMessage); ok { lastHeightFound = m.Height if m.Height == height { // found - wal.Logger.Debug("Found", "height", height, "index", index) + wal.Logger.Info("Found", "height", height, "index", index) return gr, true, nil } } @@ -275,25 +290,25 @@ func (dec *WALDecoder) Decode() (*TimedWALMessage, error) { return nil, err } if err != nil { - return nil, fmt.Errorf("failed to read checksum: %v", err) + return nil, DataCorruptionError{fmt.Errorf("failed to read checksum: %v", err)} } crc := binary.BigEndian.Uint32(b) b = make([]byte, 4) _, err = dec.rd.Read(b) if err != nil { - return nil, fmt.Errorf("failed to read length: %v", err) + return nil, DataCorruptionError{fmt.Errorf("failed to read length: %v", err)} } length := binary.BigEndian.Uint32(b) if length > maxMsgSizeBytes { - return nil, fmt.Errorf("length %d exceeded maximum possible value of %d bytes", length, maxMsgSizeBytes) + return nil, DataCorruptionError{fmt.Errorf("length %d exceeded maximum possible value of %d bytes", length, maxMsgSizeBytes)} } data := make([]byte, length) _, err = dec.rd.Read(data) if err != nil { - return nil, fmt.Errorf("failed to read data: %v", err) + return nil, DataCorruptionError{fmt.Errorf("failed to read data: %v", err)} } // check checksum before decoding data diff --git a/consensus/wal_generator.go b/consensus/wal_generator.go index 980a4489..83861d3e 100644 --- a/consensus/wal_generator.go +++ b/consensus/wal_generator.go @@ -38,9 +38,11 @@ func WALGenerateNBlocks(wr io.Writer, numBlocks int) (err error) { ///////////////////////////////////////////////////////////////////////////// // COPY PASTE FROM node.go WITH A FEW MODIFICATIONS - // NOTE: we can't import node package because of circular dependency - privValidatorFile := config.PrivValidatorFile() - privValidator := privval.LoadOrGenFilePV(privValidatorFile) + // NOTE: we can't import node package because of circular dependency. + // NOTE: we don't do handshake so need to set state.Version.Consensus.App directly. + privValidatorKeyFile := config.PrivValidatorKeyFile() + privValidatorStateFile := config.PrivValidatorStateFile() + privValidator := privval.LoadOrGenFilePV(privValidatorKeyFile, privValidatorStateFile) genDoc, err := types.GenesisDocFromFile(config.GenesisFile()) if err != nil { return errors.Wrap(err, "failed to read genesis file") @@ -51,6 +53,7 @@ func WALGenerateNBlocks(wr io.Writer, numBlocks int) (err error) { if err != nil { return errors.Wrap(err, "failed to make genesis state") } + state.Version.Consensus.App = kvstore.ProtocolVersion blockStore := bc.NewBlockStore(blockStoreDB) proxyApp := proxy.NewAppConns(proxy.NewLocalClientCreator(app)) proxyApp.SetLogger(logger.With("module", "proxy")) diff --git a/consensus/wal_test.go b/consensus/wal_test.go index c45f6ace..93beb68b 100644 --- a/consensus/wal_test.go +++ b/consensus/wal_test.go @@ -14,6 +14,7 @@ import ( "github.com/tendermint/tendermint/consensus/types" "github.com/tendermint/tendermint/libs/autofile" + "github.com/tendermint/tendermint/libs/log" tmtypes "github.com/tendermint/tendermint/types" tmtime "github.com/tendermint/tendermint/types/time" @@ -23,29 +24,32 @@ import ( func TestWALTruncate(t *testing.T) { walDir, err := ioutil.TempDir("", "wal") - if err != nil { - panic(fmt.Errorf("failed to create temp WAL file: %v", err)) - } + require.NoError(t, err) defer os.RemoveAll(walDir) walFile := filepath.Join(walDir, "wal") //this magic number 4K can truncate the content when RotateFile. defaultHeadSizeLimit(10M) is hard to simulate. //this magic number 1 * time.Millisecond make RotateFile check frequently. defaultGroupCheckDuration(5s) is hard to simulate. - wal, err := NewWAL(walFile, autofile.GroupHeadSizeLimit(4096), autofile.GroupCheckDuration(1*time.Millisecond)) - if err != nil { - t.Fatal(err) - } - - wal.Start() - defer wal.Stop() + wal, err := NewWAL(walFile, + autofile.GroupHeadSizeLimit(4096), + autofile.GroupCheckDuration(1*time.Millisecond), + ) + require.NoError(t, err) + wal.SetLogger(log.TestingLogger()) + err = wal.Start() + require.NoError(t, err) + defer func() { + wal.Stop() + // wait for the wal to finish shutting down so we + // can safely remove the directory + wal.Wait() + }() //60 block's size nearly 70K, greater than group's headBuf size(4096 * 10), when headBuf is full, truncate content will Flush to the file. //at this time, RotateFile is called, truncate content exist in each file. err = WALGenerateNBlocks(wal.Group(), 60) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) time.Sleep(1 * time.Millisecond) //wait groupCheckDuration, make sure RotateFile run @@ -69,8 +73,8 @@ func TestWALTruncate(t *testing.T) { func TestWALEncoderDecoder(t *testing.T) { now := tmtime.Now() msgs := []TimedWALMessage{ - TimedWALMessage{Time: now, Msg: EndHeightMessage{0}}, - TimedWALMessage{Time: now, Msg: timeoutInfo{Duration: time.Second, Height: 1, Round: 1, Step: types.RoundStepPropose}}, + {Time: now, Msg: EndHeightMessage{0}}, + {Time: now, Msg: timeoutInfo{Duration: time.Second, Height: 1, Round: 1, Step: types.RoundStepPropose}}, } b := new(bytes.Buffer) @@ -99,9 +103,8 @@ func TestWALSearchForEndHeight(t *testing.T) { walFile := tempWALWithData(walBody) wal, err := NewWAL(walFile) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) + wal.SetLogger(log.TestingLogger()) h := int64(3) gr, found, err := wal.SearchForEndHeight(h, &WALSearchOptions{}) diff --git a/consensus/wire.go b/consensus/wire.go index 567e6095..ecd092a1 100644 --- a/consensus/wire.go +++ b/consensus/wire.go @@ -1,7 +1,7 @@ package consensus import ( - "github.com/tendermint/go-amino" + amino "github.com/tendermint/go-amino" "github.com/tendermint/tendermint/types" ) diff --git a/crypto/armor/armor.go b/crypto/armor/armor.go index e3b29a97..c15d070e 100644 --- a/crypto/armor/armor.go +++ b/crypto/armor/armor.go @@ -5,7 +5,7 @@ import ( "fmt" "io/ioutil" - "golang.org/x/crypto/openpgp/armor" // forked to github.com/tendermint/crypto + "golang.org/x/crypto/openpgp/armor" ) func EncodeArmor(blockType string, headers map[string]string, data []byte) string { diff --git a/crypto/crypto.go b/crypto/crypto.go index 09c12ff7..b3526f88 100644 --- a/crypto/crypto.go +++ b/crypto/crypto.go @@ -1,21 +1,24 @@ package crypto import ( + "github.com/tendermint/tendermint/crypto/tmhash" cmn "github.com/tendermint/tendermint/libs/common" ) -type PrivKey interface { - Bytes() []byte - Sign(msg []byte) ([]byte, error) - PubKey() PubKey - Equals(PrivKey) bool -} +const ( + // AddressSize is the size of a pubkey address. + AddressSize = tmhash.TruncatedSize +) // An address is a []byte, but hex-encoded even in JSON. // []byte leaves us the option to change the address length. // Use an alias so Unmarshal methods (with ptr receivers) are available too. type Address = cmn.HexBytes +func AddressHash(bz []byte) Address { + return Address(tmhash.SumTruncated(bz)) +} + type PubKey interface { Address() Address Bytes() []byte @@ -23,6 +26,13 @@ type PubKey interface { Equals(PubKey) bool } +type PrivKey interface { + Bytes() []byte + Sign(msg []byte) ([]byte, error) + PubKey() PubKey + Equals(PrivKey) bool +} + type Symmetric interface { Keygen() []byte Encrypt(plaintext []byte, secret []byte) (ciphertext []byte) diff --git a/crypto/ed25519/ed25519.go b/crypto/ed25519/ed25519.go index c2bed6ab..bc60838d 100644 --- a/crypto/ed25519/ed25519.go +++ b/crypto/ed25519/ed25519.go @@ -7,7 +7,7 @@ import ( "io" amino "github.com/tendermint/go-amino" - "golang.org/x/crypto/ed25519" // forked to github.com/tendermint/crypto + "golang.org/x/crypto/ed25519" "github.com/tendermint/tendermint/crypto" "github.com/tendermint/tendermint/crypto/tmhash" @@ -18,8 +18,8 @@ import ( var _ crypto.PrivKey = PrivKeyEd25519{} const ( - PrivKeyAminoRoute = "tendermint/PrivKeyEd25519" - PubKeyAminoRoute = "tendermint/PubKeyEd25519" + PrivKeyAminoName = "tendermint/PrivKeyEd25519" + PubKeyAminoName = "tendermint/PubKeyEd25519" // Size of an Edwards25519 signature. Namely the size of a compressed // Edwards25519 point, and a field element. Both of which are 32 bytes. SignatureSize = 64 @@ -30,11 +30,11 @@ var cdc = amino.NewCodec() func init() { cdc.RegisterInterface((*crypto.PubKey)(nil), nil) cdc.RegisterConcrete(PubKeyEd25519{}, - PubKeyAminoRoute, nil) + PubKeyAminoName, nil) cdc.RegisterInterface((*crypto.PrivKey)(nil), nil) cdc.RegisterConcrete(PrivKeyEd25519{}, - PrivKeyAminoRoute, nil) + PrivKeyAminoName, nil) } // PrivKeyEd25519 implements crypto.PrivKey. @@ -46,6 +46,12 @@ func (privKey PrivKeyEd25519) Bytes() []byte { } // Sign produces a signature on the provided message. +// This assumes the privkey is wellformed in the golang format. +// The first 32 bytes should be random, +// corresponding to the normal ed25519 private key. +// The latter 32 bytes should be the compressed public key. +// If these conditions aren't met, Sign will panic or produce an +// incorrect signature. func (privKey PrivKeyEd25519) Sign(msg []byte) ([]byte, error) { signatureBytes := ed25519.Sign(privKey[:], msg) return signatureBytes[:], nil @@ -130,7 +136,7 @@ type PubKeyEd25519 [PubKeyEd25519Size]byte // Address is the SHA256-20 of the raw pubkey bytes. func (pubKey PubKeyEd25519) Address() crypto.Address { - return crypto.Address(tmhash.Sum(pubKey[:])) + return crypto.Address(tmhash.SumTruncated(pubKey[:])) } // Bytes marshals the PubKey using amino encoding. diff --git a/crypto/encoding/amino/amino.go b/crypto/encoding/amino/amino.go index d66ecd9b..f7be3a20 100644 --- a/crypto/encoding/amino/amino.go +++ b/crypto/encoding/amino/amino.go @@ -12,11 +12,11 @@ import ( var cdc = amino.NewCodec() -// routeTable is used to map public key concrete types back -// to their amino routes. This should eventually be handled +// nameTable is used to map public key concrete types back +// to their registered amino names. This should eventually be handled // by amino. Example usage: -// routeTable[reflect.TypeOf(ed25519.PubKeyEd25519{})] = ed25519.PubKeyAminoRoute -var routeTable = make(map[reflect.Type]string, 3) +// nameTable[reflect.TypeOf(ed25519.PubKeyEd25519{})] = ed25519.PubKeyAminoName +var nameTable = make(map[reflect.Type]string, 3) func init() { // NOTE: It's important that there be no conflicts here, @@ -29,16 +29,16 @@ func init() { // TODO: Have amino provide a way to go from concrete struct to route directly. // Its currently a private API - routeTable[reflect.TypeOf(ed25519.PubKeyEd25519{})] = ed25519.PubKeyAminoRoute - routeTable[reflect.TypeOf(secp256k1.PubKeySecp256k1{})] = secp256k1.PubKeyAminoRoute - routeTable[reflect.TypeOf(&multisig.PubKeyMultisigThreshold{})] = multisig.PubKeyMultisigThresholdAminoRoute + nameTable[reflect.TypeOf(ed25519.PubKeyEd25519{})] = ed25519.PubKeyAminoName + nameTable[reflect.TypeOf(secp256k1.PubKeySecp256k1{})] = secp256k1.PubKeyAminoName + nameTable[reflect.TypeOf(multisig.PubKeyMultisigThreshold{})] = multisig.PubKeyMultisigThresholdAminoRoute } -// PubkeyAminoRoute returns the amino route of a pubkey +// PubkeyAminoName returns the amino route of a pubkey // cdc is currently passed in, as eventually this will not be using // a package level codec. -func PubkeyAminoRoute(cdc *amino.Codec, key crypto.PubKey) (string, bool) { - route, found := routeTable[reflect.TypeOf(key)] +func PubkeyAminoName(cdc *amino.Codec, key crypto.PubKey) (string, bool) { + route, found := nameTable[reflect.TypeOf(key)] return route, found } @@ -47,17 +47,17 @@ func RegisterAmino(cdc *amino.Codec) { // These are all written here instead of cdc.RegisterInterface((*crypto.PubKey)(nil), nil) cdc.RegisterConcrete(ed25519.PubKeyEd25519{}, - ed25519.PubKeyAminoRoute, nil) + ed25519.PubKeyAminoName, nil) cdc.RegisterConcrete(secp256k1.PubKeySecp256k1{}, - secp256k1.PubKeyAminoRoute, nil) + secp256k1.PubKeyAminoName, nil) cdc.RegisterConcrete(multisig.PubKeyMultisigThreshold{}, multisig.PubKeyMultisigThresholdAminoRoute, nil) cdc.RegisterInterface((*crypto.PrivKey)(nil), nil) cdc.RegisterConcrete(ed25519.PrivKeyEd25519{}, - ed25519.PrivKeyAminoRoute, nil) + ed25519.PrivKeyAminoName, nil) cdc.RegisterConcrete(secp256k1.PrivKeySecp256k1{}, - secp256k1.PrivKeyAminoRoute, nil) + secp256k1.PrivKeyAminoName, nil) } func PrivKeyFromBytes(privKeyBytes []byte) (privKey crypto.PrivKey, err error) { diff --git a/crypto/encoding/amino/encode_test.go b/crypto/encoding/amino/encode_test.go index 056dbec4..95510306 100644 --- a/crypto/encoding/amino/encode_test.go +++ b/crypto/encoding/amino/encode_test.go @@ -25,9 +25,8 @@ func checkAminoBinary(t *testing.T, src, dst interface{}, size int) { assert.Equal(t, byterSrc.Bytes(), bz, "Amino binary vs Bytes() mismatch") } // Make sure we have the expected length. - if size != -1 { - assert.Equal(t, size, len(bz), "Amino binary size mismatch") - } + assert.Equal(t, size, len(bz), "Amino binary size mismatch") + // Unmarshal. err = cdc.UnmarshalBinaryBare(bz, dst) require.Nil(t, err, "%+v", err) @@ -128,18 +127,18 @@ func TestPubKeyInvalidDataProperReturnsEmpty(t *testing.T) { require.Nil(t, pk) } -func TestPubkeyAminoRoute(t *testing.T) { +func TestPubkeyAminoName(t *testing.T) { tests := []struct { key crypto.PubKey want string found bool }{ - {ed25519.PubKeyEd25519{}, ed25519.PubKeyAminoRoute, true}, - {secp256k1.PubKeySecp256k1{}, secp256k1.PubKeyAminoRoute, true}, - {&multisig.PubKeyMultisigThreshold{}, multisig.PubKeyMultisigThresholdAminoRoute, true}, + {ed25519.PubKeyEd25519{}, ed25519.PubKeyAminoName, true}, + {secp256k1.PubKeySecp256k1{}, secp256k1.PubKeyAminoName, true}, + {multisig.PubKeyMultisigThreshold{}, multisig.PubKeyMultisigThresholdAminoRoute, true}, } for i, tc := range tests { - got, found := PubkeyAminoRoute(cdc, tc.key) + got, found := PubkeyAminoName(cdc, tc.key) require.Equal(t, tc.found, found, "not equal on tc %d", i) if tc.found { require.Equal(t, tc.want, got, "not equal on tc %d", i) diff --git a/crypto/hash.go b/crypto/hash.go index a384bbb5..c1fb41f7 100644 --- a/crypto/hash.go +++ b/crypto/hash.go @@ -3,7 +3,7 @@ package crypto import ( "crypto/sha256" - "golang.org/x/crypto/ripemd160" // forked to github.com/tendermint/crypto + "golang.org/x/crypto/ripemd160" ) func Sha256(bytes []byte) []byte { diff --git a/crypto/merkle/hash.go b/crypto/merkle/hash.go new file mode 100644 index 00000000..4e24046a --- /dev/null +++ b/crypto/merkle/hash.go @@ -0,0 +1,21 @@ +package merkle + +import ( + "github.com/tendermint/tendermint/crypto/tmhash" +) + +// TODO: make these have a large predefined capacity +var ( + leafPrefix = []byte{0} + innerPrefix = []byte{1} +) + +// returns tmhash(0x00 || leaf) +func leafHash(leaf []byte) []byte { + return tmhash.Sum(append(leafPrefix, leaf...)) +} + +// returns tmhash(0x01 || left || right) +func innerHash(left []byte, right []byte) []byte { + return tmhash.Sum(append(innerPrefix, append(left, right...)...)) +} diff --git a/crypto/merkle/proof.go b/crypto/merkle/proof.go index 3059ed3b..5e2a3ab1 100644 --- a/crypto/merkle/proof.go +++ b/crypto/merkle/proof.go @@ -43,10 +43,14 @@ func (poz ProofOperators) Verify(root []byte, keypath string, args [][]byte) (er for i, op := range poz { key := op.GetKey() if len(key) != 0 { - if !bytes.Equal(keys[0], key) { - return cmn.NewError("Key mismatch on operation #%d: expected %+v but %+v", i, []byte(keys[0]), []byte(key)) + if len(keys) == 0 { + return cmn.NewError("Key path has insufficient # of parts: expected no more keys but got %+v", string(key)) } - keys = keys[1:] + lastKey := keys[len(keys)-1] + if !bytes.Equal(lastKey, key) { + return cmn.NewError("Key mismatch on operation #%d: expected %+v but got %+v", i, string(lastKey), string(key)) + } + keys = keys[:len(keys)-1] } args, err = op.Run(args) if err != nil { @@ -54,7 +58,7 @@ func (poz ProofOperators) Verify(root []byte, keypath string, args [][]byte) (er } } if !bytes.Equal(root, args[0]) { - return cmn.NewError("Calculated root hash is invalid: expected %+v but %+v", root, args[0]) + return cmn.NewError("Calculated root hash is invalid: expected %+v but got %+v", root, args[0]) } if len(keys) != 0 { return cmn.NewError("Keypath not consumed all") @@ -94,7 +98,7 @@ func (prt *ProofRuntime) Decode(pop ProofOp) (ProofOperator, error) { } func (prt *ProofRuntime) DecodeProof(proof *Proof) (ProofOperators, error) { - var poz ProofOperators + poz := make(ProofOperators, 0, len(proof.Ops)) for _, pop := range proof.Ops { operator, err := prt.Decode(pop) if err != nil { diff --git a/crypto/merkle/proof_simple_value.go b/crypto/merkle/proof_simple_value.go index 5b7b5232..247921ad 100644 --- a/crypto/merkle/proof_simple_value.go +++ b/crypto/merkle/proof_simple_value.go @@ -42,7 +42,7 @@ func SimpleValueOpDecoder(pop ProofOp) (ProofOperator, error) { return nil, cmn.NewError("unexpected ProofOp.Type; got %v, want %v", pop.Type, ProofOpSimpleValue) } var op SimpleValueOp // a bit strange as we'll discard this, but it works. - err := cdc.UnmarshalBinary(pop.Data, &op) + err := cdc.UnmarshalBinaryLengthPrefixed(pop.Data, &op) if err != nil { return nil, cmn.ErrorWrap(err, "decoding ProofOp.Data into SimpleValueOp") } @@ -50,7 +50,7 @@ func SimpleValueOpDecoder(pop ProofOp) (ProofOperator, error) { } func (op SimpleValueOp) ProofOp() ProofOp { - bz := cdc.MustMarshalBinary(op) + bz := cdc.MustMarshalBinaryLengthPrefixed(op) return ProofOp{ Type: ProofOpSimpleValue, Key: op.key, @@ -71,11 +71,11 @@ func (op SimpleValueOp) Run(args [][]byte) ([][]byte, error) { hasher.Write(value) // does not error vhash := hasher.Sum(nil) + bz := new(bytes.Buffer) // Wrap to hash the KVPair. - hasher = tmhash.New() - encodeByteSlice(hasher, []byte(op.key)) // does not error - encodeByteSlice(hasher, []byte(vhash)) // does not error - kvhash := hasher.Sum(nil) + encodeByteSlice(bz, []byte(op.key)) // does not error + encodeByteSlice(bz, []byte(vhash)) // does not error + kvhash := leafHash(bz.Bytes()) if !bytes.Equal(kvhash, op.Proof.LeafHash) { return nil, cmn.NewError("leaf hash mismatch: want %X got %X", op.Proof.LeafHash, kvhash) diff --git a/crypto/merkle/proof_test.go b/crypto/merkle/proof_test.go new file mode 100644 index 00000000..4de3246f --- /dev/null +++ b/crypto/merkle/proof_test.go @@ -0,0 +1,141 @@ +package merkle + +import ( + "testing" + + "github.com/stretchr/testify/assert" + amino "github.com/tendermint/go-amino" + cmn "github.com/tendermint/tendermint/libs/common" +) + +const ProofOpDomino = "test:domino" + +// Expects given input, produces given output. +// Like the game dominos. +type DominoOp struct { + key string // unexported, may be empty + Input string + Output string +} + +func NewDominoOp(key, input, output string) DominoOp { + return DominoOp{ + key: key, + Input: input, + Output: output, + } +} + +//nolint:unused +func DominoOpDecoder(pop ProofOp) (ProofOperator, error) { + if pop.Type != ProofOpDomino { + panic("unexpected proof op type") + } + var op DominoOp // a bit strange as we'll discard this, but it works. + err := amino.UnmarshalBinaryLengthPrefixed(pop.Data, &op) + if err != nil { + return nil, cmn.ErrorWrap(err, "decoding ProofOp.Data into SimpleValueOp") + } + return NewDominoOp(string(pop.Key), op.Input, op.Output), nil +} + +func (dop DominoOp) ProofOp() ProofOp { + bz := amino.MustMarshalBinaryLengthPrefixed(dop) + return ProofOp{ + Type: ProofOpDomino, + Key: []byte(dop.key), + Data: bz, + } +} + +func (dop DominoOp) Run(input [][]byte) (output [][]byte, err error) { + if len(input) != 1 { + return nil, cmn.NewError("Expected input of length 1") + } + if string(input[0]) != dop.Input { + return nil, cmn.NewError("Expected input %v, got %v", + dop.Input, string(input[0])) + } + return [][]byte{[]byte(dop.Output)}, nil +} + +func (dop DominoOp) GetKey() []byte { + return []byte(dop.key) +} + +//---------------------------------------- + +func TestProofOperators(t *testing.T) { + var err error + + // ProofRuntime setup + // TODO test this somehow. + // prt := NewProofRuntime() + // prt.RegisterOpDecoder(ProofOpDomino, DominoOpDecoder) + + // ProofOperators setup + op1 := NewDominoOp("KEY1", "INPUT1", "INPUT2") + op2 := NewDominoOp("KEY2", "INPUT2", "INPUT3") + op3 := NewDominoOp("", "INPUT3", "INPUT4") + op4 := NewDominoOp("KEY4", "INPUT4", "OUTPUT4") + + // Good + popz := ProofOperators([]ProofOperator{op1, op2, op3, op4}) + err = popz.Verify(bz("OUTPUT4"), "/KEY4/KEY2/KEY1", [][]byte{bz("INPUT1")}) + assert.Nil(t, err) + err = popz.VerifyValue(bz("OUTPUT4"), "/KEY4/KEY2/KEY1", bz("INPUT1")) + assert.Nil(t, err) + + // BAD INPUT + err = popz.Verify(bz("OUTPUT4"), "/KEY4/KEY2/KEY1", [][]byte{bz("INPUT1_WRONG")}) + assert.NotNil(t, err) + err = popz.VerifyValue(bz("OUTPUT4"), "/KEY4/KEY2/KEY1", bz("INPUT1_WRONG")) + assert.NotNil(t, err) + + // BAD KEY 1 + err = popz.Verify(bz("OUTPUT4"), "/KEY3/KEY2/KEY1", [][]byte{bz("INPUT1")}) + assert.NotNil(t, err) + + // BAD KEY 2 + err = popz.Verify(bz("OUTPUT4"), "KEY4/KEY2/KEY1", [][]byte{bz("INPUT1")}) + assert.NotNil(t, err) + + // BAD KEY 3 + err = popz.Verify(bz("OUTPUT4"), "/KEY4/KEY2/KEY1/", [][]byte{bz("INPUT1")}) + assert.NotNil(t, err) + + // BAD KEY 4 + err = popz.Verify(bz("OUTPUT4"), "//KEY4/KEY2/KEY1", [][]byte{bz("INPUT1")}) + assert.NotNil(t, err) + + // BAD KEY 5 + err = popz.Verify(bz("OUTPUT4"), "/KEY2/KEY1", [][]byte{bz("INPUT1")}) + assert.NotNil(t, err) + + // BAD OUTPUT 1 + err = popz.Verify(bz("OUTPUT4_WRONG"), "/KEY4/KEY2/KEY1", [][]byte{bz("INPUT1")}) + assert.NotNil(t, err) + + // BAD OUTPUT 2 + err = popz.Verify(bz(""), "/KEY4/KEY2/KEY1", [][]byte{bz("INPUT1")}) + assert.NotNil(t, err) + + // BAD POPZ 1 + popz = []ProofOperator{op1, op2, op4} + err = popz.Verify(bz("OUTPUT4"), "/KEY4/KEY2/KEY1", [][]byte{bz("INPUT1")}) + assert.NotNil(t, err) + + // BAD POPZ 2 + popz = []ProofOperator{op4, op3, op2, op1} + err = popz.Verify(bz("OUTPUT4"), "/KEY4/KEY2/KEY1", [][]byte{bz("INPUT1")}) + assert.NotNil(t, err) + + // BAD POPZ 3 + popz = []ProofOperator{} + err = popz.Verify(bz("OUTPUT4"), "/KEY4/KEY2/KEY1", [][]byte{bz("INPUT1")}) + assert.NotNil(t, err) +} + +func bz(s string) []byte { + return []byte(s) +} diff --git a/crypto/merkle/rfc6962_test.go b/crypto/merkle/rfc6962_test.go new file mode 100644 index 00000000..52eab422 --- /dev/null +++ b/crypto/merkle/rfc6962_test.go @@ -0,0 +1,97 @@ +package merkle + +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// These tests were taken from https://github.com/google/trillian/blob/master/merkle/rfc6962/rfc6962_test.go, +// and consequently fall under the above license. +import ( + "bytes" + "encoding/hex" + "testing" + + "github.com/tendermint/tendermint/crypto/tmhash" +) + +func TestRFC6962Hasher(t *testing.T) { + _, leafHashTrail := trailsFromByteSlices([][]byte{[]byte("L123456")}) + leafHash := leafHashTrail.Hash + _, leafHashTrail = trailsFromByteSlices([][]byte{{}}) + emptyLeafHash := leafHashTrail.Hash + for _, tc := range []struct { + desc string + got []byte + want string + }{ + // Since creating a merkle tree of no leaves is unsupported here, we skip + // the corresponding trillian test vector. + + // Check that the empty hash is not the same as the hash of an empty leaf. + // echo -n 00 | xxd -r -p | sha256sum + { + desc: "RFC6962 Empty Leaf", + want: "6e340b9cffb37a989ca544e6bb780a2c78901d3fb33738768511a30617afa01d"[:tmhash.Size*2], + got: emptyLeafHash, + }, + // echo -n 004C313233343536 | xxd -r -p | sha256sum + { + desc: "RFC6962 Leaf", + want: "395aa064aa4c29f7010acfe3f25db9485bbd4b91897b6ad7ad547639252b4d56"[:tmhash.Size*2], + got: leafHash, + }, + // echo -n 014E3132334E343536 | xxd -r -p | sha256sum + { + desc: "RFC6962 Node", + want: "aa217fe888e47007fa15edab33c2b492a722cb106c64667fc2b044444de66bbb"[:tmhash.Size*2], + got: innerHash([]byte("N123"), []byte("N456")), + }, + } { + t.Run(tc.desc, func(t *testing.T) { + wantBytes, err := hex.DecodeString(tc.want) + if err != nil { + t.Fatalf("hex.DecodeString(%x): %v", tc.want, err) + } + if got, want := tc.got, wantBytes; !bytes.Equal(got, want) { + t.Errorf("got %x, want %x", got, want) + } + }) + } +} + +func TestRFC6962HasherCollisions(t *testing.T) { + // Check that different leaves have different hashes. + leaf1, leaf2 := []byte("Hello"), []byte("World") + _, leafHashTrail := trailsFromByteSlices([][]byte{leaf1}) + hash1 := leafHashTrail.Hash + _, leafHashTrail = trailsFromByteSlices([][]byte{leaf2}) + hash2 := leafHashTrail.Hash + if bytes.Equal(hash1, hash2) { + t.Errorf("Leaf hashes should differ, but both are %x", hash1) + } + // Compute an intermediate subtree hash. + _, subHash1Trail := trailsFromByteSlices([][]byte{hash1, hash2}) + subHash1 := subHash1Trail.Hash + // Check that this is not the same as a leaf hash of their concatenation. + preimage := append(hash1, hash2...) + _, forgedHashTrail := trailsFromByteSlices([][]byte{preimage}) + forgedHash := forgedHashTrail.Hash + if bytes.Equal(subHash1, forgedHash) { + t.Errorf("Hasher is not second-preimage resistant") + } + // Swap the order of nodes and check that the hash is different. + _, subHash2Trail := trailsFromByteSlices([][]byte{hash2, hash1}) + subHash2 := subHash2Trail.Hash + if bytes.Equal(subHash1, subHash2) { + t.Errorf("Subtree hash does not depend on the order of leaves") + } +} diff --git a/crypto/merkle/simple_map_test.go b/crypto/merkle/simple_map_test.go index bc095c00..366d9f39 100644 --- a/crypto/merkle/simple_map_test.go +++ b/crypto/merkle/simple_map_test.go @@ -13,14 +13,14 @@ func TestSimpleMap(t *testing.T) { values []string // each string gets converted to []byte in test want string }{ - {[]string{"key1"}, []string{"value1"}, "fa9bc106ffd932d919bee935ceb6cf2b3dd72d8f"}, - {[]string{"key1"}, []string{"value2"}, "e00e7dcfe54e9fafef5111e813a587f01ba9c3e8"}, + {[]string{"key1"}, []string{"value1"}, "a44d3cc7daba1a4600b00a2434b30f8b970652169810d6dfa9fb1793a2189324"}, + {[]string{"key1"}, []string{"value2"}, "0638e99b3445caec9d95c05e1a3fc1487b4ddec6a952ff337080360b0dcc078c"}, // swap order with 2 keys - {[]string{"key1", "key2"}, []string{"value1", "value2"}, "eff12d1c703a1022ab509287c0f196130123d786"}, - {[]string{"key2", "key1"}, []string{"value2", "value1"}, "eff12d1c703a1022ab509287c0f196130123d786"}, + {[]string{"key1", "key2"}, []string{"value1", "value2"}, "8fd19b19e7bb3f2b3ee0574027d8a5a4cec370464ea2db2fbfa5c7d35bb0cff3"}, + {[]string{"key2", "key1"}, []string{"value2", "value1"}, "8fd19b19e7bb3f2b3ee0574027d8a5a4cec370464ea2db2fbfa5c7d35bb0cff3"}, // swap order with 3 keys - {[]string{"key1", "key2", "key3"}, []string{"value1", "value2", "value3"}, "b2c62a277c08dbd2ad73ca53cd1d6bfdf5830d26"}, - {[]string{"key1", "key3", "key2"}, []string{"value1", "value3", "value2"}, "b2c62a277c08dbd2ad73ca53cd1d6bfdf5830d26"}, + {[]string{"key1", "key2", "key3"}, []string{"value1", "value2", "value3"}, "1dd674ec6782a0d586a903c9c63326a41cbe56b3bba33ed6ff5b527af6efb3dc"}, + {[]string{"key1", "key3", "key2"}, []string{"value1", "value3", "value2"}, "1dd674ec6782a0d586a903c9c63326a41cbe56b3bba33ed6ff5b527af6efb3dc"}, } for i, tc := range tests { db := newSimpleMap() diff --git a/crypto/merkle/simple_proof.go b/crypto/merkle/simple_proof.go index d2cbb126..f01dcdca 100644 --- a/crypto/merkle/simple_proof.go +++ b/crypto/merkle/simple_proof.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" - "github.com/tendermint/tendermint/crypto/tmhash" cmn "github.com/tendermint/tendermint/libs/common" ) @@ -67,7 +66,8 @@ func SimpleProofsFromMap(m map[string][]byte) (rootHash []byte, proofs map[strin // Verify that the SimpleProof proves the root hash. // Check sp.Index/sp.Total manually if needed -func (sp *SimpleProof) Verify(rootHash []byte, leafHash []byte) error { +func (sp *SimpleProof) Verify(rootHash []byte, leaf []byte) error { + leafHash := leafHash(leaf) if sp.Total < 0 { return errors.New("Proof total must be positive") } @@ -128,19 +128,19 @@ func computeHashFromAunts(index int, total int, leafHash []byte, innerHashes [][ if len(innerHashes) == 0 { return nil } - numLeft := (total + 1) / 2 + numLeft := getSplitPoint(total) if index < numLeft { leftHash := computeHashFromAunts(index, numLeft, leafHash, innerHashes[:len(innerHashes)-1]) if leftHash == nil { return nil } - return SimpleHashFromTwoHashes(leftHash, innerHashes[len(innerHashes)-1]) + return innerHash(leftHash, innerHashes[len(innerHashes)-1]) } rightHash := computeHashFromAunts(index-numLeft, total-numLeft, leafHash, innerHashes[:len(innerHashes)-1]) if rightHash == nil { return nil } - return SimpleHashFromTwoHashes(innerHashes[len(innerHashes)-1], rightHash) + return innerHash(innerHashes[len(innerHashes)-1], rightHash) } } @@ -182,12 +182,13 @@ func trailsFromByteSlices(items [][]byte) (trails []*SimpleProofNode, root *Simp case 0: return nil, nil case 1: - trail := &SimpleProofNode{tmhash.Sum(items[0]), nil, nil, nil} + trail := &SimpleProofNode{leafHash(items[0]), nil, nil, nil} return []*SimpleProofNode{trail}, trail default: - lefts, leftRoot := trailsFromByteSlices(items[:(len(items)+1)/2]) - rights, rightRoot := trailsFromByteSlices(items[(len(items)+1)/2:]) - rootHash := SimpleHashFromTwoHashes(leftRoot.Hash, rightRoot.Hash) + k := getSplitPoint(len(items)) + lefts, leftRoot := trailsFromByteSlices(items[:k]) + rights, rightRoot := trailsFromByteSlices(items[k:]) + rootHash := innerHash(leftRoot.Hash, rightRoot.Hash) root := &SimpleProofNode{rootHash, nil, nil, nil} leftRoot.Parent = root leftRoot.Right = rightRoot diff --git a/crypto/merkle/simple_tree.go b/crypto/merkle/simple_tree.go index 9677aef4..5de514b5 100644 --- a/crypto/merkle/simple_tree.go +++ b/crypto/merkle/simple_tree.go @@ -1,32 +1,23 @@ package merkle import ( - "github.com/tendermint/tendermint/crypto/tmhash" + "math/bits" ) -// SimpleHashFromTwoHashes is the basic operation of the Merkle tree: Hash(left | right). -func SimpleHashFromTwoHashes(left, right []byte) []byte { - var hasher = tmhash.New() - err := encodeByteSlice(hasher, left) - if err != nil { - panic(err) - } - err = encodeByteSlice(hasher, right) - if err != nil { - panic(err) - } - return hasher.Sum(nil) -} - // SimpleHashFromByteSlices computes a Merkle tree where the leaves are the byte slice, // in the provided order. func SimpleHashFromByteSlices(items [][]byte) []byte { - hashes := make([][]byte, len(items)) - for i, item := range items { - hash := tmhash.Sum(item) - hashes[i] = hash + switch len(items) { + case 0: + return nil + case 1: + return leafHash(items[0]) + default: + k := getSplitPoint(len(items)) + left := SimpleHashFromByteSlices(items[:k]) + right := SimpleHashFromByteSlices(items[k:]) + return innerHash(left, right) } - return simpleHashFromHashes(hashes) } // SimpleHashFromMap computes a Merkle tree from sorted map. @@ -41,19 +32,16 @@ func SimpleHashFromMap(m map[string][]byte) []byte { return sm.Hash() } -//---------------------------------------------------------------- - -// Expects hashes! -func simpleHashFromHashes(hashes [][]byte) []byte { - // Recursive impl. - switch len(hashes) { - case 0: - return nil - case 1: - return hashes[0] - default: - left := simpleHashFromHashes(hashes[:(len(hashes)+1)/2]) - right := simpleHashFromHashes(hashes[(len(hashes)+1)/2:]) - return SimpleHashFromTwoHashes(left, right) +// getSplitPoint returns the largest power of 2 less than length +func getSplitPoint(length int) int { + if length < 1 { + panic("Trying to split a tree with size < 1") } + uLength := uint(length) + bitlen := bits.Len(uLength) + k := 1 << uint(bitlen-1) + if k == length { + k >>= 1 + } + return k } diff --git a/crypto/merkle/simple_tree_test.go b/crypto/merkle/simple_tree_test.go index 32edc652..9abe321c 100644 --- a/crypto/merkle/simple_tree_test.go +++ b/crypto/merkle/simple_tree_test.go @@ -34,7 +34,6 @@ func TestSimpleProof(t *testing.T) { // For each item, check the trail. for i, item := range items { - itemHash := tmhash.Sum(item) proof := proofs[i] // Check total/index @@ -43,30 +42,53 @@ func TestSimpleProof(t *testing.T) { require.Equal(t, proof.Total, total, "Unmatched totals: %d vs %d", proof.Total, total) // Verify success - err := proof.Verify(rootHash, itemHash) - require.NoError(t, err, "Verificatior failed: %v.", err) + err := proof.Verify(rootHash, item) + require.NoError(t, err, "Verification failed: %v.", err) // Trail too long should make it fail origAunts := proof.Aunts proof.Aunts = append(proof.Aunts, cmn.RandBytes(32)) - err = proof.Verify(rootHash, itemHash) + err = proof.Verify(rootHash, item) require.Error(t, err, "Expected verification to fail for wrong trail length") proof.Aunts = origAunts // Trail too short should make it fail proof.Aunts = proof.Aunts[0 : len(proof.Aunts)-1] - err = proof.Verify(rootHash, itemHash) + err = proof.Verify(rootHash, item) require.Error(t, err, "Expected verification to fail for wrong trail length") proof.Aunts = origAunts // Mutating the itemHash should make it fail. - err = proof.Verify(rootHash, MutateByteSlice(itemHash)) + err = proof.Verify(rootHash, MutateByteSlice(item)) require.Error(t, err, "Expected verification to fail for mutated leaf hash") // Mutating the rootHash should make it fail. - err = proof.Verify(MutateByteSlice(rootHash), itemHash) + err = proof.Verify(MutateByteSlice(rootHash), item) require.Error(t, err, "Expected verification to fail for mutated root hash") } } + +func Test_getSplitPoint(t *testing.T) { + tests := []struct { + length int + want int + }{ + {1, 0}, + {2, 1}, + {3, 2}, + {4, 2}, + {5, 4}, + {10, 8}, + {20, 16}, + {100, 64}, + {255, 128}, + {256, 128}, + {257, 256}, + } + for _, tt := range tests { + got := getSplitPoint(tt.length) + require.Equal(t, tt.want, got, "getSplitPoint(%d) = %v, want %v", tt.length, got, tt.want) + } +} diff --git a/crypto/merkle/wire.go b/crypto/merkle/wire.go index c20ec9aa..2b6ee350 100644 --- a/crypto/merkle/wire.go +++ b/crypto/merkle/wire.go @@ -1,7 +1,7 @@ package merkle import ( - "github.com/tendermint/go-amino" + amino "github.com/tendermint/go-amino" ) var cdc *amino.Codec diff --git a/crypto/multisig/threshold_pubkey.go b/crypto/multisig/threshold_pubkey.go index ca8d4230..234d420f 100644 --- a/crypto/multisig/threshold_pubkey.go +++ b/crypto/multisig/threshold_pubkey.go @@ -2,7 +2,6 @@ package multisig import ( "github.com/tendermint/tendermint/crypto" - "github.com/tendermint/tendermint/crypto/tmhash" ) // PubKeyMultisigThreshold implements a K of N threshold multisig. @@ -11,7 +10,7 @@ type PubKeyMultisigThreshold struct { PubKeys []crypto.PubKey `json:"pubkeys"` } -var _ crypto.PubKey = &PubKeyMultisigThreshold{} +var _ crypto.PubKey = PubKeyMultisigThreshold{} // NewPubKeyMultisigThreshold returns a new PubKeyMultisigThreshold. // Panics if len(pubkeys) < k or 0 >= k. @@ -22,7 +21,7 @@ func NewPubKeyMultisigThreshold(k int, pubkeys []crypto.PubKey) crypto.PubKey { if len(pubkeys) < k { panic("threshold k of n multisignature: len(pubkeys) < k") } - return &PubKeyMultisigThreshold{uint(k), pubkeys} + return PubKeyMultisigThreshold{uint(k), pubkeys} } // VerifyBytes expects sig to be an amino encoded version of a MultiSignature. @@ -31,8 +30,8 @@ func NewPubKeyMultisigThreshold(k int, pubkeys []crypto.PubKey) crypto.PubKey { // and all signatures are valid. (Not just k of the signatures) // The multisig uses a bitarray, so multiple signatures for the same key is not // a concern. -func (pk *PubKeyMultisigThreshold) VerifyBytes(msg []byte, marshalledSig []byte) bool { - var sig *Multisignature +func (pk PubKeyMultisigThreshold) VerifyBytes(msg []byte, marshalledSig []byte) bool { + var sig Multisignature err := cdc.UnmarshalBinaryBare(marshalledSig, &sig) if err != nil { return false @@ -64,19 +63,19 @@ func (pk *PubKeyMultisigThreshold) VerifyBytes(msg []byte, marshalledSig []byte) } // Bytes returns the amino encoded version of the PubKeyMultisigThreshold -func (pk *PubKeyMultisigThreshold) Bytes() []byte { +func (pk PubKeyMultisigThreshold) Bytes() []byte { return cdc.MustMarshalBinaryBare(pk) } // Address returns tmhash(PubKeyMultisigThreshold.Bytes()) -func (pk *PubKeyMultisigThreshold) Address() crypto.Address { - return crypto.Address(tmhash.Sum(pk.Bytes())) +func (pk PubKeyMultisigThreshold) Address() crypto.Address { + return crypto.AddressHash(pk.Bytes()) } // Equals returns true iff pk and other both have the same number of keys, and // all constituent keys are the same, and in the same order. -func (pk *PubKeyMultisigThreshold) Equals(other crypto.PubKey) bool { - otherKey, sameType := other.(*PubKeyMultisigThreshold) +func (pk PubKeyMultisigThreshold) Equals(other crypto.PubKey) bool { + otherKey, sameType := other.(PubKeyMultisigThreshold) if !sameType { return false } diff --git a/crypto/multisig/threshold_pubkey_test.go b/crypto/multisig/threshold_pubkey_test.go index bfc874eb..2d2632ab 100644 --- a/crypto/multisig/threshold_pubkey_test.go +++ b/crypto/multisig/threshold_pubkey_test.go @@ -82,7 +82,7 @@ func TestMultiSigPubKeyEquality(t *testing.T) { msg := []byte{1, 2, 3, 4} pubkeys, _ := generatePubKeysAndSignatures(5, msg) multisigKey := NewPubKeyMultisigThreshold(2, pubkeys) - var unmarshalledMultisig *PubKeyMultisigThreshold + var unmarshalledMultisig PubKeyMultisigThreshold cdc.MustUnmarshalBinaryBare(multisigKey.Bytes(), &unmarshalledMultisig) require.True(t, multisigKey.Equals(unmarshalledMultisig)) @@ -95,6 +95,29 @@ func TestMultiSigPubKeyEquality(t *testing.T) { require.False(t, multisigKey.Equals(multisigKey2)) } +func TestAddress(t *testing.T) { + msg := []byte{1, 2, 3, 4} + pubkeys, _ := generatePubKeysAndSignatures(5, msg) + multisigKey := NewPubKeyMultisigThreshold(2, pubkeys) + require.Len(t, multisigKey.Address().Bytes(), 20) +} + +func TestPubKeyMultisigThresholdAminoToIface(t *testing.T) { + msg := []byte{1, 2, 3, 4} + pubkeys, _ := generatePubKeysAndSignatures(5, msg) + multisigKey := NewPubKeyMultisigThreshold(2, pubkeys) + + ab, err := cdc.MarshalBinaryLengthPrefixed(multisigKey) + require.NoError(t, err) + // like other crypto.Pubkey implementations (e.g. ed25519.PubKeyEd25519), + // PubKeyMultisigThreshold should be deserializable into a crypto.PubKey: + var pubKey crypto.PubKey + err = cdc.UnmarshalBinaryLengthPrefixed(ab, &pubKey) + require.NoError(t, err) + + require.Equal(t, multisigKey, pubKey) +} + func generatePubKeysAndSignatures(n int, msg []byte) (pubkeys []crypto.PubKey, signatures [][]byte) { pubkeys = make([]crypto.PubKey, n) signatures = make([][]byte, n) diff --git a/crypto/multisig/wire.go b/crypto/multisig/wire.go index 68b84fbf..71e0db14 100644 --- a/crypto/multisig/wire.go +++ b/crypto/multisig/wire.go @@ -20,7 +20,7 @@ func init() { cdc.RegisterConcrete(PubKeyMultisigThreshold{}, PubKeyMultisigThresholdAminoRoute, nil) cdc.RegisterConcrete(ed25519.PubKeyEd25519{}, - ed25519.PubKeyAminoRoute, nil) + ed25519.PubKeyAminoName, nil) cdc.RegisterConcrete(secp256k1.PubKeySecp256k1{}, - secp256k1.PubKeyAminoRoute, nil) + secp256k1.PubKeyAminoName, nil) } diff --git a/crypto/random.go b/crypto/random.go index af328642..275fb104 100644 --- a/crypto/random.go +++ b/crypto/random.go @@ -1,55 +1,24 @@ package crypto import ( - "crypto/cipher" crand "crypto/rand" - "crypto/sha256" "encoding/hex" "io" - "sync" - - "golang.org/x/crypto/chacha20poly1305" - - . "github.com/tendermint/tendermint/libs/common" ) -// The randomness here is derived from xoring a chacha20 keystream with -// output from crypto/rand's OS Entropy Reader. (Due to fears of the OS' -// entropy being backdoored) -// -// For forward secrecy of produced randomness, the internal chacha key is hashed -// and thereby rotated after each call. -var gRandInfo *randInfo - -func init() { - gRandInfo = &randInfo{} - gRandInfo.MixEntropy(randBytes(32)) // Init -} - -// Mix additional bytes of randomness, e.g. from hardware, user-input, etc. -// It is OK to call it multiple times. It does not diminish security. -func MixEntropy(seedBytes []byte) { - gRandInfo.MixEntropy(seedBytes) -} - // This only uses the OS's randomness func randBytes(numBytes int) []byte { b := make([]byte, numBytes) _, err := crand.Read(b) if err != nil { - PanicCrisis(err) + panic(err) } return b } -// This uses the OS and the Seed(s). +// This only uses the OS's randomness func CRandBytes(numBytes int) []byte { - b := make([]byte, numBytes) - _, err := gRandInfo.Read(b) - if err != nil { - PanicCrisis(err) - } - return b + return randBytes(numBytes) } // CRandHex returns a hex encoded string that's floor(numDigits/2) * 2 long. @@ -60,60 +29,7 @@ func CRandHex(numDigits int) string { return hex.EncodeToString(CRandBytes(numDigits / 2)) } -// Returns a crand.Reader mixed with user-supplied entropy +// Returns a crand.Reader. func CReader() io.Reader { - return gRandInfo -} - -//-------------------------------------------------------------------------------- - -type randInfo struct { - mtx sync.Mutex - seedBytes [chacha20poly1305.KeySize]byte - chacha cipher.AEAD - reader io.Reader -} - -// You can call this as many times as you'd like. -// XXX TODO review -func (ri *randInfo) MixEntropy(seedBytes []byte) { - ri.mtx.Lock() - defer ri.mtx.Unlock() - // Make new ri.seedBytes using passed seedBytes and current ri.seedBytes: - // ri.seedBytes = sha256( seedBytes || ri.seedBytes ) - h := sha256.New() - h.Write(seedBytes) - h.Write(ri.seedBytes[:]) - hashBytes := h.Sum(nil) - copy(ri.seedBytes[:], hashBytes) - chacha, err := chacha20poly1305.New(ri.seedBytes[:]) - if err != nil { - panic("Initializing chacha20 failed") - } - ri.chacha = chacha - // Create new reader - ri.reader = &cipher.StreamReader{S: ri, R: crand.Reader} -} - -func (ri *randInfo) XORKeyStream(dst, src []byte) { - // nonce being 0 is safe due to never re-using a key. - emptyNonce := make([]byte, 12) - tmpDst := ri.chacha.Seal([]byte{}, emptyNonce, src, []byte{0}) - // this removes the poly1305 tag as well, since chacha is a stream cipher - // and we truncate at input length. - copy(dst, tmpDst[:len(src)]) - // hash seedBytes for forward secrecy, and initialize new chacha instance - newSeed := sha256.Sum256(ri.seedBytes[:]) - chacha, err := chacha20poly1305.New(newSeed[:]) - if err != nil { - panic("Initializing chacha20 failed") - } - ri.chacha = chacha -} - -func (ri *randInfo) Read(b []byte) (n int, err error) { - ri.mtx.Lock() - n, err = ri.reader.Read(b) - ri.mtx.Unlock() - return + return crand.Reader } diff --git a/crypto/secp256k1/secp256k1.go b/crypto/secp256k1/secp256k1.go index 784409f3..78857c45 100644 --- a/crypto/secp256k1/secp256k1.go +++ b/crypto/secp256k1/secp256k1.go @@ -7,17 +7,19 @@ import ( "fmt" "io" - secp256k1 "github.com/tendermint/btcd/btcec" + "golang.org/x/crypto/ripemd160" + + secp256k1 "github.com/btcsuite/btcd/btcec" + amino "github.com/tendermint/go-amino" - "golang.org/x/crypto/ripemd160" // forked to github.com/tendermint/crypto "github.com/tendermint/tendermint/crypto" ) //------------------------------------- const ( - PrivKeyAminoRoute = "tendermint/PrivKeySecp256k1" - PubKeyAminoRoute = "tendermint/PubKeySecp256k1" + PrivKeyAminoName = "tendermint/PrivKeySecp256k1" + PubKeyAminoName = "tendermint/PubKeySecp256k1" ) var cdc = amino.NewCodec() @@ -25,11 +27,11 @@ var cdc = amino.NewCodec() func init() { cdc.RegisterInterface((*crypto.PubKey)(nil), nil) cdc.RegisterConcrete(PubKeySecp256k1{}, - PubKeyAminoRoute, nil) + PubKeyAminoName, nil) cdc.RegisterInterface((*crypto.PrivKey)(nil), nil) cdc.RegisterConcrete(PrivKeySecp256k1{}, - PrivKeyAminoRoute, nil) + PrivKeyAminoName, nil) } //------------------------------------- @@ -44,16 +46,6 @@ func (privKey PrivKeySecp256k1) Bytes() []byte { return cdc.MustMarshalBinaryBare(privKey) } -// Sign creates an ECDSA signature on curve Secp256k1, using SHA256 on the msg. -func (privKey PrivKeySecp256k1) Sign(msg []byte) ([]byte, error) { - priv, _ := secp256k1.PrivKeyFromBytes(secp256k1.S256(), privKey[:]) - sig, err := priv.Sign(crypto.Sha256(msg)) - if err != nil { - return nil, err - } - return sig.Serialize(), nil -} - // PubKey performs the point-scalar multiplication from the privKey on the // generator point to get the pubkey. func (privKey PrivKeySecp256k1) PubKey() crypto.PubKey { @@ -137,20 +129,6 @@ func (pubKey PubKeySecp256k1) Bytes() []byte { return bz } -func (pubKey PubKeySecp256k1) VerifyBytes(msg []byte, sig []byte) bool { - pub, err := secp256k1.ParsePubKey(pubKey[:], secp256k1.S256()) - if err != nil { - return false - } - parsedSig, err := secp256k1.ParseSignature(sig[:], secp256k1.S256()) - if err != nil { - return false - } - // Underlying library ensures that this signature is in canonical form, to - // prevent Secp256k1 malleability from altering the sign of the s term. - return parsedSig.Verify(crypto.Sha256(msg), pub) -} - func (pubKey PubKeySecp256k1) String() string { return fmt.Sprintf("PubKeySecp256k1{%X}", pubKey[:]) } diff --git a/crypto/secp256k1/secp256k1_cgo.go b/crypto/secp256k1/secp256k1_cgo.go new file mode 100644 index 00000000..3e5b1ddd --- /dev/null +++ b/crypto/secp256k1/secp256k1_cgo.go @@ -0,0 +1,24 @@ +// +build libsecp256k1 + +package secp256k1 + +import ( + "github.com/ethereum/go-ethereum/crypto/secp256k1" + + "github.com/tendermint/tendermint/crypto" +) + +// Sign creates an ECDSA signature on curve Secp256k1, using SHA256 on the msg. +func (privKey PrivKeySecp256k1) Sign(msg []byte) ([]byte, error) { + rsv, err := secp256k1.Sign(crypto.Sha256(msg), privKey[:]) + if err != nil { + return nil, err + } + // we do not need v in r||s||v: + rs := rsv[:len(rsv)-1] + return rs, nil +} + +func (pubKey PubKeySecp256k1) VerifyBytes(msg []byte, sig []byte) bool { + return secp256k1.VerifySignature(pubKey[:], crypto.Sha256(msg), sig) +} diff --git a/crypto/secp256k1/secp256k1_nocgo.go b/crypto/secp256k1/secp256k1_nocgo.go new file mode 100644 index 00000000..cd1655a5 --- /dev/null +++ b/crypto/secp256k1/secp256k1_nocgo.go @@ -0,0 +1,70 @@ +// +build !libsecp256k1 + +package secp256k1 + +import ( + "math/big" + + secp256k1 "github.com/btcsuite/btcd/btcec" + + "github.com/tendermint/tendermint/crypto" +) + +// used to reject malleable signatures +// see: +// - https://github.com/ethereum/go-ethereum/blob/f9401ae011ddf7f8d2d95020b7446c17f8d98dc1/crypto/signature_nocgo.go#L90-L93 +// - https://github.com/ethereum/go-ethereum/blob/f9401ae011ddf7f8d2d95020b7446c17f8d98dc1/crypto/crypto.go#L39 +var secp256k1halfN = new(big.Int).Rsh(secp256k1.S256().N, 1) + +// Sign creates an ECDSA signature on curve Secp256k1, using SHA256 on the msg. +// The returned signature will be of the form R || S (in lower-S form). +func (privKey PrivKeySecp256k1) Sign(msg []byte) ([]byte, error) { + priv, _ := secp256k1.PrivKeyFromBytes(secp256k1.S256(), privKey[:]) + sig, err := priv.Sign(crypto.Sha256(msg)) + if err != nil { + return nil, err + } + sigBytes := serializeSig(sig) + return sigBytes, nil +} + +// VerifyBytes verifies a signature of the form R || S. +// It rejects signatures which are not in lower-S form. +func (pubKey PubKeySecp256k1) VerifyBytes(msg []byte, sigStr []byte) bool { + if len(sigStr) != 64 { + return false + } + pub, err := secp256k1.ParsePubKey(pubKey[:], secp256k1.S256()) + if err != nil { + return false + } + // parse the signature: + signature := signatureFromBytes(sigStr) + // Reject malleable signatures. libsecp256k1 does this check but btcec doesn't. + // see: https://github.com/ethereum/go-ethereum/blob/f9401ae011ddf7f8d2d95020b7446c17f8d98dc1/crypto/signature_nocgo.go#L90-L93 + if signature.S.Cmp(secp256k1halfN) > 0 { + return false + } + return signature.Verify(crypto.Sha256(msg), pub) +} + +// Read Signature struct from R || S. Caller needs to ensure +// that len(sigStr) == 64. +func signatureFromBytes(sigStr []byte) *secp256k1.Signature { + return &secp256k1.Signature{ + new(big.Int).SetBytes(sigStr[:32]), + new(big.Int).SetBytes(sigStr[32:64]), + } +} + +// Serialize signature to R || S. +// R, S are padded to 32 bytes respectively. +func serializeSig(sig *secp256k1.Signature) []byte { + rBytes := sig.R.Bytes() + sBytes := sig.S.Bytes() + sigBytes := make([]byte, 64) + // 0 pad the byte arrays from the left if they aren't big enough. + copy(sigBytes[32-len(rBytes):32], rBytes) + copy(sigBytes[64-len(sBytes):64], sBytes) + return sigBytes +} diff --git a/crypto/secp256k1/secp256k1_nocgo_test.go b/crypto/secp256k1/secp256k1_nocgo_test.go new file mode 100644 index 00000000..a06a0e3d --- /dev/null +++ b/crypto/secp256k1/secp256k1_nocgo_test.go @@ -0,0 +1,39 @@ +// +build !libsecp256k1 + +package secp256k1 + +import ( + "testing" + + secp256k1 "github.com/btcsuite/btcd/btcec" + + "github.com/stretchr/testify/require" +) + +// Ensure that signature verification works, and that +// non-canonical signatures fail. +// Note: run with CGO_ENABLED=0 or go test -tags !cgo. +func TestSignatureVerificationAndRejectUpperS(t *testing.T) { + msg := []byte("We have lingered long enough on the shores of the cosmic ocean.") + for i := 0; i < 500; i++ { + priv := GenPrivKey() + sigStr, err := priv.Sign(msg) + require.NoError(t, err) + sig := signatureFromBytes(sigStr) + require.False(t, sig.S.Cmp(secp256k1halfN) > 0) + + pub := priv.PubKey() + require.True(t, pub.VerifyBytes(msg, sigStr)) + + // malleate: + sig.S.Sub(secp256k1.S256().CurveParams.N, sig.S) + require.True(t, sig.S.Cmp(secp256k1halfN) > 0) + malSigStr := serializeSig(sig) + + require.False(t, pub.VerifyBytes(msg, malSigStr), + "VerifyBytes incorrect with malleated & invalid S. sig=%v, key=%v", + sig, + priv, + ) + } +} diff --git a/crypto/secp256k1/secpk256k1_test.go b/crypto/secp256k1/secpk256k1_test.go index 2fa48301..0f0b5adc 100644 --- a/crypto/secp256k1/secpk256k1_test.go +++ b/crypto/secp256k1/secpk256k1_test.go @@ -11,7 +11,7 @@ import ( "github.com/tendermint/tendermint/crypto" "github.com/tendermint/tendermint/crypto/secp256k1" - underlyingSecp256k1 "github.com/tendermint/btcd/btcec" + underlyingSecp256k1 "github.com/btcsuite/btcd/btcec" ) type keyData struct { diff --git a/crypto/tmhash/hash.go b/crypto/tmhash/hash.go index 1b29d868..f9b95824 100644 --- a/crypto/tmhash/hash.go +++ b/crypto/tmhash/hash.go @@ -6,10 +6,27 @@ import ( ) const ( - Size = 20 + Size = sha256.Size BlockSize = sha256.BlockSize ) +// New returns a new hash.Hash. +func New() hash.Hash { + return sha256.New() +} + +// Sum returns the SHA256 of the bz. +func Sum(bz []byte) []byte { + h := sha256.Sum256(bz) + return h[:] +} + +//------------------------------------------------------------- + +const ( + TruncatedSize = 20 +) + type sha256trunc struct { sha256 hash.Hash } @@ -19,7 +36,7 @@ func (h sha256trunc) Write(p []byte) (n int, err error) { } func (h sha256trunc) Sum(b []byte) []byte { shasum := h.sha256.Sum(b) - return shasum[:Size] + return shasum[:TruncatedSize] } func (h sha256trunc) Reset() { @@ -27,22 +44,22 @@ func (h sha256trunc) Reset() { } func (h sha256trunc) Size() int { - return Size + return TruncatedSize } func (h sha256trunc) BlockSize() int { return h.sha256.BlockSize() } -// New returns a new hash.Hash. -func New() hash.Hash { +// NewTruncated returns a new hash.Hash. +func NewTruncated() hash.Hash { return sha256trunc{ sha256: sha256.New(), } } -// Sum returns the first 20 bytes of SHA256 of the bz. -func Sum(bz []byte) []byte { +// SumTruncated returns the first 20 bytes of SHA256 of the bz. +func SumTruncated(bz []byte) []byte { hash := sha256.Sum256(bz) - return hash[:Size] + return hash[:TruncatedSize] } diff --git a/crypto/tmhash/hash_test.go b/crypto/tmhash/hash_test.go index 27938039..89a77980 100644 --- a/crypto/tmhash/hash_test.go +++ b/crypto/tmhash/hash_test.go @@ -14,10 +14,29 @@ func TestHash(t *testing.T) { hasher.Write(testVector) bz := hasher.Sum(nil) + bz2 := tmhash.Sum(testVector) + hasher = sha256.New() hasher.Write(testVector) - bz2 := hasher.Sum(nil) - bz2 = bz2[:20] + bz3 := hasher.Sum(nil) assert.Equal(t, bz, bz2) + assert.Equal(t, bz, bz3) +} + +func TestHashTruncated(t *testing.T) { + testVector := []byte("abc") + hasher := tmhash.NewTruncated() + hasher.Write(testVector) + bz := hasher.Sum(nil) + + bz2 := tmhash.SumTruncated(testVector) + + hasher = sha256.New() + hasher.Write(testVector) + bz3 := hasher.Sum(nil) + bz3 = bz3[:tmhash.TruncatedSize] + + assert.Equal(t, bz, bz2) + assert.Equal(t, bz, bz3) } diff --git a/crypto/xchacha20poly1305/xchachapoly.go b/crypto/xchacha20poly1305/xchachapoly.go index 115c9190..c7a175b5 100644 --- a/crypto/xchacha20poly1305/xchachapoly.go +++ b/crypto/xchacha20poly1305/xchachapoly.go @@ -8,7 +8,7 @@ import ( "errors" "fmt" - "golang.org/x/crypto/chacha20poly1305" // forked to github.com/tendermint/crypto + "golang.org/x/crypto/chacha20poly1305" ) // Implements crypto.AEAD diff --git a/crypto/xsalsa20symmetric/symmetric.go b/crypto/xsalsa20symmetric/symmetric.go index c51e2459..10a0f6f3 100644 --- a/crypto/xsalsa20symmetric/symmetric.go +++ b/crypto/xsalsa20symmetric/symmetric.go @@ -4,7 +4,7 @@ import ( "errors" "fmt" - "golang.org/x/crypto/nacl/secretbox" // forked to github.com/tendermint/crypto + "golang.org/x/crypto/nacl/secretbox" "github.com/tendermint/tendermint/crypto" cmn "github.com/tendermint/tendermint/libs/common" @@ -17,7 +17,6 @@ const secretLen = 32 // secret must be 32 bytes long. Use something like Sha256(Bcrypt(passphrase)) // The ciphertext is (secretbox.Overhead + 24) bytes longer than the plaintext. -// NOTE: call crypto.MixEntropy() first. func EncryptSymmetric(plaintext []byte, secret []byte) (ciphertext []byte) { if len(secret) != secretLen { cmn.PanicSanity(fmt.Sprintf("Secret must be 32 bytes long, got len %v", len(secret))) diff --git a/crypto/xsalsa20symmetric/symmetric_test.go b/crypto/xsalsa20symmetric/symmetric_test.go index e9adf728..160d49a9 100644 --- a/crypto/xsalsa20symmetric/symmetric_test.go +++ b/crypto/xsalsa20symmetric/symmetric_test.go @@ -6,15 +6,13 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "golang.org/x/crypto/bcrypt" // forked to github.com/tendermint/crypto + "golang.org/x/crypto/bcrypt" "github.com/tendermint/tendermint/crypto" ) func TestSimple(t *testing.T) { - crypto.MixEntropy([]byte("someentropy")) - plaintext := []byte("sometext") secret := []byte("somesecretoflengththirtytwo===32") ciphertext := EncryptSymmetric(plaintext, secret) @@ -26,13 +24,9 @@ func TestSimple(t *testing.T) { func TestSimpleWithKDF(t *testing.T) { - crypto.MixEntropy([]byte("someentropy")) - plaintext := []byte("sometext") secretPass := []byte("somesecret") - salt := []byte("somesaltsomesalt") // len 16 - // NOTE: we use a fork of x/crypto so we can inject our own randomness for salt - secret, err := bcrypt.GenerateFromPassword(salt, secretPass, 12) + secret, err := bcrypt.GenerateFromPassword(secretPass, 12) if err != nil { t.Error(err) } diff --git a/docs/.vuepress/config.js b/docs/.vuepress/config.js index 342c5eac..fb3620e3 100644 --- a/docs/.vuepress/config.js +++ b/docs/.vuepress/config.js @@ -1,14 +1,28 @@ module.exports = { - title: "Tendermint Core", - description: "Documentation for Tendermint Core", + title: "Tendermint Documentation", + description: "Documentation for Tendermint Core.", + ga: "UA-51029217-1", dest: "./dist/docs", base: "/docs/", markdown: { lineNumbers: true }, themeConfig: { - lastUpdated: "Last Updated", - nav: [{ text: "Back to Tendermint", link: "https://tendermint.com" }], + repo: "tendermint/tendermint", + editLinks: true, + docsDir: "docs", + docsBranch: "develop", + editLinkText: 'Edit this page on Github', + lastUpdated: true, + algolia: { + apiKey: '59f0e2deb984aa9cdf2b3a5fd24ac501', + indexName: 'tendermint', + debug: false + }, + nav: [ + { text: "Back to Tendermint", link: "https://tendermint.com" }, + { text: "RPC", link: "https://tendermint.com/rpc/" } + ], sidebar: [ { title: "Introduction", @@ -20,6 +34,20 @@ module.exports = { "/introduction/what-is-tendermint" ] }, + { + title: "Apps", + collapsable: false, + children: [ + "/app-dev/getting-started", + "/app-dev/abci-cli", + "/app-dev/app-architecture", + "/app-dev/app-development", + "/app-dev/subscribing-to-events-via-websocket", + "/app-dev/indexing-transactions", + "/app-dev/abci-spec", + "/app-dev/ecosystem" + ] + }, { title: "Tendermint Core", collapsable: false, @@ -38,15 +66,6 @@ module.exports = { "/tendermint-core/validators" ] }, - { - title: "Tools", - collapsable: false, - children: [ - "/tools/", - "/tools/benchmarking", - "/tools/monitoring" - ] - }, { title: "Networks", collapsable: false, @@ -57,17 +76,13 @@ module.exports = { ] }, { - title: "Apps", + title: "Tools", collapsable: false, - children: [ - "/app-dev/getting-started", - "/app-dev/abci-cli", - "/app-dev/app-architecture", - "/app-dev/app-development", - "/app-dev/subscribing-to-events-via-websocket", - "/app-dev/indexing-transactions", - "/app-dev/abci-spec", - "/app-dev/ecosystem" + children: [ + "/tools/", + "/tools/benchmarking", + "/tools/monitoring", + "/tools/remote-signer-validation" ] }, { diff --git a/docs/DOCS_README.md b/docs/DOCS_README.md index a7671c36..c91d0391 100644 --- a/docs/DOCS_README.md +++ b/docs/DOCS_README.md @@ -12,10 +12,10 @@ respectively. ## How It Works -There is a Jenkins job listening for changes in the `/docs` directory, on both +There is a CircleCI job listening for changes in the `/docs` directory, on both the `master` and `develop` branches. Any updates to files in this directory on those branches will automatically trigger a website deployment. Under the hood, -a private website repository has make targets consumed by a standard Jenkins task. +the private website repository has a `make build-docs` target consumed by a CircleCI job in that repo. ## README @@ -93,6 +93,10 @@ python -m SimpleHTTPServer 8080 then navigate to localhost:8080 in your browser. +## Search + +We are using [Algolia](https://www.algolia.com) to power full-text search. This uses a public API search-only key in the `config.js` as well as a [tendermint.json](https://github.com/algolia/docsearch-configs/blob/master/configs/tendermint.json) configuration file that we can update with PRs. + ## Consistency Because the build processes are identical (as is the information contained herein), this file should be kept in sync as diff --git a/docs/README.md b/docs/README.md index c3293547..ae4b4731 100644 --- a/docs/README.md +++ b/docs/README.md @@ -21,7 +21,7 @@ For more details on using Tendermint, see the respective documentation for ## Contribute -To contribute to the documentation, see [this file](./DOCS_README.md) for details of the build process and +To contribute to the documentation, see [this file](https://github.com/tendermint/tendermint/blob/master/docs/DOCS_README.md) for details of the build process and considerations when making changes. ## Version diff --git a/docs/app-dev/abci-cli.md b/docs/app-dev/abci-cli.md index 263c2c5e..ba6f0589 100644 --- a/docs/app-dev/abci-cli.md +++ b/docs/app-dev/abci-cli.md @@ -11,13 +11,10 @@ Make sure you [have Go installed](https://golang.org/doc/install). Next, install the `abci-cli` tool and example applications: ``` -go get github.com/tendermint/tendermint -``` - -to get vendored dependencies: - -``` -cd $GOPATH/src/github.com/tendermint/tendermint +mkdir -p $GOPATH/src/github.com/tendermint +cd $GOPATH/src/github.com/tendermint +git clone https://github.com/tendermint/tendermint.git +cd tendermint make get_tools make get_vendor_deps make install_abci diff --git a/docs/app-dev/app-architecture.md b/docs/app-dev/app-architecture.md index b141c0f3..b9c8d2e9 100644 --- a/docs/app-dev/app-architecture.md +++ b/docs/app-dev/app-architecture.md @@ -5,7 +5,7 @@ Tendermint blockchain application. The following diagram provides a superb example: - +![](../imgs/cosmos-tendermint-stack-4k.jpg) The end-user application here is the Cosmos Voyager, at the bottom left. Voyager communicates with a REST API exposed by a local Light-Client @@ -45,6 +45,6 @@ Tendermint. See the following for more extensive documentation: - [Interchain Standard for the Light-Client REST API](https://github.com/cosmos/cosmos-sdk/pull/1028) -- [Tendermint RPC Docs](https://tendermint.github.io/slate/) +- [Tendermint RPC Docs](https://tendermint.com/rpc/) - [Tendermint in Production](../tendermint-core/running-in-production.md) - [ABCI spec](./abci-spec.md) diff --git a/docs/app-dev/app-development.md b/docs/app-dev/app-development.md index 2618bb1e..d157ce37 100644 --- a/docs/app-dev/app-development.md +++ b/docs/app-dev/app-development.md @@ -47,90 +47,6 @@ The mempool and consensus logic act as clients, and each maintains an open ABCI connection with the application, which hosts an ABCI server. Shown are the request and response types sent on each connection. -## Message Protocol - -The message protocol consists of pairs of requests and responses. Some -messages have no fields, while others may include byte-arrays, strings, -or integers. See the `message Request` and `message Response` -definitions in [the protobuf definition -file](https://github.com/tendermint/tendermint/blob/develop/abci/types/types.proto), -and the [protobuf -documentation](https://developers.google.com/protocol-buffers/docs/overview) -for more details. - -For each request, a server should respond with the corresponding -response, where order of requests is preserved in the order of -responses. - -## Server - -To use ABCI in your programming language of choice, there must be a ABCI -server in that language. Tendermint supports two kinds of implementation -of the server: - -- Asynchronous, raw socket server (Tendermint Socket Protocol, also - known as TSP or Teaspoon) -- GRPC - -Both can be tested using the `abci-cli` by setting the `--abci` flag -appropriately (ie. to `socket` or `grpc`). - -See examples, in various stages of maintenance, in -[Go](https://github.com/tendermint/tendermint/tree/develop/abci/server), -[JavaScript](https://github.com/tendermint/js-abci), -[Python](https://github.com/tendermint/tendermint/tree/develop/abci/example/python3/abci), -[C++](https://github.com/mdyring/cpp-tmsp), and -[Java](https://github.com/jTendermint/jabci). - -### GRPC - -If GRPC is available in your language, this is the easiest approach, -though it will have significant performance overhead. - -To get started with GRPC, copy in the [protobuf -file](https://github.com/tendermint/tendermint/blob/develop/abci/types/types.proto) -and compile it using the GRPC plugin for your language. For instance, -for golang, the command is `protoc --go_out=plugins=grpc:. types.proto`. -See the [grpc documentation for more details](http://www.grpc.io/docs/). -`protoc` will autogenerate all the necessary code for ABCI client and -server in your language, including whatever interface your application -must satisfy to be used by the ABCI server for handling requests. - -### TSP - -If GRPC is not available in your language, or you require higher -performance, or otherwise enjoy programming, you may implement your own -ABCI server using the Tendermint Socket Protocol, known affectionately -as Teaspoon. The first step is still to auto-generate the relevant data -types and codec in your language using `protoc`. Messages coming over -the socket are proto3 encoded, but additionally length-prefixed to -facilitate use as a streaming protocol. proto3 doesn't have an -official length-prefix standard, so we use our own. The first byte in -the prefix represents the length of the Big Endian encoded length. The -remaining bytes in the prefix are the Big Endian encoded length. - -For example, if the proto3 encoded ABCI message is 0xDEADBEEF (4 -bytes), the length-prefixed message is 0x0104DEADBEEF. If the proto3 -encoded ABCI message is 65535 bytes long, the length-prefixed message -would be like 0x02FFFF.... - -Note this prefixing does not apply for grpc. - -An ABCI server must also be able to support multiple connections, as -Tendermint uses three connections. - -## Client - -There are currently two use-cases for an ABCI client. One is a testing -tool, as in the `abci-cli`, which allows ABCI requests to be sent via -command line. The other is a consensus engine, such as Tendermint Core, -which makes requests to the application every time a new transaction is -received or a block is committed. - -It is unlikely that you will need to implement a client. For details of -our client, see -[here](https://github.com/tendermint/tendermint/tree/develop/abci/client). - Most of the examples below are from [kvstore application](https://github.com/tendermint/tendermint/blob/develop/abci/example/kvstore/kvstore.go), which is a part of the abci repo. [persistent_kvstore diff --git a/docs/app-dev/ecosystem.json b/docs/app-dev/ecosystem.json index 67aca2ef..9e264af2 100644 --- a/docs/app-dev/ecosystem.json +++ b/docs/app-dev/ecosystem.json @@ -63,6 +63,13 @@ "author": "Zach Balder", "description": "Public service reporting and tracking" }, + { + "name": "ParadigmCore", + "url": "https://github.com/ParadigmFoundation/ParadigmCore", + "language": "TypeScript", + "author": "Paradigm Labs", + "description": "Reference implementation of the Paradigm Protocol, and OrderStream network client." + }, { "name": "Passchain", "url": "https://github.com/trusch/passchain", @@ -122,7 +129,7 @@ ], "abciServers": [ { - "name": "abci", + "name": "go-abci", "url": "https://github.com/tendermint/tendermint/tree/master/abci", "language": "Go", "author": "Tendermint" @@ -133,6 +140,12 @@ "language": "Javascript", "author": "Tendermint" }, + { + "name": "rust-tsp", + "url": "https://github.com/tendermint/rust-tsp", + "language": "Rust", + "author": "Tendermint" + }, { "name": "cpp-tmsp", "url": "https://github.com/mdyring/cpp-tmsp", @@ -163,6 +176,12 @@ "language": "Python", "author": "Dave Bryson" }, + { + "name": "tm-abci (fork of py-abci with async IO)", + "url": "https://github.com/SoftblocksCo/tm-abci", + "language": "Python", + "author": "Softblocks" + }, { "name": "Spearmint", "url": "https://github.com/dennismckinnon/spearmint", @@ -170,34 +189,12 @@ "author": "Dennis McKinnon" } ], - "deploymentTools": [ + "aminoLibraries": [ { - "name": "mintnet-kubernetes", - "url": "https://github.com/tendermint/tools", - "technology": "Docker and Kubernetes", - "author": "Tendermint", - "description": "Deploy a Tendermint test network using Google's kubernetes" - }, - { - "name": "terraforce", - "url": "https://github.com/tendermint/tools", - "technology": "Terraform", - "author": "Tendermint", - "description": "Terraform + our custom terraforce tool; deploy a production Tendermint network with load balancing over multiple AWS availability zones" - }, - { - "name": "ansible-tendermint", - "url": "https://github.com/tendermint/tools", - "technology": "Ansible", - "author": "Tendermint", - "description": "Ansible playbooks + Tendermint" - }, - { - "name": "brooklyn-tendermint", - "url": "https://github.com/cloudsoft/brooklyn-tendermint", - "technology": "Clocker for Apache Brooklyn ", - "author": "Cloudsoft", - "description": "Deploy a tendermint test network in docker containers " + "name": "JS-Amino", + "url": "https://github.com/TanNgocDo/Js-Amino", + "language": "Javascript", + "author": "TanNgocDo" } ] } diff --git a/docs/app-dev/ecosystem.md b/docs/app-dev/ecosystem.md index 7960e6c0..c87d3658 100644 --- a/docs/app-dev/ecosystem.md +++ b/docs/app-dev/ecosystem.md @@ -1,21 +1,9 @@ # Ecosystem The growing list of applications built using various pieces of the -Tendermint stack can be found at: +Tendermint stack can be found at the [ecosystem page](https://tendermint.com/ecosystem). -- https://tendermint.com/ecosystem - -We thank the community for their contributions thus far and welcome the +We thank the community for their contributions and welcome the addition of new projects. A pull request can be submitted to [this file](https://github.com/tendermint/tendermint/blob/master/docs/app-dev/ecosystem.json) to include your project. - -## Other Tools - -See [deploy testnets](./deploy-testnets) for information about all -the tools built by Tendermint. We have Kubernetes, Ansible, and -Terraform integrations. - -For upgrading from older to newer versions of tendermint and to migrate -your chain data, see [tm-migrator](https://github.com/hxzqlh/tm-tools) -written by @hxzqlh. diff --git a/docs/app-dev/getting-started.md b/docs/app-dev/getting-started.md index 14aa0dc3..5509a701 100644 --- a/docs/app-dev/getting-started.md +++ b/docs/app-dev/getting-started.md @@ -252,14 +252,12 @@ we'll run a Javascript version of the `counter`. To run it, you'll need to [install node](https://nodejs.org/en/download/). You'll also need to fetch the relevant repository, from -[here](https://github.com/tendermint/js-abci) then install it. As go -devs, we keep all our code under the `$GOPATH`, so run: +[here](https://github.com/tendermint/js-abci), then install it: ``` -go get github.com/tendermint/js-abci &> /dev/null -cd $GOPATH/src/github.com/tendermint/js-abci/example -npm install -cd .. +git clone https://github.com/tendermint/js-abci.git +cd js-abci +npm install abci ``` Kill the previous `counter` and `tendermint` processes. Now run the app: @@ -276,13 +274,16 @@ tendermint node ``` Once again, you should see blocks streaming by - but now, our -application is written in javascript! Try sending some transactions, and +application is written in Javascript! Try sending some transactions, and like before - the results should be the same: ``` -curl localhost:26657/broadcast_tx_commit?tx=0x00 # ok -curl localhost:26657/broadcast_tx_commit?tx=0x05 # invalid nonce -curl localhost:26657/broadcast_tx_commit?tx=0x01 # ok +# ok +curl localhost:26657/broadcast_tx_commit?tx=0x00 +# invalid nonce +curl localhost:26657/broadcast_tx_commit?tx=0x05 +# ok +curl localhost:26657/broadcast_tx_commit?tx=0x01 ``` Neat, eh? diff --git a/docs/app-dev/indexing-transactions.md b/docs/app-dev/indexing-transactions.md index 61c959ca..de8336a4 100644 --- a/docs/app-dev/indexing-transactions.md +++ b/docs/app-dev/indexing-transactions.md @@ -78,7 +78,7 @@ endpoint: curl "localhost:26657/tx_search?query=\"account.name='igor'\"&prove=true" ``` -Check out [API docs](https://tendermint.github.io/slate/?shell#txsearch) +Check out [API docs](https://tendermint.com/rpc/#txsearch) for more information on query syntax and other options. ## Subscribing to transactions @@ -97,5 +97,5 @@ by providing a query to `/subscribe` RPC endpoint. } ``` -Check out [API docs](https://tendermint.github.io/slate/#subscribe) for +Check out [API docs](https://tendermint.com/rpc/#subscribe) for more information on query syntax and other options. diff --git a/docs/app-dev/subscribing-to-events-via-websocket.md b/docs/app-dev/subscribing-to-events-via-websocket.md index 49954809..d745769c 100644 --- a/docs/app-dev/subscribing-to-events-via-websocket.md +++ b/docs/app-dev/subscribing-to-events-via-websocket.md @@ -54,7 +54,7 @@ Response: "value": "ww0z4WaZ0Xg+YI10w43wTWbBmM3dpVza4mmSQYsd0ck=" }, "voting_power": "10", - "accum": "0" + "proposer_priority": "0" } ] } diff --git a/docs/architecture/adr-001-logging.md b/docs/architecture/adr-001-logging.md index a11a49e1..77e5d39a 100644 --- a/docs/architecture/adr-001-logging.md +++ b/docs/architecture/adr-001-logging.md @@ -52,13 +52,13 @@ On top of this interface, we will need to implement a stdout logger, which will Many people say that they like the current output, so let's stick with it. ``` -NOTE[04-25|14:45:08] ABCI Replay Blocks module=consensus appHeight=0 storeHeight=0 stateHeight=0 +NOTE[2017-04-25|14:45:08] ABCI Replay Blocks module=consensus appHeight=0 storeHeight=0 stateHeight=0 ``` Couple of minor changes: ``` -I[04-25|14:45:08.322] ABCI Replay Blocks module=consensus appHeight=0 storeHeight=0 stateHeight=0 +I[2017-04-25|14:45:08.322] ABCI Replay Blocks module=consensus appHeight=0 storeHeight=0 stateHeight=0 ``` Notice the level is encoded using only one char plus milliseconds. @@ -155,14 +155,14 @@ Important keyvals should go first. Example: ``` correct -I[04-25|14:45:08.322] ABCI Replay Blocks module=consensus instance=1 appHeight=0 storeHeight=0 stateHeight=0 +I[2017-04-25|14:45:08.322] ABCI Replay Blocks module=consensus instance=1 appHeight=0 storeHeight=0 stateHeight=0 ``` not ``` wrong -I[04-25|14:45:08.322] ABCI Replay Blocks module=consensus appHeight=0 storeHeight=0 stateHeight=0 instance=1 +I[2017-04-25|14:45:08.322] ABCI Replay Blocks module=consensus appHeight=0 storeHeight=0 stateHeight=0 instance=1 ``` for that in most cases you'll need to add `instance` field to a logger upon creating, not when u log a particular message: diff --git a/docs/architecture/adr-008-priv-validator.md b/docs/architecture/adr-008-priv-validator.md index 94e882af..a8499465 100644 --- a/docs/architecture/adr-008-priv-validator.md +++ b/docs/architecture/adr-008-priv-validator.md @@ -5,14 +5,17 @@ implementations: - FilePV uses an unencrypted private key in a "priv_validator.json" file - no configuration required (just `tendermint init`). -- SocketPV uses a socket to send signing requests to another process - user is - responsible for starting that process themselves. +- TCPVal and IPCVal use TCP and Unix sockets respectively to send signing requests + to another process - the user is responsible for starting that process themselves. -The SocketPV address can be provided via flags at the command line - doing so -will cause Tendermint to ignore any "priv_validator.json" file and to listen on -the given address for incoming connections from an external priv_validator -process. It will halt any operation until at least one external process -succesfully connected. +Both TCPVal and IPCVal addresses can be provided via flags at the command line +or in the configuration file; TCPVal addresses must be of the form +`tcp://:` and IPCVal addresses `unix:///path/to/file.sock` - +doing so will cause Tendermint to ignore any private validator files. + +TCPVal will listen on the given address for incoming connections from an external +private validator process. It will halt any operation until at least one external +process successfully connected. The external priv_validator process will dial the address to connect to Tendermint, and then Tendermint will send requests on the ensuing connection to @@ -21,6 +24,9 @@ but the Tendermint process makes all requests. In a later stage we're going to support multiple validators for fault tolerance. To prevent double signing they need to be synced, which is deferred to an external solution (see #1185). +Conversely, IPCVal will make an outbound connection to an existing socket opened +by the external validator process. + In addition, Tendermint will provide implementations that can be run in that external process. These include: diff --git a/docs/architecture/adr-016-protocol-versions.md b/docs/architecture/adr-016-protocol-versions.md index 1ae1f467..3a2351a5 100644 --- a/docs/architecture/adr-016-protocol-versions.md +++ b/docs/architecture/adr-016-protocol-versions.md @@ -96,7 +96,7 @@ Each component of the software is independently versioned in a modular way and i ## Proposal -Each of BlockVersion, AppVersion, P2PVersion, is a monotonically increasing int64. +Each of BlockVersion, AppVersion, P2PVersion, is a monotonically increasing uint64. To use these versions, we need to update the block Header, the p2p NodeInfo, and the ABCI. @@ -106,8 +106,8 @@ Block Header should include a `Version` struct as its first field like: ``` type Version struct { - Block int64 - App int64 + Block uint64 + App uint64 } ``` @@ -130,9 +130,9 @@ NodeInfo should include a Version struct as its first field like: ``` type Version struct { - P2P int64 - Block int64 - App int64 + P2P uint64 + Block uint64 + App uint64 Other []string } @@ -168,9 +168,9 @@ RequestInfo should add support for protocol versions like: ``` message RequestInfo { - string software_version - int64 block_version - int64 p2p_version + string version + uint64 block_version + uint64 p2p_version } ``` @@ -180,39 +180,46 @@ Similarly, ResponseInfo should return the versions: message ResponseInfo { string data - string software_version - int64 app_version + string version + uint64 app_version int64 last_block_height bytes last_block_app_hash } ``` +The existing `version` fields should be called `software_version` but we leave +them for now to reduce the number of breaking changes. + #### EndBlock Updating the version could be done either with new fields or by using the existing `tags`. Since we're trying to communicate information that will be included in Tendermint block Headers, it should be native to the ABCI, and not -something embedded through some scheme in the tags. +something embedded through some scheme in the tags. Thus, version updates should +be communicated through EndBlock. -ResponseEndBlock will include a new field `version_updates`: +EndBlock already contains `ConsensusParams`. We can add version information to +the ConsensusParams as well: ``` -message ResponseEndBlock { - repeated Validator validator_updates - ConsensusParams consensus_param_updates - repeated common.KVPair tags +message ConsensusParams { - VersionUpdate version_update + BlockSize block_size + EvidenceParams evidence_params + VersionParams version } -message VersionUpdate { - int64 app_version +message VersionParams { + uint64 block_version + uint64 app_version } ``` -Tendermint will use the information in VersionUpdate for the next block it -proposes. +For now, the `block_version` will be ignored, as we do not allow block version +to be updated live. If the `app_version` is set, it signals that the app's +protocol version has changed, and the new `app_version` will be included in the +`Block.Header.Version.App` for the next block. ### BlockVersion diff --git a/docs/architecture/adr-020-block-size.md b/docs/architecture/adr-020-block-size.md index aebf3069..39385789 100644 --- a/docs/architecture/adr-020-block-size.md +++ b/docs/architecture/adr-020-block-size.md @@ -7,6 +7,7 @@ 28-08-2018: Third version after Ethan's comments 30-08-2018: AminoOverheadForBlock => MaxAminoOverheadForBlock 31-08-2018: Bounding evidence and chain ID +13-01-2019: Add section on MaxBytes vs MaxDataBytes ## Context @@ -20,6 +21,32 @@ We should just remove MaxTxs all together and stick with MaxBytes, and have a But we can't just reap BlockSize.MaxBytes, since MaxBytes is for the entire block, not for the txs inside the block. There's extra amino overhead + the actual headers on top of the actual transactions + evidence + last commit. +We could also consider using a MaxDataBytes instead of or in addition to MaxBytes. + +## MaxBytes vs MaxDataBytes + +The [PR #3045](https://github.com/tendermint/tendermint/pull/3045) suggested +additional clarity/justification was necessary here, wither respect to the use +of MaxDataBytes in addition to, or instead of, MaxBytes. + +MaxBytes provides a clear limit on the total size of a block that requires no +additional calculation if you want to use it to bound resource usage, and there +has been considerable discussions about optimizing tendermint around 1MB blocks. +Regardless, we need some maximum on the size of a block so we can avoid +unmarshalling blocks that are too big during the consensus, and it seems more +straightforward to provide a single fixed number for this rather than a +computation of "MaxDataBytes + everything else you need to make room for +(signatures, evidence, header)". MaxBytes provides a simple bound so we can +always say "blocks are less than X MB". + +Having both MaxBytes and MaxDataBytes feels like unnecessary complexity. It's +not particularly surprising for MaxBytes to imply the maximum size of the +entire block (not just txs), one just has to know that a block includes header, +txs, evidence, votes. For more fine grained control over the txs included in the +block, there is the MaxGas. In practice, the MaxGas may be expected to do most of +the tx throttling, and the MaxBytes to just serve as an upper bound on the total +size. Applications can use MaxGas as a MaxDataBytes by just taking the gas for +every tx to be its size in bytes. ## Proposed solution @@ -61,7 +88,7 @@ MaxXXX stayed the same. ## Status -Proposed. +Accepted. ## Consequences diff --git a/docs/architecture/adr-030-consensus-refactor.md b/docs/architecture/adr-030-consensus-refactor.md index d48cfe10..5c8c3d75 100644 --- a/docs/architecture/adr-030-consensus-refactor.md +++ b/docs/architecture/adr-030-consensus-refactor.md @@ -126,6 +126,312 @@ func TestConsensusXXX(t *testing.T) { } ``` + +## Consensus Executor + +## Consensus Core + +```go +type Event interface{} + +type EventNewHeight struct { + Height int64 + ValidatorId int +} + +type EventNewRound HeightAndRound + +type EventProposal struct { + Height int64 + Round int + Timestamp Time + BlockID BlockID + POLRound int + Sender int +} + +type Majority23PrevotesBlock struct { + Height int64 + Round int + BlockID BlockID +} + +type Majority23PrecommitBlock struct { + Height int64 + Round int + BlockID BlockID +} + +type HeightAndRound struct { + Height int64 + Round int +} + +type Majority23PrevotesAny HeightAndRound +type Majority23PrecommitAny HeightAndRound +type TimeoutPropose HeightAndRound +type TimeoutPrevotes HeightAndRound +type TimeoutPrecommit HeightAndRound + + +type Message interface{} + +type MessageProposal struct { + Height int64 + Round int + BlockID BlockID + POLRound int +} + +type VoteType int + +const ( + VoteTypeUnknown VoteType = iota + Prevote + Precommit +) + + +type MessageVote struct { + Height int64 + Round int + BlockID BlockID + Type VoteType +} + + +type MessageDecision struct { + Height int64 + Round int + BlockID BlockID +} + +type TriggerTimeout struct { + Height int64 + Round int + Duration Duration +} + + +type RoundStep int + +const ( + RoundStepUnknown RoundStep = iota + RoundStepPropose + RoundStepPrevote + RoundStepPrecommit + RoundStepCommit +) + +type State struct { + Height int64 + Round int + Step RoundStep + LockedValue BlockID + LockedRound int + ValidValue BlockID + ValidRound int + ValidatorId int + ValidatorSetSize int +} + +func proposer(height int64, round int) int {} +func getValue() BlockID {} + +func Consensus(event Event, state State) (State, Message, TriggerTimeout) { + msg = nil + timeout = nil + switch event := event.(type) { + case EventNewHeight: + if event.Height > state.Height { + state.Height = event.Height + state.Round = -1 + state.Step = RoundStepPropose + state.LockedValue = nil + state.LockedRound = -1 + state.ValidValue = nil + state.ValidRound = -1 + state.ValidatorId = event.ValidatorId + } + return state, msg, timeout + + case EventNewRound: + if event.Height == state.Height and event.Round > state.Round { + state.Round = eventRound + state.Step = RoundStepPropose + if proposer(state.Height, state.Round) == state.ValidatorId { + proposal = state.ValidValue + if proposal == nil { + proposal = getValue() + } + msg = MessageProposal { state.Height, state.Round, proposal, state.ValidRound } + } + timeout = TriggerTimeout { state.Height, state.Round, timeoutPropose(state.Round) } + } + return state, msg, timeout + + case EventProposal: + if event.Height == state.Height and event.Round == state.Round and + event.Sender == proposal(state.Height, state.Round) and state.Step == RoundStepPropose { + if event.POLRound >= state.LockedRound or event.BlockID == state.BlockID or state.LockedRound == -1 { + msg = MessageVote { state.Height, state.Round, event.BlockID, Prevote } + } + state.Step = RoundStepPrevote + } + return state, msg, timeout + + case TimeoutPropose: + if event.Height == state.Height and event.Round == state.Round and state.Step == RoundStepPropose { + msg = MessageVote { state.Height, state.Round, nil, Prevote } + state.Step = RoundStepPrevote + } + return state, msg, timeout + + case Majority23PrevotesBlock: + if event.Height == state.Height and event.Round == state.Round and state.Step >= RoundStepPrevote and event.Round > state.ValidRound { + state.ValidRound = event.Round + state.ValidValue = event.BlockID + if state.Step == RoundStepPrevote { + state.LockedRound = event.Round + state.LockedValue = event.BlockID + msg = MessageVote { state.Height, state.Round, event.BlockID, Precommit } + state.Step = RoundStepPrecommit + } + } + return state, msg, timeout + + case Majority23PrevotesAny: + if event.Height == state.Height and event.Round == state.Round and state.Step == RoundStepPrevote { + timeout = TriggerTimeout { state.Height, state.Round, timeoutPrevote(state.Round) } + } + return state, msg, timeout + + case TimeoutPrevote: + if event.Height == state.Height and event.Round == state.Round and state.Step == RoundStepPrevote { + msg = MessageVote { state.Height, state.Round, nil, Precommit } + state.Step = RoundStepPrecommit + } + return state, msg, timeout + + case Majority23PrecommitBlock: + if event.Height == state.Height { + state.Step = RoundStepCommit + state.LockedValue = event.BlockID + } + return state, msg, timeout + + case Majority23PrecommitAny: + if event.Height == state.Height and event.Round == state.Round { + timeout = TriggerTimeout { state.Height, state.Round, timeoutPrecommit(state.Round) } + } + return state, msg, timeout + + case TimeoutPrecommit: + if event.Height == state.Height and event.Round == state.Round { + state.Round = state.Round + 1 + } + return state, msg, timeout + } +} + +func ConsensusExecutor() { + proposal = nil + votes = HeightVoteSet { Height: 1 } + state = State { + Height: 1 + Round: 0 + Step: RoundStepPropose + LockedValue: nil + LockedRound: -1 + ValidValue: nil + ValidRound: -1 + } + + event = EventNewHeight {1, id} + state, msg, timeout = Consensus(event, state) + + event = EventNewRound {state.Height, 0} + state, msg, timeout = Consensus(event, state) + + if msg != nil { + send msg + } + + if timeout != nil { + trigger timeout + } + + for { + select { + case message := <- msgCh: + switch msg := message.(type) { + case MessageProposal: + + case MessageVote: + if msg.Height == state.Height { + newVote = votes.AddVote(msg) + if newVote { + switch msg.Type { + case Prevote: + prevotes = votes.Prevotes(msg.Round) + if prevotes.WeakCertificate() and msg.Round > state.Round { + event = EventNewRound { msg.Height, msg.Round } + state, msg, timeout = Consensus(event, state) + state = handleStateChange(state, msg, timeout) + } + + if blockID, ok = prevotes.TwoThirdsMajority(); ok and blockID != nil { + if msg.Round == state.Round and hasBlock(blockID) { + event = Majority23PrevotesBlock { msg.Height, msg.Round, blockID } + state, msg, timeout = Consensus(event, state) + state = handleStateChange(state, msg, timeout) + } + if proposal != nil and proposal.POLRound == msg.Round and hasBlock(blockID) { + event = EventProposal { + Height: state.Height + Round: state.Round + BlockID: blockID + POLRound: proposal.POLRound + Sender: message.Sender + } + state, msg, timeout = Consensus(event, state) + state = handleStateChange(state, msg, timeout) + } + } + + if prevotes.HasTwoThirdsAny() and msg.Round == state.Round { + event = Majority23PrevotesAny { msg.Height, msg.Round, blockID } + state, msg, timeout = Consensus(event, state) + state = handleStateChange(state, msg, timeout) + } + + case Precommit: + + } + } + } + case timeout := <- timeoutCh: + + case block := <- blockCh: + + } + } +} + +func handleStateChange(state, msg, timeout) State { + if state.Step == Commit { + state = ExecuteBlock(state.LockedValue) + } + if msg != nil { + send msg + } + if timeout != nil { + trigger timeout + } +} + +``` + ### Implementation roadmap * implement proposed implementation diff --git a/docs/architecture/adr-033-pubsub.md b/docs/architecture/adr-033-pubsub.md new file mode 100644 index 00000000..88922646 --- /dev/null +++ b/docs/architecture/adr-033-pubsub.md @@ -0,0 +1,247 @@ +# ADR 033: pubsub 2.0 + +Author: Anton Kaliaev (@melekes) + +## Changelog + +02-10-2018: Initial draft + +16-01-2019: Second version based on our conversation with Jae + +17-01-2019: Third version explaining how new design solves current issues + +25-01-2019: Fourth version to treat buffered and unbuffered channels differently + +## Context + +Since the initial version of the pubsub, there's been a number of issues +raised: [#951], [#1879], [#1880]. Some of them are high-level issues questioning the +core design choices made. Others are minor and mostly about the interface of +`Subscribe()` / `Publish()` functions. + +### Sync vs Async + +Now, when publishing a message to subscribers, we can do it in a goroutine: + +_using channels for data transmission_ +```go +for each subscriber { + out := subscriber.outc + go func() { + out <- msg + } +} +``` + +_by invoking callback functions_ +```go +for each subscriber { + go subscriber.callbackFn() +} +``` + +This gives us greater performance and allows us to avoid "slow client problem" +(when other subscribers have to wait for a slow subscriber). A pool of +goroutines can be used to avoid uncontrolled memory growth. + +In certain cases, this is what you want. But in our case, because we need +strict ordering of events (if event A was published before B, the guaranteed +delivery order will be A -> B), we can't publish msg in a new goroutine every time. + +We can also have a goroutine per subscriber, although we'd need to be careful +with the number of subscribers. It's more difficult to implement as well + +unclear if we'll benefit from it (cause we'd be forced to create N additional +channels to distribute msg to these goroutines). + +### Non-blocking send + +There is also a question whenever we should have a non-blocking send. +Currently, sends are blocking, so publishing to one client can block on +publishing to another. This means a slow or unresponsive client can halt the +system. Instead, we can use a non-blocking send: + +```go +for each subscriber { + out := subscriber.outc + select { + case out <- msg: + default: + log("subscriber %v buffer is full, skipping...") + } +} +``` + +This fixes the "slow client problem", but there is no way for a slow client to +know if it had missed a message. We could return a second channel and close it +to indicate subscription termination. On the other hand, if we're going to +stick with blocking send, **devs must always ensure subscriber's handling code +does not block**, which is a hard task to put on their shoulders. + +The interim option is to run goroutines pool for a single message, wait for all +goroutines to finish. This will solve "slow client problem", but we'd still +have to wait `max(goroutine_X_time)` before we can publish the next message. + +### Channels vs Callbacks + +Yet another question is whether we should use channels for message transmission or +call subscriber-defined callback functions. Callback functions give subscribers +more flexibility - you can use mutexes in there, channels, spawn goroutines, +anything you really want. But they also carry local scope, which can result in +memory leaks and/or memory usage increase. + +Go channels are de-facto standard for carrying data between goroutines. + +### Why `Subscribe()` accepts an `out` channel? + +Because in our tests, we create buffered channels (cap: 1). Alternatively, we +can make capacity an argument and return a channel. + +## Decision + +### MsgAndTags + +Use a `MsgAndTags` struct on the subscription channel to indicate what tags the +msg matched. + +```go +type MsgAndTags struct { + Msg interface{} + Tags TagMap +} +``` + +### Subscription Struct + + +Change `Subscribe()` function to return a `Subscription` struct: + +```go +type Subscription struct { + // private fields +} + +func (s *Subscription) Out() <-chan MsgAndTags +func (s *Subscription) Cancelled() <-chan struct{} +func (s *Subscription) Err() error +``` + +`Out()` returns a channel onto which messages and tags are published. +`Unsubscribe`/`UnsubscribeAll` does not close the channel to avoid clients from +receiving a nil message. + +`Cancelled()` returns a channel that's closed when the subscription is terminated +and supposed to be used in a select statement. + +If the channel returned by `Cancelled()` is not closed yet, `Err()` returns nil. +If the channel is closed, `Err()` returns a non-nil error explaining why: +`ErrUnsubscribed` if the subscriber choose to unsubscribe, +`ErrOutOfCapacity` if the subscriber is not pulling messages fast enough and the channel returned by `Out()` became full. +After `Err()` returns a non-nil error, successive calls to `Err() return the same error. + +```go +subscription, err := pubsub.Subscribe(...) +if err != nil { + // ... +} +for { +select { + case msgAndTags <- subscription.Out(): + // ... + case <-subscription.Cancelled(): + return subscription.Err() +} +``` + +### Capacity and Subscriptions + +Make the `Out()` channel buffered (with capacity 1) by default. In most cases, we want to +terminate the slow subscriber. Only in rare cases, we want to block the pubsub +(e.g. when debugging consensus). This should lower the chances of the pubsub +being frozen. + +```go +// outCap can be used to set capacity of Out channel +// (1 by default, must be greater than 0). +Subscribe(ctx context.Context, clientID string, query Query, outCap... int) (Subscription, error) { +``` + +Use a different function for an unbuffered channel: + +```go +// Subscription uses an unbuffered channel. Publishing will block. +SubscribeUnbuffered(ctx context.Context, clientID string, query Query) (Subscription, error) { +``` + +SubscribeUnbuffered should not be exposed to users. + +### Blocking/Nonblocking + +The publisher should treat these kinds of channels separately. +It should block on unbuffered channels (for use with internal consensus events +in the consensus tests) and not block on the buffered ones. If a client is too +slow to keep up with it's messages, it's subscription is terminated: + +for each subscription { + out := subscription.outChan + if cap(out) == 0 { + // block on unbuffered channel + out <- msg + } else { + // don't block on buffered channels + select { + case out <- msg: + default: + // set the error, notify on the cancel chan + subscription.err = fmt.Errorf("client is too slow for msg) + close(subscription.cancelChan) + + // ... unsubscribe and close out + } + } +} + +### How this new design solves the current issues? + +[#951] ([#1880]): + +Because of non-blocking send, situation where we'll deadlock is not possible +anymore. If the client stops reading messages, it will be removed. + +[#1879]: + +MsgAndTags is used now instead of a plain message. + +### Future problems and their possible solutions + +[#2826] + +One question I am still pondering about: how to prevent pubsub from slowing +down consensus. We can increase the pubsub queue size (which is 0 now). Also, +it's probably a good idea to limit the total number of subscribers. + +This can be made automatically. Say we set queue size to 1000 and, when it's >= +80% full, refuse new subscriptions. + +## Status + +In review + +## Consequences + +### Positive + +- more idiomatic interface +- subscribers know what tags msg was published with +- subscribers aware of the reason their subscription was cancelled + +### Negative + +- (since v1) no concurrency when it comes to publishing messages + +### Neutral + + +[#951]: https://github.com/tendermint/tendermint/issues/951 +[#1879]: https://github.com/tendermint/tendermint/issues/1879 +[#1880]: https://github.com/tendermint/tendermint/issues/1880 +[#2826]: https://github.com/tendermint/tendermint/issues/2826 diff --git a/docs/architecture/adr-034-priv-validator-file-structure.md b/docs/architecture/adr-034-priv-validator-file-structure.md new file mode 100644 index 00000000..83160bfb --- /dev/null +++ b/docs/architecture/adr-034-priv-validator-file-structure.md @@ -0,0 +1,72 @@ +# ADR 034: PrivValidator file structure + +## Changelog + +03-11-2018: Initial Draft + +## Context + +For now, the PrivValidator file `priv_validator.json` contains mutable and immutable parts. +Even in an insecure mode which does not encrypt private key on disk, it is reasonable to separate +the mutable part and immutable part. + +References: +[#1181](https://github.com/tendermint/tendermint/issues/1181) +[#2657](https://github.com/tendermint/tendermint/issues/2657) +[#2313](https://github.com/tendermint/tendermint/issues/2313) + +## Proposed Solution + +We can split mutable and immutable parts with two structs: +```go +// FilePVKey stores the immutable part of PrivValidator +type FilePVKey struct { + Address types.Address `json:"address"` + PubKey crypto.PubKey `json:"pub_key"` + PrivKey crypto.PrivKey `json:"priv_key"` + + filePath string +} + +// FilePVState stores the mutable part of PrivValidator +type FilePVLastSignState struct { + Height int64 `json:"height"` + Round int `json:"round"` + Step int8 `json:"step"` + Signature []byte `json:"signature,omitempty"` + SignBytes cmn.HexBytes `json:"signbytes,omitempty"` + + filePath string + mtx sync.Mutex +} +``` + +Then we can combine `FilePVKey` with `FilePVLastSignState` and will get the original `FilePV`. + +```go +type FilePV struct { + Key FilePVKey + LastSignState FilePVLastSignState +} +``` + +As discussed, `FilePV` should be located in `config`, and `FilePVLastSignState` should be stored in `data`. The +store path of each file should be specified in `config.yml`. + +What we need to do next is changing the methods of `FilePV`. + +## Status + +Draft. + +## Consequences + +### Positive + +- separate the mutable and immutable of PrivValidator + +### Negative + +- need to add more config for file path + +### Neutral diff --git a/docs/architecture/adr-035-documentation.md b/docs/architecture/adr-035-documentation.md new file mode 100644 index 00000000..92cb0791 --- /dev/null +++ b/docs/architecture/adr-035-documentation.md @@ -0,0 +1,40 @@ +# ADR 035: Documentation + +Author: @zramsay (Zach Ramsay) + +## Changelog + +### November 2nd 2018 + +- initial write-up + +## Context + +The Tendermint documentation has undergone several changes until settling on the current model. Originally, the documentation was hosted on the website and had to be updated asynchronously from the code. Along with the other repositories requiring documentation, the whole stack moved to using Read The Docs to automatically generate, publish, and host the documentation. This, however, was insufficient; the RTD site had advertisement, it wasn't easily accessible to devs, didn't collect metrics, was another set of external links, etc. + +## Decision + +For two reasons, the decision was made to use VuePress: + +1) ability to get metrics (implemented on both Tendermint and SDK) +2) host the documentation on the website as a `/docs` endpoint. + +This is done while maintaining synchrony between the docs and code, i.e., the website is built whenever the docs are updated. + +## Status + +The two points above have been implemented; the `config.js` has a Google Analytics identifier and the documentation workflow has been up and running largely without problems for several months. Details about the documentation build & workflow can be found [here](../DOCS_README.md) + +## Consequences + +Because of the organizational seperation between Tendermint & Cosmos, there is a challenge of "what goes where" for certain aspects of documentation. + +### Positive + +This architecture is largely positive relative to prior docs arrangements. + +### Negative + +A significant portion of the docs automation / build process is in private repos with limited access/visibility to devs. However, these tasks are handled by the SRE team. + +### Neutral diff --git a/docs/architecture/adr-template.md b/docs/architecture/adr-template.md index d47c7f55..28a5ecfb 100644 --- a/docs/architecture/adr-template.md +++ b/docs/architecture/adr-template.md @@ -1,15 +1,36 @@ -# ADR 000: Template for an ADR +# ADR {ADR-NUMBER}: {TITLE} + +## Changelog +* {date}: {changelog} ## Context +> This section contains all the context one needs to understand the current state, and why there is a problem. It should be as succinct as possible and introduce the high level idea behind the solution. ## Decision +> This section explains all of the details of the proposed solution, including implementation details. +It should also describe affects / corollary items that may need to be changed as a part of this. +If the proposed change will be large, please also indicate a way to do the change to maximize ease of review. +(e.g. the optimal split of things to do between separate PR's) + ## Status +> A decision may be "proposed" if it hasn't been agreed upon yet, or "accepted" once it is agreed upon. If a later ADR changes or reverses a decision, it may be marked as "deprecated" or "superseded" with a reference to its replacement. + +{Deprecated|Proposed|Accepted} + ## Consequences +> This section describes the consequences, after applying the decision. All consequences should be summarized here, not just the "positive" ones. + ### Positive ### Negative ### Neutral + +## References + +> Are there any relevant PR comments, issues that led up to this, or articles referrenced for why we made the given design choice? If so link them here! + +* {reference link} diff --git a/docs/imgs/cosmos-tendermint-stack-4k.jpg b/docs/imgs/cosmos-tendermint-stack-4k.jpg new file mode 100644 index 00000000..b10ffa6a Binary files /dev/null and b/docs/imgs/cosmos-tendermint-stack-4k.jpg differ diff --git a/docs/introduction/install.md b/docs/introduction/install.md index f3498514..3005a734 100644 --- a/docs/introduction/install.md +++ b/docs/introduction/install.md @@ -79,11 +79,9 @@ make install Install [LevelDB](https://github.com/google/leveldb) (minimum version is 1.7). -Build Tendermint with C libraries: `make build_c`. - ### Ubuntu -Install LevelDB with snappy: +Install LevelDB with snappy (optionally): ``` sudo apt-get update @@ -95,9 +93,9 @@ wget https://github.com/google/leveldb/archive/v1.20.tar.gz && \ tar -zxvf v1.20.tar.gz && \ cd leveldb-1.20/ && \ make && \ - cp -r out-static/lib* out-shared/lib* /usr/local/lib/ && \ + sudo cp -r out-static/lib* out-shared/lib* /usr/local/lib/ && \ cd include/ && \ - cp -r leveldb /usr/local/include/ && \ + sudo cp -r leveldb /usr/local/include/ && \ sudo ldconfig && \ rm -f v1.20.tar.gz ``` @@ -109,8 +107,16 @@ Set database backend to cleveldb: db_backend = "cleveldb" ``` -To build Tendermint, run +To install Tendermint, run ``` -CGO_LDFLAGS="-lsnappy" go build -ldflags "-X github.com/tendermint/tendermint/version.GitCommit=`git rev-parse --short=8 HEAD`" -tags "tendermint gcc" -o build/tendermint ./cmd/tendermint/ +CGO_LDFLAGS="-lsnappy" make install_c ``` + +or run + +``` +CGO_LDFLAGS="-lsnappy" make build_c +``` + +to put the binary in `./build`. diff --git a/docs/introduction/quick-start.md b/docs/introduction/quick-start.md index 05facadf..b77e7273 100644 --- a/docs/introduction/quick-start.md +++ b/docs/introduction/quick-start.md @@ -40,7 +40,11 @@ These files are found in `$HOME/.tendermint`: ``` $ ls $HOME/.tendermint -config.toml data genesis.json priv_validator.json +config data + +$ ls $HOME/.tendermint/config/ + +config.toml genesis.json node_key.json priv_validator.json ``` For a single, local node, no further configuration is required. @@ -110,7 +114,18 @@ source ~/.profile This will install `go` and other dependencies, get the Tendermint source code, then compile the `tendermint` binary. -Next, use the `tendermint testnet` command to create four directories of config files (found in `./mytestnet`) and copy each directory to the relevant machine in the cloud, so that each machine has `$HOME/mytestnet/node[0-3]` directory. Then from each machine, run: +Next, use the `tendermint testnet` command to create four directories of config files (found in `./mytestnet`) and copy each directory to the relevant machine in the cloud, so that each machine has `$HOME/mytestnet/node[0-3]` directory. + +Before you can start the network, you'll need peers identifiers (IPs are not enough and can change). We'll refer to them as ID1, ID2, ID3, ID4. + +``` +tendermint show_node_id --home ./mytestnet/node0 +tendermint show_node_id --home ./mytestnet/node1 +tendermint show_node_id --home ./mytestnet/node2 +tendermint show_node_id --home ./mytestnet/node3 +``` + +Finally, from each machine, run: ``` tendermint node --home ./mytestnet/node0 --proxy_app=kvstore --p2p.persistent_peers="ID1@IP1:26656,ID2@IP2:26656,ID3@IP3:26656,ID4@IP4:26656" @@ -121,6 +136,6 @@ tendermint node --home ./mytestnet/node3 --proxy_app=kvstore --p2p.persistent_pe Note that after the third node is started, blocks will start to stream in because >2/3 of validators (defined in the `genesis.json`) have come online. -Seeds can also be specified in the `config.toml`. See [here](../tendermint-core/configuration.md) for more information about configuration options. +Persistent peers can also be specified in the `config.toml`. See [here](../tendermint-core/configuration.md) for more information about configuration options. Transactions can then be sent as covered in the single, local node example above. diff --git a/docs/introduction/what-is-tendermint.md b/docs/introduction/what-is-tendermint.md index 389bf965..a35dd9ec 100644 --- a/docs/introduction/what-is-tendermint.md +++ b/docs/introduction/what-is-tendermint.md @@ -70,10 +70,6 @@ Tendermint is in essence similar software, but with two key differences: the application logic that's right for them, from key-value store to cryptocurrency to e-voting platform and beyond. -The layout of this Tendermint website content is also ripped directly -and without shame from [consul.io](https://www.consul.io/) and the other -[Hashicorp sites](https://www.hashicorp.com/#tools). - ### Bitcoin, Ethereum, etc. Tendermint emerged in the tradition of cryptocurrencies like Bitcoin, diff --git a/docs/networks/docker-compose.md b/docs/networks/docker-compose.md index a1924eb9..7e4adde8 100644 --- a/docs/networks/docker-compose.md +++ b/docs/networks/docker-compose.md @@ -1,20 +1,17 @@ # Docker Compose -With Docker Compose, we can spin up local testnets in a single command: - -``` -make localnet-start -``` +With Docker Compose, you can spin up local testnets with a single command. ## Requirements -- [Install tendermint](/docs/install.md) -- [Install docker](https://docs.docker.com/engine/installation/) -- [Install docker-compose](https://docs.docker.com/compose/install/) +1. [Install tendermint](/docs/introduction/install.md) +2. [Install docker](https://docs.docker.com/engine/installation/) +3. [Install docker-compose](https://docs.docker.com/compose/install/) ## Build -Build the `tendermint` binary and the `tendermint/localnode` docker image. +Build the `tendermint` binary and, optionally, the `tendermint/localnode` +docker image. Note the binary will be mounted into the container so it can be updated without rebuilding the image. @@ -25,11 +22,10 @@ cd $GOPATH/src/github.com/tendermint/tendermint # Build the linux binary in ./build make build-linux -# Build tendermint/localnode image +# (optionally) Build tendermint/localnode image make build-docker-localnode ``` - ## Run a testnet To start a 4 node testnet run: @@ -38,9 +34,13 @@ To start a 4 node testnet run: make localnet-start ``` -The nodes bind their RPC servers to ports 26657, 26660, 26662, and 26664 on the host. +The nodes bind their RPC servers to ports 26657, 26660, 26662, and 26664 on the +host. + This file creates a 4-node network using the localnode image. -The nodes of the network expose their P2P and RPC endpoints to the host machine on ports 26656-26657, 26659-26660, 26661-26662, and 26663-26664 respectively. + +The nodes of the network expose their P2P and RPC endpoints to the host machine +on ports 26656-26657, 26659-26660, 26661-26662, and 26663-26664 respectively. To update the binary, just rebuild it and restart the nodes: @@ -52,34 +52,112 @@ make localnet-start ## Configuration -The `make localnet-start` creates files for a 4-node testnet in `./build` by calling the `tendermint testnet` command. +The `make localnet-start` creates files for a 4-node testnet in `./build` by +calling the `tendermint testnet` command. -The `./build` directory is mounted to the `/tendermint` mount point to attach the binary and config files to the container. +The `./build` directory is mounted to the `/tendermint` mount point to attach +the binary and config files to the container. -For instance, to create a single node testnet: +To change the number of validators / non-validators change the `localnet-start` Makefile target: + +``` +localnet-start: localnet-stop + @if ! [ -f build/node0/config/genesis.json ]; then docker run --rm -v $(CURDIR)/build:/tendermint:Z tendermint/localnode testnet --v 5 --n 3 --o . --populate-persistent-peers --starting-ip-address 192.167.10.2 ; fi + docker-compose up +``` + +The command now will generate config files for 5 validators and 3 +non-validators network. + +Before running it, don't forget to cleanup the old files: ``` cd $GOPATH/src/github.com/tendermint/tendermint # Clear the build folder -rm -rf ./build +rm -rf ./build/node* +``` -# Build binary -make build-linux +## Configuring abci containers -# Create configuration -docker run -e LOG="stdout" -v `pwd`/build:/tendermint tendermint/localnode testnet --o . --v 1 +To use your own abci applications with 4-node setup edit the [docker-compose.yaml](https://github.com/tendermint/tendermint/blob/develop/docker-compose.yml) file and add image to your abci application. -#Run the node -docker run -v `pwd`/build:/tendermint tendermint/localnode +``` + abci0: + container_name: abci0 + image: "abci-image" + build: + context: . + dockerfile: abci.Dockerfile + command: + networks: + localnet: + ipv4_address: 192.167.10.6 + + abci1: + container_name: abci1 + image: "abci-image" + build: + context: . + dockerfile: abci.Dockerfile + command: + networks: + localnet: + ipv4_address: 192.167.10.7 + + abci2: + container_name: abci2 + image: "abci-image" + build: + context: . + dockerfile: abci.Dockerfile + command: + networks: + localnet: + ipv4_address: 192.167.10.8 + + abci3: + container_name: abci3 + image: "abci-image" + build: + context: . + dockerfile: abci.Dockerfile + command: + networks: + localnet: + ipv4_address: 192.167.10.9 ``` +Override the [command](https://github.com/tendermint/tendermint/blob/master/networks/local/localnode/Dockerfile#L12) in each node to connect to it's abci. + +``` + node0: + container_name: node0 + image: "tendermint/localnode" + ports: + - "26656-26657:26656-26657" + environment: + - ID=0 + - LOG=$${LOG:-tendermint.log} + volumes: + - ./build:/tendermint:Z + command: node --proxy_app=tcp://abci0:26658 + networks: + localnet: + ipv4_address: 192.167.10.2 +``` + +Similarly do for node1, node2 and node3 then [run testnet](https://github.com/tendermint/tendermint/blob/master/docs/networks/docker-compose.md#run-a-testnet) + ## Logging -Log is saved under the attached volume, in the `tendermint.log` file. If the `LOG` environment variable is set to `stdout` at start, the log is not saved, but printed on the screen. +Log is saved under the attached volume, in the `tendermint.log` file. If the +`LOG` environment variable is set to `stdout` at start, the log is not saved, +but printed on the screen. ## Special binaries -If you have multiple binaries with different names, you can specify which one to run with the BINARY environment variable. The path of the binary is relative to the attached volume. - +If you have multiple binaries with different names, you can specify which one +to run with the `BINARY` environment variable. The path of the binary is relative +to the attached volume. diff --git a/docs/package-lock.json b/docs/package-lock.json deleted file mode 100644 index 3449eda1..00000000 --- a/docs/package-lock.json +++ /dev/null @@ -1,4670 +0,0 @@ -{ - "name": "tendermint", - "version": "0.0.1", - "lockfileVersion": 1, - "requires": true, - "dependencies": { - "@azu/format-text": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/@azu/format-text/-/format-text-1.0.1.tgz", - "integrity": "sha1-aWc1CpRkD2sChVFpvYl85U1s6+I=" - }, - "@azu/style-format": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/@azu/style-format/-/style-format-1.0.0.tgz", - "integrity": "sha1-5wGH+Khi4ZGxvObAJo8TrNOlayA=", - "requires": { - "@azu/format-text": "^1.0.1" - } - }, - "@sindresorhus/is": { - "version": "0.7.0", - "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-0.7.0.tgz", - "integrity": "sha512-ONhaKPIufzzrlNbqtWFFd+jlnemX6lJAgq9ZeiZtS7I1PIf/la7CW4m83rTXRnVnsMbW2k56pGYu7AUFJD9Pow==" - }, - "@textlint/ast-node-types": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/@textlint/ast-node-types/-/ast-node-types-4.0.3.tgz", - "integrity": "sha512-mkkqbuxZkCESmMCrVN5QEgmFqBJAcoAGIaZaQfziqKAyCQBLLgKVJzeFuup9mDm9mvCTKekhLk9yIaEFc8EFxA==" - }, - "@textlint/ast-traverse": { - "version": "2.0.9", - "resolved": "https://registry.npmjs.org/@textlint/ast-traverse/-/ast-traverse-2.0.9.tgz", - "integrity": "sha512-E2neVj65wyadt3hr9R+DHW01dG4dNOMmFRab7Bph/rkDDeK85w/6RNJgIt9vBCPtt7a4bndTj1oZrK6wDZAEtQ==", - "requires": { - "@textlint/ast-node-types": "^4.0.3" - } - }, - "@textlint/feature-flag": { - "version": "3.0.5", - "resolved": "https://registry.npmjs.org/@textlint/feature-flag/-/feature-flag-3.0.5.tgz", - "integrity": "sha512-hXTDGvltgiUtJs7QhALSILNE+g0cdY4CyqHR2r5+EmiYbS3NuqWVLn3GZYUPWXl9rVDky/IpR+6DF0uLJF8m8Q==", - "requires": { - "map-like": "^2.0.0" - } - }, - "@textlint/fixer-formatter": { - "version": "3.0.8", - "resolved": "https://registry.npmjs.org/@textlint/fixer-formatter/-/fixer-formatter-3.0.8.tgz", - "integrity": "sha512-LTHcCLTyESdz90NGYzrYC0juSqLzGBc5VMMRO8Xvz3fapBya/Sn5ncgvsHqnKY0OIbV/IdOT54G2F46D8R6P9Q==", - "requires": { - "@textlint/kernel": "^3.0.0", - "chalk": "^1.1.3", - "debug": "^2.1.0", - "diff": "^2.2.2", - "interop-require": "^1.0.0", - "is-file": "^1.0.0", - "string-width": "^1.0.1", - "text-table": "^0.2.0", - "try-resolve": "^1.0.1" - }, - "dependencies": { - "@textlint/kernel": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@textlint/kernel/-/kernel-3.0.0.tgz", - "integrity": "sha512-SxHWr6VAD/SdqTCy1uB03bFLbGYbhZeQTeUuIJE6s1pD7wtQ1+Y1n8nx9I9m7nqGZi5eYuVA6WnpvCq10USz+w==", - "requires": { - "@textlint/ast-node-types": "^4.0.3", - "@textlint/ast-traverse": "^2.0.9", - "@textlint/feature-flag": "^3.0.5", - "@types/bluebird": "^3.5.18", - "bluebird": "^3.5.1", - "debug": "^2.6.6", - "deep-equal": "^1.0.1", - "map-like": "^2.0.0", - "object-assign": "^4.1.1", - "structured-source": "^3.0.2" - } - }, - "ansi-styles": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz", - "integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4=" - }, - "chalk": { - "version": "1.1.3", - "resolved": "http://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", - "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=", - "requires": { - "ansi-styles": "^2.2.1", - "escape-string-regexp": "^1.0.2", - "has-ansi": "^2.0.0", - "strip-ansi": "^3.0.0", - "supports-color": "^2.0.0" - } - }, - "supports-color": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", - "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=" - } - } - }, - "@textlint/kernel": { - "version": "2.0.9", - "resolved": "https://registry.npmjs.org/@textlint/kernel/-/kernel-2.0.9.tgz", - "integrity": "sha512-0237/9yDIlSVaH0pcVAxm0rV1xF96UpjXUXoBRdciWnf2+O0tWQEeBC9B2/B2jLw9Ha0zGlK+q+bLREpXB97Cw==", - "requires": { - "@textlint/ast-node-types": "^4.0.2", - "@textlint/ast-traverse": "^2.0.8", - "@textlint/feature-flag": "^3.0.4", - "@types/bluebird": "^3.5.18", - "bluebird": "^3.5.1", - "debug": "^2.6.6", - "deep-equal": "^1.0.1", - "object-assign": "^4.1.1", - "structured-source": "^3.0.2" - } - }, - "@textlint/linter-formatter": { - "version": "3.0.8", - "resolved": "https://registry.npmjs.org/@textlint/linter-formatter/-/linter-formatter-3.0.8.tgz", - "integrity": "sha512-hayZi4ybj01Km9Soi34cT8EkmEcqGgQKHu1tvPQVd8S2zaE3m/8nmf6qhwAo/HAwMzbIj0XxdV8nVuiUfz8ADQ==", - "requires": { - "@azu/format-text": "^1.0.1", - "@azu/style-format": "^1.0.0", - "@textlint/kernel": "^3.0.0", - "chalk": "^1.0.0", - "concat-stream": "^1.5.1", - "js-yaml": "^3.2.4", - "optionator": "^0.8.1", - "pluralize": "^2.0.0", - "string-width": "^1.0.1", - "string.prototype.padstart": "^3.0.0", - "strip-ansi": "^3.0.1", - "table": "^3.7.8", - "text-table": "^0.2.0", - "try-resolve": "^1.0.1", - "xml-escape": "^1.0.0" - }, - "dependencies": { - "@textlint/kernel": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@textlint/kernel/-/kernel-3.0.0.tgz", - "integrity": "sha512-SxHWr6VAD/SdqTCy1uB03bFLbGYbhZeQTeUuIJE6s1pD7wtQ1+Y1n8nx9I9m7nqGZi5eYuVA6WnpvCq10USz+w==", - "requires": { - "@textlint/ast-node-types": "^4.0.3", - "@textlint/ast-traverse": "^2.0.9", - "@textlint/feature-flag": "^3.0.5", - "@types/bluebird": "^3.5.18", - "bluebird": "^3.5.1", - "debug": "^2.6.6", - "deep-equal": "^1.0.1", - "map-like": "^2.0.0", - "object-assign": "^4.1.1", - "structured-source": "^3.0.2" - } - }, - "ansi-styles": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz", - "integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4=" - }, - "chalk": { - "version": "1.1.3", - "resolved": "http://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", - "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=", - "requires": { - "ansi-styles": "^2.2.1", - "escape-string-regexp": "^1.0.2", - "has-ansi": "^2.0.0", - "strip-ansi": "^3.0.0", - "supports-color": "^2.0.0" - } - }, - "supports-color": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", - "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=" - } - } - }, - "@textlint/markdown-to-ast": { - "version": "6.0.9", - "resolved": "https://registry.npmjs.org/@textlint/markdown-to-ast/-/markdown-to-ast-6.0.9.tgz", - "integrity": "sha512-hfAWBvTeUGh5t5kTn2U3uP3qOSM1BSrxzl1jF3nn0ywfZXpRBZr5yRjXnl4DzIYawCtZOshmRi/tI3/x4TE1jQ==", - "requires": { - "@textlint/ast-node-types": "^4.0.3", - "debug": "^2.1.3", - "remark-frontmatter": "^1.2.0", - "remark-parse": "^5.0.0", - "structured-source": "^3.0.2", - "traverse": "^0.6.6", - "unified": "^6.1.6" - } - }, - "@textlint/text-to-ast": { - "version": "3.0.9", - "resolved": "https://registry.npmjs.org/@textlint/text-to-ast/-/text-to-ast-3.0.9.tgz", - "integrity": "sha512-0Vycl2XtGv3pUtUNkBn9M/e3jBAtmlh7STUa3GuiyATXg49PsqqX7c8NxGPrNqMvDYCJ3ZubBx8GSEyra6ZWFw==", - "requires": { - "@textlint/ast-node-types": "^4.0.3" - } - }, - "@textlint/textlint-plugin-markdown": { - "version": "4.0.10", - "resolved": "https://registry.npmjs.org/@textlint/textlint-plugin-markdown/-/textlint-plugin-markdown-4.0.10.tgz", - "integrity": "sha512-HIV2UAhjnt9/tJQbuXkrD3CRiEFRtNpYoQEZCNCwd1nBMWUypAFthL9jT1KJ8tagOF7wEiGMB19QfDxiNQ+6mw==", - "requires": { - "@textlint/markdown-to-ast": "^6.0.8" - } - }, - "@textlint/textlint-plugin-text": { - "version": "3.0.10", - "resolved": "https://registry.npmjs.org/@textlint/textlint-plugin-text/-/textlint-plugin-text-3.0.10.tgz", - "integrity": "sha512-GSw9vsuKt7E85jDSFEXT0VYZo4C3e8XFFrSWYqXlwPKl/oQ/WHQfMg7GM288uGoEaMzbKEfBtpdwdZqTjGHOQA==", - "requires": { - "@textlint/text-to-ast": "^3.0.8" - } - }, - "@types/bluebird": { - "version": "3.5.24", - "resolved": "https://registry.npmjs.org/@types/bluebird/-/bluebird-3.5.24.tgz", - "integrity": "sha512-YeQoDpq4Lm8ppSBqAnAeF/xy1cYp/dMTif2JFcvmAbETMRlvKHT2iLcWu+WyYiJO3b3Ivokwo7EQca/xfLVJmg==" - }, - "adverb-where": { - "version": "0.0.9", - "resolved": "https://registry.npmjs.org/adverb-where/-/adverb-where-0.0.9.tgz", - "integrity": "sha1-CcXN3Y1QO5/l924LjcXHCo8ZPjQ=" - }, - "aggregate-error": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-1.0.0.tgz", - "integrity": "sha1-iINE2tAiCnLjr1CQYRf0h3GSX6w=", - "requires": { - "clean-stack": "^1.0.0", - "indent-string": "^3.0.0" - } - }, - "ajv": { - "version": "5.5.2", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-5.5.2.tgz", - "integrity": "sha1-c7Xuyj+rZT49P5Qis0GtQiBdyWU=", - "requires": { - "co": "^4.6.0", - "fast-deep-equal": "^1.0.0", - "fast-json-stable-stringify": "^2.0.0", - "json-schema-traverse": "^0.3.0" - } - }, - "ajv-keywords": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-1.5.1.tgz", - "integrity": "sha1-MU3QpLM2j609/NxU7eYXG4htrzw=" - }, - "ansi-regex": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", - "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=" - }, - "ansi-styles": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", - "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", - "requires": { - "color-convert": "^1.9.0" - } - }, - "anymatch": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-1.3.2.tgz", - "integrity": "sha512-0XNayC8lTHQ2OI8aljNCN3sSx6hsr/1+rlcDAotXJR7C1oZZHCNsfpbKwMjRA3Uqb5tF1Rae2oloTr4xpq+WjA==", - "requires": { - "micromatch": "^2.1.5", - "normalize-path": "^2.0.0" - } - }, - "argparse": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", - "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", - "requires": { - "sprintf-js": "~1.0.2" - } - }, - "arr-diff": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/arr-diff/-/arr-diff-2.0.0.tgz", - "integrity": "sha1-jzuCf5Vai9ZpaX5KQlasPOrjVs8=", - "requires": { - "arr-flatten": "^1.0.1" - } - }, - "arr-flatten": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/arr-flatten/-/arr-flatten-1.1.0.tgz", - "integrity": "sha512-L3hKV5R/p5o81R7O02IGnwpDmkp6E982XhtbuwSe3O4qOtMMMtodicASA1Cny2U+aCXcNpml+m4dPsvsJ3jatg==" - }, - "arr-union": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/arr-union/-/arr-union-3.1.0.tgz", - "integrity": "sha1-45sJrqne+Gao8gbiiK9jkZuuOcQ=" - }, - "array-union": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/array-union/-/array-union-1.0.2.tgz", - "integrity": "sha1-mjRBDk9OPaI96jdb5b5w8kd47Dk=", - "requires": { - "array-uniq": "^1.0.1" - } - }, - "array-uniq": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/array-uniq/-/array-uniq-1.0.3.tgz", - "integrity": "sha1-r2rId6Jcx/dOBYiUdThY39sk/bY=" - }, - "array-unique": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/array-unique/-/array-unique-0.2.1.tgz", - "integrity": "sha1-odl8yvy8JiXMcPrc6zalDFiwGlM=" - }, - "arrify": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/arrify/-/arrify-1.0.1.tgz", - "integrity": "sha1-iYUI2iIm84DfkEcoRWhJwVAaSw0=" - }, - "asn1": { - "version": "0.2.4", - "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz", - "integrity": "sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg==", - "requires": { - "safer-buffer": "~2.1.0" - } - }, - "assert-plus": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", - "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" - }, - "assign-symbols": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/assign-symbols/-/assign-symbols-1.0.0.tgz", - "integrity": "sha1-WWZ/QfrdTyDMvCu5a41Pf3jsA2c=" - }, - "async-each": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/async-each/-/async-each-1.0.1.tgz", - "integrity": "sha1-GdOGodntxufByF04iu28xW0zYC0=" - }, - "asynckit": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", - "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=" - }, - "atob": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/atob/-/atob-2.1.2.tgz", - "integrity": "sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg==" - }, - "aws-sign2": { - "version": "0.7.0", - "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz", - "integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=" - }, - "aws4": { - "version": "1.8.0", - "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.8.0.tgz", - "integrity": "sha512-ReZxvNHIOv88FlT7rxcXIIC0fPt4KZqZbOlivyWtXLt8ESx84zd3kMC6iK5jVeS2qt+g7ftS7ye4fi06X5rtRQ==" - }, - "bail": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/bail/-/bail-1.0.3.tgz", - "integrity": "sha512-1X8CnjFVQ+a+KW36uBNMTU5s8+v5FzeqrP7hTG5aTb4aPreSbZJlhwPon9VKMuEVgV++JM+SQrALY3kr7eswdg==" - }, - "balanced-match": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", - "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" - }, - "base": { - "version": "0.11.2", - "resolved": "https://registry.npmjs.org/base/-/base-0.11.2.tgz", - "integrity": "sha512-5T6P4xPgpp0YDFvSWwEZ4NoE3aM4QBQXDzmVbraCkFj8zHM+mba8SyqB5DbZWyR7mYHo6Y7BdQo3MoA4m0TeQg==", - "requires": { - "cache-base": "^1.0.1", - "class-utils": "^0.3.5", - "component-emitter": "^1.2.1", - "define-property": "^1.0.0", - "isobject": "^3.0.1", - "mixin-deep": "^1.2.0", - "pascalcase": "^0.1.1" - }, - "dependencies": { - "define-property": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/define-property/-/define-property-1.0.0.tgz", - "integrity": "sha1-dp66rz9KY6rTr56NMEybvnm/sOY=", - "requires": { - "is-descriptor": "^1.0.0" - } - }, - "is-accessor-descriptor": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz", - "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", - "requires": { - "kind-of": "^6.0.0" - } - }, - "is-data-descriptor": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz", - "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", - "requires": { - "kind-of": "^6.0.0" - } - }, - "is-descriptor": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz", - "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==", - "requires": { - "is-accessor-descriptor": "^1.0.0", - "is-data-descriptor": "^1.0.0", - "kind-of": "^6.0.2" - } - }, - "isobject": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz", - "integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8=" - }, - "kind-of": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.2.tgz", - "integrity": "sha512-s5kLOcnH0XqDO+FvuaLX8DDjZ18CGFk7VygH40QoKPUQhW4e2rvM0rwUq0t8IQDOwYSeLK01U90OjzBTme2QqA==" - } - } - }, - "bcrypt-pbkdf": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", - "integrity": "sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4=", - "optional": true, - "requires": { - "tweetnacl": "^0.14.3" - } - }, - "binary-extensions": { - "version": "1.12.0", - "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-1.12.0.tgz", - "integrity": "sha512-DYWGk01lDcxeS/K9IHPGWfT8PsJmbXRtRd2Sx72Tnb8pcYZQFF1oSDb8hJtS1vhp212q1Rzi5dUf9+nq0o9UIg==" - }, - "bluebird": { - "version": "3.5.2", - "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.2.tgz", - "integrity": "sha512-dhHTWMI7kMx5whMQntl7Vr9C6BvV10lFXDAasnqnrMYhXVCzzk6IO9Fo2L75jXHT07WrOngL1WDXOp+yYS91Yg==" - }, - "boundary": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/boundary/-/boundary-1.0.1.tgz", - "integrity": "sha1-TWfcJgLAzBbdm85+v4fpSCkPWBI=" - }, - "brace-expansion": { - "version": "1.1.11", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", - "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", - "requires": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" - } - }, - "braces": { - "version": "1.8.5", - "resolved": "https://registry.npmjs.org/braces/-/braces-1.8.5.tgz", - "integrity": "sha1-uneWLhLf+WnWt2cR6RS3N4V79qc=", - "requires": { - "expand-range": "^1.8.1", - "preserve": "^0.2.0", - "repeat-element": "^1.1.2" - } - }, - "buffer-from": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", - "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==" - }, - "builtin-modules": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-1.1.1.tgz", - "integrity": "sha1-Jw8HbFpywC9bZaR9+Uxf46J4iS8=" - }, - "cache-base": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/cache-base/-/cache-base-1.0.1.tgz", - "integrity": "sha512-AKcdTnFSWATd5/GCPRxr2ChwIJ85CeyrEyjRHlKxQ56d4XJMGym0uAiKn0xbLOGOl3+yRpOTi484dVCEc5AUzQ==", - "requires": { - "collection-visit": "^1.0.0", - "component-emitter": "^1.2.1", - "get-value": "^2.0.6", - "has-value": "^1.0.0", - "isobject": "^3.0.1", - "set-value": "^2.0.0", - "to-object-path": "^0.3.0", - "union-value": "^1.0.0", - "unset-value": "^1.0.0" - }, - "dependencies": { - "isobject": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz", - "integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8=" - } - } - }, - "cacheable-request": { - "version": "2.1.4", - "resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-2.1.4.tgz", - "integrity": "sha1-DYCIAbY0KtM8kd+dC0TcCbkeXD0=", - "requires": { - "clone-response": "1.0.2", - "get-stream": "3.0.0", - "http-cache-semantics": "3.8.1", - "keyv": "3.0.0", - "lowercase-keys": "1.0.0", - "normalize-url": "2.0.1", - "responselike": "1.0.2" - }, - "dependencies": { - "lowercase-keys": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-1.0.0.tgz", - "integrity": "sha1-TjNms55/VFfjXxMkvfb4jQv8cwY=" - } - } - }, - "camelcase": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-4.1.0.tgz", - "integrity": "sha1-1UVjW+HjPFQmScaRc+Xeas+uNN0=" - }, - "capture-stack-trace": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/capture-stack-trace/-/capture-stack-trace-1.0.1.tgz", - "integrity": "sha512-mYQLZnx5Qt1JgB1WEiMCf2647plpGeQ2NMR/5L0HNZzGQo4fuSPnK+wjfPnKZV0aiJDgzmWqqkV/g7JD+DW0qw==" - }, - "caseless": { - "version": "0.12.0", - "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", - "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=" - }, - "ccount": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/ccount/-/ccount-1.0.3.tgz", - "integrity": "sha512-Jt9tIBkRc9POUof7QA/VwWd+58fKkEEfI+/t1/eOlxKM7ZhrczNzMFefge7Ai+39y1pR/pP6cI19guHy3FSLmw==" - }, - "chalk": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.1.tgz", - "integrity": "sha512-ObN6h1v2fTJSmUXoS3nMQ92LbDK9be4TV+6G+omQlGJFdcUX5heKi1LZ1YnRMIgwTLEj3E24bT6tYni50rlCfQ==", - "requires": { - "ansi-styles": "^3.2.1", - "escape-string-regexp": "^1.0.5", - "supports-color": "^5.3.0" - } - }, - "character-entities": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/character-entities/-/character-entities-1.2.2.tgz", - "integrity": "sha512-sMoHX6/nBiy3KKfC78dnEalnpn0Az0oSNvqUWYTtYrhRI5iUIYsROU48G+E+kMFQzqXaJ8kHJZ85n7y6/PHgwQ==" - }, - "character-entities-html4": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/character-entities-html4/-/character-entities-html4-1.1.2.tgz", - "integrity": "sha512-sIrXwyna2+5b0eB9W149izTPJk/KkJTg6mEzDGibwBUkyH1SbDa+nf515Ppdi3MaH35lW0JFJDWeq9Luzes1Iw==" - }, - "character-entities-legacy": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/character-entities-legacy/-/character-entities-legacy-1.1.2.tgz", - "integrity": "sha512-9NB2VbXtXYWdXzqrvAHykE/f0QJxzaKIpZ5QzNZrrgQ7Iyxr2vnfS8fCBNVW9nUEZE0lo57nxKRqnzY/dKrwlA==" - }, - "character-reference-invalid": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/character-reference-invalid/-/character-reference-invalid-1.1.2.tgz", - "integrity": "sha512-7I/xceXfKyUJmSAn/jw8ve/9DyOP7XxufNYLI9Px7CmsKgEUaZLUTax6nZxGQtaoiZCjpu6cHPj20xC/vqRReQ==" - }, - "charenc": { - "version": "0.0.2", - "resolved": "https://registry.npmjs.org/charenc/-/charenc-0.0.2.tgz", - "integrity": "sha1-wKHS86cJLgN3S/qD8UwPxXkKhmc=" - }, - "chokidar": { - "version": "1.7.0", - "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-1.7.0.tgz", - "integrity": "sha1-eY5ol3gVHIB2tLNg5e3SjNortGg=", - "requires": { - "anymatch": "^1.3.0", - "async-each": "^1.0.0", - "fsevents": "^1.0.0", - "glob-parent": "^2.0.0", - "inherits": "^2.0.1", - "is-binary-path": "^1.0.0", - "is-glob": "^2.0.0", - "path-is-absolute": "^1.0.0", - "readdirp": "^2.0.0" - } - }, - "circular-json": { - "version": "0.3.3", - "resolved": "https://registry.npmjs.org/circular-json/-/circular-json-0.3.3.tgz", - "integrity": "sha512-UZK3NBx2Mca+b5LsG7bY183pHWt5Y1xts4P3Pz7ENTwGVnJOUWbRb3ocjvX7hx9tq/yTAdclXm9sZ38gNuem4A==" - }, - "class-utils": { - "version": "0.3.6", - "resolved": "https://registry.npmjs.org/class-utils/-/class-utils-0.3.6.tgz", - "integrity": "sha512-qOhPa/Fj7s6TY8H8esGu5QNpMMQxz79h+urzrNYN6mn+9BnxlDGf5QZ+XeCDsxSjPqsSR56XOZOJmpeurnLMeg==", - "requires": { - "arr-union": "^3.1.0", - "define-property": "^0.2.5", - "isobject": "^3.0.0", - "static-extend": "^0.1.1" - }, - "dependencies": { - "define-property": { - "version": "0.2.5", - "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz", - "integrity": "sha1-w1se+RjsPJkPmlvFe+BKrOxcgRY=", - "requires": { - "is-descriptor": "^0.1.0" - } - }, - "isobject": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz", - "integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8=" - } - } - }, - "clean-stack": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/clean-stack/-/clean-stack-1.3.0.tgz", - "integrity": "sha1-noIVAa6XmYbEax1m0tQy2y/UrjE=" - }, - "clone-response": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/clone-response/-/clone-response-1.0.2.tgz", - "integrity": "sha1-0dyXOSAxTfZ/vrlCI7TuNQI56Ws=", - "requires": { - "mimic-response": "^1.0.0" - } - }, - "co": { - "version": "4.6.0", - "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", - "integrity": "sha1-bqa989hTrlTMuOR7+gvz+QMfsYQ=" - }, - "code-point-at": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz", - "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=" - }, - "collapse-white-space": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/collapse-white-space/-/collapse-white-space-1.0.4.tgz", - "integrity": "sha512-YfQ1tAUZm561vpYD+5eyWN8+UsceQbSrqqlc/6zDY2gtAE+uZLSdkkovhnGpmCThsvKBFakq4EdY/FF93E8XIw==" - }, - "collection-visit": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/collection-visit/-/collection-visit-1.0.0.tgz", - "integrity": "sha1-S8A3PBZLwykbTTaMgpzxqApZ3KA=", - "requires": { - "map-visit": "^1.0.0", - "object-visit": "^1.0.0" - } - }, - "color-convert": { - "version": "1.9.3", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", - "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", - "requires": { - "color-name": "1.1.3" - } - }, - "color-name": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", - "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" - }, - "combined-stream": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.7.tgz", - "integrity": "sha512-brWl9y6vOB1xYPZcpZde3N9zDByXTosAeMDo4p1wzo6UMOX4vumB+TP1RZ76sfE6Md68Q0NJSrE/gbezd4Ul+w==", - "requires": { - "delayed-stream": "~1.0.0" - } - }, - "component-emitter": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.2.1.tgz", - "integrity": "sha1-E3kY1teCg/ffemt8WmPhQOaUJeY=" - }, - "concat-map": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", - "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" - }, - "concat-stream": { - "version": "1.6.2", - "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz", - "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==", - "requires": { - "buffer-from": "^1.0.0", - "inherits": "^2.0.3", - "readable-stream": "^2.2.2", - "typedarray": "^0.0.6" - } - }, - "copy-descriptor": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/copy-descriptor/-/copy-descriptor-0.1.1.tgz", - "integrity": "sha1-Z29us8OZl8LuGsOpJP1hJHSPV40=" - }, - "core-util-is": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", - "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" - }, - "create-error-class": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/create-error-class/-/create-error-class-3.0.2.tgz", - "integrity": "sha1-Br56vvlHo/FKMP1hBnHUAbyot7Y=", - "requires": { - "capture-stack-trace": "^1.0.0" - } - }, - "crypt": { - "version": "0.0.2", - "resolved": "https://registry.npmjs.org/crypt/-/crypt-0.0.2.tgz", - "integrity": "sha1-iNf/fsDfuG9xPch7u0LQRNPmxBs=" - }, - "dashdash": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", - "integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=", - "requires": { - "assert-plus": "^1.0.0" - } - }, - "debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "requires": { - "ms": "2.0.0" - } - }, - "decode-uri-component": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.2.0.tgz", - "integrity": "sha1-6zkTMzRYd1y4TNGh+uBiEGu4dUU=" - }, - "decompress-response": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-3.3.0.tgz", - "integrity": "sha1-gKTdMjdIOEv6JICDYirt7Jgq3/M=", - "requires": { - "mimic-response": "^1.0.0" - } - }, - "deep-equal": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/deep-equal/-/deep-equal-1.0.1.tgz", - "integrity": "sha1-9dJgKStmDghO/0zbyfCK0yR0SLU=" - }, - "deep-extend": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", - "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==" - }, - "deep-is": { - "version": "0.1.3", - "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.3.tgz", - "integrity": "sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ=" - }, - "define-properties": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz", - "integrity": "sha512-3MqfYKj2lLzdMSf8ZIZE/V+Zuy+BgD6f164e8K2w7dgnpKArBDerGYpM46IYYcjnkdPNMjPk9A6VFB8+3SKlXQ==", - "requires": { - "object-keys": "^1.0.12" - } - }, - "define-property": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/define-property/-/define-property-2.0.2.tgz", - "integrity": "sha512-jwK2UV4cnPpbcG7+VRARKTZPUWowwXA8bzH5NP6ud0oeAxyYPuGZUAC7hMugpCdz4BeSZl2Dl9k66CHJ/46ZYQ==", - "requires": { - "is-descriptor": "^1.0.2", - "isobject": "^3.0.1" - }, - "dependencies": { - "is-accessor-descriptor": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz", - "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", - "requires": { - "kind-of": "^6.0.0" - } - }, - "is-data-descriptor": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz", - "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", - "requires": { - "kind-of": "^6.0.0" - } - }, - "is-descriptor": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz", - "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==", - "requires": { - "is-accessor-descriptor": "^1.0.0", - "is-data-descriptor": "^1.0.0", - "kind-of": "^6.0.2" - } - }, - "isobject": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz", - "integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8=" - }, - "kind-of": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.2.tgz", - "integrity": "sha512-s5kLOcnH0XqDO+FvuaLX8DDjZ18CGFk7VygH40QoKPUQhW4e2rvM0rwUq0t8IQDOwYSeLK01U90OjzBTme2QqA==" - } - } - }, - "del": { - "version": "2.2.2", - "resolved": "https://registry.npmjs.org/del/-/del-2.2.2.tgz", - "integrity": "sha1-wSyYHQZ4RshLyvhiz/kw2Qf/0ag=", - "requires": { - "globby": "^5.0.0", - "is-path-cwd": "^1.0.0", - "is-path-in-cwd": "^1.0.0", - "object-assign": "^4.0.1", - "pify": "^2.0.0", - "pinkie-promise": "^2.0.0", - "rimraf": "^2.2.8" - }, - "dependencies": { - "pify": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", - "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=" - } - } - }, - "delayed-stream": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", - "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=" - }, - "diff": { - "version": "2.2.3", - "resolved": "https://registry.npmjs.org/diff/-/diff-2.2.3.tgz", - "integrity": "sha1-YOr9DSjukG5Oj/ClLBIpUhAzv5k=" - }, - "dns-packet": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/dns-packet/-/dns-packet-1.3.1.tgz", - "integrity": "sha512-0UxfQkMhYAUaZI+xrNZOz/as5KgDU0M/fQ9b6SpkyLbk3GEswDi6PADJVaYJradtRVsRIlF1zLyOodbcTCDzUg==", - "requires": { - "ip": "^1.1.0", - "safe-buffer": "^5.0.1" - } - }, - "dns-socket": { - "version": "1.6.3", - "resolved": "https://registry.npmjs.org/dns-socket/-/dns-socket-1.6.3.tgz", - "integrity": "sha512-/mUy3VGqIP69dAZjh2xxHXcpK9wk2Len1Dxz8mWAdrIgFC8tnR/aQAyU4a+UTXzOcTvEvGBdp1zFiwnpWKaXng==", - "requires": { - "dns-packet": "^1.1.0" - } - }, - "duplexer3": { - "version": "0.1.4", - "resolved": "https://registry.npmjs.org/duplexer3/-/duplexer3-0.1.4.tgz", - "integrity": "sha1-7gHdHKwO08vH/b6jfcCo8c4ALOI=" - }, - "e-prime": { - "version": "0.10.2", - "resolved": "https://registry.npmjs.org/e-prime/-/e-prime-0.10.2.tgz", - "integrity": "sha1-6pN165hWNt6IATx6n7EprZ4V7/g=" - }, - "ecc-jsbn": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", - "integrity": "sha1-OoOpBOVDUyh4dMVkt1SThoSamMk=", - "optional": true, - "requires": { - "jsbn": "~0.1.0", - "safer-buffer": "^2.1.0" - } - }, - "error-ex": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz", - "integrity": "sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==", - "requires": { - "is-arrayish": "^0.2.1" - } - }, - "es-abstract": { - "version": "1.12.0", - "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.12.0.tgz", - "integrity": "sha512-C8Fx/0jFmV5IPoMOFPA9P9G5NtqW+4cOPit3MIuvR2t7Ag2K15EJTpxnHAYTzL+aYQJIESYeXZmDBfOBE1HcpA==", - "requires": { - "es-to-primitive": "^1.1.1", - "function-bind": "^1.1.1", - "has": "^1.0.1", - "is-callable": "^1.1.3", - "is-regex": "^1.0.4" - } - }, - "es-to-primitive": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.1.1.tgz", - "integrity": "sha1-RTVSSKiJeQNLZ5Lhm7gfK3l13Q0=", - "requires": { - "is-callable": "^1.1.1", - "is-date-object": "^1.0.1", - "is-symbol": "^1.0.1" - } - }, - "escape-string-regexp": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=" - }, - "esprima": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", - "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==" - }, - "expand-brackets": { - "version": "0.1.5", - "resolved": "https://registry.npmjs.org/expand-brackets/-/expand-brackets-0.1.5.tgz", - "integrity": "sha1-3wcoTjQqgHzXM6xa9yQR5YHRF3s=", - "requires": { - "is-posix-bracket": "^0.1.0" - } - }, - "expand-range": { - "version": "1.8.2", - "resolved": "https://registry.npmjs.org/expand-range/-/expand-range-1.8.2.tgz", - "integrity": "sha1-opnv/TNf4nIeuujiV+x5ZE/IUzc=", - "requires": { - "fill-range": "^2.1.0" - } - }, - "extend": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", - "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" - }, - "extend-shallow": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-3.0.2.tgz", - "integrity": "sha1-Jqcarwc7OfshJxcnRhMcJwQCjbg=", - "requires": { - "assign-symbols": "^1.0.0", - "is-extendable": "^1.0.1" - }, - "dependencies": { - "is-extendable": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-1.0.1.tgz", - "integrity": "sha512-arnXMxT1hhoKo9k1LZdmlNyJdDDfy2v0fXjFlmok4+i8ul/6WlbVge9bhM74OpNPQPMGUToDtz+KXa1PneJxOA==", - "requires": { - "is-plain-object": "^2.0.4" - } - } - } - }, - "extglob": { - "version": "0.3.2", - "resolved": "https://registry.npmjs.org/extglob/-/extglob-0.3.2.tgz", - "integrity": "sha1-Lhj/PS9JqydlzskCPwEdqo2DSaE=", - "requires": { - "is-extglob": "^1.0.0" - } - }, - "extsprintf": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz", - "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=" - }, - "fast-deep-equal": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-1.1.0.tgz", - "integrity": "sha1-wFNHeBfIa1HaqFPIHgWbcz0CNhQ=" - }, - "fast-json-stable-stringify": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz", - "integrity": "sha1-1RQsDK7msRifh9OnYREGT4bIu/I=" - }, - "fast-levenshtein": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", - "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=" - }, - "fault": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/fault/-/fault-1.0.2.tgz", - "integrity": "sha512-o2eo/X2syzzERAtN5LcGbiVQ0WwZSlN3qLtadwAz3X8Bu+XWD16dja/KMsjZLiQr+BLGPDnHGkc4yUJf1Xpkpw==", - "requires": { - "format": "^0.2.2" - } - }, - "file-entry-cache": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-2.0.0.tgz", - "integrity": "sha1-w5KZDD5oR4PYOLjISkXYoEhFg2E=", - "requires": { - "flat-cache": "^1.2.1", - "object-assign": "^4.0.1" - } - }, - "filename-regex": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/filename-regex/-/filename-regex-2.0.1.tgz", - "integrity": "sha1-wcS5vuPglyXdsQa3XB4wH+LxiyY=" - }, - "fill-range": { - "version": "2.2.4", - "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-2.2.4.tgz", - "integrity": "sha512-cnrcCbj01+j2gTG921VZPnHbjmdAf8oQV/iGeV2kZxGSyfYjjTyY79ErsK1WJWMpw6DaApEX72binqJE+/d+5Q==", - "requires": { - "is-number": "^2.1.0", - "isobject": "^2.0.0", - "randomatic": "^3.0.0", - "repeat-element": "^1.1.2", - "repeat-string": "^1.5.2" - } - }, - "find-up": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/find-up/-/find-up-2.1.0.tgz", - "integrity": "sha1-RdG35QbHF93UgndaK3eSCjwMV6c=", - "requires": { - "locate-path": "^2.0.0" - } - }, - "flat-cache": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-1.3.0.tgz", - "integrity": "sha1-0wMLMrOBVPTjt+nHCfSQ9++XxIE=", - "requires": { - "circular-json": "^0.3.1", - "del": "^2.0.2", - "graceful-fs": "^4.1.2", - "write": "^0.2.1" - } - }, - "fn-name": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/fn-name/-/fn-name-2.0.1.tgz", - "integrity": "sha1-UhTXU3pNBqSjAcDMJi/rhBiAAuc=" - }, - "for-in": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/for-in/-/for-in-1.0.2.tgz", - "integrity": "sha1-gQaNKVqBQuwKxybG4iAMMPttXoA=" - }, - "for-own": { - "version": "0.1.5", - "resolved": "https://registry.npmjs.org/for-own/-/for-own-0.1.5.tgz", - "integrity": "sha1-UmXGgaTylNq78XyVCbZ2OqhFEM4=", - "requires": { - "for-in": "^1.0.1" - } - }, - "forever-agent": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz", - "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=" - }, - "form-data": { - "version": "2.3.2", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.2.tgz", - "integrity": "sha1-SXBJi+YEwgwAXU9cI67NIda0kJk=", - "requires": { - "asynckit": "^0.4.0", - "combined-stream": "1.0.6", - "mime-types": "^2.1.12" - }, - "dependencies": { - "combined-stream": { - "version": "1.0.6", - "resolved": "http://registry.npmjs.org/combined-stream/-/combined-stream-1.0.6.tgz", - "integrity": "sha1-cj599ugBrFYTETp+RFqbactjKBg=", - "requires": { - "delayed-stream": "~1.0.0" - } - } - } - }, - "format": { - "version": "0.2.2", - "resolved": "https://registry.npmjs.org/format/-/format-0.2.2.tgz", - "integrity": "sha1-1hcBB+nv3E7TDJ3DkBbflCtctYs=" - }, - "fragment-cache": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/fragment-cache/-/fragment-cache-0.2.1.tgz", - "integrity": "sha1-QpD60n8T6Jvn8zeZxrxaCr//DRk=", - "requires": { - "map-cache": "^0.2.2" - } - }, - "from2": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/from2/-/from2-2.3.0.tgz", - "integrity": "sha1-i/tVAr3kpNNs/e6gB/zKIdfjgq8=", - "requires": { - "inherits": "^2.0.1", - "readable-stream": "^2.0.0" - } - }, - "fs.realpath": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", - "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=" - }, - "fsevents": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-1.2.4.tgz", - "integrity": "sha512-z8H8/diyk76B7q5wg+Ud0+CqzcAF3mBBI/bA5ne5zrRUUIvNkJY//D3BqyH571KuAC4Nr7Rw7CjWX4r0y9DvNg==", - "optional": true, - "requires": { - "nan": "^2.9.2", - "node-pre-gyp": "^0.10.0" - }, - "dependencies": { - "abbrev": { - "version": "1.1.1", - "bundled": true, - "optional": true - }, - "ansi-regex": { - "version": "2.1.1", - "bundled": true - }, - "aproba": { - "version": "1.2.0", - "bundled": true, - "optional": true - }, - "are-we-there-yet": { - "version": "1.1.4", - "bundled": true, - "optional": true, - "requires": { - "delegates": "^1.0.0", - "readable-stream": "^2.0.6" - } - }, - "balanced-match": { - "version": "1.0.0", - "bundled": true - }, - "brace-expansion": { - "version": "1.1.11", - "bundled": true, - "requires": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" - } - }, - "chownr": { - "version": "1.0.1", - "bundled": true, - "optional": true - }, - "code-point-at": { - "version": "1.1.0", - "bundled": true - }, - "concat-map": { - "version": "0.0.1", - "bundled": true - }, - "console-control-strings": { - "version": "1.1.0", - "bundled": true - }, - "core-util-is": { - "version": "1.0.2", - "bundled": true, - "optional": true - }, - "debug": { - "version": "2.6.9", - "bundled": true, - "optional": true, - "requires": { - "ms": "2.0.0" - } - }, - "deep-extend": { - "version": "0.5.1", - "bundled": true, - "optional": true - }, - "delegates": { - "version": "1.0.0", - "bundled": true, - "optional": true - }, - "detect-libc": { - "version": "1.0.3", - "bundled": true, - "optional": true - }, - "fs-minipass": { - "version": "1.2.5", - "bundled": true, - "optional": true, - "requires": { - "minipass": "^2.2.1" - } - }, - "fs.realpath": { - "version": "1.0.0", - "bundled": true, - "optional": true - }, - "gauge": { - "version": "2.7.4", - "bundled": true, - "optional": true, - "requires": { - "aproba": "^1.0.3", - "console-control-strings": "^1.0.0", - "has-unicode": "^2.0.0", - "object-assign": "^4.1.0", - "signal-exit": "^3.0.0", - "string-width": "^1.0.1", - "strip-ansi": "^3.0.1", - "wide-align": "^1.1.0" - } - }, - "glob": { - "version": "7.1.2", - "bundled": true, - "optional": true, - "requires": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.0.4", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" - } - }, - "has-unicode": { - "version": "2.0.1", - "bundled": true, - "optional": true - }, - "iconv-lite": { - "version": "0.4.21", - "bundled": true, - "optional": true, - "requires": { - "safer-buffer": "^2.1.0" - } - }, - "ignore-walk": { - "version": "3.0.1", - "bundled": true, - "optional": true, - "requires": { - "minimatch": "^3.0.4" - } - }, - "inflight": { - "version": "1.0.6", - "bundled": true, - "optional": true, - "requires": { - "once": "^1.3.0", - "wrappy": "1" - } - }, - "inherits": { - "version": "2.0.3", - "bundled": true - }, - "ini": { - "version": "1.3.5", - "bundled": true, - "optional": true - }, - "is-fullwidth-code-point": { - "version": "1.0.0", - "bundled": true, - "requires": { - "number-is-nan": "^1.0.0" - } - }, - "isarray": { - "version": "1.0.0", - "bundled": true, - "optional": true - }, - "minimatch": { - "version": "3.0.4", - "bundled": true, - "requires": { - "brace-expansion": "^1.1.7" - } - }, - "minimist": { - "version": "0.0.8", - "bundled": true - }, - "minipass": { - "version": "2.2.4", - "bundled": true, - "requires": { - "safe-buffer": "^5.1.1", - "yallist": "^3.0.0" - } - }, - "minizlib": { - "version": "1.1.0", - "bundled": true, - "optional": true, - "requires": { - "minipass": "^2.2.1" - } - }, - "mkdirp": { - "version": "0.5.1", - "bundled": true, - "requires": { - "minimist": "0.0.8" - } - }, - "ms": { - "version": "2.0.0", - "bundled": true, - "optional": true - }, - "needle": { - "version": "2.2.0", - "bundled": true, - "optional": true, - "requires": { - "debug": "^2.1.2", - "iconv-lite": "^0.4.4", - "sax": "^1.2.4" - } - }, - "node-pre-gyp": { - "version": "0.10.0", - "bundled": true, - "optional": true, - "requires": { - "detect-libc": "^1.0.2", - "mkdirp": "^0.5.1", - "needle": "^2.2.0", - "nopt": "^4.0.1", - "npm-packlist": "^1.1.6", - "npmlog": "^4.0.2", - "rc": "^1.1.7", - "rimraf": "^2.6.1", - "semver": "^5.3.0", - "tar": "^4" - } - }, - "nopt": { - "version": "4.0.1", - "bundled": true, - "optional": true, - "requires": { - "abbrev": "1", - "osenv": "^0.1.4" - } - }, - "npm-bundled": { - "version": "1.0.3", - "bundled": true, - "optional": true - }, - "npm-packlist": { - "version": "1.1.10", - "bundled": true, - "optional": true, - "requires": { - "ignore-walk": "^3.0.1", - "npm-bundled": "^1.0.1" - } - }, - "npmlog": { - "version": "4.1.2", - "bundled": true, - "optional": true, - "requires": { - "are-we-there-yet": "~1.1.2", - "console-control-strings": "~1.1.0", - "gauge": "~2.7.3", - "set-blocking": "~2.0.0" - } - }, - "number-is-nan": { - "version": "1.0.1", - "bundled": true - }, - "object-assign": { - "version": "4.1.1", - "bundled": true, - "optional": true - }, - "once": { - "version": "1.4.0", - "bundled": true, - "requires": { - "wrappy": "1" - } - }, - "os-homedir": { - "version": "1.0.2", - "bundled": true, - "optional": true - }, - "os-tmpdir": { - "version": "1.0.2", - "bundled": true, - "optional": true - }, - "osenv": { - "version": "0.1.5", - "bundled": true, - "optional": true, - "requires": { - "os-homedir": "^1.0.0", - "os-tmpdir": "^1.0.0" - } - }, - "path-is-absolute": { - "version": "1.0.1", - "bundled": true, - "optional": true - }, - "process-nextick-args": { - "version": "2.0.0", - "bundled": true, - "optional": true - }, - "rc": { - "version": "1.2.7", - "bundled": true, - "optional": true, - "requires": { - "deep-extend": "^0.5.1", - "ini": "~1.3.0", - "minimist": "^1.2.0", - "strip-json-comments": "~2.0.1" - }, - "dependencies": { - "minimist": { - "version": "1.2.0", - "bundled": true, - "optional": true - } - } - }, - "readable-stream": { - "version": "2.3.6", - "bundled": true, - "optional": true, - "requires": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.3", - "isarray": "~1.0.0", - "process-nextick-args": "~2.0.0", - "safe-buffer": "~5.1.1", - "string_decoder": "~1.1.1", - "util-deprecate": "~1.0.1" - } - }, - "rimraf": { - "version": "2.6.2", - "bundled": true, - "optional": true, - "requires": { - "glob": "^7.0.5" - } - }, - "safe-buffer": { - "version": "5.1.1", - "bundled": true - }, - "safer-buffer": { - "version": "2.1.2", - "bundled": true, - "optional": true - }, - "sax": { - "version": "1.2.4", - "bundled": true, - "optional": true - }, - "semver": { - "version": "5.5.0", - "bundled": true, - "optional": true - }, - "set-blocking": { - "version": "2.0.0", - "bundled": true, - "optional": true - }, - "signal-exit": { - "version": "3.0.2", - "bundled": true, - "optional": true - }, - "string-width": { - "version": "1.0.2", - "bundled": true, - "requires": { - "code-point-at": "^1.0.0", - "is-fullwidth-code-point": "^1.0.0", - "strip-ansi": "^3.0.0" - } - }, - "string_decoder": { - "version": "1.1.1", - "bundled": true, - "optional": true, - "requires": { - "safe-buffer": "~5.1.0" - } - }, - "strip-ansi": { - "version": "3.0.1", - "bundled": true, - "requires": { - "ansi-regex": "^2.0.0" - } - }, - "strip-json-comments": { - "version": "2.0.1", - "bundled": true, - "optional": true - }, - "tar": { - "version": "4.4.1", - "bundled": true, - "optional": true, - "requires": { - "chownr": "^1.0.1", - "fs-minipass": "^1.2.5", - "minipass": "^2.2.4", - "minizlib": "^1.1.0", - "mkdirp": "^0.5.0", - "safe-buffer": "^5.1.1", - "yallist": "^3.0.2" - } - }, - "util-deprecate": { - "version": "1.0.2", - "bundled": true, - "optional": true - }, - "wide-align": { - "version": "1.1.2", - "bundled": true, - "optional": true, - "requires": { - "string-width": "^1.0.2" - } - }, - "wrappy": { - "version": "1.0.2", - "bundled": true - }, - "yallist": { - "version": "3.0.2", - "bundled": true - } - } - }, - "function-bind": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", - "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" - }, - "get-stdin": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-5.0.1.tgz", - "integrity": "sha1-Ei4WFZHiH/TFJTAwVpPyDmOTo5g=" - }, - "get-stream": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", - "integrity": "sha1-jpQ9E1jcN1VQVOy+LtsFqhdO3hQ=" - }, - "get-value": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/get-value/-/get-value-2.0.6.tgz", - "integrity": "sha1-3BXKHGcjh8p2vTesCjlbogQqLCg=" - }, - "getpass": { - "version": "0.1.7", - "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", - "integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=", - "requires": { - "assert-plus": "^1.0.0" - } - }, - "glob": { - "version": "7.1.3", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.3.tgz", - "integrity": "sha512-vcfuiIxogLV4DlGBHIUOwI0IbrJ8HWPc4MU7HzviGeNho/UJDfi6B5p3sHeWIQ0KGIU0Jpxi5ZHxemQfLkkAwQ==", - "requires": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.0.4", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" - } - }, - "glob-base": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/glob-base/-/glob-base-0.3.0.tgz", - "integrity": "sha1-27Fk9iIbHAscz4Kuoyi0l98Oo8Q=", - "requires": { - "glob-parent": "^2.0.0", - "is-glob": "^2.0.0" - } - }, - "glob-parent": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-2.0.0.tgz", - "integrity": "sha1-gTg9ctsFT8zPUzbaqQLxgvbtuyg=", - "requires": { - "is-glob": "^2.0.0" - } - }, - "globby": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/globby/-/globby-5.0.0.tgz", - "integrity": "sha1-69hGZ8oNuzMLmbz8aOrCvFQ3Dg0=", - "requires": { - "array-union": "^1.0.1", - "arrify": "^1.0.0", - "glob": "^7.0.3", - "object-assign": "^4.0.1", - "pify": "^2.0.0", - "pinkie-promise": "^2.0.0" - }, - "dependencies": { - "pify": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", - "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=" - } - } - }, - "got": { - "version": "6.7.1", - "resolved": "http://registry.npmjs.org/got/-/got-6.7.1.tgz", - "integrity": "sha1-JAzQV4WpoY5WHcG0S0HHY+8ejbA=", - "requires": { - "create-error-class": "^3.0.0", - "duplexer3": "^0.1.4", - "get-stream": "^3.0.0", - "is-redirect": "^1.0.0", - "is-retry-allowed": "^1.0.0", - "is-stream": "^1.0.0", - "lowercase-keys": "^1.0.0", - "safe-buffer": "^5.0.1", - "timed-out": "^4.0.0", - "unzip-response": "^2.0.1", - "url-parse-lax": "^1.0.0" - } - }, - "graceful-fs": { - "version": "4.1.11", - "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.11.tgz", - "integrity": "sha1-Dovf5NHduIVNZOBOp8AOKgJuVlg=" - }, - "har-schema": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz", - "integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI=" - }, - "har-validator": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.0.tgz", - "integrity": "sha512-+qnmNjI4OfH2ipQ9VQOw23bBd/ibtfbVdK2fYbY4acTDqKTW/YDp9McimZdDbG8iV9fZizUqQMD5xvriB146TA==", - "requires": { - "ajv": "^5.3.0", - "har-schema": "^2.0.0" - } - }, - "has": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", - "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", - "requires": { - "function-bind": "^1.1.1" - } - }, - "has-ansi": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz", - "integrity": "sha1-NPUEnOHs3ysGSa8+8k5F7TVBbZE=", - "requires": { - "ansi-regex": "^2.0.0" - } - }, - "has-flag": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", - "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=" - }, - "has-symbol-support-x": { - "version": "1.4.2", - "resolved": "https://registry.npmjs.org/has-symbol-support-x/-/has-symbol-support-x-1.4.2.tgz", - "integrity": "sha512-3ToOva++HaW+eCpgqZrCfN51IPB+7bJNVT6CUATzueB5Heb8o6Nam0V3HG5dlDvZU1Gn5QLcbahiKw/XVk5JJw==" - }, - "has-symbols": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.0.tgz", - "integrity": "sha1-uhqPGvKg/DllD1yFA2dwQSIGO0Q=" - }, - "has-to-string-tag-x": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/has-to-string-tag-x/-/has-to-string-tag-x-1.4.1.tgz", - "integrity": "sha512-vdbKfmw+3LoOYVr+mtxHaX5a96+0f3DljYd8JOqvOLsf5mw2Otda2qCDT9qRqLAhrjyQ0h7ual5nOiASpsGNFw==", - "requires": { - "has-symbol-support-x": "^1.4.1" - } - }, - "has-value": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/has-value/-/has-value-1.0.0.tgz", - "integrity": "sha1-GLKB2lhbHFxR3vJMkw7SmgvmsXc=", - "requires": { - "get-value": "^2.0.6", - "has-values": "^1.0.0", - "isobject": "^3.0.0" - }, - "dependencies": { - "isobject": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz", - "integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8=" - } - } - }, - "has-values": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/has-values/-/has-values-1.0.0.tgz", - "integrity": "sha1-lbC2P+whRmGab+V/51Yo1aOe/k8=", - "requires": { - "is-number": "^3.0.0", - "kind-of": "^4.0.0" - }, - "dependencies": { - "is-number": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-number/-/is-number-3.0.0.tgz", - "integrity": "sha1-JP1iAaR4LPUFYcgQJ2r8fRLXEZU=", - "requires": { - "kind-of": "^3.0.2" - }, - "dependencies": { - "kind-of": { - "version": "3.2.2", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", - "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=", - "requires": { - "is-buffer": "^1.1.5" - } - } - } - }, - "kind-of": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-4.0.0.tgz", - "integrity": "sha1-IIE989cSkosgc3hpGkUGb65y3Vc=", - "requires": { - "is-buffer": "^1.1.5" - } - } - } - }, - "hosted-git-info": { - "version": "2.7.1", - "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.7.1.tgz", - "integrity": "sha512-7T/BxH19zbcCTa8XkMlbK5lTo1WtgkFi3GvdWEyNuc4Vex7/9Dqbnpsf4JMydcfj9HCg4zUWFTL3Za6lapg5/w==" - }, - "http-cache-semantics": { - "version": "3.8.1", - "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-3.8.1.tgz", - "integrity": "sha512-5ai2iksyV8ZXmnZhHH4rWPoxxistEexSi5936zIQ1bnNTW5VnA85B6P/VpXiRM017IgRvb2kKo1a//y+0wSp3w==" - }, - "http-signature": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", - "integrity": "sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=", - "requires": { - "assert-plus": "^1.0.0", - "jsprim": "^1.2.2", - "sshpk": "^1.7.0" - } - }, - "ignore": { - "version": "3.3.10", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-3.3.10.tgz", - "integrity": "sha512-Pgs951kaMm5GXP7MOvxERINe3gsaVjUWFm+UZPSq9xYriQAksyhg0csnS0KXSNRD5NmNdapXEpjxG49+AKh/ug==" - }, - "indent-string": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-3.2.0.tgz", - "integrity": "sha1-Sl/W0nzDMvN+VBmlBNu4NxBckok=" - }, - "inflight": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", - "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", - "requires": { - "once": "^1.3.0", - "wrappy": "1" - } - }, - "inherits": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", - "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" - }, - "ini": { - "version": "1.3.5", - "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz", - "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==" - }, - "interop-require": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/interop-require/-/interop-require-1.0.0.tgz", - "integrity": "sha1-5TEDZ5lEyI1+YQW2Kp9EdceDlx4=" - }, - "into-stream": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/into-stream/-/into-stream-3.1.0.tgz", - "integrity": "sha1-lvsKk2wSur1v8XUqF9BWFqvQlMY=", - "requires": { - "from2": "^2.1.1", - "p-is-promise": "^1.1.0" - } - }, - "ip": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/ip/-/ip-1.1.5.tgz", - "integrity": "sha1-vd7XARQpCCjAoDnnLvJfWq7ENUo=" - }, - "ip-regex": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/ip-regex/-/ip-regex-2.1.0.tgz", - "integrity": "sha1-+ni/XS5pE8kRzp+BnuUUa7bYROk=" - }, - "is-absolute-url": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/is-absolute-url/-/is-absolute-url-2.1.0.tgz", - "integrity": "sha1-UFMN+4T8yap9vnhS6Do3uTufKqY=" - }, - "is-accessor-descriptor": { - "version": "0.1.6", - "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz", - "integrity": "sha1-qeEss66Nh2cn7u84Q/igiXtcmNY=", - "requires": { - "kind-of": "^3.0.2" - } - }, - "is-alphabetical": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/is-alphabetical/-/is-alphabetical-1.0.2.tgz", - "integrity": "sha512-V0xN4BYezDHcBSKb1QHUFMlR4as/XEuCZBzMJUU4n7+Cbt33SmUnSol+pnXFvLxSHNq2CemUXNdaXV6Flg7+xg==" - }, - "is-alphanumeric": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-alphanumeric/-/is-alphanumeric-1.0.0.tgz", - "integrity": "sha1-Spzvcdr0wAHB2B1j0UDPU/1oifQ=" - }, - "is-alphanumerical": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/is-alphanumerical/-/is-alphanumerical-1.0.2.tgz", - "integrity": "sha512-pyfU/0kHdISIgslFfZN9nfY1Gk3MquQgUm1mJTjdkEPpkAKNWuBTSqFwewOpR7N351VkErCiyV71zX7mlQQqsg==", - "requires": { - "is-alphabetical": "^1.0.0", - "is-decimal": "^1.0.0" - } - }, - "is-arrayish": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", - "integrity": "sha1-d8mYQFJ6qOyxqLppe4BkWnqSap0=" - }, - "is-binary-path": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-1.0.1.tgz", - "integrity": "sha1-dfFmQrSA8YenEcgUFh/TpKdlWJg=", - "requires": { - "binary-extensions": "^1.0.0" - } - }, - "is-buffer": { - "version": "1.1.6", - "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", - "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==" - }, - "is-builtin-module": { - "version": "1.0.0", - "resolved": "http://registry.npmjs.org/is-builtin-module/-/is-builtin-module-1.0.0.tgz", - "integrity": "sha1-VAVy0096wxGfj3bDDLwbHgN6/74=", - "requires": { - "builtin-modules": "^1.0.0" - } - }, - "is-callable": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.1.4.tgz", - "integrity": "sha512-r5p9sxJjYnArLjObpjA4xu5EKI3CuKHkJXMhT7kwbpUyIFD1n5PMAsoPvWnvtZiNz7LjkYDRZhd7FlI0eMijEA==" - }, - "is-data-descriptor": { - "version": "0.1.4", - "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz", - "integrity": "sha1-C17mSDiOLIYCgueT8YVv7D8wG1Y=", - "requires": { - "kind-of": "^3.0.2" - } - }, - "is-date-object": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.1.tgz", - "integrity": "sha1-mqIOtq7rv/d/vTPnTKAbM1gdOhY=" - }, - "is-decimal": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/is-decimal/-/is-decimal-1.0.2.tgz", - "integrity": "sha512-TRzl7mOCchnhchN+f3ICUCzYvL9ul7R+TYOsZ8xia++knyZAJfv/uA1FvQXsAnYIl1T3B2X5E/J7Wb1QXiIBXg==" - }, - "is-descriptor": { - "version": "0.1.6", - "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-0.1.6.tgz", - "integrity": "sha512-avDYr0SB3DwO9zsMov0gKCESFYqCnE4hq/4z3TdUlukEy5t9C0YRq7HLrsN52NAcqXKaepeCD0n+B0arnVG3Hg==", - "requires": { - "is-accessor-descriptor": "^0.1.6", - "is-data-descriptor": "^0.1.4", - "kind-of": "^5.0.0" - }, - "dependencies": { - "kind-of": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-5.1.0.tgz", - "integrity": "sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw==" - } - } - }, - "is-dotfile": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/is-dotfile/-/is-dotfile-1.0.3.tgz", - "integrity": "sha1-pqLzL/0t+wT1yiXs0Pa4PPeYoeE=" - }, - "is-empty": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/is-empty/-/is-empty-1.2.0.tgz", - "integrity": "sha1-3pu1snhzigWgsJpX4ftNSjQan2s=" - }, - "is-equal-shallow": { - "version": "0.1.3", - "resolved": "https://registry.npmjs.org/is-equal-shallow/-/is-equal-shallow-0.1.3.tgz", - "integrity": "sha1-IjgJj8Ih3gvPpdnqxMRdY4qhxTQ=", - "requires": { - "is-primitive": "^2.0.0" - } - }, - "is-extendable": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz", - "integrity": "sha1-YrEQ4omkcUGOPsNqYX1HLjAd/Ik=" - }, - "is-extglob": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-1.0.0.tgz", - "integrity": "sha1-rEaBd8SUNAWgkvyPKXYMb/xiBsA=" - }, - "is-file": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-file/-/is-file-1.0.0.tgz", - "integrity": "sha1-KKRM+9nT2xkwRfIrZfzo7fliBZY=" - }, - "is-fullwidth-code-point": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz", - "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=", - "requires": { - "number-is-nan": "^1.0.0" - } - }, - "is-glob": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-2.0.1.tgz", - "integrity": "sha1-0Jb5JqPe1WAPP9/ZEZjLCIjC2GM=", - "requires": { - "is-extglob": "^1.0.0" - } - }, - "is-hexadecimal": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/is-hexadecimal/-/is-hexadecimal-1.0.2.tgz", - "integrity": "sha512-but/G3sapV3MNyqiDBLrOi4x8uCIw0RY3o/Vb5GT0sMFHrVV7731wFSVy41T5FO1og7G0gXLJh0MkgPRouko/A==" - }, - "is-hidden": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/is-hidden/-/is-hidden-1.1.1.tgz", - "integrity": "sha512-175UKecS8+U4hh2PSY0j4xnm2GKYzvSKnbh+naC93JjuBA7LgIo6YxlbcsSo6seFBdQO3RuIcH980yvqqD/2cA==" - }, - "is-ip": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/is-ip/-/is-ip-2.0.0.tgz", - "integrity": "sha1-aO6gfooKCpTC0IDdZ0xzGrKkYas=", - "requires": { - "ip-regex": "^2.0.0" - } - }, - "is-number": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/is-number/-/is-number-2.1.0.tgz", - "integrity": "sha1-Afy7s5NGOlSPL0ZszhbezknbkI8=", - "requires": { - "kind-of": "^3.0.2" - } - }, - "is-object": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/is-object/-/is-object-1.0.1.tgz", - "integrity": "sha1-iVJojF7C/9awPsyF52ngKQMINHA=" - }, - "is-online": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/is-online/-/is-online-7.0.0.tgz", - "integrity": "sha1-fiQIwK4efje6jVC9sjcmDTK/2W4=", - "requires": { - "got": "^6.7.1", - "p-any": "^1.0.0", - "p-timeout": "^1.0.0", - "public-ip": "^2.3.0" - } - }, - "is-path-cwd": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-path-cwd/-/is-path-cwd-1.0.0.tgz", - "integrity": "sha1-0iXsIxMuie3Tj9p2dHLmLmXxEG0=" - }, - "is-path-in-cwd": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/is-path-in-cwd/-/is-path-in-cwd-1.0.1.tgz", - "integrity": "sha512-FjV1RTW48E7CWM7eE/J2NJvAEEVektecDBVBE5Hh3nM1Jd0kvhHtX68Pr3xsDf857xt3Y4AkwVULK1Vku62aaQ==", - "requires": { - "is-path-inside": "^1.0.0" - } - }, - "is-path-inside": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-1.0.1.tgz", - "integrity": "sha1-jvW33lBDej/cprToZe96pVy0gDY=", - "requires": { - "path-is-inside": "^1.0.1" - } - }, - "is-plain-obj": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-1.1.0.tgz", - "integrity": "sha1-caUMhCnfync8kqOQpKA7OfzVHT4=" - }, - "is-plain-object": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz", - "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==", - "requires": { - "isobject": "^3.0.1" - }, - "dependencies": { - "isobject": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz", - "integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8=" - } - } - }, - "is-posix-bracket": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/is-posix-bracket/-/is-posix-bracket-0.1.1.tgz", - "integrity": "sha1-MzTceXdDaOkvAW5vvAqI9c1ua8Q=" - }, - "is-primitive": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/is-primitive/-/is-primitive-2.0.0.tgz", - "integrity": "sha1-IHurkWOEmcB7Kt8kCkGochADRXU=" - }, - "is-redirect": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-redirect/-/is-redirect-1.0.0.tgz", - "integrity": "sha1-HQPd7VO9jbDzDCbk+V02/HyH3CQ=" - }, - "is-regex": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.0.4.tgz", - "integrity": "sha1-VRdIm1RwkbCTDglWVM7SXul+lJE=", - "requires": { - "has": "^1.0.1" - } - }, - "is-relative-url": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/is-relative-url/-/is-relative-url-2.0.0.tgz", - "integrity": "sha1-cpAtf+BLPUeS59sV+duEtyBMnO8=", - "requires": { - "is-absolute-url": "^2.0.0" - } - }, - "is-retry-allowed": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/is-retry-allowed/-/is-retry-allowed-1.1.0.tgz", - "integrity": "sha1-EaBgVotnM5REAz0BJaYaINVk+zQ=" - }, - "is-stream": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz", - "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ=" - }, - "is-symbol": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.1.tgz", - "integrity": "sha1-PMWfAAJRlLarLjjbrmaJJWtmBXI=" - }, - "is-typedarray": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", - "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=" - }, - "is-utf8": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/is-utf8/-/is-utf8-0.2.1.tgz", - "integrity": "sha1-Sw2hRCEE0bM2NA6AeX6GXPOffXI=" - }, - "is-whitespace-character": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/is-whitespace-character/-/is-whitespace-character-1.0.2.tgz", - "integrity": "sha512-SzM+T5GKUCtLhlHFKt2SDAX2RFzfS6joT91F2/WSi9LxgFdsnhfPK/UIA+JhRR2xuyLdrCys2PiFDrtn1fU5hQ==" - }, - "is-windows": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/is-windows/-/is-windows-1.0.2.tgz", - "integrity": "sha512-eXK1UInq2bPmjyX6e3VHIzMLobc4J94i4AWn+Hpq3OU5KkrRC96OAcR3PRJ/pGu6m8TRnBHP9dkXQVsT/COVIA==" - }, - "is-word-character": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/is-word-character/-/is-word-character-1.0.2.tgz", - "integrity": "sha512-T3FlsX8rCHAH8e7RE7PfOPZVFQlcV3XRF9eOOBQ1uf70OxO7CjjSOjeImMPCADBdYWcStAbVbYvJ1m2D3tb+EA==" - }, - "isarray": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" - }, - "isemail": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/isemail/-/isemail-3.1.3.tgz", - "integrity": "sha512-5xbsG5wYADIcB+mfLsd+nst1V/D+I7EU7LEZPo2GOIMu4JzfcRs5yQoypP4avA7QtUqgxYLKBYNv4IdzBmbhdw==", - "requires": { - "punycode": "2.x.x" - } - }, - "isobject": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/isobject/-/isobject-2.1.0.tgz", - "integrity": "sha1-8GVWEJaj8dou9GJy+BXIQNh+DIk=", - "requires": { - "isarray": "1.0.0" - } - }, - "isstream": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", - "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=" - }, - "isurl": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/isurl/-/isurl-1.0.0.tgz", - "integrity": "sha512-1P/yWsxPlDtn7QeRD+ULKQPaIaN6yF368GZ2vDfv0AL0NwpStafjWCDDdn0k8wgFMWpVAqG7oJhxHnlud42i9w==", - "requires": { - "has-to-string-tag-x": "^1.2.0", - "is-object": "^1.0.1" - } - }, - "js-yaml": { - "version": "3.12.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.12.0.tgz", - "integrity": "sha512-PIt2cnwmPfL4hKNwqeiuz4bKfnzHTBv6HyVgjahA6mPLwPDzjDWrplJBMjHUFxku/N3FlmrbyPclad+I+4mJ3A==", - "requires": { - "argparse": "^1.0.7", - "esprima": "^4.0.0" - } - }, - "jsbn": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz", - "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=", - "optional": true - }, - "json-buffer": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.0.tgz", - "integrity": "sha1-Wx85evx11ne96Lz8Dkfh+aPZqJg=" - }, - "json-parse-better-errors": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/json-parse-better-errors/-/json-parse-better-errors-1.0.2.tgz", - "integrity": "sha512-mrqyZKfX5EhL7hvqcV6WG1yYjnjeuYDzDhhcAAUrq8Po85NBQBJP+ZDUT75qZQ98IkUoBqdkExkukOU7Ts2wrw==" - }, - "json-schema": { - "version": "0.2.3", - "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz", - "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=" - }, - "json-schema-traverse": { - "version": "0.3.1", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.3.1.tgz", - "integrity": "sha1-NJptRMU6Ud6JtAgFxdXlm0F9M0A=" - }, - "json-stable-stringify": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/json-stable-stringify/-/json-stable-stringify-1.0.1.tgz", - "integrity": "sha1-mnWdOcXy/1A/1TAGRu1EX4jE+a8=", - "requires": { - "jsonify": "~0.0.0" - } - }, - "json-stringify-safe": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", - "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=" - }, - "json5": { - "version": "0.5.1", - "resolved": "https://registry.npmjs.org/json5/-/json5-0.5.1.tgz", - "integrity": "sha1-Hq3nrMASA0rYTiOWdn6tn6VJWCE=" - }, - "jsonify": { - "version": "0.0.0", - "resolved": "https://registry.npmjs.org/jsonify/-/jsonify-0.0.0.tgz", - "integrity": "sha1-LHS27kHZPKUbe1qu6PUDYx0lKnM=" - }, - "jsprim": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz", - "integrity": "sha1-MT5mvB5cwG5Di8G3SZwuXFastqI=", - "requires": { - "assert-plus": "1.0.0", - "extsprintf": "1.3.0", - "json-schema": "0.2.3", - "verror": "1.10.0" - } - }, - "keyv": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/keyv/-/keyv-3.0.0.tgz", - "integrity": "sha512-eguHnq22OE3uVoSYG0LVWNP+4ppamWr9+zWBe1bsNcovIMy6huUJFPgy4mGwCd/rnl3vOLGW1MTlu4c57CT1xA==", - "requires": { - "json-buffer": "3.0.0" - } - }, - "kind-of": { - "version": "3.2.2", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", - "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=", - "requires": { - "is-buffer": "^1.1.5" - } - }, - "levn": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz", - "integrity": "sha1-OwmSTt+fCDwEkP3UwLxEIeBHZO4=", - "requires": { - "prelude-ls": "~1.1.2", - "type-check": "~0.3.2" - } - }, - "link-check": { - "version": "4.4.4", - "resolved": "https://registry.npmjs.org/link-check/-/link-check-4.4.4.tgz", - "integrity": "sha512-yvowNBZEMOFH9nGLiJ5/YV68PBMVTo4opC2SzcACO8g4gSPTB9Rwa5GIziOX9Z5Er3Yf01DHoOyVV2LeApIw8w==", - "requires": { - "is-relative-url": "^2.0.0", - "isemail": "^3.1.2", - "ms": "^2.1.1", - "request": "^2.87.0" - }, - "dependencies": { - "ms": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", - "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==" - } - } - }, - "load-json-file": { - "version": "1.1.0", - "resolved": "http://registry.npmjs.org/load-json-file/-/load-json-file-1.1.0.tgz", - "integrity": "sha1-lWkFcI1YtLq0wiYbBPWfMcmTdMA=", - "requires": { - "graceful-fs": "^4.1.2", - "parse-json": "^2.2.0", - "pify": "^2.0.0", - "pinkie-promise": "^2.0.0", - "strip-bom": "^2.0.0" - }, - "dependencies": { - "parse-json": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-2.2.0.tgz", - "integrity": "sha1-9ID0BDTvgHQfhGkJn43qGPVaTck=", - "requires": { - "error-ex": "^1.2.0" - } - }, - "pify": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", - "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=" - } - } - }, - "load-plugin": { - "version": "2.2.2", - "resolved": "https://registry.npmjs.org/load-plugin/-/load-plugin-2.2.2.tgz", - "integrity": "sha512-FYzamtURIJefQykZGtiClYuZkJBUKzmx8Tc74y8JGAulDzbzVm/C+w/MbAljHRr+REL0cRzy3WgnHE+T8gce5g==", - "requires": { - "npm-prefix": "^1.2.0", - "resolve-from": "^4.0.0" - } - }, - "locate-path": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-2.0.0.tgz", - "integrity": "sha1-K1aLJl7slExtnA3pw9u7ygNUzY4=", - "requires": { - "p-locate": "^2.0.0", - "path-exists": "^3.0.0" - } - }, - "lodash": { - "version": "4.17.11", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz", - "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==" - }, - "log-symbols": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-1.0.2.tgz", - "integrity": "sha1-N2/3tY6jCGoPCfrMdGF+ylAeGhg=", - "requires": { - "chalk": "^1.0.0" - }, - "dependencies": { - "ansi-styles": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz", - "integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4=" - }, - "chalk": { - "version": "1.1.3", - "resolved": "http://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", - "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=", - "requires": { - "ansi-styles": "^2.2.1", - "escape-string-regexp": "^1.0.2", - "has-ansi": "^2.0.0", - "strip-ansi": "^3.0.0", - "supports-color": "^2.0.0" - } - }, - "supports-color": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", - "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=" - } - } - }, - "longest-streak": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/longest-streak/-/longest-streak-2.0.2.tgz", - "integrity": "sha512-TmYTeEYxiAmSVdpbnQDXGtvYOIRsCMg89CVZzwzc2o7GFL1CjoiRPjH5ec0NFAVlAx3fVof9dX/t6KKRAo2OWA==" - }, - "lowercase-keys": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-1.0.1.tgz", - "integrity": "sha512-G2Lj61tXDnVFFOi8VZds+SoQjtQC3dgokKdDG2mTm1tx4m50NUHBOZSBwQQHyy0V12A0JTG4icfZQH+xPyh8VA==" - }, - "map-cache": { - "version": "0.2.2", - "resolved": "https://registry.npmjs.org/map-cache/-/map-cache-0.2.2.tgz", - "integrity": "sha1-wyq9C9ZSXZsFFkW7TyasXcmKDb8=" - }, - "map-like": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/map-like/-/map-like-2.0.0.tgz", - "integrity": "sha1-lEltSa0zPA3DI0snrbvR6FNZU7Q=" - }, - "map-visit": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/map-visit/-/map-visit-1.0.0.tgz", - "integrity": "sha1-7Nyo8TFE5mDxtb1B8S80edmN+48=", - "requires": { - "object-visit": "^1.0.0" - } - }, - "markdown-escapes": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/markdown-escapes/-/markdown-escapes-1.0.2.tgz", - "integrity": "sha512-lbRZ2mE3Q9RtLjxZBZ9+IMl68DKIXaVAhwvwn9pmjnPLS0h/6kyBMgNhqi1xFJ/2yv6cSyv0jbiZavZv93JkkA==" - }, - "markdown-extensions": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/markdown-extensions/-/markdown-extensions-1.1.1.tgz", - "integrity": "sha512-WWC0ZuMzCyDHYCasEGs4IPvLyTGftYwh6wIEOULOF0HXcqZlhwRzrK0w2VUlxWA98xnvb/jszw4ZSkJ6ADpM6Q==" - }, - "markdown-table": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/markdown-table/-/markdown-table-1.1.2.tgz", - "integrity": "sha512-NcWuJFHDA8V3wkDgR/j4+gZx+YQwstPgfQDV8ndUeWWzta3dnDTBxpVzqS9lkmJAuV5YX35lmyojl6HO5JXAgw==" - }, - "math-random": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/math-random/-/math-random-1.0.1.tgz", - "integrity": "sha1-izqsWIuKZuSXXjzepn97sylgH6w=" - }, - "md5": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/md5/-/md5-2.2.1.tgz", - "integrity": "sha1-U6s41f48iJG6RlMp6iP6wFQBJvk=", - "requires": { - "charenc": "~0.0.1", - "crypt": "~0.0.1", - "is-buffer": "~1.1.1" - } - }, - "mdast-util-compact": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/mdast-util-compact/-/mdast-util-compact-1.0.2.tgz", - "integrity": "sha512-d2WS98JSDVbpSsBfVvD9TaDMlqPRz7ohM/11G0rp5jOBb5q96RJ6YLszQ/09AAixyzh23FeIpCGqfaamEADtWg==", - "requires": { - "unist-util-visit": "^1.1.0" - } - }, - "micromatch": { - "version": "2.3.11", - "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-2.3.11.tgz", - "integrity": "sha1-hmd8l9FyCzY0MdBNDRUpO9OMFWU=", - "requires": { - "arr-diff": "^2.0.0", - "array-unique": "^0.2.1", - "braces": "^1.8.2", - "expand-brackets": "^0.1.4", - "extglob": "^0.3.1", - "filename-regex": "^2.0.0", - "is-extglob": "^1.0.0", - "is-glob": "^2.0.1", - "kind-of": "^3.0.2", - "normalize-path": "^2.0.1", - "object.omit": "^2.0.0", - "parse-glob": "^3.0.4", - "regex-cache": "^0.4.2" - } - }, - "mime-db": { - "version": "1.36.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.36.0.tgz", - "integrity": "sha512-L+xvyD9MkoYMXb1jAmzI/lWYAxAMCPvIBSWur0PZ5nOf5euahRLVqH//FKW9mWp2lkqUgYiXPgkzfMUFi4zVDw==" - }, - "mime-types": { - "version": "2.1.20", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.20.tgz", - "integrity": "sha512-HrkrPaP9vGuWbLK1B1FfgAkbqNjIuy4eHlIYnFi7kamZyLLrGlo2mpcx0bBmNpKqBtYtAfGbodDddIgddSJC2A==", - "requires": { - "mime-db": "~1.36.0" - } - }, - "mimic-response": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-1.0.1.tgz", - "integrity": "sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ==" - }, - "minimatch": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", - "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", - "requires": { - "brace-expansion": "^1.1.7" - } - }, - "minimist": { - "version": "1.2.0", - "resolved": "http://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", - "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=" - }, - "mixin-deep": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/mixin-deep/-/mixin-deep-1.3.1.tgz", - "integrity": "sha512-8ZItLHeEgaqEvd5lYBXfm4EZSFCX29Jb9K+lAHhDKzReKBQKj3R+7NOF6tjqYi9t4oI8VUfaWITJQm86wnXGNQ==", - "requires": { - "for-in": "^1.0.2", - "is-extendable": "^1.0.1" - }, - "dependencies": { - "is-extendable": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-1.0.1.tgz", - "integrity": "sha512-arnXMxT1hhoKo9k1LZdmlNyJdDDfy2v0fXjFlmok4+i8ul/6WlbVge9bhM74OpNPQPMGUToDtz+KXa1PneJxOA==", - "requires": { - "is-plain-object": "^2.0.4" - } - } - } - }, - "mkdirp": { - "version": "0.5.1", - "resolved": "http://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", - "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", - "requires": { - "minimist": "0.0.8" - }, - "dependencies": { - "minimist": { - "version": "0.0.8", - "resolved": "http://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", - "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=" - } - } - }, - "ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" - }, - "nan": { - "version": "2.11.0", - "resolved": "https://registry.npmjs.org/nan/-/nan-2.11.0.tgz", - "integrity": "sha512-F4miItu2rGnV2ySkXOQoA8FKz/SR2Q2sWP0sbTxNxz/tuokeC8WxOhPMcwi0qIyGtVn/rrSeLbvVkznqCdwYnw==", - "optional": true - }, - "nanomatch": { - "version": "1.2.13", - "resolved": "https://registry.npmjs.org/nanomatch/-/nanomatch-1.2.13.tgz", - "integrity": "sha512-fpoe2T0RbHwBTBUOftAfBPaDEi06ufaUai0mE6Yn1kacc3SnTErfb/h+X94VXzI64rKFHYImXSvdwGGCmwOqCA==", - "requires": { - "arr-diff": "^4.0.0", - "array-unique": "^0.3.2", - "define-property": "^2.0.2", - "extend-shallow": "^3.0.2", - "fragment-cache": "^0.2.1", - "is-windows": "^1.0.2", - "kind-of": "^6.0.2", - "object.pick": "^1.3.0", - "regex-not": "^1.0.0", - "snapdragon": "^0.8.1", - "to-regex": "^3.0.1" - }, - "dependencies": { - "arr-diff": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/arr-diff/-/arr-diff-4.0.0.tgz", - "integrity": "sha1-1kYQdP6/7HHn4VI1dhoyml3HxSA=" - }, - "array-unique": { - "version": "0.3.2", - "resolved": "https://registry.npmjs.org/array-unique/-/array-unique-0.3.2.tgz", - "integrity": "sha1-qJS3XUvE9s1nnvMkSp/Y9Gri1Cg=" - }, - "kind-of": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.2.tgz", - "integrity": "sha512-s5kLOcnH0XqDO+FvuaLX8DDjZ18CGFk7VygH40QoKPUQhW4e2rvM0rwUq0t8IQDOwYSeLK01U90OjzBTme2QqA==" - } - } - }, - "nlcst-to-string": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/nlcst-to-string/-/nlcst-to-string-2.0.2.tgz", - "integrity": "sha512-DV7wVvMcAsmZ5qEwvX1JUNF4lKkAAKbChwNlIH7NLsPR7LWWoeIt53YlZ5CQH5KDXEXQ9Xa3mw0PbPewymrtew==" - }, - "no-cliches": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/no-cliches/-/no-cliches-0.1.0.tgz", - "integrity": "sha1-9OuBpVH+zegT+MYR415kpRGNw4w=" - }, - "normalize-package-data": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-2.4.0.tgz", - "integrity": "sha512-9jjUFbTPfEy3R/ad/2oNbKtW9Hgovl5O1FvFWKkKblNXoN/Oou6+9+KKohPK13Yc3/TyunyWhJp6gvRNR/PPAw==", - "requires": { - "hosted-git-info": "^2.1.4", - "is-builtin-module": "^1.0.0", - "semver": "2 || 3 || 4 || 5", - "validate-npm-package-license": "^3.0.1" - } - }, - "normalize-path": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-2.1.1.tgz", - "integrity": "sha1-GrKLVW4Zg2Oowab35vogE3/mrtk=", - "requires": { - "remove-trailing-separator": "^1.0.1" - } - }, - "normalize-url": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-2.0.1.tgz", - "integrity": "sha512-D6MUW4K/VzoJ4rJ01JFKxDrtY1v9wrgzCX5f2qj/lzH1m/lW6MhUZFKerVsnyjOhOsYzI9Kqqak+10l4LvLpMw==", - "requires": { - "prepend-http": "^2.0.0", - "query-string": "^5.0.1", - "sort-keys": "^2.0.0" - }, - "dependencies": { - "prepend-http": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/prepend-http/-/prepend-http-2.0.0.tgz", - "integrity": "sha1-6SQ0v6XqjBn0HN/UAddBo8gZ2Jc=" - } - } - }, - "npm-prefix": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/npm-prefix/-/npm-prefix-1.2.0.tgz", - "integrity": "sha1-5hlFX3B0ulTMZtbQ033Z8b5ry8A=", - "requires": { - "rc": "^1.1.0", - "shellsubstitute": "^1.1.0", - "untildify": "^2.1.0" - } - }, - "number-is-nan": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz", - "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0=" - }, - "oauth-sign": { - "version": "0.9.0", - "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz", - "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==" - }, - "object-assign": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", - "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=" - }, - "object-copy": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/object-copy/-/object-copy-0.1.0.tgz", - "integrity": "sha1-fn2Fi3gb18mRpBupde04EnVOmYw=", - "requires": { - "copy-descriptor": "^0.1.0", - "define-property": "^0.2.5", - "kind-of": "^3.0.3" - }, - "dependencies": { - "define-property": { - "version": "0.2.5", - "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz", - "integrity": "sha1-w1se+RjsPJkPmlvFe+BKrOxcgRY=", - "requires": { - "is-descriptor": "^0.1.0" - } - } - } - }, - "object-keys": { - "version": "1.0.12", - "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.0.12.tgz", - "integrity": "sha512-FTMyFUm2wBcGHnH2eXmz7tC6IwlqQZ6mVZ+6dm6vZ4IQIHjs6FdNsQBuKGPuUUUY6NfJw2PshC08Tn6LzLDOag==" - }, - "object-visit": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/object-visit/-/object-visit-1.0.1.tgz", - "integrity": "sha1-95xEk68MU3e1n+OdOV5BBC3QRbs=", - "requires": { - "isobject": "^3.0.0" - }, - "dependencies": { - "isobject": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz", - "integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8=" - } - } - }, - "object.assign": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.0.tgz", - "integrity": "sha512-exHJeq6kBKj58mqGyTQ9DFvrZC/eR6OwxzoM9YRoGBqrXYonaFyGiFMuc9VZrXf7DarreEwMpurG3dd+CNyW5w==", - "requires": { - "define-properties": "^1.1.2", - "function-bind": "^1.1.1", - "has-symbols": "^1.0.0", - "object-keys": "^1.0.11" - } - }, - "object.omit": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/object.omit/-/object.omit-2.0.1.tgz", - "integrity": "sha1-Gpx0SCnznbuFjHbKNXmuKlTr0fo=", - "requires": { - "for-own": "^0.1.4", - "is-extendable": "^0.1.1" - } - }, - "object.pick": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/object.pick/-/object.pick-1.3.0.tgz", - "integrity": "sha1-h6EKxMFpS9Lhy/U1kaZhQftd10c=", - "requires": { - "isobject": "^3.0.1" - }, - "dependencies": { - "isobject": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz", - "integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8=" - } - } - }, - "once": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", - "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", - "requires": { - "wrappy": "1" - } - }, - "optionator": { - "version": "0.8.2", - "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.2.tgz", - "integrity": "sha1-NkxeQJ0/TWMB1sC0wFu6UBgK62Q=", - "requires": { - "deep-is": "~0.1.3", - "fast-levenshtein": "~2.0.4", - "levn": "~0.3.0", - "prelude-ls": "~1.1.2", - "type-check": "~0.3.2", - "wordwrap": "~1.0.0" - } - }, - "os-homedir": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz", - "integrity": "sha1-/7xJiDNuDoM94MFox+8VISGqf7M=" - }, - "p-any": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/p-any/-/p-any-1.1.0.tgz", - "integrity": "sha512-Ef0tVa4CZ5pTAmKn+Cg3w8ABBXh+hHO1aV8281dKOoUHfX+3tjG2EaFcC+aZyagg9b4EYGsHEjz21DnEE8Og2g==", - "requires": { - "p-some": "^2.0.0" - } - }, - "p-cancelable": { - "version": "0.4.1", - "resolved": "http://registry.npmjs.org/p-cancelable/-/p-cancelable-0.4.1.tgz", - "integrity": "sha512-HNa1A8LvB1kie7cERyy21VNeHb2CWJJYqyyC2o3klWFfMGlFmWv2Z7sFgZH8ZiaYL95ydToKTFVXgMV/Os0bBQ==" - }, - "p-finally": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", - "integrity": "sha1-P7z7FbiZpEEjs0ttzBi3JDNqLK4=" - }, - "p-is-promise": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/p-is-promise/-/p-is-promise-1.1.0.tgz", - "integrity": "sha1-nJRWmJ6fZYgBewQ01WCXZ1w9oF4=" - }, - "p-limit": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-1.3.0.tgz", - "integrity": "sha512-vvcXsLAJ9Dr5rQOPk7toZQZJApBl2K4J6dANSsEuh6QI41JYcsS/qhTGa9ErIUUgK3WNQoJYvylxvjqmiqEA9Q==", - "requires": { - "p-try": "^1.0.0" - } - }, - "p-locate": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-2.0.0.tgz", - "integrity": "sha1-IKAQOyIqcMj9OcwuWAaA893l7EM=", - "requires": { - "p-limit": "^1.1.0" - } - }, - "p-some": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/p-some/-/p-some-2.0.1.tgz", - "integrity": "sha1-Zdh8ixVO289SIdFnd4ttLhUPbwY=", - "requires": { - "aggregate-error": "^1.0.0" - } - }, - "p-timeout": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-1.2.1.tgz", - "integrity": "sha1-XrOzU7f86Z8QGhA4iAuwVOu+o4Y=", - "requires": { - "p-finally": "^1.0.0" - } - }, - "p-try": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/p-try/-/p-try-1.0.0.tgz", - "integrity": "sha1-y8ec26+P1CKOE/Yh8rGiN8GyB7M=" - }, - "parse-entities": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/parse-entities/-/parse-entities-1.1.2.tgz", - "integrity": "sha512-5N9lmQ7tmxfXf+hO3X6KRG6w7uYO/HL9fHalSySTdyn63C3WNvTM/1R8tn1u1larNcEbo3Slcy2bsVDQqvEpUg==", - "requires": { - "character-entities": "^1.0.0", - "character-entities-legacy": "^1.0.0", - "character-reference-invalid": "^1.0.0", - "is-alphanumerical": "^1.0.0", - "is-decimal": "^1.0.0", - "is-hexadecimal": "^1.0.0" - } - }, - "parse-glob": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/parse-glob/-/parse-glob-3.0.4.tgz", - "integrity": "sha1-ssN2z7EfNVE7rdFz7wu246OIORw=", - "requires": { - "glob-base": "^0.3.0", - "is-dotfile": "^1.0.0", - "is-extglob": "^1.0.0", - "is-glob": "^2.0.0" - } - }, - "parse-json": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-4.0.0.tgz", - "integrity": "sha1-vjX1Qlvh9/bHRxhPmKeIy5lHfuA=", - "requires": { - "error-ex": "^1.3.1", - "json-parse-better-errors": "^1.0.1" - } - }, - "pascalcase": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/pascalcase/-/pascalcase-0.1.1.tgz", - "integrity": "sha1-s2PlXoAGym/iF4TS2yK9FdeRfxQ=" - }, - "passive-voice": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/passive-voice/-/passive-voice-0.1.0.tgz", - "integrity": "sha1-Fv+RrkC6DpLEPmcXY/3IQqcCcLE=" - }, - "path-exists": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", - "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=" - }, - "path-is-absolute": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", - "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=" - }, - "path-is-inside": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/path-is-inside/-/path-is-inside-1.0.2.tgz", - "integrity": "sha1-NlQX3t5EQw0cEa9hAn+s8HS9/FM=" - }, - "path-to-glob-pattern": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/path-to-glob-pattern/-/path-to-glob-pattern-1.0.2.tgz", - "integrity": "sha1-Rz5qOikqnRP7rj7czuctO6uoxhk=" - }, - "path-type": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/path-type/-/path-type-1.1.0.tgz", - "integrity": "sha1-WcRPfuSR2nBNpBXaWkBwuk+P5EE=", - "requires": { - "graceful-fs": "^4.1.2", - "pify": "^2.0.0", - "pinkie-promise": "^2.0.0" - }, - "dependencies": { - "pify": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", - "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=" - } - } - }, - "performance-now": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", - "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=" - }, - "pify": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz", - "integrity": "sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY=" - }, - "pinkie": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/pinkie/-/pinkie-2.0.4.tgz", - "integrity": "sha1-clVrgM+g1IqXToDnckjoDtT3+HA=" - }, - "pinkie-promise": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/pinkie-promise/-/pinkie-promise-2.0.1.tgz", - "integrity": "sha1-ITXW36ejWMBprJsXh3YogihFD/o=", - "requires": { - "pinkie": "^2.0.0" - } - }, - "pluralize": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/pluralize/-/pluralize-2.0.0.tgz", - "integrity": "sha1-crcmqm+sHt7uQiVsfY3CVrM1Z38=" - }, - "posix-character-classes": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/posix-character-classes/-/posix-character-classes-0.1.1.tgz", - "integrity": "sha1-AerA/jta9xoqbAL+q7jB/vfgDqs=" - }, - "prelude-ls": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz", - "integrity": "sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ=" - }, - "prepend-http": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/prepend-http/-/prepend-http-1.0.4.tgz", - "integrity": "sha1-1PRWKwzjaW5BrFLQ4ALlemNdxtw=" - }, - "preserve": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/preserve/-/preserve-0.2.0.tgz", - "integrity": "sha1-gV7R9uvGWSb4ZbMQwHE7yzMVzks=" - }, - "prettier": { - "version": "1.14.3", - "resolved": "https://registry.npmjs.org/prettier/-/prettier-1.14.3.tgz", - "integrity": "sha512-qZDVnCrnpsRJJq5nSsiHCE3BYMED2OtsI+cmzIzF1QIfqm5ALf8tEJcO27zV1gKNKRPdhjO0dNWnrzssDQ1tFg==" - }, - "process-nextick-args": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz", - "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==" - }, - "psl": { - "version": "1.1.29", - "resolved": "https://registry.npmjs.org/psl/-/psl-1.1.29.tgz", - "integrity": "sha512-AeUmQ0oLN02flVHXWh9sSJF7mcdFq0ppid/JkErufc3hGIV/AMa8Fo9VgDo/cT2jFdOWoFvHp90qqBH54W+gjQ==" - }, - "public-ip": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/public-ip/-/public-ip-2.4.0.tgz", - "integrity": "sha512-74cIy+T2cDmt+Z71AfVipH2q6qqZITPyNGszKV86OGDYIRvti1m8zg4GOaiTPCLgEIWnToKYXbhEnMiZWHPEUA==", - "requires": { - "dns-socket": "^1.6.2", - "got": "^8.0.0", - "is-ip": "^2.0.0", - "pify": "^3.0.0" - }, - "dependencies": { - "got": { - "version": "8.3.2", - "resolved": "https://registry.npmjs.org/got/-/got-8.3.2.tgz", - "integrity": "sha512-qjUJ5U/hawxosMryILofZCkm3C84PLJS/0grRIpjAwu+Lkxxj5cxeCU25BG0/3mDSpXKTyZr8oh8wIgLaH0QCw==", - "requires": { - "@sindresorhus/is": "^0.7.0", - "cacheable-request": "^2.1.1", - "decompress-response": "^3.3.0", - "duplexer3": "^0.1.4", - "get-stream": "^3.0.0", - "into-stream": "^3.1.0", - "is-retry-allowed": "^1.1.0", - "isurl": "^1.0.0-alpha5", - "lowercase-keys": "^1.0.0", - "mimic-response": "^1.0.0", - "p-cancelable": "^0.4.0", - "p-timeout": "^2.0.1", - "pify": "^3.0.0", - "safe-buffer": "^5.1.1", - "timed-out": "^4.0.1", - "url-parse-lax": "^3.0.0", - "url-to-options": "^1.0.1" - } - }, - "p-timeout": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-2.0.1.tgz", - "integrity": "sha512-88em58dDVB/KzPEx1X0N3LwFfYZPyDc4B6eF38M1rk9VTZMbxXXgjugz8mmwpS9Ox4BDZ+t6t3QP5+/gazweIA==", - "requires": { - "p-finally": "^1.0.0" - } - }, - "prepend-http": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/prepend-http/-/prepend-http-2.0.0.tgz", - "integrity": "sha1-6SQ0v6XqjBn0HN/UAddBo8gZ2Jc=" - }, - "url-parse-lax": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/url-parse-lax/-/url-parse-lax-3.0.0.tgz", - "integrity": "sha1-FrXK/Afb42dsGxmZF3gj1lA6yww=", - "requires": { - "prepend-http": "^2.0.0" - } - } - } - }, - "punycode": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", - "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==" - }, - "qs": { - "version": "6.5.2", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz", - "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==" - }, - "query-string": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/query-string/-/query-string-5.1.1.tgz", - "integrity": "sha512-gjWOsm2SoGlgLEdAGt7a6slVOk9mGiXmPFMqrEhLQ68rhQuBnpfs3+EmlvqKyxnCo9/PPlF+9MtY02S1aFg+Jw==", - "requires": { - "decode-uri-component": "^0.2.0", - "object-assign": "^4.1.0", - "strict-uri-encode": "^1.0.0" - } - }, - "randomatic": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/randomatic/-/randomatic-3.1.0.tgz", - "integrity": "sha512-KnGPVE0lo2WoXxIZ7cPR8YBpiol4gsSuOwDSg410oHh80ZMp5EiypNqL2K4Z77vJn6lB5rap7IkAmcUlalcnBQ==", - "requires": { - "is-number": "^4.0.0", - "kind-of": "^6.0.0", - "math-random": "^1.0.1" - }, - "dependencies": { - "is-number": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/is-number/-/is-number-4.0.0.tgz", - "integrity": "sha512-rSklcAIlf1OmFdyAqbnWTLVelsQ58uvZ66S/ZyawjWqIviTWCjg2PzVGw8WUA+nNuPTqb4wgA+NszrJ+08LlgQ==" - }, - "kind-of": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.2.tgz", - "integrity": "sha512-s5kLOcnH0XqDO+FvuaLX8DDjZ18CGFk7VygH40QoKPUQhW4e2rvM0rwUq0t8IQDOwYSeLK01U90OjzBTme2QqA==" - } - } - }, - "rc": { - "version": "1.2.8", - "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", - "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", - "requires": { - "deep-extend": "^0.6.0", - "ini": "~1.3.0", - "minimist": "^1.2.0", - "strip-json-comments": "~2.0.1" - } - }, - "rc-config-loader": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/rc-config-loader/-/rc-config-loader-2.0.2.tgz", - "integrity": "sha512-Nx9SNM47eNRqe0TdntOY600qWb8NDh+xU9sv5WnTscEtzfTB0ukihlqwuCLPteyJksvZ0sEVPoySNE01TKrmTQ==", - "requires": { - "debug": "^3.1.0", - "js-yaml": "^3.12.0", - "json5": "^1.0.1", - "object-assign": "^4.1.0", - "object-keys": "^1.0.12", - "path-exists": "^3.0.0", - "require-from-string": "^2.0.2" - }, - "dependencies": { - "debug": { - "version": "3.2.5", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.5.tgz", - "integrity": "sha512-D61LaDQPQkxJ5AUM2mbSJRbPkNs/TmdmOeLAi1hgDkpDfIfetSrjmWhccwtuResSwMbACjx/xXQofvM9CE/aeg==", - "requires": { - "ms": "^2.1.1" - } - }, - "json5": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/json5/-/json5-1.0.1.tgz", - "integrity": "sha512-aKS4WQjPenRxiQsC93MNfjx+nbF4PAdYzmd/1JIj8HYzqfbu86beTuNgXDzPknWk0n0uARlyewZo4s++ES36Ow==", - "requires": { - "minimist": "^1.2.0" - } - }, - "ms": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", - "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==" - } - } - }, - "read-pkg": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-1.1.0.tgz", - "integrity": "sha1-9f+qXs0pyzHAR0vKfXVra7KePyg=", - "requires": { - "load-json-file": "^1.0.0", - "normalize-package-data": "^2.3.2", - "path-type": "^1.0.0" - } - }, - "read-pkg-up": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/read-pkg-up/-/read-pkg-up-3.0.0.tgz", - "integrity": "sha1-PtSWaF26D4/hGNBpHcUfSh/5bwc=", - "requires": { - "find-up": "^2.0.0", - "read-pkg": "^3.0.0" - }, - "dependencies": { - "load-json-file": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-4.0.0.tgz", - "integrity": "sha1-L19Fq5HjMhYjT9U62rZo607AmTs=", - "requires": { - "graceful-fs": "^4.1.2", - "parse-json": "^4.0.0", - "pify": "^3.0.0", - "strip-bom": "^3.0.0" - } - }, - "path-type": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/path-type/-/path-type-3.0.0.tgz", - "integrity": "sha512-T2ZUsdZFHgA3u4e5PfPbjd7HDDpxPnQb5jN0SrDsjNSuVXHJqtwTnWqG0B1jZrgmJ/7lj1EmVIByWt1gxGkWvg==", - "requires": { - "pify": "^3.0.0" - } - }, - "read-pkg": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-3.0.0.tgz", - "integrity": "sha1-nLxoaXj+5l0WwA4rGcI3/Pbjg4k=", - "requires": { - "load-json-file": "^4.0.0", - "normalize-package-data": "^2.3.2", - "path-type": "^3.0.0" - } - }, - "strip-bom": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz", - "integrity": "sha1-IzTBjpx1n3vdVv3vfprj1YjmjtM=" - } - } - }, - "readable-stream": { - "version": "2.3.6", - "resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", - "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", - "requires": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.3", - "isarray": "~1.0.0", - "process-nextick-args": "~2.0.0", - "safe-buffer": "~5.1.1", - "string_decoder": "~1.1.1", - "util-deprecate": "~1.0.1" - } - }, - "readdirp": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-2.2.1.tgz", - "integrity": "sha512-1JU/8q+VgFZyxwrJ+SVIOsh+KywWGpds3NTqikiKpDMZWScmAYyKIgqkO+ARvNWJfXeXR1zxz7aHF4u4CyH6vQ==", - "requires": { - "graceful-fs": "^4.1.11", - "micromatch": "^3.1.10", - "readable-stream": "^2.0.2" - }, - "dependencies": { - "arr-diff": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/arr-diff/-/arr-diff-4.0.0.tgz", - "integrity": "sha1-1kYQdP6/7HHn4VI1dhoyml3HxSA=" - }, - "array-unique": { - "version": "0.3.2", - "resolved": "https://registry.npmjs.org/array-unique/-/array-unique-0.3.2.tgz", - "integrity": "sha1-qJS3XUvE9s1nnvMkSp/Y9Gri1Cg=" - }, - "braces": { - "version": "2.3.2", - "resolved": "https://registry.npmjs.org/braces/-/braces-2.3.2.tgz", - "integrity": "sha512-aNdbnj9P8PjdXU4ybaWLK2IF3jc/EoDYbC7AazW6to3TRsfXxscC9UXOB5iDiEQrkyIbWp2SLQda4+QAa7nc3w==", - "requires": { - "arr-flatten": "^1.1.0", - "array-unique": "^0.3.2", - "extend-shallow": "^2.0.1", - "fill-range": "^4.0.0", - "isobject": "^3.0.1", - "repeat-element": "^1.1.2", - "snapdragon": "^0.8.1", - "snapdragon-node": "^2.0.1", - "split-string": "^3.0.2", - "to-regex": "^3.0.1" - }, - "dependencies": { - "extend-shallow": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", - "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=", - "requires": { - "is-extendable": "^0.1.0" - } - } - } - }, - "expand-brackets": { - "version": "2.1.4", - "resolved": "https://registry.npmjs.org/expand-brackets/-/expand-brackets-2.1.4.tgz", - "integrity": "sha1-t3c14xXOMPa27/D4OwQVGiJEliI=", - "requires": { - "debug": "^2.3.3", - "define-property": "^0.2.5", - "extend-shallow": "^2.0.1", - "posix-character-classes": "^0.1.0", - "regex-not": "^1.0.0", - "snapdragon": "^0.8.1", - "to-regex": "^3.0.1" - }, - "dependencies": { - "define-property": { - "version": "0.2.5", - "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz", - "integrity": "sha1-w1se+RjsPJkPmlvFe+BKrOxcgRY=", - "requires": { - "is-descriptor": "^0.1.0" - } - }, - "extend-shallow": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", - "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=", - "requires": { - "is-extendable": "^0.1.0" - } - }, - "is-accessor-descriptor": { - "version": "0.1.6", - "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz", - "integrity": "sha1-qeEss66Nh2cn7u84Q/igiXtcmNY=", - "requires": { - "kind-of": "^3.0.2" - }, - "dependencies": { - "kind-of": { - "version": "3.2.2", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", - "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=", - "requires": { - "is-buffer": "^1.1.5" - } - } - } - }, - "is-data-descriptor": { - "version": "0.1.4", - "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz", - "integrity": "sha1-C17mSDiOLIYCgueT8YVv7D8wG1Y=", - "requires": { - "kind-of": "^3.0.2" - }, - "dependencies": { - "kind-of": { - "version": "3.2.2", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", - "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=", - "requires": { - "is-buffer": "^1.1.5" - } - } - } - }, - "is-descriptor": { - "version": "0.1.6", - "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-0.1.6.tgz", - "integrity": "sha512-avDYr0SB3DwO9zsMov0gKCESFYqCnE4hq/4z3TdUlukEy5t9C0YRq7HLrsN52NAcqXKaepeCD0n+B0arnVG3Hg==", - "requires": { - "is-accessor-descriptor": "^0.1.6", - "is-data-descriptor": "^0.1.4", - "kind-of": "^5.0.0" - } - }, - "kind-of": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-5.1.0.tgz", - "integrity": "sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw==" - } - } - }, - "extglob": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/extglob/-/extglob-2.0.4.tgz", - "integrity": "sha512-Nmb6QXkELsuBr24CJSkilo6UHHgbekK5UiZgfE6UHD3Eb27YC6oD+bhcT+tJ6cl8dmsgdQxnWlcry8ksBIBLpw==", - "requires": { - "array-unique": "^0.3.2", - "define-property": "^1.0.0", - "expand-brackets": "^2.1.4", - "extend-shallow": "^2.0.1", - "fragment-cache": "^0.2.1", - "regex-not": "^1.0.0", - "snapdragon": "^0.8.1", - "to-regex": "^3.0.1" - }, - "dependencies": { - "define-property": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/define-property/-/define-property-1.0.0.tgz", - "integrity": "sha1-dp66rz9KY6rTr56NMEybvnm/sOY=", - "requires": { - "is-descriptor": "^1.0.0" - } - }, - "extend-shallow": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", - "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=", - "requires": { - "is-extendable": "^0.1.0" - } - } - } - }, - "fill-range": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-4.0.0.tgz", - "integrity": "sha1-1USBHUKPmOsGpj3EAtJAPDKMOPc=", - "requires": { - "extend-shallow": "^2.0.1", - "is-number": "^3.0.0", - "repeat-string": "^1.6.1", - "to-regex-range": "^2.1.0" - }, - "dependencies": { - "extend-shallow": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", - "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=", - "requires": { - "is-extendable": "^0.1.0" - } - } - } - }, - "is-accessor-descriptor": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz", - "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", - "requires": { - "kind-of": "^6.0.0" - } - }, - "is-data-descriptor": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz", - "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", - "requires": { - "kind-of": "^6.0.0" - } - }, - "is-descriptor": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz", - "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==", - "requires": { - "is-accessor-descriptor": "^1.0.0", - "is-data-descriptor": "^1.0.0", - "kind-of": "^6.0.2" - } - }, - "is-number": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-number/-/is-number-3.0.0.tgz", - "integrity": "sha1-JP1iAaR4LPUFYcgQJ2r8fRLXEZU=", - "requires": { - "kind-of": "^3.0.2" - }, - "dependencies": { - "kind-of": { - "version": "3.2.2", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", - "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=", - "requires": { - "is-buffer": "^1.1.5" - } - } - } - }, - "isobject": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz", - "integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8=" - }, - "kind-of": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.2.tgz", - "integrity": "sha512-s5kLOcnH0XqDO+FvuaLX8DDjZ18CGFk7VygH40QoKPUQhW4e2rvM0rwUq0t8IQDOwYSeLK01U90OjzBTme2QqA==" - }, - "micromatch": { - "version": "3.1.10", - "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-3.1.10.tgz", - "integrity": "sha512-MWikgl9n9M3w+bpsY3He8L+w9eF9338xRl8IAO5viDizwSzziFEyUzo2xrrloB64ADbTf8uA8vRqqttDTOmccg==", - "requires": { - "arr-diff": "^4.0.0", - "array-unique": "^0.3.2", - "braces": "^2.3.1", - "define-property": "^2.0.2", - "extend-shallow": "^3.0.2", - "extglob": "^2.0.4", - "fragment-cache": "^0.2.1", - "kind-of": "^6.0.2", - "nanomatch": "^1.2.9", - "object.pick": "^1.3.0", - "regex-not": "^1.0.0", - "snapdragon": "^0.8.1", - "to-regex": "^3.0.2" - } - } - } - }, - "regex-cache": { - "version": "0.4.4", - "resolved": "https://registry.npmjs.org/regex-cache/-/regex-cache-0.4.4.tgz", - "integrity": "sha512-nVIZwtCjkC9YgvWkpM55B5rBhBYRZhAaJbgcFYXXsHnbZ9UZI9nnVWYZpBlCqv9ho2eZryPnWrZGsOdPwVWXWQ==", - "requires": { - "is-equal-shallow": "^0.1.3" - } - }, - "regex-not": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/regex-not/-/regex-not-1.0.2.tgz", - "integrity": "sha512-J6SDjUgDxQj5NusnOtdFxDwN/+HWykR8GELwctJ7mdqhcyy1xEc4SRFHUXvxTp661YaVKAjfRLZ9cCqS6tn32A==", - "requires": { - "extend-shallow": "^3.0.2", - "safe-regex": "^1.1.0" - } - }, - "remark": { - "version": "9.0.0", - "resolved": "https://registry.npmjs.org/remark/-/remark-9.0.0.tgz", - "integrity": "sha512-amw8rGdD5lHbMEakiEsllmkdBP+/KpjW/PRK6NSGPZKCQowh0BT4IWXDAkRMyG3SB9dKPXWMviFjNusXzXNn3A==", - "requires": { - "remark-parse": "^5.0.0", - "remark-stringify": "^5.0.0", - "unified": "^6.0.0" - } - }, - "remark-cli": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/remark-cli/-/remark-cli-5.0.0.tgz", - "integrity": "sha512-+j0tza5XZ/XHfity3mg5GJFezRt5hS+ybC7/LDItmOAA8u8gRgB51B+/m5U3yT6RLlhefdqkMGKZnZMcamnvsQ==", - "requires": { - "markdown-extensions": "^1.1.0", - "remark": "^9.0.0", - "unified-args": "^5.0.0" - } - }, - "remark-frontmatter": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/remark-frontmatter/-/remark-frontmatter-1.2.1.tgz", - "integrity": "sha512-PEXZFO3jrB+E0G6ZIsV8GOED1gPHQF5hgedJQJ8SbsLRQv4KKrFj3A+huaeu0qtzTScdxPeDTacQ9gkV4vIarA==", - "requires": { - "fault": "^1.0.1", - "xtend": "^4.0.1" - } - }, - "remark-lint-no-dead-urls": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/remark-lint-no-dead-urls/-/remark-lint-no-dead-urls-0.3.0.tgz", - "integrity": "sha512-eG+vVrNui7zeBmU6fsjIi8rwXriuyNhNcmJDQ7M5oaxCluWbH5bt6Yi/JNsabYE39dFdlVbw9JM3cLjaJv2hQw==", - "requires": { - "is-online": "^7.0.0", - "is-relative-url": "^2.0.0", - "link-check": "^4.1.0", - "unified-lint-rule": "^1.0.1", - "unist-util-visit": "^1.1.3" - } - }, - "remark-lint-write-good": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/remark-lint-write-good/-/remark-lint-write-good-1.0.3.tgz", - "integrity": "sha512-d4D4VrAklAx2ONhpXoQnt0YrJFpJBE5XEeCyDGjPhm4DkIoLOmHWZEjxl1HvdrpGXLb/KfYU4lJPeyxlKiDhVA==", - "requires": { - "nlcst-to-string": "^2.0.0", - "unified-lint-rule": "^1.0.1", - "unist-util-visit": "^1.1.1", - "write-good": "^0.11.1" - } - }, - "remark-parse": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/remark-parse/-/remark-parse-5.0.0.tgz", - "integrity": "sha512-b3iXszZLH1TLoyUzrATcTQUZrwNl1rE70rVdSruJFlDaJ9z5aMkhrG43Pp68OgfHndL/ADz6V69Zow8cTQu+JA==", - "requires": { - "collapse-white-space": "^1.0.2", - "is-alphabetical": "^1.0.0", - "is-decimal": "^1.0.0", - "is-whitespace-character": "^1.0.0", - "is-word-character": "^1.0.0", - "markdown-escapes": "^1.0.0", - "parse-entities": "^1.1.0", - "repeat-string": "^1.5.4", - "state-toggle": "^1.0.0", - "trim": "0.0.1", - "trim-trailing-lines": "^1.0.0", - "unherit": "^1.0.4", - "unist-util-remove-position": "^1.0.0", - "vfile-location": "^2.0.0", - "xtend": "^4.0.1" - } - }, - "remark-stringify": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/remark-stringify/-/remark-stringify-5.0.0.tgz", - "integrity": "sha512-Ws5MdA69ftqQ/yhRF9XhVV29mhxbfGhbz0Rx5bQH+oJcNhhSM6nCu1EpLod+DjrFGrU0BMPs+czVmJZU7xiS7w==", - "requires": { - "ccount": "^1.0.0", - "is-alphanumeric": "^1.0.0", - "is-decimal": "^1.0.0", - "is-whitespace-character": "^1.0.0", - "longest-streak": "^2.0.1", - "markdown-escapes": "^1.0.0", - "markdown-table": "^1.1.0", - "mdast-util-compact": "^1.0.0", - "parse-entities": "^1.0.2", - "repeat-string": "^1.5.4", - "state-toggle": "^1.0.0", - "stringify-entities": "^1.0.1", - "unherit": "^1.0.4", - "xtend": "^4.0.1" - } - }, - "remove-trailing-separator": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/remove-trailing-separator/-/remove-trailing-separator-1.1.0.tgz", - "integrity": "sha1-wkvOKig62tW8P1jg1IJJuSN52O8=" - }, - "repeat-element": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/repeat-element/-/repeat-element-1.1.3.tgz", - "integrity": "sha512-ahGq0ZnV5m5XtZLMb+vP76kcAM5nkLqk0lpqAuojSKGgQtn4eRi4ZZGm2olo2zKFH+sMsWaqOCW1dqAnOru72g==" - }, - "repeat-string": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/repeat-string/-/repeat-string-1.6.1.tgz", - "integrity": "sha1-jcrkcOHIirwtYA//Sndihtp15jc=" - }, - "replace-ext": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/replace-ext/-/replace-ext-1.0.0.tgz", - "integrity": "sha1-3mMSg3P8v3w8z6TeWkgMRaZ5WOs=" - }, - "request": { - "version": "2.88.0", - "resolved": "https://registry.npmjs.org/request/-/request-2.88.0.tgz", - "integrity": "sha512-NAqBSrijGLZdM0WZNsInLJpkJokL72XYjUpnB0iwsRgxh7dB6COrHnTBNwN0E+lHDAJzu7kLAkDeY08z2/A0hg==", - "requires": { - "aws-sign2": "~0.7.0", - "aws4": "^1.8.0", - "caseless": "~0.12.0", - "combined-stream": "~1.0.6", - "extend": "~3.0.2", - "forever-agent": "~0.6.1", - "form-data": "~2.3.2", - "har-validator": "~5.1.0", - "http-signature": "~1.2.0", - "is-typedarray": "~1.0.0", - "isstream": "~0.1.2", - "json-stringify-safe": "~5.0.1", - "mime-types": "~2.1.19", - "oauth-sign": "~0.9.0", - "performance-now": "^2.1.0", - "qs": "~6.5.2", - "safe-buffer": "^5.1.2", - "tough-cookie": "~2.4.3", - "tunnel-agent": "^0.6.0", - "uuid": "^3.3.2" - } - }, - "require-from-string": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", - "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==" - }, - "resolve-from": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", - "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==" - }, - "resolve-url": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/resolve-url/-/resolve-url-0.2.1.tgz", - "integrity": "sha1-LGN/53yJOv0qZj/iGqkIAGjiBSo=" - }, - "responselike": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/responselike/-/responselike-1.0.2.tgz", - "integrity": "sha1-kYcg7ztjHFZCvgaPFa3lpG9Loec=", - "requires": { - "lowercase-keys": "^1.0.0" - } - }, - "ret": { - "version": "0.1.15", - "resolved": "https://registry.npmjs.org/ret/-/ret-0.1.15.tgz", - "integrity": "sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg==" - }, - "rimraf": { - "version": "2.6.2", - "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.2.tgz", - "integrity": "sha512-lreewLK/BlghmxtfH36YYVg1i8IAce4TI7oao75I1g245+6BctqTVQiBP3YUJ9C6DQOXJmkYR9X9fCLtCOJc5w==", - "requires": { - "glob": "^7.0.5" - } - }, - "safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" - }, - "safe-regex": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/safe-regex/-/safe-regex-1.1.0.tgz", - "integrity": "sha1-QKNmnzsHfR6UPURinhV91IAjvy4=", - "requires": { - "ret": "~0.1.10" - } - }, - "safer-buffer": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", - "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" - }, - "semver": { - "version": "5.5.1", - "resolved": "https://registry.npmjs.org/semver/-/semver-5.5.1.tgz", - "integrity": "sha512-PqpAxfrEhlSUWge8dwIp4tZnQ25DIOthpiaHNIthsjEFQD6EvqUKUDM7L8O2rShkFccYo1VjJR0coWfNkCubRw==" - }, - "set-value": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/set-value/-/set-value-2.0.0.tgz", - "integrity": "sha512-hw0yxk9GT/Hr5yJEYnHNKYXkIA8mVJgd9ditYZCe16ZczcaELYYcfvaXesNACk2O8O0nTiPQcQhGUQj8JLzeeg==", - "requires": { - "extend-shallow": "^2.0.1", - "is-extendable": "^0.1.1", - "is-plain-object": "^2.0.3", - "split-string": "^3.0.1" - }, - "dependencies": { - "extend-shallow": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", - "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=", - "requires": { - "is-extendable": "^0.1.0" - } - } - } - }, - "shellsubstitute": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/shellsubstitute/-/shellsubstitute-1.2.0.tgz", - "integrity": "sha1-5PcCpQxRiw9v6YRRiQ1wWvKba3A=" - }, - "slice-ansi": { - "version": "0.0.4", - "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-0.0.4.tgz", - "integrity": "sha1-7b+JA/ZvfOL46v1s7tZeJkyDGzU=" - }, - "sliced": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/sliced/-/sliced-1.0.1.tgz", - "integrity": "sha1-CzpmK10Ewxd7GSa+qCsD+Dei70E=" - }, - "snapdragon": { - "version": "0.8.2", - "resolved": "https://registry.npmjs.org/snapdragon/-/snapdragon-0.8.2.tgz", - "integrity": "sha512-FtyOnWN/wCHTVXOMwvSv26d+ko5vWlIDD6zoUJ7LW8vh+ZBC8QdljveRP+crNrtBwioEUWy/4dMtbBjA4ioNlg==", - "requires": { - "base": "^0.11.1", - "debug": "^2.2.0", - "define-property": "^0.2.5", - "extend-shallow": "^2.0.1", - "map-cache": "^0.2.2", - "source-map": "^0.5.6", - "source-map-resolve": "^0.5.0", - "use": "^3.1.0" - }, - "dependencies": { - "define-property": { - "version": "0.2.5", - "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz", - "integrity": "sha1-w1se+RjsPJkPmlvFe+BKrOxcgRY=", - "requires": { - "is-descriptor": "^0.1.0" - } - }, - "extend-shallow": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", - "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=", - "requires": { - "is-extendable": "^0.1.0" - } - } - } - }, - "snapdragon-node": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/snapdragon-node/-/snapdragon-node-2.1.1.tgz", - "integrity": "sha512-O27l4xaMYt/RSQ5TR3vpWCAB5Kb/czIcqUFOM/C4fYcLnbZUc1PkjTAMjof2pBWaSTwOUd6qUHcFGVGj7aIwnw==", - "requires": { - "define-property": "^1.0.0", - "isobject": "^3.0.0", - "snapdragon-util": "^3.0.1" - }, - "dependencies": { - "define-property": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/define-property/-/define-property-1.0.0.tgz", - "integrity": "sha1-dp66rz9KY6rTr56NMEybvnm/sOY=", - "requires": { - "is-descriptor": "^1.0.0" - } - }, - "is-accessor-descriptor": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz", - "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", - "requires": { - "kind-of": "^6.0.0" - } - }, - "is-data-descriptor": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz", - "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", - "requires": { - "kind-of": "^6.0.0" - } - }, - "is-descriptor": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz", - "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==", - "requires": { - "is-accessor-descriptor": "^1.0.0", - "is-data-descriptor": "^1.0.0", - "kind-of": "^6.0.2" - } - }, - "isobject": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz", - "integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8=" - }, - "kind-of": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.2.tgz", - "integrity": "sha512-s5kLOcnH0XqDO+FvuaLX8DDjZ18CGFk7VygH40QoKPUQhW4e2rvM0rwUq0t8IQDOwYSeLK01U90OjzBTme2QqA==" - } - } - }, - "snapdragon-util": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/snapdragon-util/-/snapdragon-util-3.0.1.tgz", - "integrity": "sha512-mbKkMdQKsjX4BAL4bRYTj21edOf8cN7XHdYUJEe+Zn99hVEYcMvKPct1IqNe7+AZPirn8BCDOQBHQZknqmKlZQ==", - "requires": { - "kind-of": "^3.2.0" - } - }, - "sort-keys": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/sort-keys/-/sort-keys-2.0.0.tgz", - "integrity": "sha1-ZYU1WEhh7JfXMNbPQYIuH1ZoQSg=", - "requires": { - "is-plain-obj": "^1.0.0" - } - }, - "source-map": { - "version": "0.5.7", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz", - "integrity": "sha1-igOdLRAh0i0eoUyA2OpGi6LvP8w=" - }, - "source-map-resolve": { - "version": "0.5.2", - "resolved": "https://registry.npmjs.org/source-map-resolve/-/source-map-resolve-0.5.2.tgz", - "integrity": "sha512-MjqsvNwyz1s0k81Goz/9vRBe9SZdB09Bdw+/zYyO+3CuPk6fouTaxscHkgtE8jKvf01kVfl8riHzERQ/kefaSA==", - "requires": { - "atob": "^2.1.1", - "decode-uri-component": "^0.2.0", - "resolve-url": "^0.2.1", - "source-map-url": "^0.4.0", - "urix": "^0.1.0" - } - }, - "source-map-url": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/source-map-url/-/source-map-url-0.4.0.tgz", - "integrity": "sha1-PpNdfd1zYxuXZZlW1VEo6HtQhKM=" - }, - "spdx-correct": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/spdx-correct/-/spdx-correct-3.0.0.tgz", - "integrity": "sha512-N19o9z5cEyc8yQQPukRCZ9EUmb4HUpnrmaL/fxS2pBo2jbfcFRVuFZ/oFC+vZz0MNNk0h80iMn5/S6qGZOL5+g==", - "requires": { - "spdx-expression-parse": "^3.0.0", - "spdx-license-ids": "^3.0.0" - } - }, - "spdx-exceptions": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/spdx-exceptions/-/spdx-exceptions-2.1.0.tgz", - "integrity": "sha512-4K1NsmrlCU1JJgUrtgEeTVyfx8VaYea9J9LvARxhbHtVtohPs/gFGG5yy49beySjlIMhhXZ4QqujIZEfS4l6Cg==" - }, - "spdx-expression-parse": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/spdx-expression-parse/-/spdx-expression-parse-3.0.0.tgz", - "integrity": "sha512-Yg6D3XpRD4kkOmTpdgbUiEJFKghJH03fiC1OPll5h/0sO6neh2jqRDVHOQ4o/LMea0tgCkbMgea5ip/e+MkWyg==", - "requires": { - "spdx-exceptions": "^2.1.0", - "spdx-license-ids": "^3.0.0" - } - }, - "spdx-license-ids": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.1.tgz", - "integrity": "sha512-TfOfPcYGBB5sDuPn3deByxPhmfegAhpDYKSOXZQN81Oyrrif8ZCodOLzK3AesELnCx03kikhyDwh0pfvvQvF8w==" - }, - "split-lines": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/split-lines/-/split-lines-2.0.0.tgz", - "integrity": "sha512-gaIdhbqxkB5/VflPXsJwZvEzh/kdwiRPF9iqpkxX4us+lzB8INedFwjCyo6vwuz5x2Ddlnav2zh270CEjCG8mA==" - }, - "split-string": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/split-string/-/split-string-3.1.0.tgz", - "integrity": "sha512-NzNVhJDYpwceVVii8/Hu6DKfD2G+NrQHlS/V/qgv763EYudVwEcMQNxd2lh+0VrUByXN/oJkl5grOhYWvQUYiw==", - "requires": { - "extend-shallow": "^3.0.0" - } - }, - "sprintf-js": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", - "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=" - }, - "sshpk": { - "version": "1.14.2", - "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.14.2.tgz", - "integrity": "sha1-xvxhZIo9nE52T9P8306hBeSSupg=", - "requires": { - "asn1": "~0.2.3", - "assert-plus": "^1.0.0", - "bcrypt-pbkdf": "^1.0.0", - "dashdash": "^1.12.0", - "ecc-jsbn": "~0.1.1", - "getpass": "^0.1.1", - "jsbn": "~0.1.0", - "safer-buffer": "^2.0.2", - "tweetnacl": "~0.14.0" - } - }, - "state-toggle": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/state-toggle/-/state-toggle-1.0.1.tgz", - "integrity": "sha512-Qe8QntFrrpWTnHwvwj2FZTgv+PKIsp0B9VxLzLLbSpPXWOgRgc5LVj/aTiSfK1RqIeF9jeC1UeOH8Q8y60A7og==" - }, - "static-extend": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/static-extend/-/static-extend-0.1.2.tgz", - "integrity": "sha1-YICcOcv/VTNyJv1eC1IPNB8ftcY=", - "requires": { - "define-property": "^0.2.5", - "object-copy": "^0.1.0" - }, - "dependencies": { - "define-property": { - "version": "0.2.5", - "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz", - "integrity": "sha1-w1se+RjsPJkPmlvFe+BKrOxcgRY=", - "requires": { - "is-descriptor": "^0.1.0" - } - } - } - }, - "strict-uri-encode": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/strict-uri-encode/-/strict-uri-encode-1.1.0.tgz", - "integrity": "sha1-J5siXfHVgrH1TmWt3UNS4Y+qBxM=" - }, - "string-width": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", - "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=", - "requires": { - "code-point-at": "^1.0.0", - "is-fullwidth-code-point": "^1.0.0", - "strip-ansi": "^3.0.0" - } - }, - "string.prototype.padstart": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/string.prototype.padstart/-/string.prototype.padstart-3.0.0.tgz", - "integrity": "sha1-W8+tOfRkm7LQMSkuGbzwtRDUskI=", - "requires": { - "define-properties": "^1.1.2", - "es-abstract": "^1.4.3", - "function-bind": "^1.0.2" - } - }, - "string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "requires": { - "safe-buffer": "~5.1.0" - } - }, - "stringify-entities": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/stringify-entities/-/stringify-entities-1.3.2.tgz", - "integrity": "sha512-nrBAQClJAPN2p+uGCVJRPIPakKeKWZ9GtBCmormE7pWOSlHat7+x5A8gx85M7HM5Dt0BP3pP5RhVW77WdbJJ3A==", - "requires": { - "character-entities-html4": "^1.0.0", - "character-entities-legacy": "^1.0.0", - "is-alphanumerical": "^1.0.0", - "is-hexadecimal": "^1.0.0" - } - }, - "strip-ansi": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", - "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", - "requires": { - "ansi-regex": "^2.0.0" - } - }, - "strip-bom": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-2.0.0.tgz", - "integrity": "sha1-YhmoVhZSBJHzV4i9vxRHqZx+aw4=", - "requires": { - "is-utf8": "^0.2.0" - } - }, - "strip-json-comments": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", - "integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo=" - }, - "structured-source": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/structured-source/-/structured-source-3.0.2.tgz", - "integrity": "sha1-3YAkJeD1PcSm56yjdSkBoczaevU=", - "requires": { - "boundary": "^1.0.1" - } - }, - "supports-color": { - "version": "5.5.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", - "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", - "requires": { - "has-flag": "^3.0.0" - } - }, - "table": { - "version": "3.8.3", - "resolved": "http://registry.npmjs.org/table/-/table-3.8.3.tgz", - "integrity": "sha1-K7xULw/amGGnVdOUf+/Ys/UThV8=", - "requires": { - "ajv": "^4.7.0", - "ajv-keywords": "^1.0.0", - "chalk": "^1.1.1", - "lodash": "^4.0.0", - "slice-ansi": "0.0.4", - "string-width": "^2.0.0" - }, - "dependencies": { - "ajv": { - "version": "4.11.8", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-4.11.8.tgz", - "integrity": "sha1-gv+wKynmYq5TvcIK8VlHcGc5xTY=", - "requires": { - "co": "^4.6.0", - "json-stable-stringify": "^1.0.1" - } - }, - "ansi-regex": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz", - "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=" - }, - "ansi-styles": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz", - "integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4=" - }, - "chalk": { - "version": "1.1.3", - "resolved": "http://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", - "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=", - "requires": { - "ansi-styles": "^2.2.1", - "escape-string-regexp": "^1.0.2", - "has-ansi": "^2.0.0", - "strip-ansi": "^3.0.0", - "supports-color": "^2.0.0" - } - }, - "is-fullwidth-code-point": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", - "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=" - }, - "string-width": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", - "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==", - "requires": { - "is-fullwidth-code-point": "^2.0.0", - "strip-ansi": "^4.0.0" - }, - "dependencies": { - "strip-ansi": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", - "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=", - "requires": { - "ansi-regex": "^3.0.0" - } - } - } - }, - "supports-color": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", - "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=" - } - } - }, - "text-table": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", - "integrity": "sha1-f17oI66AUgfACvLfSoTsP8+lcLQ=" - }, - "textlint": { - "version": "10.2.1", - "resolved": "https://registry.npmjs.org/textlint/-/textlint-10.2.1.tgz", - "integrity": "sha512-tjSvxRZ7iewPmw0ShIA5IIZNJM9m157K1hGXE9wGxALcSb+xOZ0oLPv1HN7z0UzqOuMNqYyeN7mi4N0IplLkYA==", - "requires": { - "@textlint/ast-node-types": "^4.0.2", - "@textlint/ast-traverse": "^2.0.8", - "@textlint/feature-flag": "^3.0.4", - "@textlint/fixer-formatter": "^3.0.7", - "@textlint/kernel": "^2.0.9", - "@textlint/linter-formatter": "^3.0.7", - "@textlint/textlint-plugin-markdown": "^4.0.10", - "@textlint/textlint-plugin-text": "^3.0.10", - "@types/bluebird": "^3.5.18", - "bluebird": "^3.0.5", - "debug": "^2.1.0", - "deep-equal": "^1.0.1", - "file-entry-cache": "^2.0.0", - "get-stdin": "^5.0.1", - "glob": "^7.1.1", - "interop-require": "^1.0.0", - "is-file": "^1.0.0", - "log-symbols": "^1.0.2", - "map-like": "^2.0.0", - "md5": "^2.2.1", - "mkdirp": "^0.5.0", - "object-assign": "^4.0.1", - "optionator": "^0.8.0", - "path-to-glob-pattern": "^1.0.2", - "rc-config-loader": "^2.0.1", - "read-pkg": "^1.1.0", - "read-pkg-up": "^3.0.0", - "structured-source": "^3.0.2", - "try-resolve": "^1.0.1", - "unique-concat": "^0.2.2" - } - }, - "textlint-rule-helper": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/textlint-rule-helper/-/textlint-rule-helper-2.0.0.tgz", - "integrity": "sha1-lctGlslcQljS4zienmS4SflyE4I=", - "requires": { - "unist-util-visit": "^1.1.0" - } - }, - "textlint-rule-stop-words": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/textlint-rule-stop-words/-/textlint-rule-stop-words-1.0.5.tgz", - "integrity": "sha512-sttfqpFX3ji4AD4eF3gpiCH+csqsaztO0V2koWVYhrHyPjUL4cPlB1I/H4Fa7G3Ik35dBA0q5Tf+88A0vO9erQ==", - "requires": { - "lodash": "^4.17.10", - "split-lines": "^2.0.0", - "textlint-rule-helper": "^2.0.0" - } - }, - "timed-out": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/timed-out/-/timed-out-4.0.1.tgz", - "integrity": "sha1-8y6srFoXW+ol1/q1Zas+2HQe9W8=" - }, - "to-object-path": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/to-object-path/-/to-object-path-0.3.0.tgz", - "integrity": "sha1-KXWIt7Dn4KwI4E5nL4XB9JmeF68=", - "requires": { - "kind-of": "^3.0.2" - } - }, - "to-regex": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/to-regex/-/to-regex-3.0.2.tgz", - "integrity": "sha512-FWtleNAtZ/Ki2qtqej2CXTOayOH9bHDQF+Q48VpWyDXjbYxA4Yz8iDB31zXOBUlOHHKidDbqGVrTUvQMPmBGBw==", - "requires": { - "define-property": "^2.0.2", - "extend-shallow": "^3.0.2", - "regex-not": "^1.0.2", - "safe-regex": "^1.1.0" - } - }, - "to-regex-range": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-2.1.1.tgz", - "integrity": "sha1-fIDBe53+vlmeJzZ+DU3VWQFB2zg=", - "requires": { - "is-number": "^3.0.0", - "repeat-string": "^1.6.1" - }, - "dependencies": { - "is-number": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-number/-/is-number-3.0.0.tgz", - "integrity": "sha1-JP1iAaR4LPUFYcgQJ2r8fRLXEZU=", - "requires": { - "kind-of": "^3.0.2" - } - } - } - }, - "to-vfile": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/to-vfile/-/to-vfile-2.2.0.tgz", - "integrity": "sha512-saGC8/lWdGrEoBMLUtgzhRHWAkQMP8gdldA3MOAUhBwTGEb1RSMVcflHGSx4ZJsdEZ9o1qDBCPp47LCPrbZWow==", - "requires": { - "is-buffer": "^1.1.4", - "vfile": "^2.0.0", - "x-is-function": "^1.0.4" - } - }, - "too-wordy": { - "version": "0.1.4", - "resolved": "https://registry.npmjs.org/too-wordy/-/too-wordy-0.1.4.tgz", - "integrity": "sha1-jnsgp7ek2Pw3WfTgDEkpmT0bEvA=" - }, - "tough-cookie": { - "version": "2.4.3", - "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.4.3.tgz", - "integrity": "sha512-Q5srk/4vDM54WJsJio3XNn6K2sCG+CQ8G5Wz6bZhRZoAe/+TxjWB/GlFAnYEbkYVlON9FMk/fE3h2RLpPXo4lQ==", - "requires": { - "psl": "^1.1.24", - "punycode": "^1.4.1" - }, - "dependencies": { - "punycode": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", - "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=" - } - } - }, - "traverse": { - "version": "0.6.6", - "resolved": "https://registry.npmjs.org/traverse/-/traverse-0.6.6.tgz", - "integrity": "sha1-y99WD9e5r2MlAv7UD5GMFX6pcTc=" - }, - "trim": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/trim/-/trim-0.0.1.tgz", - "integrity": "sha1-WFhUf2spB1fulczMZm+1AITEYN0=" - }, - "trim-trailing-lines": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/trim-trailing-lines/-/trim-trailing-lines-1.1.1.tgz", - "integrity": "sha512-bWLv9BbWbbd7mlqqs2oQYnLD/U/ZqeJeJwbO0FG2zA1aTq+HTvxfHNKFa/HGCVyJpDiioUYaBhfiT6rgk+l4mg==" - }, - "trough": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/trough/-/trough-1.0.3.tgz", - "integrity": "sha512-fwkLWH+DimvA4YCy+/nvJd61nWQQ2liO/nF/RjkTpiOGi+zxZzVkhb1mvbHIIW4b/8nDsYI8uTmAlc0nNkRMOw==" - }, - "try-resolve": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/try-resolve/-/try-resolve-1.0.1.tgz", - "integrity": "sha1-z95vq9ctY+V5fPqrhzq76OcA6RI=" - }, - "tunnel-agent": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", - "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=", - "requires": { - "safe-buffer": "^5.0.1" - } - }, - "tweetnacl": { - "version": "0.14.5", - "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", - "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=", - "optional": true - }, - "type-check": { - "version": "0.3.2", - "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.3.2.tgz", - "integrity": "sha1-WITKtRLPHTVeP7eE8wgEsrUg23I=", - "requires": { - "prelude-ls": "~1.1.2" - } - }, - "typedarray": { - "version": "0.0.6", - "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", - "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=" - }, - "unherit": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/unherit/-/unherit-1.1.1.tgz", - "integrity": "sha512-+XZuV691Cn4zHsK0vkKYwBEwB74T3IZIcxrgn2E4rKwTfFyI1zCh7X7grwh9Re08fdPlarIdyWgI8aVB3F5A5g==", - "requires": { - "inherits": "^2.0.1", - "xtend": "^4.0.1" - } - }, - "unified": { - "version": "6.2.0", - "resolved": "https://registry.npmjs.org/unified/-/unified-6.2.0.tgz", - "integrity": "sha512-1k+KPhlVtqmG99RaTbAv/usu85fcSRu3wY8X+vnsEhIxNP5VbVIDiXnLqyKIG+UMdyTg0ZX9EI6k2AfjJkHPtA==", - "requires": { - "bail": "^1.0.0", - "extend": "^3.0.0", - "is-plain-obj": "^1.1.0", - "trough": "^1.0.0", - "vfile": "^2.0.0", - "x-is-string": "^0.1.0" - } - }, - "unified-args": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/unified-args/-/unified-args-5.1.0.tgz", - "integrity": "sha512-IR8bS/qrfOMuIYrLlaXt+3L6cvDHv5YbBfYNVGBLbShUjE9vpbnUiPFMc/XKtH6oAGrD/m8lvVwCHDsFGBBzJA==", - "requires": { - "camelcase": "^4.0.0", - "chalk": "^2.0.0", - "chokidar": "^1.5.1", - "json5": "^0.5.1", - "minimist": "^1.2.0", - "text-table": "^0.2.0", - "unified-engine": "^5.1.0" - } - }, - "unified-engine": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/unified-engine/-/unified-engine-5.1.0.tgz", - "integrity": "sha512-N7b7HG6doQUtkWr+kH35tfUhfc9QiYeiZGG6TcZlexSURf4xRUpYKBbc2f67qJF5oPmn6mMkImkdhr31Q6saoA==", - "requires": { - "concat-stream": "^1.5.1", - "debug": "^3.1.0", - "fault": "^1.0.0", - "fn-name": "^2.0.1", - "glob": "^7.0.3", - "ignore": "^3.2.0", - "is-empty": "^1.0.0", - "is-hidden": "^1.0.1", - "is-object": "^1.0.1", - "js-yaml": "^3.6.1", - "load-plugin": "^2.0.0", - "parse-json": "^4.0.0", - "to-vfile": "^2.0.0", - "trough": "^1.0.0", - "unist-util-inspect": "^4.1.2", - "vfile-reporter": "^4.0.0", - "vfile-statistics": "^1.1.0", - "x-is-function": "^1.0.4", - "x-is-string": "^0.1.0", - "xtend": "^4.0.1" - }, - "dependencies": { - "debug": { - "version": "3.2.5", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.5.tgz", - "integrity": "sha512-D61LaDQPQkxJ5AUM2mbSJRbPkNs/TmdmOeLAi1hgDkpDfIfetSrjmWhccwtuResSwMbACjx/xXQofvM9CE/aeg==", - "requires": { - "ms": "^2.1.1" - } - }, - "ms": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", - "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==" - } - } - }, - "unified-lint-rule": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/unified-lint-rule/-/unified-lint-rule-1.0.3.tgz", - "integrity": "sha512-6z+HH3mtlFdj/w3MaQpObrZAd9KRiro370GxBFh13qkV8LYR21lLozA4iQiZPhe7KuX/lHewoGOEgQ4AWrAR3Q==", - "requires": { - "wrapped": "^1.0.1" - } - }, - "union-value": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/union-value/-/union-value-1.0.0.tgz", - "integrity": "sha1-XHHDTLW61dzr4+oM0IIHulqhrqQ=", - "requires": { - "arr-union": "^3.1.0", - "get-value": "^2.0.6", - "is-extendable": "^0.1.1", - "set-value": "^0.4.3" - }, - "dependencies": { - "extend-shallow": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", - "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=", - "requires": { - "is-extendable": "^0.1.0" - } - }, - "set-value": { - "version": "0.4.3", - "resolved": "https://registry.npmjs.org/set-value/-/set-value-0.4.3.tgz", - "integrity": "sha1-fbCPnT0i3H945Trzw79GZuzfzPE=", - "requires": { - "extend-shallow": "^2.0.1", - "is-extendable": "^0.1.1", - "is-plain-object": "^2.0.1", - "to-object-path": "^0.3.0" - } - } - } - }, - "unique-concat": { - "version": "0.2.2", - "resolved": "https://registry.npmjs.org/unique-concat/-/unique-concat-0.2.2.tgz", - "integrity": "sha1-khD5vcqsxeHjkpSQ18AZ35bxhxI=" - }, - "unist-util-inspect": { - "version": "4.1.3", - "resolved": "https://registry.npmjs.org/unist-util-inspect/-/unist-util-inspect-4.1.3.tgz", - "integrity": "sha512-Fv9R88ZBbDp7mHN+wsbxS1r8VW3unyhZh/F18dcJRQsg0+g3DxNQnMS+AEG/uotB8Md+HMK/TfzSU5lUDWxkZg==", - "requires": { - "is-empty": "^1.0.0" - } - }, - "unist-util-is": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/unist-util-is/-/unist-util-is-2.1.2.tgz", - "integrity": "sha512-YkXBK/H9raAmG7KXck+UUpnKiNmUdB+aBGrknfQ4EreE1banuzrKABx3jP6Z5Z3fMSPMQQmeXBlKpCbMwBkxVw==" - }, - "unist-util-remove-position": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/unist-util-remove-position/-/unist-util-remove-position-1.1.2.tgz", - "integrity": "sha512-XxoNOBvq1WXRKXxgnSYbtCF76TJrRoe5++pD4cCBsssSiWSnPEktyFrFLE8LTk3JW5mt9hB0Sk5zn4x/JeWY7Q==", - "requires": { - "unist-util-visit": "^1.1.0" - } - }, - "unist-util-stringify-position": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/unist-util-stringify-position/-/unist-util-stringify-position-1.1.2.tgz", - "integrity": "sha512-pNCVrk64LZv1kElr0N1wPiHEUoXNVFERp+mlTg/s9R5Lwg87f9bM/3sQB99w+N9D/qnM9ar3+AKDBwo/gm/iQQ==" - }, - "unist-util-visit": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/unist-util-visit/-/unist-util-visit-1.4.0.tgz", - "integrity": "sha512-FiGu34ziNsZA3ZUteZxSFaczIjGmksfSgdKqBfOejrrfzyUy5b7YrlzT1Bcvi+djkYDituJDy2XB7tGTeBieKw==", - "requires": { - "unist-util-visit-parents": "^2.0.0" - } - }, - "unist-util-visit-parents": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/unist-util-visit-parents/-/unist-util-visit-parents-2.0.1.tgz", - "integrity": "sha512-6B0UTiMfdWql4cQ03gDTCSns+64Zkfo2OCbK31Ov0uMizEz+CJeAp0cgZVb5Fhmcd7Bct2iRNywejT0orpbqUA==", - "requires": { - "unist-util-is": "^2.1.2" - } - }, - "unset-value": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/unset-value/-/unset-value-1.0.0.tgz", - "integrity": "sha1-g3aHP30jNRef+x5vw6jtDfyKtVk=", - "requires": { - "has-value": "^0.3.1", - "isobject": "^3.0.0" - }, - "dependencies": { - "has-value": { - "version": "0.3.1", - "resolved": "https://registry.npmjs.org/has-value/-/has-value-0.3.1.tgz", - "integrity": "sha1-ex9YutpiyoJ+wKIHgCVlSEWZXh8=", - "requires": { - "get-value": "^2.0.3", - "has-values": "^0.1.4", - "isobject": "^2.0.0" - }, - "dependencies": { - "isobject": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/isobject/-/isobject-2.1.0.tgz", - "integrity": "sha1-8GVWEJaj8dou9GJy+BXIQNh+DIk=", - "requires": { - "isarray": "1.0.0" - } - } - } - }, - "has-values": { - "version": "0.1.4", - "resolved": "https://registry.npmjs.org/has-values/-/has-values-0.1.4.tgz", - "integrity": "sha1-bWHeldkd/Km5oCCJrThL/49it3E=" - }, - "isobject": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz", - "integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8=" - } - } - }, - "untildify": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/untildify/-/untildify-2.1.0.tgz", - "integrity": "sha1-F+soB5h/dpUunASF/DEdBqgmouA=", - "requires": { - "os-homedir": "^1.0.0" - } - }, - "unzip-response": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/unzip-response/-/unzip-response-2.0.1.tgz", - "integrity": "sha1-0vD3N9FrBhXnKmk17QQhRXLVb5c=" - }, - "urix": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/urix/-/urix-0.1.0.tgz", - "integrity": "sha1-2pN/emLiH+wf0Y1Js1wpNQZ6bHI=" - }, - "url-parse-lax": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/url-parse-lax/-/url-parse-lax-1.0.0.tgz", - "integrity": "sha1-evjzA2Rem9eaJy56FKxovAYJ2nM=", - "requires": { - "prepend-http": "^1.0.1" - } - }, - "url-to-options": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/url-to-options/-/url-to-options-1.0.1.tgz", - "integrity": "sha1-FQWgOiiaSMvXpDTvuu7FBV9WM6k=" - }, - "use": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/use/-/use-3.1.1.tgz", - "integrity": "sha512-cwESVXlO3url9YWlFW/TA9cshCEhtu7IKJ/p5soJ/gGpj7vbvFrAY/eIioQ6Dw23KjZhYgiIo8HOs1nQ2vr/oQ==" - }, - "util-deprecate": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" - }, - "uuid": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", - "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==" - }, - "validate-npm-package-license": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/validate-npm-package-license/-/validate-npm-package-license-3.0.4.tgz", - "integrity": "sha512-DpKm2Ui/xN7/HQKCtpZxoRWBhZ9Z0kqtygG8XCgNQ8ZlDnxuQmWhj566j8fN4Cu3/JmbhsDo7fcAJq4s9h27Ew==", - "requires": { - "spdx-correct": "^3.0.0", - "spdx-expression-parse": "^3.0.0" - } - }, - "verror": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz", - "integrity": "sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=", - "requires": { - "assert-plus": "^1.0.0", - "core-util-is": "1.0.2", - "extsprintf": "^1.2.0" - } - }, - "vfile": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/vfile/-/vfile-2.3.0.tgz", - "integrity": "sha512-ASt4mBUHcTpMKD/l5Q+WJXNtshlWxOogYyGYYrg4lt/vuRjC1EFQtlAofL5VmtVNIZJzWYFJjzGWZ0Gw8pzW1w==", - "requires": { - "is-buffer": "^1.1.4", - "replace-ext": "1.0.0", - "unist-util-stringify-position": "^1.0.0", - "vfile-message": "^1.0.0" - } - }, - "vfile-location": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/vfile-location/-/vfile-location-2.0.3.tgz", - "integrity": "sha512-zM5/l4lfw1CBoPx3Jimxoc5RNDAHHpk6AM6LM0pTIkm5SUSsx8ZekZ0PVdf0WEZ7kjlhSt7ZlqbRL6Cd6dBs6A==" - }, - "vfile-message": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/vfile-message/-/vfile-message-1.0.1.tgz", - "integrity": "sha512-vSGCkhNvJzO6VcWC6AlJW4NtYOVtS+RgCaqFIYUjoGIlHnFL+i0LbtYvonDWOMcB97uTPT4PRsyYY7REWC9vug==", - "requires": { - "unist-util-stringify-position": "^1.1.1" - } - }, - "vfile-reporter": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/vfile-reporter/-/vfile-reporter-4.0.0.tgz", - "integrity": "sha1-6m8K4TQvSEFXOYXgX5QXNvJ96do=", - "requires": { - "repeat-string": "^1.5.0", - "string-width": "^1.0.0", - "supports-color": "^4.1.0", - "unist-util-stringify-position": "^1.0.0", - "vfile-statistics": "^1.1.0" - }, - "dependencies": { - "has-flag": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-2.0.0.tgz", - "integrity": "sha1-6CB68cx7MNRGzHC3NLXovhj4jVE=" - }, - "supports-color": { - "version": "4.5.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-4.5.0.tgz", - "integrity": "sha1-vnoN5ITexcXN34s9WRJQRJEvY1s=", - "requires": { - "has-flag": "^2.0.0" - } - } - } - }, - "vfile-statistics": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/vfile-statistics/-/vfile-statistics-1.1.1.tgz", - "integrity": "sha512-dxUM6IYvGChHuwMT3dseyU5BHprNRXzAV0OHx1A769lVGsTiT50kU7BbpRFV+IE6oWmU+PwHdsTKfXhnDIRIgQ==" - }, - "weasel-words": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/weasel-words/-/weasel-words-0.1.1.tgz", - "integrity": "sha1-cTeUZYXHP+RIggE4U70ADF1oek4=" - }, - "wordwrap": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz", - "integrity": "sha1-J1hIEIkUVqQXHI0CJkQa3pDLyus=" - }, - "wrapped": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/wrapped/-/wrapped-1.0.1.tgz", - "integrity": "sha1-x4PZ2Aeyc+mwHoUWgKk4yHyQckI=", - "requires": { - "co": "3.1.0", - "sliced": "^1.0.1" - }, - "dependencies": { - "co": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/co/-/co-3.1.0.tgz", - "integrity": "sha1-TqVOpaCJOBUxheFSEMaNkJK8G3g=" - } - } - }, - "wrappy": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" - }, - "write": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/write/-/write-0.2.1.tgz", - "integrity": "sha1-X8A4KOJkzqP+kUVUdvejxWbLB1c=", - "requires": { - "mkdirp": "^0.5.1" - } - }, - "write-good": { - "version": "0.11.3", - "resolved": "https://registry.npmjs.org/write-good/-/write-good-0.11.3.tgz", - "integrity": "sha512-fDKIHO5wCzTLCOGNJl1rzzJrZlTIzfZl8msOoJQZzRhYo0X/tFTm4+2B1zTibFYK01Nnd1kLZBjj4xjcFLePNQ==", - "requires": { - "adverb-where": "0.0.9", - "e-prime": "^0.10.2", - "no-cliches": "^0.1.0", - "object.assign": "^4.0.4", - "passive-voice": "^0.1.0", - "too-wordy": "^0.1.4", - "weasel-words": "^0.1.1" - } - }, - "x-is-function": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/x-is-function/-/x-is-function-1.0.4.tgz", - "integrity": "sha1-XSlNw9Joy90GJYDgxd93o5HR+h4=" - }, - "x-is-string": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/x-is-string/-/x-is-string-0.1.0.tgz", - "integrity": "sha1-R0tQhlrzpJqcRlfwWs0UVFj3fYI=" - }, - "xml-escape": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/xml-escape/-/xml-escape-1.1.0.tgz", - "integrity": "sha1-OQTBQ/qOs6ADDsZG0pAqLxtwbEQ=" - }, - "xtend": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz", - "integrity": "sha1-pcbVMr5lbiPbgg77lDofBJmNY68=" - } - } -} diff --git a/docs/package.json b/docs/package.json deleted file mode 100644 index d45ba539..00000000 --- a/docs/package.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "dependencies": { - "prettier": "^1.13.7", - "remark-cli": "^5.0.0", - "remark-lint-no-dead-urls": "^0.3.0", - "remark-lint-write-good": "^1.0.3", - "textlint": "^10.2.1", - "textlint-rule-stop-words": "^1.0.3" - }, - "name": "tendermint", - "description": "Tendermint Core Documentation", - "version": "0.0.1", - "main": "README.md", - "devDependencies": {}, - "scripts": { - "lint:json": "prettier \"**/*.json\" --write", - "lint:md": "prettier \"**/*.md\" --write && remark . && textlint \"md/**\"", - "lint": "yarn lint:json && yarn lint:md" - }, - "repository": { - "type": "git", - "url": "git+https://github.com/tendermint/tendermint.git" - }, - "keywords": [ - "tendermint", - "blockchain" - ], - "author": "Tendermint", - "license": "ISC", - "bugs": { - "url": "https://github.com/tendermint/tendermint/issues" - }, - "homepage": "https://tendermint.com/docs/", - "remarkConfig": { - "plugins": [ - "remark-lint-no-dead-urls", - "remark-lint-write-good" - ] - } -} diff --git a/docs/spec/README.md b/docs/spec/README.md index 3e2c2bcd..7ec9387c 100644 --- a/docs/spec/README.md +++ b/docs/spec/README.md @@ -14,31 +14,31 @@ please submit them to our [bug bounty](https://tendermint.com/security)! ### Data Structures -- [Encoding and Digests](https://github.com/tendermint/tendermint/blob/master/docs/spec/blockchain/encoding.md) -- [Blockchain](https://github.com/tendermint/tendermint/blob/master/docs/spec/blockchain/blockchain.md) -- [State](https://github.com/tendermint/tendermint/blob/master/docs/spec/blockchain/state.md) +- [Encoding and Digests](./blockchain/encoding.md) +- [Blockchain](./blockchain/blockchain.md) +- [State](./blockchain/state.md) ### Consensus Protocol -- [Consensus Algorithm](/docs/spec/consensus/consensus.md) -- [Creating a proposal](/docs/spec/consensus/creating-proposal.md) -- [Time](/docs/spec/consensus/bft-time.md) -- [Light-Client](/docs/spec/consensus/light-client.md) +- [Consensus Algorithm](./consensus/consensus.md) +- [Creating a proposal](./consensus/creating-proposal.md) +- [Time](./consensus/bft-time.md) +- [Light-Client](./consensus/light-client.md) ### P2P and Network Protocols -- [The Base P2P Layer](https://github.com/tendermint/tendermint/tree/master/docs/spec/p2p): multiplex the protocols ("reactors") on authenticated and encrypted TCP connections -- [Peer Exchange (PEX)](https://github.com/tendermint/tendermint/tree/master/docs/spec/reactors/pex): gossip known peer addresses so peers can find each other -- [Block Sync](https://github.com/tendermint/tendermint/tree/master/docs/spec/reactors/block_sync): gossip blocks so peers can catch up quickly -- [Consensus](https://github.com/tendermint/tendermint/tree/master/docs/spec/reactors/consensus): gossip votes and block parts so new blocks can be committed -- [Mempool](https://github.com/tendermint/tendermint/tree/master/docs/spec/reactors/mempool): gossip transactions so they get included in blocks -- Evidence: Forthcoming, see [this issue](https://github.com/tendermint/tendermint/issues/2329). +- [The Base P2P Layer](./p2p/): multiplex the protocols ("reactors") on authenticated and encrypted TCP connections +- [Peer Exchange (PEX)](./reactors/pex/): gossip known peer addresses so peers can find each other +- [Block Sync](./reactors/block_sync/): gossip blocks so peers can catch up quickly +- [Consensus](./reactors/consensus/): gossip votes and block parts so new blocks can be committed +- [Mempool](./reactors/mempool/): gossip transactions so they get included in blocks +- [Evidence](./reactors/evidence/): sending invalid evidence will stop the peer ### Software -- [ABCI](/docs/spec/software/abci.md): Details about interactions between the +- [ABCI](./software/abci.md): Details about interactions between the application and consensus engine over ABCI -- [Write-Ahead Log](/docs/spec/software/wal.md): Details about how the consensus +- [Write-Ahead Log](./software/wal.md): Details about how the consensus engine preserves data and recovers from crash failures ## Overview diff --git a/docs/spec/abci/abci.md b/docs/spec/abci/abci.md index 15e24624..b9dc744d 100644 --- a/docs/spec/abci/abci.md +++ b/docs/spec/abci/abci.md @@ -45,7 +45,9 @@ include a `Tags` field in their `Response*`. Each tag is key-value pair denoting something about what happened during the methods execution. Tags can be used to index transactions and blocks according to what happened -during their execution. +during their execution. Note that the set of tags returned for a block from +`BeginBlock` and `EndBlock` are merged. In case both methods return the same +tag, only the value defined in `EndBlock` is used. Keys and values in tags must be UTF-8 encoded strings (e.g. "account.owner": "Bob", "balance": "100.0", @@ -134,10 +136,13 @@ Commit are included in the header of the next block. ### Info - **Request**: - - `Version (string)`: The Tendermint version + - `Version (string)`: The Tendermint software semantic version + - `BlockVersion (uint64)`: The Tendermint Block Protocol version + - `P2PVersion (uint64)`: The Tendermint P2P Protocol version - **Response**: - `Data (string)`: Some arbitrary information - - `Version (Version)`: Version information + - `Version (string)`: The application software semantic version + - `AppVersion (uint64)`: The application protocol version - `LastBlockHeight (int64)`: Latest block for which the app has called Commit - `LastBlockAppHash ([]byte)`: Latest result of Commit @@ -145,6 +150,7 @@ Commit are included in the header of the next block. - Return information about the application state. - Used to sync Tendermint with the application during a handshake that happens on startup. + - The returned `AppVersion` will be included in the Header of every block. - Tendermint expects `LastBlockAppHash` and `LastBlockHeight` to be updated during `Commit`, ensuring that `Commit` is never called twice for the same block height. @@ -338,6 +344,7 @@ Commit are included in the header of the next block. ### Header - **Fields**: + - `Version (Version)`: Version of the blockchain and the application - `ChainID (string)`: ID of the blockchain - `Height (int64)`: Height of the block in the chain - `Time (google.protobuf.Timestamp)`: Time of the block. It is the proposer's @@ -363,6 +370,15 @@ Commit are included in the header of the next block. - Provides the proposer of the current block, for use in proposer-based reward mechanisms. +### Version + +- **Fields**: + - `Block (uint64)`: Protocol version of the blockchain data structures. + - `App (uint64)`: Protocol version of the application. +- **Usage**: + - Block version should be static in the life of a blockchain. + - App version may be updated over time by the application. + ### Validator - **Fields**: @@ -427,11 +443,12 @@ Commit are included in the header of the next block. ### ConsensusParams - **Fields**: - - `BlockSize (BlockSize)`: Parameters limiting the size of a block. - - `EvidenceParams (EvidenceParams)`: Parameters limiting the validity of + - `BlockSize (BlockSizeParams)`: Parameters limiting the size of a block. + - `Evidence (EvidenceParams)`: Parameters limiting the validity of evidence of byzantine behaviour. + - `Validator (ValidatorParams)`: Parameters limitng the types of pubkeys validators can use. -### BlockSize +### BlockSizeParams - **Fields**: - `MaxBytes (int64)`: Max size of a block, in bytes. @@ -449,6 +466,12 @@ Commit are included in the header of the next block. similar mechanism for handling Nothing-At-Stake attacks. - NOTE: this should change to time (instead of blocks)! +### ValidatorParams + +- **Fields**: + - `PubKeyTypes ([]string)`: List of accepted pubkey types. Uses same + naming as `PubKey.Type`. + ### Proof - **Fields**: diff --git a/docs/spec/abci/apps.md b/docs/spec/abci/apps.md index acf2c4e6..a3b34240 100644 --- a/docs/spec/abci/apps.md +++ b/docs/spec/abci/apps.md @@ -166,6 +166,11 @@ the tags will be hashed into the next block header. The application may set the validator set during InitChain, and update it during EndBlock. +Note that the maximum total power of the validator set is bounded by +`MaxTotalVotingPower = MaxInt64 / 8`. Applications are responsible for ensuring +they do not make changes to the validator set that cause it to exceed this +limit. + ### InitChain ResponseInitChain can return a list of validators. @@ -206,6 +211,7 @@ following rules: - if the validator does not already exist, it will be added to the validator set with the given power - if the validator does already exist, its power will be adjusted to the given power +- the total power of the new validator set must not exceed MaxTotalVotingPower Note the updates returned in block `H` will only take effect at block `H+2`. @@ -407,21 +413,22 @@ If `storeBlockHeight == stateBlockHeight && appBlockHeight < storeBlockHeight`, replay all blocks in full from `appBlockHeight` to `storeBlockHeight`. This happens if we completed processing the block, but the app forgot its height. -If `storeBlockHeight == stateBlockHeight && appBlockHeight == storeBlockHeight`, we're done +If `storeBlockHeight == stateBlockHeight && appBlockHeight == storeBlockHeight`, we're done. This happens if we crashed at an opportune spot. If `storeBlockHeight == stateBlockHeight+1` This happens if we started processing the block but didn't finish. - If `appBlockHeight < stateBlockHeight` - replay all blocks in full from `appBlockHeight` to `storeBlockHeight-1`, - and replay the block at `storeBlockHeight` using the WAL. - This happens if the app forgot the last block it committed. +If `appBlockHeight < stateBlockHeight` + replay all blocks in full from `appBlockHeight` to `storeBlockHeight-1`, + and replay the block at `storeBlockHeight` using the WAL. +This happens if the app forgot the last block it committed. - If `appBlockHeight == stateBlockHeight`, - replay the last block (storeBlockHeight) in full. - This happens if we crashed before the app finished Commit +If `appBlockHeight == stateBlockHeight`, + replay the last block (storeBlockHeight) in full. +This happens if we crashed before the app finished Commit + +If `appBlockHeight == storeBlockHeight` + update the state using the saved ABCI responses but dont run the block against the real app. +This happens if we crashed after the app finished Commit but before Tendermint saved the state. - If appBlockHeight == storeBlockHeight { - update the state using the saved ABCI responses but dont run the block against the real app. - This happens if we crashed after the app finished Commit but before Tendermint saved the state. diff --git a/docs/spec/abci/client-server.md b/docs/spec/abci/client-server.md index 822bfd1f..5ac7b3eb 100644 --- a/docs/spec/abci/client-server.md +++ b/docs/spec/abci/client-server.md @@ -3,12 +3,8 @@ This section is for those looking to implement their own ABCI Server, perhaps in a new programming language. -You are expected to have read [ABCI Methods and Types](abci.md) and [ABCI -Applications](apps.md). - -See additional details in the [ABCI -readme](https://github.com/tendermint/tendermint/blob/develop/abci/README.md)(TODO: deduplicate -those details). +You are expected to have read [ABCI Methods and Types](./abci.md) and [ABCI +Applications](./apps.md). ## Message Protocol @@ -24,17 +20,16 @@ For each request, a server should respond with the corresponding response, where the order of requests is preserved in the order of responses. -## Server +## Server Implementations To use ABCI in your programming language of choice, there must be a ABCI -server in that language. Tendermint supports two kinds of implementation -of the server: +server in that language. Tendermint supports three implementations of the ABCI, written in Go: -- Asynchronous, raw socket server (Tendermint Socket Protocol, also - known as TSP or Teaspoon) +- In-process (Golang only) +- ABCI-socket - GRPC -Both can be tested using the `abci-cli` by setting the `--abci` flag +The latter two can be tested using the `abci-cli` by setting the `--abci` flag appropriately (ie. to `socket` or `grpc`). See examples, in various stages of maintenance, in @@ -44,6 +39,12 @@ See examples, in various stages of maintenance, in [C++](https://github.com/mdyring/cpp-tmsp), and [Java](https://github.com/jTendermint/jabci). +### In Process + +The simplest implementation uses function calls within Golang. +This means ABCI applications written in Golang can be compiled with TendermintCore and run as a single binary. + + ### GRPC If GRPC is available in your language, this is the easiest approach, @@ -58,15 +59,18 @@ See the [grpc documentation for more details](http://www.grpc.io/docs/). server in your language, including whatever interface your application must satisfy to be used by the ABCI server for handling requests. +Note the length-prefixing used in the socket implementation (TSP) does not apply for GRPC. + ### TSP +Tendermint Socket Protocol is an asynchronous, raw socket server which provides ordered message passing over unix or tcp. +Messages are serialized using Protobuf3 and length-prefixed with a [signed Varint](https://developers.google.com/protocol-buffers/docs/encoding?csw=1#signed-integers) + If GRPC is not available in your language, or you require higher performance, or otherwise enjoy programming, you may implement your own -ABCI server using the Tendermint Socket Protocol, known affectionately -as Teaspoon. The first step is still to auto-generate the relevant data -types and codec in your language using `protoc`. Messages coming over -the socket are proto3 encoded, but additionally length-prefixed to -facilitate use as a streaming protocol. proto3 doesn't have an +ABCI server using the Tendermint Socket Protocol. The first step is still to auto-generate the relevant data +types and codec in your language using `protoc`. In addition to being proto3 encoded, messages coming over +the socket are length-prefixed to facilitate use as a streaming protocol. proto3 doesn't have an official length-prefix standard, so we use our own. The first byte in the prefix represents the length of the Big Endian encoded length. The remaining bytes in the prefix are the Big Endian encoded length. @@ -76,12 +80,14 @@ bytes), the length-prefixed message is 0x0104DEADBEEF. If the proto3 encoded ABCI message is 65535 bytes long, the length-prefixed message would be like 0x02FFFF.... -Note this prefixing does not apply for grpc. +The benefit of using this `varint` encoding over the old version (where integers were encoded as `` is that +it is the standard way to encode integers in Protobuf. It is also generally shorter. + +As noted above, this prefixing does not apply for GRPC. An ABCI server must also be able to support multiple connections, as Tendermint uses three connections. - ### Async vs Sync The main ABCI server (ie. non-GRPC) provides ordered asynchronous messages. diff --git a/docs/spec/blockchain/blockchain.md b/docs/spec/blockchain/blockchain.md index bd0af70a..00cccfc2 100644 --- a/docs/spec/blockchain/blockchain.md +++ b/docs/spec/blockchain/blockchain.md @@ -8,6 +8,7 @@ The Tendermint blockchains consists of a short list of basic data types: - `Block` - `Header` +- `Version` - `BlockID` - `Time` - `Data` (for transactions) @@ -38,6 +39,7 @@ the data in the current block, the previous block, and the results returned by t ```go type Header struct { // basic block info + Version Version ChainID string Height int64 Time Time @@ -49,7 +51,7 @@ type Header struct { // hashes of block data LastCommitHash []byte // commit from validators from the last block - DataHash []byte // Merkle root of transactions + DataHash []byte // MerkleRoot of transaction hashes // hashes from the app output from the prev block ValidatorsHash []byte // validators for the current block @@ -65,28 +67,43 @@ type Header struct { Further details on each of these fields is described below. +## Version + +The `Version` contains the protocol version for the blockchain and the +application as two `uint64` values: + +```go +type Version struct { + Block uint64 + App uint64 +} +``` + + ## BlockID The `BlockID` contains two distinct Merkle roots of the block. -The first, used as the block's main hash, is the Merkle root -of all the fields in the header. The second, used for secure gossipping of -the block during consensus, is the Merkle root of the complete serialized block -cut into parts. The `BlockID` includes these two hashes, as well as the number of -parts. +The first, used as the block's main hash, is the MerkleRoot +of all the fields in the header (ie. `MerkleRoot(header)`. +The second, used for secure gossipping of the block during consensus, +is the MerkleRoot of the complete serialized block +cut into parts (ie. `MerkleRoot(MakeParts(block))`). +The `BlockID` includes these two hashes, as well as the number of +parts (ie. `len(MakeParts(block))`) ```go type BlockID struct { Hash []byte - Parts PartsHeader + PartsHeader PartSetHeader } -type PartsHeader struct { - Hash []byte +type PartSetHeader struct { Total int32 + Hash []byte } ``` -TODO: link to details of merkle sums. +See [MerkleRoot](/docs/spec/blockchain/encoding.md#MerkleRoot) for details. ## Time @@ -94,10 +111,6 @@ Tendermint uses the [Google.Protobuf.WellKnownTypes.Timestamp](https://developers.google.com/protocol-buffers/docs/reference/csharp/class/google/protobuf/well-known-types/timestamp) format, which uses two integers, one for Seconds and for Nanoseconds. -NOTE: there is currently a small divergence between Tendermint and the -Google.Protobuf.WellKnownTypes.Timestamp that should be resolved. See [this -issue](https://github.com/tendermint/go-amino/issues/223) for details. - ## Data Data is just a wrapper for a list of transactions, where transactions are @@ -131,14 +144,14 @@ The vote includes information about the validator signing it. ```go type Vote struct { - ValidatorAddress []byte - ValidatorIndex int - Height int64 - Round int - Timestamp Time - Type int8 - BlockID BlockID - Signature []byte + Type byte + Height int64 + Round int + BlockID BlockID + Timestamp Time + ValidatorAddress []byte + ValidatorIndex int + Signature []byte } ``` @@ -149,8 +162,8 @@ a _precommit_ has `vote.Type == 2`. ## Signature Signatures in Tendermint are raw bytes representing the underlying signature. -The only signature scheme currently supported for Tendermint validators is -ED25519. The signature is the raw 64-byte ED25519 signature. + +See the [signature spec](/docs/spec/blockchain/encoding.md#key-types) for more. ## EvidenceData @@ -177,6 +190,8 @@ type DuplicateVoteEvidence struct { } ``` +See the [pubkey spec](/docs/spec/blockchain/encoding.md#key-types) for more. + ## Validation Here we describe the validation rules for every element in a block. @@ -194,19 +209,28 @@ the current version of the `state` corresponds to the state after executing transactions from the `prevBlock`. Elements of an object are accessed as expected, ie. `block.Header`. -See [here](https://github.com/tendermint/tendermint/blob/master/docs/spec/blockchain/state.md) for the definition of `state`. +See the [definition of `State`](/docs/spec/blockchain/state.md). ### Header A Header is valid if its corresponding fields are valid. +### Version + +``` +block.Version.Block == state.Version.Block +block.Version.App == state.Version.App +``` + +The block version must match the state version. + ### ChainID ``` len(block.ChainID) < 50 ``` -ChainID must be maximum 50 UTF-8 symbols. +ChainID must be less than 50 bytes. ### Height @@ -230,6 +254,15 @@ It must equal the weighted median of the timestamps of the valid votes in the bl Note: the timestamp of a vote must be greater by at least one millisecond than that of the block being voted on. +The timestamp of the first block must be equal to the genesis time (since +there's no votes to compute the median). + +``` +if block.Header.Height == 1 { + block.Header.Timestamp == genesisTime +} +``` + See the section on [BFT time](../consensus/bft-time.md) for more details. ### NumTxs @@ -255,28 +288,25 @@ The first block has `block.Header.TotalTxs = block.Header.NumberTxs`. LastBlockID is the previous block's BlockID: ```go -prevBlockParts := MakeParts(prevBlock, state.LastConsensusParams.BlockGossip.BlockPartSize) +prevBlockParts := MakeParts(prevBlock) block.Header.LastBlockID == BlockID { - Hash: SimpleMerkleRoot(prevBlock.Header), + Hash: MerkleRoot(prevBlock.Header), PartsHeader{ - Hash: SimpleMerkleRoot(prevBlockParts), + Hash: MerkleRoot(prevBlockParts), Total: len(prevBlockParts), }, } ``` -Note: it depends on the ConsensusParams, -which are held in the `state` and may be updated by the application. - The first block has `block.Header.LastBlockID == BlockID{}`. ### LastCommitHash ```go -block.Header.LastCommitHash == SimpleMerkleRoot(block.LastCommit) +block.Header.LastCommitHash == MerkleRoot(block.LastCommit.Precommits) ``` -Simple Merkle root of the votes included in the block. +MerkleRoot of the votes included in the block. These are the votes that committed the previous block. The first block has `block.Header.LastCommitHash == []byte{}` @@ -284,37 +314,42 @@ The first block has `block.Header.LastCommitHash == []byte{}` ### DataHash ```go -block.Header.DataHash == SimpleMerkleRoot(block.Txs.Txs) +block.Header.DataHash == MerkleRoot(Hashes(block.Txs.Txs)) ``` -Simple Merkle root of the transactions included in the block. +MerkleRoot of the hashes of transactions included in the block. + +Note the transactions are hashed before being included in the Merkle tree, +so the leaves of the Merkle tree are the hashes, not the transactions +themselves. This is because transaction hashes are regularly used as identifiers for +transactions. ### ValidatorsHash ```go -block.ValidatorsHash == SimpleMerkleRoot(state.Validators) +block.ValidatorsHash == MerkleRoot(state.Validators) ``` -Simple Merkle root of the current validator set that is committing the block. +MerkleRoot of the current validator set that is committing the block. This can be used to validate the `LastCommit` included in the next block. ### NextValidatorsHash ```go -block.NextValidatorsHash == SimpleMerkleRoot(state.NextValidators) +block.NextValidatorsHash == MerkleRoot(state.NextValidators) ``` -Simple Merkle root of the next validator set that will be the validator set that commits the next block. +MerkleRoot of the next validator set that will be the validator set that commits the next block. This is included so that the current validator set gets a chance to sign the next validator sets Merkle root. -### ConsensusParamsHash +### ConsensusHash ```go -block.ConsensusParamsHash == SimpleMerkleRoot(state.ConsensusParams) +block.ConsensusHash == state.ConsensusParams.Hash() ``` -Simple Merkle root of the consensus parameters. +Hash of the amino-encoding of a subset of the consensus parameters. ### AppHash @@ -329,20 +364,20 @@ The first block has `block.Header.AppHash == []byte{}`. ### LastResultsHash ```go -block.ResultsHash == SimpleMerkleRoot(state.LastResults) +block.ResultsHash == MerkleRoot(state.LastResults) ``` -Simple Merkle root of the results of the transactions in the previous block. +MerkleRoot of the results of the transactions in the previous block. The first block has `block.Header.ResultsHash == []byte{}`. ## EvidenceHash ```go -block.EvidenceHash == SimpleMerkleRoot(block.Evidence) +block.EvidenceHash == MerkleRoot(block.Evidence) ``` -Simple Merkle root of the evidence of Byzantine behaviour included in this block. +MerkleRoot of the evidence of Byzantine behaviour included in this block. ### ProposerAddress @@ -401,8 +436,9 @@ must be greater than 2/3 of the total voting power of the complete validator set A vote is a signed message broadcast in the consensus for a particular block at a particular height and round. When stored in the blockchain or propagated over the network, votes are encoded in Amino. -For signing, votes are represented via `CanonicalVote` and also encoded using amino (protobuf compatible) via -`Vote.SignBytes` which includes the `ChainID`. +For signing, votes are represented via `CanonicalVote` and also encoded using amino (protobuf compatible) via +`Vote.SignBytes` which includes the `ChainID`, and uses a different ordering of +the fields. We define a method `Verify` that returns `true` if the signature verifies against the pubkey for the `SignBytes` using the given ChainID: diff --git a/docs/spec/blockchain/encoding.md b/docs/spec/blockchain/encoding.md index 2ff024ce..1b999335 100644 --- a/docs/spec/blockchain/encoding.md +++ b/docs/spec/blockchain/encoding.md @@ -30,6 +30,12 @@ For example, the byte-array `[0xA, 0xB]` would be encoded as `0x020A0B`, while a byte-array containing 300 entires beginning with `[0xA, 0xB, ...]` would be encoded as `0xAC020A0B...` where `0xAC02` is the UVarint encoding of 300. +## Hashing + +Tendermint uses `SHA256` as its hash function. +Objects are always Amino encoded before being hashed. +So `SHA256(obj)` is short for `SHA256(AminoEncode(obj))`. + ## Public Key Cryptography Tendermint uses Amino to distinguish between different types of private keys, @@ -59,40 +65,36 @@ You can simply use below table and concatenate Prefix || Length (of raw bytes) | | PubKeySecp256k1 | tendermint/PubKeySecp256k1 | 0xEB5AE987 | 0x21 | | | PrivKeyEd25519 | tendermint/PrivKeyEd25519 | 0xA3288910 | 0x40 | | | PrivKeySecp256k1 | tendermint/PrivKeySecp256k1 | 0xE1B0F79B | 0x20 | | -| SignatureEd25519 | tendermint/SignatureEd25519 | 0x2031EA53 | 0x40 | | -| SignatureSecp256k1 | tendermint/SignatureSecp256k1 | 0x7FC4A495 | variable | +| PubKeyMultisigThreshold | tendermint/PubKeyMultisigThreshold | 0x22C1F7E2 | variable | | -| +### Example -### Examples - -1. For example, the 33-byte (or 0x21-byte in hex) Secp256k1 pubkey +For example, the 33-byte (or 0x21-byte in hex) Secp256k1 pubkey `020BD40F225A57ED383B440CF073BC5539D0341F5767D2BF2D78406D00475A2EE9` would be encoded as - `EB5AE98221020BD40F225A57ED383B440CF073BC5539D0341F5767D2BF2D78406D00475A2EE9` + `EB5AE98721020BD40F225A57ED383B440CF073BC5539D0341F5767D2BF2D78406D00475A2EE9` -2. For example, the variable size Secp256k1 signature (in this particular example 70 or 0x46 bytes) - `304402201CD4B8C764D2FD8AF23ECFE6666CA8A53886D47754D951295D2D311E1FEA33BF02201E0F906BB1CF2C30EAACFFB032A7129358AFF96B9F79B06ACFFB18AC90C2ADD7` - would be encoded as - `16E1FEEA46304402201CD4B8C764D2FD8AF23ECFE6666CA8A53886D47754D951295D2D311E1FEA33BF02201E0F906BB1CF2C30EAACFFB032A7129358AFF96B9F79B06ACFFB18AC90C2ADD7` +### Key Types -### Addresses - -Addresses for each public key types are computed as follows: +Each type specifies it's own pubkey, address, and signature format. #### Ed25519 -First 20-bytes of the SHA256 hash of the raw 32-byte public key: +TODO: pubkey + +The address is the first 20-bytes of the SHA256 hash of the raw 32-byte public key: ``` address = SHA256(pubkey)[:20] ``` -NOTE: before v0.22.0, this was the RIPEMD160 of the Amino encoded public key. +The signature is the raw 64-byte ED25519 signature. #### Secp256k1 -RIPEMD160 hash of the SHA256 hash of the OpenSSL compressed public key: +TODO: pubkey + +The address is the RIPEMD160 hash of the SHA256 hash of the OpenSSL compressed public key: ``` address = RIPEMD160(SHA256(pubkey)) @@ -100,12 +102,21 @@ address = RIPEMD160(SHA256(pubkey)) This is the same as Bitcoin. +The signature is the 64-byte concatenation of ECDSA `r` and `s` (ie. `r || s`), +where `s` is lexicographically less than its inverse, to prevent malleability. +This is like Ethereum, but without the extra byte for pubkey recovery, since +Tendermint assumes the pubkey is always provided anyway. + +#### Multisig + +TODO + ## Other Common Types ### BitArray -The BitArray is used in block headers and some consensus messages to signal -whether or not something was done by each validator. BitArray is represented +The BitArray is used in some consensus messages to represent votes received from +validators, or parts received in a block. It is represented with a struct containing the number of bits (`Bits`) and the bit-array itself encoded in base64 (`Elems`). @@ -127,24 +138,27 @@ representing `1` and `0`. Ie. the BitArray `10110` would be JSON encoded as Part is used to break up blocks into pieces that can be gossiped in parallel and securely verified using a Merkle tree of the parts. -Part contains the index of the part in the larger set (`Index`), the actual -underlying data of the part (`Bytes`), and a simple Merkle proof that the part is contained in -the larger set (`Proof`). +Part contains the index of the part (`Index`), the actual +underlying data of the part (`Bytes`), and a Merkle proof that the part is contained in +the set (`Proof`). ```go type Part struct { Index int - Bytes byte[] - Proof byte[] + Bytes []byte + Proof SimpleProof } ``` +See details of SimpleProof, below. + ### MakeParts Encode an object using Amino and slice it into parts. +Tendermint uses a part size of 65536 bytes. ```go -func MakeParts(obj interface{}, partSize int) []Part +func MakeParts(block Block) []Part ``` ## Merkle Trees @@ -152,12 +166,17 @@ func MakeParts(obj interface{}, partSize int) []Part For an overview of Merkle trees, see [wikipedia](https://en.wikipedia.org/wiki/Merkle_tree) -A Simple Tree is a simple compact binary tree for a static list of items. Simple Merkle trees are used in numerous places in Tendermint to compute a cryptographic digest of a data structure. In a Simple Tree, the transactions and validation signatures of a block are hashed using this simple merkle tree logic. +We use the RFC 6962 specification of a merkle tree, with sha256 as the hash function. +Merkle trees are used throughout Tendermint to compute a cryptographic digest of a data structure. +The differences between RFC 6962 and the simplest form a merkle tree are that: -If the number of items is not a power of two, the tree will not be full -and some leaf nodes will be at different levels. Simple Tree tries to -keep both sides of the tree the same size, but the left side may be one -greater, for example: +1) leaf nodes and inner nodes have different hashes. + This is for "second pre-image resistance", to prevent the proof to an inner node being valid as the proof of a leaf. + The leaf nodes are `SHA256(0x00 || leaf_data)`, and inner nodes are `SHA256(0x01 || left_hash || right_hash)`. + +2) When the number of items isn't a power of two, the left half of the tree is as big as it could be. + (The smallest power of two less than the number of items) This allows new leaves to be added with less + recomputation. For example: ``` Simple Tree with 6 items Simple Tree with 7 items @@ -171,69 +190,79 @@ greater, for example: / \ / \ / \ / \ / \ / \ / \ / \ / \ / \ / \ / \ - * h2 * h5 * * * h6 - / \ / \ / \ / \ / \ -h0 h1 h3 h4 h0 h1 h2 h3 h4 h5 + * * h4 h5 * * * h6 + / \ / \ / \ / \ / \ +h0 h1 h2 h3 h0 h1 h2 h3 h4 h5 ``` -Tendermint always uses the `TMHASH` hash function, which is the first 20-bytes -of the SHA256: +### MerkleRoot -``` -func TMHASH(bz []byte) []byte { - shasum := SHA256(bz) - return shasum[:20] -} -``` - -### Simple Merkle Root - -The function `SimpleMerkleRoot` is a simple recursive function defined as follows: +The function `MerkleRoot` is a simple recursive function defined as follows: ```go -func SimpleMerkleRoot(hashes [][]byte) []byte{ - switch len(hashes) { - case 0: - return nil - case 1: - return hashes[0] - default: - left := SimpleMerkleRoot(hashes[:(len(hashes)+1)/2]) - right := SimpleMerkleRoot(hashes[(len(hashes)+1)/2:]) - return SimpleConcatHash(left, right) - } +// SHA256(0x00 || leaf) +func leafHash(leaf []byte) []byte { + return tmhash.Sum(append(0x00, leaf...)) } -func SimpleConcatHash(left, right []byte) []byte{ - left = encodeByteSlice(left) - right = encodeByteSlice(right) - return TMHASH(append(left, right)) +// SHA256(0x01 || left || right) +func innerHash(left []byte, right []byte) []byte { + return tmhash.Sum(append(0x01, append(left, right...)...)) +} + +// largest power of 2 less than k +func getSplitPoint(k int) { ... } + +func MerkleRoot(items [][]byte) []byte{ + switch len(items) { + case 0: + return nil + case 1: + return leafHash(leafs[0]) + default: + k := getSplitPoint(len(items)) + left := MerkleRoot(items[:k]) + right := MerkleRoot(items[k:]) + return innerHash(left, right) + } } ``` -Note that the leaves are Amino encoded as byte-arrays (ie. simple Uvarint length -prefix) before being concatenated together and hashed. +Note: `MerkleRoot` operates on items which are arbitrary byte arrays, not +necessarily hashes. For items which need to be hashed first, we introduce the +`Hashes` function: -Note: we will abuse notion and invoke `SimpleMerkleRoot` with arguments of type `struct` or type `[]struct`. -For `struct` arguments, we compute a `[][]byte` containing the hash of each -field in the struct sorted by the hash of the field name. -For `[]struct` arguments, we compute a `[][]byte` by hashing the individual `struct` elements. +``` +func Hashes(items [][]byte) [][]byte { + return SHA256 of each item +} +``` + +Note: we will abuse notion and invoke `MerkleRoot` with arguments of type `struct` or type `[]struct`. +For `struct` arguments, we compute a `[][]byte` containing the amino encoding of each +field in the struct, in the same order the fields appear in the struct. +For `[]struct` arguments, we compute a `[][]byte` by amino encoding the individual `struct` elements. ### Simple Merkle Proof -Proof that a leaf is in a Merkle tree consists of a simple structure: +Proof that a leaf is in a Merkle tree is composed as follows: -``` +```golang type SimpleProof struct { + Total int + Index int + LeafHash []byte Aunts [][]byte } ``` -Which is verified using the following: +Which is verified as follows: -``` -func (proof SimpleProof) Verify(index, total int, leafHash, rootHash []byte) bool { - computedHash := computeHashFromAunts(index, total, leafHash, proof.Aunts) +```golang +func (proof SimpleProof) Verify(rootHash []byte, leaf []byte) bool { + assert(proof.LeafHash, leafHash(leaf) + + computedHash := computeHashFromAunts(proof.Index, proof.Total, proof.LeafHash, proof.Aunts) return computedHash == rootHash } @@ -247,26 +276,18 @@ func computeHashFromAunts(index, total int, leafHash []byte, innerHashes [][]byt assert(len(innerHashes) > 0) - numLeft := (total + 1) / 2 + numLeft := getSplitPoint(total) // largest power of 2 less than total if index < numLeft { leftHash := computeHashFromAunts(index, numLeft, leafHash, innerHashes[:len(innerHashes)-1]) assert(leftHash != nil) - return SimpleHashFromTwoHashes(leftHash, innerHashes[len(innerHashes)-1]) + return innerHash(leftHash, innerHashes[len(innerHashes)-1]) } rightHash := computeHashFromAunts(index-numLeft, total-numLeft, leafHash, innerHashes[:len(innerHashes)-1]) assert(rightHash != nil) - return SimpleHashFromTwoHashes(innerHashes[len(innerHashes)-1], rightHash) + return innerHash(innerHashes[len(innerHashes)-1], rightHash) } ``` -### Simple Tree with Dictionaries - -The Simple Tree is used to merkelize a list of items, so to merkelize a -(short) dictionary of key-value pairs, encode the dictionary as an -ordered list of `KVPair` structs. The block hash is such a hash -derived from all the fields of the block `Header`. The state hash is -similarly derived. - ### IAVL+ Tree Because Tendermint only uses a Simple Merkle Tree, application developers are expect to use their own Merkle tree in their applications. For example, the IAVL+ Tree - an immutable self-balancing binary tree for persisting application state is used by the [Cosmos SDK](https://github.com/cosmos/cosmos-sdk/blob/develop/docs/sdk/core/multistore.md) @@ -300,20 +321,24 @@ Where the `"value"` is the base64 encoding of the raw pubkey bytes, and the Signed messages (eg. votes, proposals) in the consensus are encoded using Amino. -When signing, the elements of a message are sorted alphabetically by key and prepended with -a `chain_id` and `type` field. +When signing, the elements of a message are re-ordered so the fixed-length fields +are first, making it easy to quickly check the type, height, and round. +The `ChainID` is also appended to the end. We call this encoding the SignBytes. For instance, SignBytes for a vote is the Amino encoding of the following struct: ```go type CanonicalVote struct { - ChainID string - Type string + Type byte + Height int64 `binary:"fixed64"` + Round int64 `binary:"fixed64"` BlockID CanonicalBlockID - Height int64 - Round int Timestamp time.Time - VoteType byte + ChainID string } ``` -NOTE: see [#1622](https://github.com/tendermint/tendermint/issues/1622) for how field ordering will change +The field ordering and the fixed sized encoding for the first three fields is optimized to ease parsing of SignBytes +in HSMs. It creates fixed offsets for relevant fields that need to be read in this context. +For more details, see the [signing spec](/docs/spec/consensus/signing.md). +Also, see the motivating discussion in +[#1622](https://github.com/tendermint/tendermint/issues/1622). diff --git a/docs/spec/blockchain/state.md b/docs/spec/blockchain/state.md index e904bb33..7df096bc 100644 --- a/docs/spec/blockchain/state.md +++ b/docs/spec/blockchain/state.md @@ -15,6 +15,7 @@ validation. ```go type State struct { + Version Version LastResults []Result AppHash []byte @@ -55,11 +56,11 @@ type Validator struct { } ``` -When hashing the Validator struct, the pubkey is not hashed, -because the address is already the hash of the pubkey. +When hashing the Validator struct, the address is not included, +because it is redundant with the pubkey. The `state.Validators`, `state.LastValidators`, and `state.NextValidators`, must always by sorted by validator address, -so that there is a canonical order for computing the SimpleMerkleRoot. +so that there is a canonical order for computing the MerkleRoot. We also define a `TotalVotingPower` function, to return the total voting power: @@ -77,32 +78,44 @@ func TotalVotingPower(vals []Validators) int64{ ConsensusParams define various limits for blockchain data structures. Like validator sets, they are set during genesis and can be updated by the application through ABCI. +When hashed, only a subset of the params are included, to allow the params to +evolve without breaking the header. -``` +```go type ConsensusParams struct { BlockSize - TxSize - BlockGossip - EvidenceParams + Evidence + Validator +} + +type hashedParams struct { + BlockMaxBytes int64 + BlockMaxGas int64 +} + +func (params ConsensusParams) Hash() []byte { + SHA256(hashedParams{ + BlockMaxBytes: params.BlockSize.MaxBytes, + BlockMaxGas: params.BlockSize.MaxGas, + }) } type BlockSize struct { - MaxBytes int + MaxBytes int64 MaxGas int64 } -type TxSize struct { - MaxBytes int - MaxGas int64 -} - -type BlockGossip struct { - BlockPartSizeBytes int -} - -type EvidenceParams struct { +type Evidence struct { MaxAge int64 } + +type Validator struct { + PubKeyTypes []string +} + +type ValidatorParams struct { + PubKeyTypes []string +} ``` #### BlockSize @@ -114,20 +127,15 @@ otherwise. Blocks should additionally be limited by the amount of "gas" consumed by the transactions in the block, though this is not yet implemented. -#### TxSize - -These parameters are not yet enforced and may disappear. See [issue -#2347](https://github.com/tendermint/tendermint/issues/2347). - -#### BlockGossip - -When gossipping blocks in the consensus, they are first split into parts. The -size of each part is `ConsensusParams.BlockGossip.BlockPartSizeBytes`. - -#### EvidenceParams +#### Evidence For evidence in a block to be valid, it must satisfy: ``` -block.Header.Height - evidence.Height < ConsensusParams.EvidenceParams.MaxAge +block.Header.Height - evidence.Height < ConsensusParams.Evidence.MaxAge ``` + +#### Validator + +Validators from genesis file and `ResponseEndBlock` must have pubkeys of type ∈ +`ConsensusParams.Validator.PubKeyTypes`. diff --git a/docs/spec/consensus/signing.md b/docs/spec/consensus/signing.md new file mode 100644 index 00000000..f97df0c6 --- /dev/null +++ b/docs/spec/consensus/signing.md @@ -0,0 +1,205 @@ +# Validator Signing + +Here we specify the rules for validating a proposal and vote before signing. +First we include some general notes on validating data structures common to both types. +We then provide specific validation rules for each. Finally, we include validation rules to prevent double-sigining. + +## SignedMsgType + +The `SignedMsgType` is a single byte that refers to the type of the message +being signed. It is defined in Go as follows: + +``` +// SignedMsgType is a type of signed message in the consensus. +type SignedMsgType byte + +const ( + // Votes + PrevoteType SignedMsgType = 0x01 + PrecommitType SignedMsgType = 0x02 + + // Proposals + ProposalType SignedMsgType = 0x20 +) +``` + +All signed messages must correspond to one of these types. + +## Timestamp + +Timestamp validation is subtle and there are currently no bounds placed on the +timestamp included in a proposal or vote. It is expected that validators will honestly +report their local clock time. The median of all timestamps +included in a commit is used as the timestamp for the next block height. + +Timestamps are expected to be strictly monotonic for a given validator, though +this is not currently enforced. + +## ChainID + +ChainID is an unstructured string with a max length of 50-bytes. +In the future, the ChainID may become structured, and may take on longer lengths. +For now, it is recommended that signers be configured for a particular ChainID, +and to only sign votes and proposals corresponding to that ChainID. + +## BlockID + +BlockID is the structure used to represent the block: + +``` +type BlockID struct { + Hash []byte + PartsHeader PartSetHeader +} + +type PartSetHeader struct { + Hash []byte + Total int +} +``` + +To be included in a valid vote or proposal, BlockID must either represent a `nil` block, or a complete one. +We introduce two methods, `BlockID.IsZero()` and `BlockID.IsComplete()` for these cases, respectively. + +`BlockID.IsZero()` returns true for BlockID `b` if each of the following +are true: + +``` +b.Hash == nil +b.PartsHeader.Total == 0 +b.PartsHeader.Hash == nil +``` + +`BlockID.IsComplete()` returns true for BlockID `b` if each of the following +are true: + +``` +len(b.Hash) == 32 +b.PartsHeader.Total > 0 +len(b.PartsHeader.Hash) == 32 +``` + +## Proposals + +The structure of a proposal for signing looks like: + +``` +type CanonicalProposal struct { + Type SignedMsgType // type alias for byte + Height int64 `binary:"fixed64"` + Round int64 `binary:"fixed64"` + POLRound int64 `binary:"fixed64"` + BlockID BlockID + Timestamp time.Time + ChainID string +} +``` + +A proposal is valid if each of the following lines evaluates to true for proposal `p`: + +``` +p.Type == 0x20 +p.Height > 0 +p.Round >= 0 +p.POLRound >= -1 +p.BlockID.IsComplete() +``` + +In other words, a proposal is valid for signing if it contains the type of a Proposal +(0x20), has a positive, non-zero height, a +non-negative round, a POLRound not less than -1, and a complete BlockID. + +## Votes + +The structure of a vote for signing looks like: + +``` +type CanonicalVote struct { + Type SignedMsgType // type alias for byte + Height int64 `binary:"fixed64"` + Round int64 `binary:"fixed64"` + BlockID BlockID + Timestamp time.Time + ChainID string +} +``` + +A vote is valid if each of the following lines evaluates to true for vote `v`: + +``` +v.Type == 0x1 || v.Type == 0x2 +v.Height > 0 +v.Round >= 0 +v.BlockID.IsZero() || v.BlockID.IsComplete() +``` + +In other words, a vote is valid for signing if it contains the type of a Prevote +or Precommit (0x1 or 0x2, respectively), has a positive, non-zero height, a +non-negative round, and an empty or valid BlockID. + +## Invalid Votes and Proposals + +Votes and proposals which do not satisfy the above rules are considered invalid. +Peers gossipping invalid votes and proposals may be disconnected from other peers on the network. +Note, however, that there is not currently any explicit mechanism to punish validators signing votes or proposals that fail +these basic validation rules. + +## Double Signing + +Signers must be careful not to sign conflicting messages, also known as "double signing" or "equivocating". +Tendermint has mechanisms to publish evidence of validators that signed conflicting votes, so they can be punished +by the application. Note Tendermint does not currently handle evidence of conflciting proposals, though it may in the future. + +### State + +To prevent such double signing, signers must track the height, round, and type of the last message signed. +Assume the signer keeps the following state, `s`: + +``` +type LastSigned struct { + Height int64 + Round int64 + Type SignedMsgType // byte +} +``` + +After signing a vote or proposal `m`, the signer sets: + +``` +s.Height = m.Height +s.Round = m.Round +s.Type = m.Type +``` + +### Proposals + +A signer should only sign a proposal `p` if any of the following lines are true: + +``` +p.Height > s.Height +p.Height == s.Height && p.Round > s.Round +``` + +In other words, a proposal should only be signed if it's at a higher height, or a higher round for the same height. +Once a proposal or vote has been signed for a given height and round, a proposal should never be signed for the same height and round. + +### Votes + +A signer should only sign a vote `v` if any of the following lines are true: + +``` +v.Height > s.Height +v.Height == s.Height && v.Round > s.Round +v.Height == s.Height && v.Round == s.Round && v.Step == 0x1 && s.Step == 0x20 +v.Height == s.Height && v.Round == s.Round && v.Step == 0x2 && s.Step != 0x2 +``` + +In other words, a vote should only be signed if it's: + +- at a higher height +- at a higher round for the same height +- a prevote for the same height and round where we haven't signed a prevote or precommit (but have signed a proposal) +- a precommit for the same height and round where we haven't signed a precommit (but have signed a proposal and/or a prevote) + +This means that once a validator signs a prevote for a given height and round, the only other message it can sign for that height and round is a precommit. +And once a validator signs a precommit for a given height and round, it must not sign any other message for that same height and round. diff --git a/docs/spec/p2p/peer.md b/docs/spec/p2p/peer.md index a1ff25d8..f5c2e7bf 100644 --- a/docs/spec/p2p/peer.md +++ b/docs/spec/p2p/peer.md @@ -75,22 +75,25 @@ The Tendermint Version Handshake allows the peers to exchange their NodeInfo: ```golang type NodeInfo struct { + Version p2p.Version ID p2p.ID ListenAddr string Network string - Version string + SoftwareVersion string Channels []int8 Moniker string Other NodeInfoOther } +type Version struct { + P2P uint64 + Block uint64 + App uint64 +} + type NodeInfoOther struct { - AminoVersion string - P2PVersion string - ConsensusVersion string - RPCVersion string TxIndex string RPCAddress string } @@ -99,8 +102,7 @@ type NodeInfoOther struct { The connection is disconnected if: - `peer.NodeInfo.ID` is not equal `peerConn.ID` -- `peer.NodeInfo.Version` is not formatted as `X.X.X` where X are integers known as Major, Minor, and Revision -- `peer.NodeInfo.Version` Major is not the same as ours +- `peer.NodeInfo.Version.Block` does not match ours - `peer.NodeInfo.Network` is not the same as ours - `peer.Channels` does not intersect with our known Channels. - `peer.NodeInfo.ListenAddr` is malformed or is a DNS host that cannot be diff --git a/docs/spec/reactors/block_sync/reactor.md b/docs/spec/reactors/block_sync/reactor.md index 045bbd40..91fd79b0 100644 --- a/docs/spec/reactors/block_sync/reactor.md +++ b/docs/spec/reactors/block_sync/reactor.md @@ -65,24 +65,24 @@ type Requester { mtx Mutex block Block height int64 - 
peerID p2p.ID - redoChannel chan struct{} + 
 peerID p2p.ID + redoChannel chan p2p.ID //redo may send multi-time; peerId is used to identify repeat } ``` -Pool is core data structure that stores last executed block (`height`), assignment of requests to peers (`requesters`), current height for each peer and number of pending requests for each peer (`peers`), maximum peer height, etc. +Pool is a core data structure that stores last executed block (`height`), assignment of requests to peers (`requesters`), current height for each peer and number of pending requests for each peer (`peers`), maximum peer height, etc. ```go type Pool { - mtx Mutex - requesters map[int64]*Requester - height int64 - peers map[p2p.ID]*Peer - maxPeerHeight int64 - numPending int32 - store BlockStore - requestsChannel chan<- BlockRequest - errorsChannel chan<- peerError + mtx Mutex + requesters map[int64]*Requester + height int64 + peers map[p2p.ID]*Peer + maxPeerHeight int64 + numPending int32 + store BlockStore + requestsChannel chan<- BlockRequest + errorsChannel chan<- peerError } ``` @@ -90,11 +90,11 @@ Peer data structure stores for each peer current `height` and number of pending ```go type Peer struct { - id p2p.ID - height int64 - numPending int32 - timeout *time.Timer - didTimeout bool + id p2p.ID + height int64 + numPending int32 + timeout *time.Timer + didTimeout bool } ``` @@ -169,11 +169,11 @@ Requester task is responsible for fetching a single block at position `height`. ```go fetchBlock(height, pool): - while true do + while true do { peerID = nil block = nil peer = pickAvailablePeer(height) - peerId = peer.id + peerID = peer.id enqueue BlockRequest(height, peerID) to pool.requestsChannel redo = false @@ -181,12 +181,15 @@ fetchBlock(height, pool): select { upon receiving Quit message do return - upon receiving message on redoChannel do - mtx.Lock() - pool.numPending++ - redo = true - mtx.UnLock() + upon receiving redo message with id on redoChannel do + if peerID == id { + mtx.Lock() + pool.numPending++ + redo = true + mtx.UnLock() + } } + } pickAvailablePeer(height): selectedPeer = nil @@ -244,7 +247,7 @@ createRequesters(pool): main(pool): create trySyncTicker with interval trySyncIntervalMS create statusUpdateTicker with interval statusUpdateIntervalSeconds - create switchToConsensusTicker with interbal switchToConsensusIntervalSeconds + create switchToConsensusTicker with interval switchToConsensusIntervalSeconds while true do select { diff --git a/docs/spec/reactors/consensus/consensus-reactor.md b/docs/spec/reactors/consensus/consensus-reactor.md index 7be35032..47c6949a 100644 --- a/docs/spec/reactors/consensus/consensus-reactor.md +++ b/docs/spec/reactors/consensus/consensus-reactor.md @@ -129,13 +129,16 @@ handleMessage(msg): Reset prs.CatchupCommitRound and prs.CatchupCommit ``` -### CommitStepMessage handler +### NewValidBlockMessage handler ``` handleMessage(msg): - if prs.Height == msg.Height then - prs.ProposalBlockPartsHeader = msg.BlockPartsHeader - prs.ProposalBlockParts = msg.BlockParts + if prs.Height != msg.Height then return + + if prs.Round != msg.Round && !msg.IsCommit then return + + prs.ProposalBlockPartsHeader = msg.BlockPartsHeader + prs.ProposalBlockParts = msg.BlockParts ``` ### HasVoteMessage handler @@ -161,8 +164,8 @@ handleMessage(msg): handleMessage(msg): if prs.Height != msg.Height || prs.Round != msg.Round || prs.Proposal then return prs.Proposal = true - prs.ProposalBlockPartsHeader = msg.BlockPartsHeader - prs.ProposalBlockParts = empty set + if prs.ProposalBlockParts == empty set then // otherwise it is set in NewValidBlockMessage handler + prs.ProposalBlockPartsHeader = msg.BlockPartsHeader prs.ProposalPOLRound = msg.POLRound prs.ProposalPOL = nil Send msg through internal peerMsgQueue to ConsensusState service @@ -335,12 +338,11 @@ BlockID has seen +2/3 votes. This routine is based on the local RoundState (`rs` ## Broadcast routine -The Broadcast routine subscribes to an internal event bus to receive new round steps, votes messages and proposal -heartbeat messages, and broadcasts messages to peers upon receiving those events. +The Broadcast routine subscribes to an internal event bus to receive new round steps and votes messages, and broadcasts messages to peers upon receiving those +events. It broadcasts `NewRoundStepMessage` or `CommitStepMessage` upon new round state event. Note that broadcasting these messages does not depend on the PeerRoundState; it is sent on the StateChannel. Upon receiving VoteMessage it broadcasts `HasVoteMessage` message to its peers on the StateChannel. -`ProposalHeartbeatMessage` is sent the same way on the StateChannel. ## Channels diff --git a/docs/spec/reactors/consensus/consensus.md b/docs/spec/reactors/consensus/consensus.md index a1cf17bc..55960874 100644 --- a/docs/spec/reactors/consensus/consensus.md +++ b/docs/spec/reactors/consensus/consensus.md @@ -26,7 +26,7 @@ only to a subset of processes called peers. By the gossiping protocol, a validat all needed information (`ProposalMessage`, `VoteMessage` and `BlockPartMessage`) so they can reach agreement on some block, and also obtain the content of the chosen block (block parts). As part of the gossiping protocol, processes also send auxiliary messages that inform peers about the -executed steps of the core consensus algorithm (`NewRoundStepMessage` and `CommitStepMessage`), and +executed steps of the core consensus algorithm (`NewRoundStepMessage` and `NewValidBlockMessage`), and also messages that inform peers what votes the process has seen (`HasVoteMessage`, `VoteSetMaj23Message` and `VoteSetBitsMessage`). These messages are then used in the gossiping protocol to determine what messages a process should send to its peers. @@ -47,25 +47,21 @@ type ProposalMessage struct { ### Proposal Proposal contains height and round for which this proposal is made, BlockID as a unique identifier -of proposed block, timestamp, and two fields (POLRound and POLBlockID) that are needed for -termination of the consensus. The message is signed by the validator private key. +of proposed block, timestamp, and POLRound (a so-called Proof-of-Lock (POL) round) that is needed for +termination of the consensus. If POLRound >= 0, then BlockID corresponds to the block that +is locked in POLRound. The message is signed by the validator private key. ```go type Proposal struct { Height int64 Round int - Timestamp Time - BlockID BlockID POLRound int - POLBlockID BlockID + BlockID BlockID + Timestamp Time Signature Signature } ``` -NOTE: In the current version of the Tendermint, the consensus value in proposal is represented with -PartSetHeader, and with BlockID in vote message. It should be aligned as suggested in this spec as -BlockID contains PartSetHeader. - ## VoteMessage VoteMessage is sent to vote for some block (or to inform others that a process does not vote in the @@ -93,33 +89,6 @@ type BlockPartMessage struct { } ``` -## ProposalHeartbeatMessage - -ProposalHeartbeatMessage is sent to signal that a node is alive and waiting for transactions -to be able to create a next block proposal. - -```go -type ProposalHeartbeatMessage struct { - Heartbeat Heartbeat -} -``` - -### Heartbeat - -Heartbeat contains validator information (address and index), -height, round and sequence number. It is signed by the private key of the validator. - -```go -type Heartbeat struct { - ValidatorAddress []byte - ValidatorIndex int - Height int64 - Round int - Sequence int - Signature Signature -} -``` - ## NewRoundStepMessage NewRoundStepMessage is sent for every step transition during the core consensus algorithm execution. @@ -136,23 +105,26 @@ type NewRoundStepMessage struct { } ``` -## CommitStepMessage +## NewValidBlockMessage -CommitStepMessage is sent when an agreement on some block is reached. It contains height for which -agreement is reached, block parts header that describes the decided block and is used to obtain all +NewValidBlockMessage is sent when a validator observes a valid block B in some round r, +i.e., there is a Proposal for block B and 2/3+ prevotes for the block B in the round r. +It contains height and round in which valid block is observed, block parts header that describes +the valid block and is used to obtain all block parts, and a bit array of the block parts a process currently has, so its peers can know what parts it is missing so they can send them. +In case the block is also committed, then IsCommit flag is set to true. ```go -type CommitStepMessage struct { +type NewValidBlockMessage struct { Height int64 - BlockID BlockID + Round int + BlockPartsHeader PartSetHeader BlockParts BitArray + IsCommit bool } ``` -TODO: We use BlockID instead of BlockPartsHeader (in current implementation) for symmetry. - ## ProposalPOLMessage ProposalPOLMessage is sent when a previous block is re-proposed. diff --git a/docs/tendermint-core/configuration.md b/docs/tendermint-core/configuration.md index 8b3c3c22..0d9a58c4 100644 --- a/docs/tendermint-core/configuration.md +++ b/docs/tendermint-core/configuration.md @@ -36,19 +36,26 @@ db_backend = "leveldb" # Database directory db_dir = "data" -# Output level for logging -log_level = "state:info,*:error" +# Output level for logging, including package level options +log_level = "main:info,state:info,*:error" + +# Output format: 'plain' (colored text) or 'json' +log_format = "plain" ##### additional base config options ##### -# The ID of the chain to join (should be signed with every transaction and vote) -chain_id = "" - # Path to the JSON file containing the initial validator set and other meta data -genesis_file = "genesis.json" +genesis_file = "config/genesis.json" # Path to the JSON file containing the private key to use as a validator in the consensus protocol -priv_validator_file = "priv_validator.json" +priv_validator_file = "config/priv_validator.json" + +# TCP or UNIX socket address for Tendermint to listen on for +# connections from an external PrivValidator process +priv_validator_laddr = "" + +# Path to the JSON file containing the private key to use for node authentication in the p2p protocol +node_key_file = "config/node_key.json" # Mechanism to connect to the ABCI application: socket | grpc abci = "socket" @@ -68,13 +75,24 @@ filter_peers = false # TCP or UNIX socket address for the RPC server to listen on laddr = "tcp://0.0.0.0:26657" +# A list of origins a cross-domain request can be executed from +# Default value '[]' disables cors support +# Use '["*"]' to allow any origin +cors_allowed_origins = [] + +# A list of methods the client is allowed to use with cross-domain requests +cors_allowed_methods = ["HEAD", "GET", "POST"] + +# A list of non simple headers the client is allowed to use with cross-domain requests +cors_allowed_headers = ["Origin", "Accept", "Content-Type", "X-Requested-With", "X-Server-Time"] + # TCP or UNIX socket address for the gRPC server to listen on # NOTE: This server only supports /broadcast_tx_commit grpc_laddr = "" # Maximum number of simultaneous connections. # Does not include RPC (HTTP&WebSocket) connections. See max_open_connections -# If you want to accept more significant number than the default, make sure +# If you want to accept a larger number than the default, make sure # you increase your OS limits. # 0 - unlimited. # Should be < {ulimit -Sn} - {MaxNumInboundPeers} - {MaxNumOutboundPeers} - {N of wal, db and other open files} @@ -86,7 +104,7 @@ unsafe = false # Maximum number of simultaneous connections (including WebSocket). # Does not include gRPC connections. See grpc_max_open_connections -# If you want to accept more significant number than the default, make sure +# If you want to accept a larger number than the default, make sure # you increase your OS limits. # 0 - unlimited. # Should be < {ulimit -Sn} - {MaxNumInboundPeers} - {MaxNumOutboundPeers} - {N of wal, db and other open files} @@ -99,6 +117,12 @@ max_open_connections = 900 # Address to listen for incoming connections laddr = "tcp://0.0.0.0:26656" +# Address to advertise to peers for them to dial +# If empty, will use the same port as the laddr, +# and will introspect on the listener or use UPnP +# to figure out the address. +external_address = "" + # Comma separated list of seed nodes to connect to seeds = "" @@ -109,7 +133,7 @@ persistent_peers = "" upnp = false # Path to address book -addr_book_file = "addrbook.json" +addr_book_file = "config/addrbook.json" # Set true for strict address routability rules # Set false for private or local networks @@ -146,7 +170,7 @@ seed_mode = false private_peer_ids = "" # Toggle to disable guard against peers connecting from the same ip. -allow_duplicate_ip = true +allow_duplicate_ip = false # Peer connection configuration. handshake_timeout = "20s" @@ -157,26 +181,26 @@ dial_timeout = "3s" recheck = true broadcast = true -wal_dir = "data/mempool.wal" +wal_dir = "" # size of the mempool -size = 100000 +size = 5000 # size of the cache (used to filter transactions we saw earlier) -cache_size = 100000 +cache_size = 10000 ##### consensus configuration options ##### [consensus] wal_file = "data/cs.wal/wal" -timeout_propose = "3000ms" +timeout_propose = "3s" timeout_propose_delta = "500ms" -timeout_prevote = "1000ms" +timeout_prevote = "1s" timeout_prevote_delta = "500ms" -timeout_precommit = "1000ms" +timeout_precommit = "1s" timeout_precommit_delta = "500ms" -timeout_commit = "1000ms" +timeout_commit = "1s" # Make progress as soon as we have all the precommits (as if TimeoutCommit = 0) skip_timeout_commit = false @@ -187,7 +211,10 @@ create_empty_blocks_interval = "0s" # Reactor sleep duration parameters peer_gossip_sleep_duration = "100ms" -peer_query_maj23_sleep_duration = "2000ms" +peer_query_maj23_sleep_duration = "2s" + +# Block time parameters. Corresponds to the minimum time increment between consecutive blocks. +blocktime_iota = "1s" ##### transactions indexer configuration options ##### [tx_index] @@ -228,7 +255,7 @@ prometheus = false prometheus_listen_addr = ":26660" # Maximum number of simultaneous connections. -# If you want to accept a more significant number than the default, make sure +# If you want to accept a larger number than the default, make sure # you increase your OS limits. # 0 - unlimited. max_open_connections = 3 diff --git a/docs/tendermint-core/rpc.md b/docs/tendermint-core/rpc.md index 7ae59f0d..4ea5ab0d 100644 --- a/docs/tendermint-core/rpc.md +++ b/docs/tendermint-core/rpc.md @@ -2,6 +2,6 @@ The RPC documentation is hosted here: -- https://tendermint.com/rpc/ +- [https://tendermint.com/rpc/](https://tendermint.com/rpc/) To update the documentation, edit the relevant `godoc` comments in the [rpc/core directory](https://github.com/tendermint/tendermint/tree/develop/rpc/core). diff --git a/docs/tendermint-core/using-tendermint.md b/docs/tendermint-core/using-tendermint.md index 5ee18361..2ca8c9e9 100644 --- a/docs/tendermint-core/using-tendermint.md +++ b/docs/tendermint-core/using-tendermint.md @@ -60,42 +60,34 @@ definition](https://github.com/tendermint/tendermint/blob/master/types/genesis.g ``` { - "genesis_time": "2018-07-09T22:43:06.255718641Z", - "chain_id": "chain-IAkWsK", + "genesis_time": "2018-11-13T18:11:50.277637Z", + "chain_id": "test-chain-s4ui7D", + "consensus_params": { + "block_size": { + "max_bytes": "22020096", + "max_gas": "-1" + }, + "evidence": { + "max_age": "100000" + }, + "validator": { + "pub_key_types": [ + "ed25519" + ] + } + }, "validators": [ { + "address": "39C04A480B54AB258A45355A5E48ADDED9956C65", "pub_key": { "type": "tendermint/PubKeyEd25519", - "value": "oX8HhKsErMluxI0QWNSR8djQMSupDvHdAYrHwP7n73k=" + "value": "DMEMMj1+thrkUCGocbvvKzXeaAtRslvX9MWtB+smuIA=" }, - "power": "1", - "name": "node0" - }, - { - "pub_key": { - "type": "tendermint/PubKeyEd25519", - "value": "UZNSJA9zmeFQj36Rs296lY+WFQ4Rt6s7snPpuKypl5I=" - }, - "power": "1", - "name": "node1" - }, - { - "pub_key": { - "type": "tendermint/PubKeyEd25519", - "value": "i9GrM6/MHB4zjCelMZBUYHNXYIzl4n0RkDCVmmLhS/o=" - }, - "power": "1", - "name": "node2" - }, - { - "pub_key": { - "type": "tendermint/PubKeyEd25519", - "value": "0qq7954l87trEqbQV9c7d1gurnjTGMxreXc848ZZ5aw=" - }, - "power": "1", - "name": "node3" + "power": "10", + "name": "" } - ] + ], + "app_hash": "" } ``` @@ -121,7 +113,7 @@ blocks are produced regularly, even if there are no transactions. See _No Empty Blocks_, below, to modify this setting. Tendermint supports in-process versions of the `counter`, `kvstore` and -`nil` apps that ship as examples with `abci-cli`. It's easy to compile +`noop` apps that ship as examples with `abci-cli`. It's easy to compile your own app in-process with Tendermint if it's written in Go. If your app is not written in Go, simply run it in another process, and use the `--proxy_app` flag to specify the address of the socket it is listening @@ -527,18 +519,16 @@ developers guide](../app-dev/app-development.md) for more details. ### Local Network -To run a network locally, say on a single machine, you must change the -`_laddr` fields in the `config.toml` (or using the flags) so that the -listening addresses of the various sockets don't conflict. Additionally, -you must set `addr_book_strict=false` in the `config.toml`, otherwise -Tendermint's p2p library will deny making connections to peers with the -same IP address. +To run a network locally, say on a single machine, you must change the `_laddr` +fields in the `config.toml` (or using the flags) so that the listening +addresses of the various sockets don't conflict. Additionally, you must set +`addr_book_strict=false` in the `config.toml`, otherwise Tendermint's p2p +library will deny making connections to peers with the same IP address. ### Upgrading -The Tendermint development cycle currently includes a lot of breaking changes. -Upgrading from an old version to a new version usually means throwing -away the chain data. Try out the -[tm-migrate](https://github.com/hxzqlh/tm-tools) tool written by -[@hxzqlh](https://github.com/hxzqlh) if you are keen to preserve the -state of your chain when upgrading to newer versions. +See the +[UPGRADING.md](https://github.com/tendermint/tendermint/blob/master/UPGRADING.md) +guide. You may need to reset your chain between major breaking releases. +Although, we expect Tendermint to have fewer breaking releases in the future +(especially after 1.0 release). diff --git a/docs/tools/README.md b/docs/tools/README.md index ef1ae7c2..0b861621 100644 --- a/docs/tools/README.md +++ b/docs/tools/README.md @@ -1,4 +1,7 @@ # Overview -Tendermint comes with some tools for [benchmarking](./benchmarking.md) -and [monitoring](./monitoring.md). +Tendermint comes with some tools for: + +* [Benchmarking](./benchmarking.md) +* [Monitoring](./monitoring.md) +* [Validation of remote signers](./remote-signer-validation.md) diff --git a/docs/tools/benchmarking.md b/docs/tools/benchmarking.md index e17c2856..67a472e4 100644 --- a/docs/tools/benchmarking.md +++ b/docs/tools/benchmarking.md @@ -2,7 +2,7 @@ Tendermint blockchain benchmarking tool: -- https://github.com/tendermint/tools/tree/master/tm-bench +- [https://github.com/tendermint/tendermint/tree/master/tools/tm-bench](https://github.com/tendermint/tendermint/tree/master/tools/tm-bench) For example, the following: diff --git a/docs/tools/monitoring.md b/docs/tools/monitoring.md index c0fa94c0..fa3901dd 100644 --- a/docs/tools/monitoring.md +++ b/docs/tools/monitoring.md @@ -3,7 +3,7 @@ Tendermint blockchain monitoring tool; watches over one or more nodes, collecting and providing various statistics to the user: -- https://github.com/tendermint/tendermint/tree/master/tools/tm-monitor +- [https://github.com/tendermint/tendermint/tree/master/tools/tm-monitor](https://github.com/tendermint/tendermint/tree/master/tools/tm-monitor) ## Quick Start diff --git a/docs/tools/remote-signer-validation.md b/docs/tools/remote-signer-validation.md new file mode 100644 index 00000000..c8a948e3 --- /dev/null +++ b/docs/tools/remote-signer-validation.md @@ -0,0 +1,146 @@ +# tm-signer-harness + +Located under the `tools/tm-signer-harness` folder in the [Tendermint +repository](https://github.com/tendermint/tendermint). + +The Tendermint remote signer test harness facilitates integration testing +between Tendermint and remote signers such as +[KMS](https://github.com/tendermint/kms). Such remote signers allow for signing +of important Tendermint messages using +[HSMs](https://en.wikipedia.org/wiki/Hardware_security_module), providing +additional security. + +When executed, `tm-signer-harness`: + +1. Runs a listener (either TCP or Unix sockets). +2. Waits for a connection from the remote signer. +3. Upon connection from the remote signer, executes a number of automated tests + to ensure compatibility. +4. Upon successful validation, the harness process exits with a 0 exit code. + Upon validation failure, it exits with a particular exit code related to the + error. + +## Prerequisites +Requires the same prerequisites as for building +[Tendermint](https://github.com/tendermint/tendermint). + +## Building +From the `tools/tm-signer-harness` directory in your Tendermint source +repository, simply run: + +```bash +make + +# To have global access to this executable +make install +``` + +## Docker Image +To build a Docker image containing the `tm-signer-harness`, also from the +`tools/tm-signer-harness` directory of your Tendermint source repo, simply run: + +```bash +make docker-image +``` + +## Running against KMS +As an example of how to use `tm-signer-harness`, the following instructions show +you how to execute its tests against [KMS](https://github.com/tendermint/kms). +For this example, we will make use of the **software signing module in KMS**, as +the hardware signing module requires a physical +[YubiHSM](https://www.yubico.com/products/yubihsm/) device. + +### Step 1: Install KMS on your local machine +See the [KMS repo](https://github.com/tendermint/kms) for details on how to set +KMS up on your local machine. + +If you have [Rust](https://www.rust-lang.org/) installed on your local machine, +you can simply install KMS by: + +```bash +cargo install tmkms +``` + +### Step 2: Make keys for KMS +The KMS software signing module needs a key with which to sign messages. In our +example, we will simply export a signing key from our local Tendermint instance. + +```bash +# Will generate all necessary Tendermint configuration files, including: +# - ~/.tendermint/config/priv_validator_key.json +# - ~/.tendermint/data/priv_validator_state.json +tendermint init + +# Extract the signing key from our local Tendermint instance +tm-signer-harness extract_key \ # Use the "extract_key" command + -tmhome ~/.tendermint \ # Where to find the Tendermint home directory + -output ./signing.key # Where to write the key +``` + +Also, because we want KMS to connect to `tm-signer-harness`, we will need to +provide a secret connection key from KMS' side: + +```bash +tmkms keygen secret_connection.key +``` + +### Step 3: Configure and run KMS +KMS needs some configuration to tell it to use the softer signing module as well +as the `signing.key` file we just generated. Save the following to a file called +`tmkms.toml`: + +```toml +[[validator]] +addr = "tcp://127.0.0.1:61219" # This is where we will find tm-signer-harness. +chain_id = "test-chain-0XwP5E" # The Tendermint chain ID for which KMS will be signing (found in ~/.tendermint/config/genesis.json). +reconnect = true # true is the default +secret_key = "./secret_connection.key" # Where to find our secret connection key. + +[[providers.softsign]] +id = "test-chain-0XwP5E" # The Tendermint chain ID for which KMS will be signing (same as validator.chain_id above). +path = "./signing.key" # The signing key we extracted earlier. +``` + +Then run KMS with this configuration: + +```bash +tmkms start -c tmkms.toml +``` + +This will start KMS, which will repeatedly try to connect to +`tcp://127.0.0.1:61219` until it is successful. + +### Step 4: Run tm-signer-harness +Now we get to run the signer test harness: + +```bash +tm-signer-harness run \ # The "run" command executes the tests + -addr tcp://127.0.0.1:61219 \ # The address we promised KMS earlier + -tmhome ~/.tendermint # Where to find our Tendermint configuration/data files. +``` + +If the current version of Tendermint and KMS are compatible, `tm-signer-harness` +should now exit with a 0 exit code. If they are somehow not compatible, it +should exit with a meaningful non-zero exit code (see the exit codes below). + +### Step 5: Shut down KMS +Simply hit Ctrl+Break on your KMS instance (or use the `kill` command in Linux) +to terminate it gracefully. + +## Exit Code Meanings +The following list shows the various exit codes from `tm-signer-harness` and +their meanings: + +| Exit Code | Description | +| --- | --- | +| 0 | Success! | +| 1 | Invalid command line parameters supplied to `tm-signer-harness` | +| 2 | Maximum number of accept retries reached (the `-accept-retries` parameter) | +| 3 | Failed to load `${TMHOME}/config/genesis.json` | +| 4 | Failed to create listener specified by `-addr` parameter | +| 5 | Failed to start listener | +| 6 | Interrupted by `SIGINT` (e.g. when hitting Ctrl+Break or Ctrl+C) | +| 7 | Other unknown error | +| 8 | Test 1 failed: public key mismatch | +| 9 | Test 2 failed: signing of proposals failed | +| 10 | Test 3 failed: signing of votes failed | diff --git a/docs/yarn.lock b/docs/yarn.lock deleted file mode 100644 index 4f453ed4..00000000 --- a/docs/yarn.lock +++ /dev/null @@ -1,2611 +0,0 @@ -# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. -# yarn lockfile v1 - - -"@azu/format-text@^1.0.1": - version "1.0.1" - resolved "https://registry.yarnpkg.com/@azu/format-text/-/format-text-1.0.1.tgz#6967350a94640f6b02855169bd897ce54d6cebe2" - -"@azu/style-format@^1.0.0": - version "1.0.0" - resolved "https://registry.yarnpkg.com/@azu/style-format/-/style-format-1.0.0.tgz#e70187f8a862e191b1bce6c0268f13acd3a56b20" - dependencies: - "@azu/format-text" "^1.0.1" - -"@sindresorhus/is@^0.7.0": - version "0.7.0" - resolved "https://registry.yarnpkg.com/@sindresorhus/is/-/is-0.7.0.tgz#9a06f4f137ee84d7df0460c1fdb1135ffa6c50fd" - -"@textlint/ast-node-types@^4.0.2", "@textlint/ast-node-types@^4.0.3": - version "4.0.3" - resolved "https://registry.yarnpkg.com/@textlint/ast-node-types/-/ast-node-types-4.0.3.tgz#b51c87bb86022323f764fbdc976b173f19261cc5" - -"@textlint/ast-traverse@^2.0.8", "@textlint/ast-traverse@^2.0.9": - version "2.0.9" - resolved "https://registry.yarnpkg.com/@textlint/ast-traverse/-/ast-traverse-2.0.9.tgz#4bf427cf01b7195013e75d27540a77ad68c363d9" - dependencies: - "@textlint/ast-node-types" "^4.0.3" - -"@textlint/feature-flag@^3.0.4", "@textlint/feature-flag@^3.0.5": - version "3.0.5" - resolved "https://registry.yarnpkg.com/@textlint/feature-flag/-/feature-flag-3.0.5.tgz#3783e0f2661053d2a74fdad775993395a2d530b4" - dependencies: - map-like "^2.0.0" - -"@textlint/fixer-formatter@^3.0.7": - version "3.0.8" - resolved "https://registry.yarnpkg.com/@textlint/fixer-formatter/-/fixer-formatter-3.0.8.tgz#90ef804c60b9e694c8c048a06febbf1f331abd49" - dependencies: - "@textlint/kernel" "^3.0.0" - chalk "^1.1.3" - debug "^2.1.0" - diff "^2.2.2" - interop-require "^1.0.0" - is-file "^1.0.0" - string-width "^1.0.1" - text-table "^0.2.0" - try-resolve "^1.0.1" - -"@textlint/kernel@^2.0.9": - version "2.0.9" - resolved "https://registry.yarnpkg.com/@textlint/kernel/-/kernel-2.0.9.tgz#a4471b7969e192551230c35ea9fae32d80128ee0" - dependencies: - "@textlint/ast-node-types" "^4.0.2" - "@textlint/ast-traverse" "^2.0.8" - "@textlint/feature-flag" "^3.0.4" - "@types/bluebird" "^3.5.18" - bluebird "^3.5.1" - debug "^2.6.6" - deep-equal "^1.0.1" - object-assign "^4.1.1" - structured-source "^3.0.2" - -"@textlint/kernel@^3.0.0": - version "3.0.0" - resolved "https://registry.yarnpkg.com/@textlint/kernel/-/kernel-3.0.0.tgz#ba10962acff64f17b9e5fce8089a40f1f8880dcd" - dependencies: - "@textlint/ast-node-types" "^4.0.3" - "@textlint/ast-traverse" "^2.0.9" - "@textlint/feature-flag" "^3.0.5" - "@types/bluebird" "^3.5.18" - bluebird "^3.5.1" - debug "^2.6.6" - deep-equal "^1.0.1" - map-like "^2.0.0" - object-assign "^4.1.1" - structured-source "^3.0.2" - -"@textlint/linter-formatter@^3.0.7": - version "3.0.8" - resolved "https://registry.yarnpkg.com/@textlint/linter-formatter/-/linter-formatter-3.0.8.tgz#030aa03ff3d85dda94ca9fa9e6bf824f9c1cb7ef" - dependencies: - "@azu/format-text" "^1.0.1" - "@azu/style-format" "^1.0.0" - "@textlint/kernel" "^3.0.0" - chalk "^1.0.0" - concat-stream "^1.5.1" - js-yaml "^3.2.4" - optionator "^0.8.1" - pluralize "^2.0.0" - string-width "^1.0.1" - string.prototype.padstart "^3.0.0" - strip-ansi "^3.0.1" - table "^3.7.8" - text-table "^0.2.0" - try-resolve "^1.0.1" - xml-escape "^1.0.0" - -"@textlint/markdown-to-ast@^6.0.8": - version "6.0.9" - resolved "https://registry.yarnpkg.com/@textlint/markdown-to-ast/-/markdown-to-ast-6.0.9.tgz#e7c89e5ad15d17dcd8e5a62758358936827658fa" - dependencies: - "@textlint/ast-node-types" "^4.0.3" - debug "^2.1.3" - remark-frontmatter "^1.2.0" - remark-parse "^5.0.0" - structured-source "^3.0.2" - traverse "^0.6.6" - unified "^6.1.6" - -"@textlint/text-to-ast@^3.0.8": - version "3.0.9" - resolved "https://registry.yarnpkg.com/@textlint/text-to-ast/-/text-to-ast-3.0.9.tgz#dcb63f09cc79ea2096fc823c3b6cd07c79a060b5" - dependencies: - "@textlint/ast-node-types" "^4.0.3" - -"@textlint/textlint-plugin-markdown@^4.0.10": - version "4.0.10" - resolved "https://registry.yarnpkg.com/@textlint/textlint-plugin-markdown/-/textlint-plugin-markdown-4.0.10.tgz#a99b4a308067597e89439a9e87bc1c4a7f4d076b" - dependencies: - "@textlint/markdown-to-ast" "^6.0.8" - -"@textlint/textlint-plugin-text@^3.0.10": - version "3.0.10" - resolved "https://registry.yarnpkg.com/@textlint/textlint-plugin-text/-/textlint-plugin-text-3.0.10.tgz#619600bdc352d33a68e7a73d77d58b0c52b2a44f" - dependencies: - "@textlint/text-to-ast" "^3.0.8" - -"@types/bluebird@^3.5.18": - version "3.5.23" - resolved "https://registry.yarnpkg.com/@types/bluebird/-/bluebird-3.5.23.tgz#e805da976b76892b2b2e50eec29e84914c730670" - -abbrev@1: - version "1.1.1" - resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.1.1.tgz#f8f2c887ad10bf67f634f005b6987fed3179aac8" - -adverb-where@0.0.9: - version "0.0.9" - resolved "https://registry.yarnpkg.com/adverb-where/-/adverb-where-0.0.9.tgz#09c5cddd8d503b9fe5f76e0b8dc5c70a8f193e34" - -aggregate-error@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/aggregate-error/-/aggregate-error-1.0.0.tgz#888344dad0220a72e3af50906117f48771925fac" - dependencies: - clean-stack "^1.0.0" - indent-string "^3.0.0" - -ajv-keywords@^1.0.0: - version "1.5.1" - resolved "https://registry.yarnpkg.com/ajv-keywords/-/ajv-keywords-1.5.1.tgz#314dd0a4b3368fad3dfcdc54ede6171b886daf3c" - -ajv@^4.7.0: - version "4.11.8" - resolved "https://registry.yarnpkg.com/ajv/-/ajv-4.11.8.tgz#82ffb02b29e662ae53bdc20af15947706739c536" - dependencies: - co "^4.6.0" - json-stable-stringify "^1.0.1" - -ajv@^5.1.0: - version "5.5.2" - resolved "https://registry.yarnpkg.com/ajv/-/ajv-5.5.2.tgz#73b5eeca3fab653e3d3f9422b341ad42205dc965" - dependencies: - co "^4.6.0" - fast-deep-equal "^1.0.0" - fast-json-stable-stringify "^2.0.0" - json-schema-traverse "^0.3.0" - -ansi-regex@^2.0.0: - version "2.1.1" - resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-2.1.1.tgz#c3b33ab5ee360d86e0e628f0468ae7ef27d654df" - -ansi-regex@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-3.0.0.tgz#ed0317c322064f79466c02966bddb605ab37d998" - -ansi-styles@^2.2.1: - version "2.2.1" - resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-2.2.1.tgz#b432dd3358b634cf75e1e4664368240533c1ddbe" - -ansi-styles@^3.2.1: - version "3.2.1" - resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-3.2.1.tgz#41fbb20243e50b12be0f04b8dedbf07520ce841d" - dependencies: - color-convert "^1.9.0" - -anymatch@^1.3.0: - version "1.3.2" - resolved "https://registry.yarnpkg.com/anymatch/-/anymatch-1.3.2.tgz#553dcb8f91e3c889845dfdba34c77721b90b9d7a" - dependencies: - micromatch "^2.1.5" - normalize-path "^2.0.0" - -aproba@^1.0.3: - version "1.2.0" - resolved "https://registry.yarnpkg.com/aproba/-/aproba-1.2.0.tgz#6802e6264efd18c790a1b0d517f0f2627bf2c94a" - -are-we-there-yet@~1.1.2: - version "1.1.5" - resolved "https://registry.yarnpkg.com/are-we-there-yet/-/are-we-there-yet-1.1.5.tgz#4b35c2944f062a8bfcda66410760350fe9ddfc21" - dependencies: - delegates "^1.0.0" - readable-stream "^2.0.6" - -argparse@^1.0.7: - version "1.0.10" - resolved "https://registry.yarnpkg.com/argparse/-/argparse-1.0.10.tgz#bcd6791ea5ae09725e17e5ad988134cd40b3d911" - dependencies: - sprintf-js "~1.0.2" - -arr-diff@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/arr-diff/-/arr-diff-2.0.0.tgz#8f3b827f955a8bd669697e4a4256ac3ceae356cf" - dependencies: - arr-flatten "^1.0.1" - -arr-flatten@^1.0.1: - version "1.1.0" - resolved "https://registry.yarnpkg.com/arr-flatten/-/arr-flatten-1.1.0.tgz#36048bbff4e7b47e136644316c99669ea5ae91f1" - -array-iterate@^1.0.0: - version "1.1.2" - resolved "https://registry.yarnpkg.com/array-iterate/-/array-iterate-1.1.2.tgz#f66a57e84426f8097f4197fbb6c051b8e5cdf7d8" - -array-union@^1.0.1: - version "1.0.2" - resolved "https://registry.yarnpkg.com/array-union/-/array-union-1.0.2.tgz#9a34410e4f4e3da23dea375be5be70f24778ec39" - dependencies: - array-uniq "^1.0.1" - -array-uniq@^1.0.1: - version "1.0.3" - resolved "https://registry.yarnpkg.com/array-uniq/-/array-uniq-1.0.3.tgz#af6ac877a25cc7f74e058894753858dfdb24fdb6" - -array-unique@^0.2.1: - version "0.2.1" - resolved "https://registry.yarnpkg.com/array-unique/-/array-unique-0.2.1.tgz#a1d97ccafcbc2625cc70fadceb36a50c58b01a53" - -arrify@^1.0.0: - version "1.0.1" - resolved "https://registry.yarnpkg.com/arrify/-/arrify-1.0.1.tgz#898508da2226f380df904728456849c1501a4b0d" - -asn1@~0.2.3: - version "0.2.4" - resolved "https://registry.yarnpkg.com/asn1/-/asn1-0.2.4.tgz#8d2475dfab553bb33e77b54e59e880bb8ce23136" - dependencies: - safer-buffer "~2.1.0" - -assert-plus@1.0.0, assert-plus@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/assert-plus/-/assert-plus-1.0.0.tgz#f12e0f3c5d77b0b1cdd9146942e4e96c1e4dd525" - -async-each@^1.0.0: - version "1.0.1" - resolved "https://registry.yarnpkg.com/async-each/-/async-each-1.0.1.tgz#19d386a1d9edc6e7c1c85d388aedbcc56d33602d" - -asynckit@^0.4.0: - version "0.4.0" - resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79" - -aws-sign2@~0.7.0: - version "0.7.0" - resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.7.0.tgz#b46e890934a9591f2d2f6f86d7e6a9f1b3fe76a8" - -aws4@^1.6.0: - version "1.8.0" - resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.8.0.tgz#f0e003d9ca9e7f59c7a508945d7b2ef9a04a542f" - -bail@^1.0.0: - version "1.0.3" - resolved "https://registry.yarnpkg.com/bail/-/bail-1.0.3.tgz#63cfb9ddbac829b02a3128cd53224be78e6c21a3" - -balanced-match@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767" - -bcrypt-pbkdf@^1.0.0: - version "1.0.2" - resolved "https://registry.yarnpkg.com/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz#a4301d389b6a43f9b67ff3ca11a3f6637e360e9e" - dependencies: - tweetnacl "^0.14.3" - -binary-extensions@^1.0.0: - version "1.11.0" - resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-1.11.0.tgz#46aa1751fb6a2f93ee5e689bb1087d4b14c6c205" - -bluebird@^3.0.5, bluebird@^3.5.1: - version "3.5.1" - resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.5.1.tgz#d9551f9de98f1fcda1e683d17ee91a0602ee2eb9" - -boundary@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/boundary/-/boundary-1.0.1.tgz#4d67dc2602c0cc16dd9bce7ebf87e948290f5812" - -brace-expansion@^1.1.7: - version "1.1.11" - resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.11.tgz#3c7fcbf529d87226f3d2f52b966ff5271eb441dd" - dependencies: - balanced-match "^1.0.0" - concat-map "0.0.1" - -braces@^1.8.2: - version "1.8.5" - resolved "https://registry.yarnpkg.com/braces/-/braces-1.8.5.tgz#ba77962e12dff969d6b76711e914b737857bf6a7" - dependencies: - expand-range "^1.8.1" - preserve "^0.2.0" - repeat-element "^1.1.2" - -buffer-from@^1.0.0: - version "1.1.1" - resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.1.tgz#32713bc028f75c02fdb710d7c7bcec1f2c6070ef" - -builtin-modules@^1.0.0: - version "1.1.1" - resolved "https://registry.yarnpkg.com/builtin-modules/-/builtin-modules-1.1.1.tgz#270f076c5a72c02f5b65a47df94c5fe3a278892f" - -cacheable-request@^2.1.1: - version "2.1.4" - resolved "https://registry.yarnpkg.com/cacheable-request/-/cacheable-request-2.1.4.tgz#0d808801b6342ad33c91df9d0b44dc09b91e5c3d" - dependencies: - clone-response "1.0.2" - get-stream "3.0.0" - http-cache-semantics "3.8.1" - keyv "3.0.0" - lowercase-keys "1.0.0" - normalize-url "2.0.1" - responselike "1.0.2" - -camelcase@^4.0.0: - version "4.1.0" - resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-4.1.0.tgz#d545635be1e33c542649c69173e5de6acfae34dd" - -capture-stack-trace@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/capture-stack-trace/-/capture-stack-trace-1.0.0.tgz#4a6fa07399c26bba47f0b2496b4d0fb408c5550d" - -caseless@~0.12.0: - version "0.12.0" - resolved "https://registry.yarnpkg.com/caseless/-/caseless-0.12.0.tgz#1b681c21ff84033c826543090689420d187151dc" - -ccount@^1.0.0: - version "1.0.3" - resolved "https://registry.yarnpkg.com/ccount/-/ccount-1.0.3.tgz#f1cec43f332e2ea5a569fd46f9f5bde4e6102aff" - -chalk@^1.0.0, chalk@^1.1.1, chalk@^1.1.3: - version "1.1.3" - resolved "https://registry.yarnpkg.com/chalk/-/chalk-1.1.3.tgz#a8115c55e4a702fe4d150abd3872822a7e09fc98" - dependencies: - ansi-styles "^2.2.1" - escape-string-regexp "^1.0.2" - has-ansi "^2.0.0" - strip-ansi "^3.0.0" - supports-color "^2.0.0" - -chalk@^2.0.0: - version "2.4.1" - resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.4.1.tgz#18c49ab16a037b6eb0152cc83e3471338215b66e" - dependencies: - ansi-styles "^3.2.1" - escape-string-regexp "^1.0.5" - supports-color "^5.3.0" - -character-entities-html4@^1.0.0: - version "1.1.2" - resolved "https://registry.yarnpkg.com/character-entities-html4/-/character-entities-html4-1.1.2.tgz#c44fdde3ce66b52e8d321d6c1bf46101f0150610" - -character-entities-legacy@^1.0.0: - version "1.1.2" - resolved "https://registry.yarnpkg.com/character-entities-legacy/-/character-entities-legacy-1.1.2.tgz#7c6defb81648498222c9855309953d05f4d63a9c" - -character-entities@^1.0.0: - version "1.2.2" - resolved "https://registry.yarnpkg.com/character-entities/-/character-entities-1.2.2.tgz#58c8f371c0774ef0ba9b2aca5f00d8f100e6e363" - -character-reference-invalid@^1.0.0: - version "1.1.2" - resolved "https://registry.yarnpkg.com/character-reference-invalid/-/character-reference-invalid-1.1.2.tgz#21e421ad3d84055952dab4a43a04e73cd425d3ed" - -charenc@~0.0.1: - version "0.0.2" - resolved "https://registry.yarnpkg.com/charenc/-/charenc-0.0.2.tgz#c0a1d2f3a7092e03774bfa83f14c0fc5790a8667" - -chokidar@^1.5.1: - version "1.7.0" - resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-1.7.0.tgz#798e689778151c8076b4b360e5edd28cda2bb468" - dependencies: - anymatch "^1.3.0" - async-each "^1.0.0" - glob-parent "^2.0.0" - inherits "^2.0.1" - is-binary-path "^1.0.0" - is-glob "^2.0.0" - path-is-absolute "^1.0.0" - readdirp "^2.0.0" - optionalDependencies: - fsevents "^1.0.0" - -chownr@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/chownr/-/chownr-1.0.1.tgz#e2a75042a9551908bebd25b8523d5f9769d79181" - -circular-json@^0.3.1: - version "0.3.3" - resolved "https://registry.yarnpkg.com/circular-json/-/circular-json-0.3.3.tgz#815c99ea84f6809529d2f45791bdf82711352d66" - -clean-stack@^1.0.0: - version "1.3.0" - resolved "https://registry.yarnpkg.com/clean-stack/-/clean-stack-1.3.0.tgz#9e821501ae979986c46b1d66d2d432db2fd4ae31" - -clone-response@1.0.2: - version "1.0.2" - resolved "https://registry.yarnpkg.com/clone-response/-/clone-response-1.0.2.tgz#d1dc973920314df67fbeb94223b4ee350239e96b" - dependencies: - mimic-response "^1.0.0" - -co@3.1.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/co/-/co-3.1.0.tgz#4ea54ea5a08938153185e15210c68d9092bc1b78" - -co@^4.6.0: - version "4.6.0" - resolved "https://registry.yarnpkg.com/co/-/co-4.6.0.tgz#6ea6bdf3d853ae54ccb8e47bfa0bf3f9031fb184" - -code-point-at@^1.0.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/code-point-at/-/code-point-at-1.1.0.tgz#0d070b4d043a5bea33a2f1a40e2edb3d9a4ccf77" - -collapse-white-space@^1.0.2: - version "1.0.4" - resolved "https://registry.yarnpkg.com/collapse-white-space/-/collapse-white-space-1.0.4.tgz#ce05cf49e54c3277ae573036a26851ba430a0091" - -color-convert@^1.9.0: - version "1.9.2" - resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-1.9.2.tgz#49881b8fba67df12a96bdf3f56c0aab9e7913147" - dependencies: - color-name "1.1.1" - -color-name@1.1.1: - version "1.1.1" - resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.1.tgz#4b1415304cf50028ea81643643bd82ea05803689" - -combined-stream@1.0.6, combined-stream@~1.0.5: - version "1.0.6" - resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.6.tgz#723e7df6e801ac5613113a7e445a9b69cb632818" - dependencies: - delayed-stream "~1.0.0" - -concat-map@0.0.1: - version "0.0.1" - resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" - -concat-stream@^1.5.1: - version "1.6.2" - resolved "https://registry.yarnpkg.com/concat-stream/-/concat-stream-1.6.2.tgz#904bdf194cd3122fc675c77fc4ac3d4ff0fd1a34" - dependencies: - buffer-from "^1.0.0" - inherits "^2.0.3" - readable-stream "^2.2.2" - typedarray "^0.0.6" - -console-control-strings@^1.0.0, console-control-strings@~1.1.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/console-control-strings/-/console-control-strings-1.1.0.tgz#3d7cf4464db6446ea644bf4b39507f9851008e8e" - -core-util-is@1.0.2, core-util-is@~1.0.0: - version "1.0.2" - resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7" - -create-error-class@^3.0.0: - version "3.0.2" - resolved "https://registry.yarnpkg.com/create-error-class/-/create-error-class-3.0.2.tgz#06be7abef947a3f14a30fd610671d401bca8b7b6" - dependencies: - capture-stack-trace "^1.0.0" - -crypt@~0.0.1: - version "0.0.2" - resolved "https://registry.yarnpkg.com/crypt/-/crypt-0.0.2.tgz#88d7ff7ec0dfb86f713dc87bbb42d044d3e6c41b" - -dashdash@^1.12.0: - version "1.14.1" - resolved "https://registry.yarnpkg.com/dashdash/-/dashdash-1.14.1.tgz#853cfa0f7cbe2fed5de20326b8dd581035f6e2f0" - dependencies: - assert-plus "^1.0.0" - -debug@^2.1.0, debug@^2.1.2, debug@^2.1.3, debug@^2.6.6: - version "2.6.9" - resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f" - dependencies: - ms "2.0.0" - -debug@^3.1.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/debug/-/debug-3.1.0.tgz#5bb5a0672628b64149566ba16819e61518c67261" - dependencies: - ms "2.0.0" - -decode-uri-component@^0.2.0: - version "0.2.0" - resolved "https://registry.yarnpkg.com/decode-uri-component/-/decode-uri-component-0.2.0.tgz#eb3913333458775cb84cd1a1fae062106bb87545" - -decompress-response@^3.3.0: - version "3.3.0" - resolved "https://registry.yarnpkg.com/decompress-response/-/decompress-response-3.3.0.tgz#80a4dd323748384bfa248083622aedec982adff3" - dependencies: - mimic-response "^1.0.0" - -deep-equal@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/deep-equal/-/deep-equal-1.0.1.tgz#f5d260292b660e084eff4cdbc9f08ad3247448b5" - -deep-extend@^0.6.0: - version "0.6.0" - resolved "https://registry.yarnpkg.com/deep-extend/-/deep-extend-0.6.0.tgz#c4fa7c95404a17a9c3e8ca7e1537312b736330ac" - -deep-is@~0.1.3: - version "0.1.3" - resolved "https://registry.yarnpkg.com/deep-is/-/deep-is-0.1.3.tgz#b369d6fb5dbc13eecf524f91b070feedc357cf34" - -define-properties@^1.1.2: - version "1.1.2" - resolved "https://registry.yarnpkg.com/define-properties/-/define-properties-1.1.2.tgz#83a73f2fea569898fb737193c8f873caf6d45c94" - dependencies: - foreach "^2.0.5" - object-keys "^1.0.8" - -del@^2.0.2: - version "2.2.2" - resolved "https://registry.yarnpkg.com/del/-/del-2.2.2.tgz#c12c981d067846c84bcaf862cff930d907ffd1a8" - dependencies: - globby "^5.0.0" - is-path-cwd "^1.0.0" - is-path-in-cwd "^1.0.0" - object-assign "^4.0.1" - pify "^2.0.0" - pinkie-promise "^2.0.0" - rimraf "^2.2.8" - -delayed-stream@~1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619" - -delegates@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/delegates/-/delegates-1.0.0.tgz#84c6e159b81904fdca59a0ef44cd870d31250f9a" - -detect-libc@^1.0.2: - version "1.0.3" - resolved "https://registry.yarnpkg.com/detect-libc/-/detect-libc-1.0.3.tgz#fa137c4bd698edf55cd5cd02ac559f91a4c4ba9b" - -diff@^2.2.2: - version "2.2.3" - resolved "https://registry.yarnpkg.com/diff/-/diff-2.2.3.tgz#60eafd0d28ee906e4e8ff0a52c1229521033bf99" - -dns-packet@^1.1.0: - version "1.3.1" - resolved "https://registry.yarnpkg.com/dns-packet/-/dns-packet-1.3.1.tgz#12aa426981075be500b910eedcd0b47dd7deda5a" - dependencies: - ip "^1.1.0" - safe-buffer "^5.0.1" - -dns-socket@^1.6.2: - version "1.6.3" - resolved "https://registry.yarnpkg.com/dns-socket/-/dns-socket-1.6.3.tgz#5268724fad4aa46ad9c5ca4ffcd16e1de5342aab" - dependencies: - dns-packet "^1.1.0" - -duplexer3@^0.1.4: - version "0.1.4" - resolved "https://registry.yarnpkg.com/duplexer3/-/duplexer3-0.1.4.tgz#ee01dd1cac0ed3cbc7fdbea37dc0a8f1ce002ce2" - -e-prime@^0.10.2: - version "0.10.2" - resolved "https://registry.yarnpkg.com/e-prime/-/e-prime-0.10.2.tgz#ea9375eb985636de88013c7a9fb129ad9e15eff8" - -ecc-jsbn@~0.1.1: - version "0.1.2" - resolved "https://registry.yarnpkg.com/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz#3a83a904e54353287874c564b7549386849a98c9" - dependencies: - jsbn "~0.1.0" - safer-buffer "^2.1.0" - -error-ex@^1.2.0, error-ex@^1.3.1: - version "1.3.2" - resolved "https://registry.yarnpkg.com/error-ex/-/error-ex-1.3.2.tgz#b4ac40648107fdcdcfae242f428bea8a14d4f1bf" - dependencies: - is-arrayish "^0.2.1" - -es-abstract@^1.4.3: - version "1.12.0" - resolved "https://registry.yarnpkg.com/es-abstract/-/es-abstract-1.12.0.tgz#9dbbdd27c6856f0001421ca18782d786bf8a6165" - dependencies: - es-to-primitive "^1.1.1" - function-bind "^1.1.1" - has "^1.0.1" - is-callable "^1.1.3" - is-regex "^1.0.4" - -es-to-primitive@^1.1.1: - version "1.1.1" - resolved "https://registry.yarnpkg.com/es-to-primitive/-/es-to-primitive-1.1.1.tgz#45355248a88979034b6792e19bb81f2b7975dd0d" - dependencies: - is-callable "^1.1.1" - is-date-object "^1.0.1" - is-symbol "^1.0.1" - -escape-string-regexp@^1.0.2, escape-string-regexp@^1.0.5: - version "1.0.5" - resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz#1b61c0562190a8dff6ae3bb2cf0200ca130b86d4" - -esprima@^4.0.0: - version "4.0.1" - resolved "https://registry.yarnpkg.com/esprima/-/esprima-4.0.1.tgz#13b04cdb3e6c5d19df91ab6987a8695619b0aa71" - -expand-brackets@^0.1.4: - version "0.1.5" - resolved "https://registry.yarnpkg.com/expand-brackets/-/expand-brackets-0.1.5.tgz#df07284e342a807cd733ac5af72411e581d1177b" - dependencies: - is-posix-bracket "^0.1.0" - -expand-range@^1.8.1: - version "1.8.2" - resolved "https://registry.yarnpkg.com/expand-range/-/expand-range-1.8.2.tgz#a299effd335fe2721ebae8e257ec79644fc85337" - dependencies: - fill-range "^2.1.0" - -extend@^3.0.0, extend@~3.0.1: - version "3.0.2" - resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.2.tgz#f8b1136b4071fbd8eb140aff858b1019ec2915fa" - -extglob@^0.3.1: - version "0.3.2" - resolved "https://registry.yarnpkg.com/extglob/-/extglob-0.3.2.tgz#2e18ff3d2f49ab2765cec9023f011daa8d8349a1" - dependencies: - is-extglob "^1.0.0" - -extsprintf@1.3.0: - version "1.3.0" - resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.3.0.tgz#96918440e3041a7a414f8c52e3c574eb3c3e1e05" - -extsprintf@^1.2.0: - version "1.4.0" - resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.4.0.tgz#e2689f8f356fad62cca65a3a91c5df5f9551692f" - -fast-deep-equal@^1.0.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/fast-deep-equal/-/fast-deep-equal-1.1.0.tgz#c053477817c86b51daa853c81e059b733d023614" - -fast-json-stable-stringify@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz#d5142c0caee6b1189f87d3a76111064f86c8bbf2" - -fast-levenshtein@~2.0.4: - version "2.0.6" - resolved "https://registry.yarnpkg.com/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz#3d8a5c66883a16a30ca8643e851f19baa7797917" - -fault@^1.0.0, fault@^1.0.1: - version "1.0.2" - resolved "https://registry.yarnpkg.com/fault/-/fault-1.0.2.tgz#c3d0fec202f172a3a4d414042ad2bb5e2a3ffbaa" - dependencies: - format "^0.2.2" - -file-entry-cache@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/file-entry-cache/-/file-entry-cache-2.0.0.tgz#c392990c3e684783d838b8c84a45d8a048458361" - dependencies: - flat-cache "^1.2.1" - object-assign "^4.0.1" - -filename-regex@^2.0.0: - version "2.0.1" - resolved "https://registry.yarnpkg.com/filename-regex/-/filename-regex-2.0.1.tgz#c1c4b9bee3e09725ddb106b75c1e301fe2f18b26" - -fill-range@^2.1.0: - version "2.2.4" - resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-2.2.4.tgz#eb1e773abb056dcd8df2bfdf6af59b8b3a936565" - dependencies: - is-number "^2.1.0" - isobject "^2.0.0" - randomatic "^3.0.0" - repeat-element "^1.1.2" - repeat-string "^1.5.2" - -find-up@^2.0.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/find-up/-/find-up-2.1.0.tgz#45d1b7e506c717ddd482775a2b77920a3c0c57a7" - dependencies: - locate-path "^2.0.0" - -flat-cache@^1.2.1: - version "1.3.0" - resolved "https://registry.yarnpkg.com/flat-cache/-/flat-cache-1.3.0.tgz#d3030b32b38154f4e3b7e9c709f490f7ef97c481" - dependencies: - circular-json "^0.3.1" - del "^2.0.2" - graceful-fs "^4.1.2" - write "^0.2.1" - -fn-name@^2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/fn-name/-/fn-name-2.0.1.tgz#5214d7537a4d06a4a301c0cc262feb84188002e7" - -for-in@^1.0.1: - version "1.0.2" - resolved "https://registry.yarnpkg.com/for-in/-/for-in-1.0.2.tgz#81068d295a8142ec0ac726c6e2200c30fb6d5e80" - -for-own@^0.1.4: - version "0.1.5" - resolved "https://registry.yarnpkg.com/for-own/-/for-own-0.1.5.tgz#5265c681a4f294dabbf17c9509b6763aa84510ce" - dependencies: - for-in "^1.0.1" - -foreach@^2.0.5: - version "2.0.5" - resolved "https://registry.yarnpkg.com/foreach/-/foreach-2.0.5.tgz#0bee005018aeb260d0a3af3ae658dd0136ec1b99" - -forever-agent@~0.6.1: - version "0.6.1" - resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91" - -form-data@~2.3.1: - version "2.3.2" - resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.3.2.tgz#4970498be604c20c005d4f5c23aecd21d6b49099" - dependencies: - asynckit "^0.4.0" - combined-stream "1.0.6" - mime-types "^2.1.12" - -format@^0.2.2: - version "0.2.2" - resolved "https://registry.yarnpkg.com/format/-/format-0.2.2.tgz#d6170107e9efdc4ed30c9dc39016df942b5cb58b" - -from2@^2.1.1: - version "2.3.0" - resolved "https://registry.yarnpkg.com/from2/-/from2-2.3.0.tgz#8bfb5502bde4a4d36cfdeea007fcca21d7e382af" - dependencies: - inherits "^2.0.1" - readable-stream "^2.0.0" - -fs-minipass@^1.2.5: - version "1.2.5" - resolved "https://registry.yarnpkg.com/fs-minipass/-/fs-minipass-1.2.5.tgz#06c277218454ec288df77ada54a03b8702aacb9d" - dependencies: - minipass "^2.2.1" - -fs.realpath@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f" - -fsevents@^1.0.0: - version "1.2.4" - resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-1.2.4.tgz#f41dcb1af2582af3692da36fc55cbd8e1041c426" - dependencies: - nan "^2.9.2" - node-pre-gyp "^0.10.0" - -function-bind@^1.0.2, function-bind@^1.1.1: - version "1.1.1" - resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.1.tgz#a56899d3ea3c9bab874bb9773b7c5ede92f4895d" - -gauge@~2.7.3: - version "2.7.4" - resolved "https://registry.yarnpkg.com/gauge/-/gauge-2.7.4.tgz#2c03405c7538c39d7eb37b317022e325fb018bf7" - dependencies: - aproba "^1.0.3" - console-control-strings "^1.0.0" - has-unicode "^2.0.0" - object-assign "^4.1.0" - signal-exit "^3.0.0" - string-width "^1.0.1" - strip-ansi "^3.0.1" - wide-align "^1.1.0" - -get-stdin@^5.0.1: - version "5.0.1" - resolved "https://registry.yarnpkg.com/get-stdin/-/get-stdin-5.0.1.tgz#122e161591e21ff4c52530305693f20e6393a398" - -get-stream@3.0.0, get-stream@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-3.0.0.tgz#8e943d1358dc37555054ecbe2edb05aa174ede14" - -getpass@^0.1.1: - version "0.1.7" - resolved "https://registry.yarnpkg.com/getpass/-/getpass-0.1.7.tgz#5eff8e3e684d569ae4cb2b1282604e8ba62149fa" - dependencies: - assert-plus "^1.0.0" - -glob-base@^0.3.0: - version "0.3.0" - resolved "https://registry.yarnpkg.com/glob-base/-/glob-base-0.3.0.tgz#dbb164f6221b1c0b1ccf82aea328b497df0ea3c4" - dependencies: - glob-parent "^2.0.0" - is-glob "^2.0.0" - -glob-parent@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/glob-parent/-/glob-parent-2.0.0.tgz#81383d72db054fcccf5336daa902f182f6edbb28" - dependencies: - is-glob "^2.0.0" - -glob@^7.0.3, glob@^7.0.5, glob@^7.1.1: - version "7.1.2" - resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.2.tgz#c19c9df9a028702d678612384a6552404c636d15" - dependencies: - fs.realpath "^1.0.0" - inflight "^1.0.4" - inherits "2" - minimatch "^3.0.4" - once "^1.3.0" - path-is-absolute "^1.0.0" - -globby@^5.0.0: - version "5.0.0" - resolved "https://registry.yarnpkg.com/globby/-/globby-5.0.0.tgz#ebd84667ca0dbb330b99bcfc68eac2bc54370e0d" - dependencies: - array-union "^1.0.1" - arrify "^1.0.0" - glob "^7.0.3" - object-assign "^4.0.1" - pify "^2.0.0" - pinkie-promise "^2.0.0" - -got@^6.7.1: - version "6.7.1" - resolved "https://registry.yarnpkg.com/got/-/got-6.7.1.tgz#240cd05785a9a18e561dc1b44b41c763ef1e8db0" - dependencies: - create-error-class "^3.0.0" - duplexer3 "^0.1.4" - get-stream "^3.0.0" - is-redirect "^1.0.0" - is-retry-allowed "^1.0.0" - is-stream "^1.0.0" - lowercase-keys "^1.0.0" - safe-buffer "^5.0.1" - timed-out "^4.0.0" - unzip-response "^2.0.1" - url-parse-lax "^1.0.0" - -got@^8.0.0: - version "8.3.2" - resolved "https://registry.yarnpkg.com/got/-/got-8.3.2.tgz#1d23f64390e97f776cac52e5b936e5f514d2e937" - dependencies: - "@sindresorhus/is" "^0.7.0" - cacheable-request "^2.1.1" - decompress-response "^3.3.0" - duplexer3 "^0.1.4" - get-stream "^3.0.0" - into-stream "^3.1.0" - is-retry-allowed "^1.1.0" - isurl "^1.0.0-alpha5" - lowercase-keys "^1.0.0" - mimic-response "^1.0.0" - p-cancelable "^0.4.0" - p-timeout "^2.0.1" - pify "^3.0.0" - safe-buffer "^5.1.1" - timed-out "^4.0.1" - url-parse-lax "^3.0.0" - url-to-options "^1.0.1" - -graceful-fs@^4.1.2: - version "4.1.11" - resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.1.11.tgz#0e8bdfe4d1ddb8854d64e04ea7c00e2a026e5658" - -har-schema@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/har-schema/-/har-schema-2.0.0.tgz#a94c2224ebcac04782a0d9035521f24735b7ec92" - -har-validator@~5.0.3: - version "5.0.3" - resolved "https://registry.yarnpkg.com/har-validator/-/har-validator-5.0.3.tgz#ba402c266194f15956ef15e0fcf242993f6a7dfd" - dependencies: - ajv "^5.1.0" - har-schema "^2.0.0" - -has-ansi@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/has-ansi/-/has-ansi-2.0.0.tgz#34f5049ce1ecdf2b0649af3ef24e45ed35416d91" - dependencies: - ansi-regex "^2.0.0" - -has-flag@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-2.0.0.tgz#e8207af1cc7b30d446cc70b734b5e8be18f88d51" - -has-flag@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-3.0.0.tgz#b5d454dc2199ae225699f3467e5a07f3b955bafd" - -has-symbol-support-x@^1.4.1: - version "1.4.2" - resolved "https://registry.yarnpkg.com/has-symbol-support-x/-/has-symbol-support-x-1.4.2.tgz#1409f98bc00247da45da67cee0a36f282ff26455" - -has-symbols@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/has-symbols/-/has-symbols-1.0.0.tgz#ba1a8f1af2a0fc39650f5c850367704122063b44" - -has-to-string-tag-x@^1.2.0: - version "1.4.1" - resolved "https://registry.yarnpkg.com/has-to-string-tag-x/-/has-to-string-tag-x-1.4.1.tgz#a045ab383d7b4b2012a00148ab0aa5f290044d4d" - dependencies: - has-symbol-support-x "^1.4.1" - -has-unicode@^2.0.0: - version "2.0.1" - resolved "https://registry.yarnpkg.com/has-unicode/-/has-unicode-2.0.1.tgz#e0e6fe6a28cf51138855e086d1691e771de2a8b9" - -has@^1.0.1: - version "1.0.3" - resolved "https://registry.yarnpkg.com/has/-/has-1.0.3.tgz#722d7cbfc1f6aa8241f16dd814e011e1f41e8796" - dependencies: - function-bind "^1.1.1" - -hosted-git-info@^2.1.4: - version "2.7.1" - resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.7.1.tgz#97f236977bd6e125408930ff6de3eec6281ec047" - -http-cache-semantics@3.8.1: - version "3.8.1" - resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-3.8.1.tgz#39b0e16add9b605bf0a9ef3d9daaf4843b4cacd2" - -http-signature@~1.2.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/http-signature/-/http-signature-1.2.0.tgz#9aecd925114772f3d95b65a60abb8f7c18fbace1" - dependencies: - assert-plus "^1.0.0" - jsprim "^1.2.2" - sshpk "^1.7.0" - -iconv-lite@^0.4.4: - version "0.4.23" - resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.23.tgz#297871f63be507adcfbfca715d0cd0eed84e9a63" - dependencies: - safer-buffer ">= 2.1.2 < 3" - -ignore-walk@^3.0.1: - version "3.0.1" - resolved "https://registry.yarnpkg.com/ignore-walk/-/ignore-walk-3.0.1.tgz#a83e62e7d272ac0e3b551aaa82831a19b69f82f8" - dependencies: - minimatch "^3.0.4" - -ignore@^3.2.0: - version "3.3.10" - resolved "https://registry.yarnpkg.com/ignore/-/ignore-3.3.10.tgz#0a97fb876986e8081c631160f8f9f389157f0043" - -indent-string@^3.0.0: - version "3.2.0" - resolved "https://registry.yarnpkg.com/indent-string/-/indent-string-3.2.0.tgz#4a5fd6d27cc332f37e5419a504dbb837105c9289" - -inflight@^1.0.4: - version "1.0.6" - resolved "https://registry.yarnpkg.com/inflight/-/inflight-1.0.6.tgz#49bd6331d7d02d0c09bc910a1075ba8165b56df9" - dependencies: - once "^1.3.0" - wrappy "1" - -inherits@2, inherits@^2.0.1, inherits@^2.0.3, inherits@~2.0.3: - version "2.0.3" - resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de" - -ini@~1.3.0: - version "1.3.5" - resolved "https://registry.yarnpkg.com/ini/-/ini-1.3.5.tgz#eee25f56db1c9ec6085e0c22778083f596abf927" - -interop-require@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/interop-require/-/interop-require-1.0.0.tgz#e53103679944c88d7e6105b62a9f4475c783971e" - -into-stream@^3.1.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/into-stream/-/into-stream-3.1.0.tgz#96fb0a936c12babd6ff1752a17d05616abd094c6" - dependencies: - from2 "^2.1.1" - p-is-promise "^1.1.0" - -ip-regex@^2.0.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/ip-regex/-/ip-regex-2.1.0.tgz#fa78bf5d2e6913c911ce9f819ee5146bb6d844e9" - -ip@^1.1.0: - version "1.1.5" - resolved "https://registry.yarnpkg.com/ip/-/ip-1.1.5.tgz#bdded70114290828c0a039e72ef25f5aaec4354a" - -is-absolute-url@^2.0.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/is-absolute-url/-/is-absolute-url-2.1.0.tgz#50530dfb84fcc9aa7dbe7852e83a37b93b9f2aa6" - -is-alphabetical@^1.0.0: - version "1.0.2" - resolved "https://registry.yarnpkg.com/is-alphabetical/-/is-alphabetical-1.0.2.tgz#1fa6e49213cb7885b75d15862fb3f3d96c884f41" - -is-alphanumeric@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/is-alphanumeric/-/is-alphanumeric-1.0.0.tgz#4a9cef71daf4c001c1d81d63d140cf53fd6889f4" - -is-alphanumerical@^1.0.0: - version "1.0.2" - resolved "https://registry.yarnpkg.com/is-alphanumerical/-/is-alphanumerical-1.0.2.tgz#1138e9ae5040158dc6ff76b820acd6b7a181fd40" - dependencies: - is-alphabetical "^1.0.0" - is-decimal "^1.0.0" - -is-arrayish@^0.2.1: - version "0.2.1" - resolved "https://registry.yarnpkg.com/is-arrayish/-/is-arrayish-0.2.1.tgz#77c99840527aa8ecb1a8ba697b80645a7a926a9d" - -is-binary-path@^1.0.0: - version "1.0.1" - resolved "https://registry.yarnpkg.com/is-binary-path/-/is-binary-path-1.0.1.tgz#75f16642b480f187a711c814161fd3a4a7655898" - dependencies: - binary-extensions "^1.0.0" - -is-buffer@^1.1.4, is-buffer@^1.1.5, is-buffer@~1.1.1: - version "1.1.6" - resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-1.1.6.tgz#efaa2ea9daa0d7ab2ea13a97b2b8ad51fefbe8be" - -is-builtin-module@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/is-builtin-module/-/is-builtin-module-1.0.0.tgz#540572d34f7ac3119f8f76c30cbc1b1e037affbe" - dependencies: - builtin-modules "^1.0.0" - -is-callable@^1.1.1, is-callable@^1.1.3: - version "1.1.4" - resolved "https://registry.yarnpkg.com/is-callable/-/is-callable-1.1.4.tgz#1e1adf219e1eeb684d691f9d6a05ff0d30a24d75" - -is-date-object@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/is-date-object/-/is-date-object-1.0.1.tgz#9aa20eb6aeebbff77fbd33e74ca01b33581d3a16" - -is-decimal@^1.0.0: - version "1.0.2" - resolved "https://registry.yarnpkg.com/is-decimal/-/is-decimal-1.0.2.tgz#894662d6a8709d307f3a276ca4339c8fa5dff0ff" - -is-dotfile@^1.0.0: - version "1.0.3" - resolved "https://registry.yarnpkg.com/is-dotfile/-/is-dotfile-1.0.3.tgz#a6a2f32ffd2dfb04f5ca25ecd0f6b83cf798a1e1" - -is-empty@^1.0.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/is-empty/-/is-empty-1.2.0.tgz#de9bb5b278738a05a0b09a57e1fb4d4a341a9f6b" - -is-equal-shallow@^0.1.3: - version "0.1.3" - resolved "https://registry.yarnpkg.com/is-equal-shallow/-/is-equal-shallow-0.1.3.tgz#2238098fc221de0bcfa5d9eac4c45d638aa1c534" - dependencies: - is-primitive "^2.0.0" - -is-extendable@^0.1.1: - version "0.1.1" - resolved "https://registry.yarnpkg.com/is-extendable/-/is-extendable-0.1.1.tgz#62b110e289a471418e3ec36a617d472e301dfc89" - -is-extglob@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/is-extglob/-/is-extglob-1.0.0.tgz#ac468177c4943405a092fc8f29760c6ffc6206c0" - -is-file@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/is-file/-/is-file-1.0.0.tgz#28a44cfbd9d3db193045f22b65fce8edf9620596" - -is-fullwidth-code-point@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz#ef9e31386f031a7f0d643af82fde50c457ef00cb" - dependencies: - number-is-nan "^1.0.0" - -is-fullwidth-code-point@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz#a3b30a5c4f199183167aaab93beefae3ddfb654f" - -is-glob@^2.0.0, is-glob@^2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/is-glob/-/is-glob-2.0.1.tgz#d096f926a3ded5600f3fdfd91198cb0888c2d863" - dependencies: - is-extglob "^1.0.0" - -is-hexadecimal@^1.0.0: - version "1.0.2" - resolved "https://registry.yarnpkg.com/is-hexadecimal/-/is-hexadecimal-1.0.2.tgz#b6e710d7d07bb66b98cb8cece5c9b4921deeb835" - -is-hidden@^1.0.1: - version "1.1.1" - resolved "https://registry.yarnpkg.com/is-hidden/-/is-hidden-1.1.1.tgz#82ee6a93aeef3fb007ad5b9457c0584d45329f38" - -is-ip@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/is-ip/-/is-ip-2.0.0.tgz#68eea07e8a0a0a94c2d080dd674c731ab2a461ab" - dependencies: - ip-regex "^2.0.0" - -is-number@^2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/is-number/-/is-number-2.1.0.tgz#01fcbbb393463a548f2f466cce16dece49db908f" - dependencies: - kind-of "^3.0.2" - -is-number@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/is-number/-/is-number-4.0.0.tgz#0026e37f5454d73e356dfe6564699867c6a7f0ff" - -is-object@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/is-object/-/is-object-1.0.1.tgz#8952688c5ec2ffd6b03ecc85e769e02903083470" - -is-online@^7.0.0: - version "7.0.0" - resolved "https://registry.yarnpkg.com/is-online/-/is-online-7.0.0.tgz#7e2408c0ae1e7e37ba8d50bdb237260d32bfd96e" - dependencies: - got "^6.7.1" - p-any "^1.0.0" - p-timeout "^1.0.0" - public-ip "^2.3.0" - -is-path-cwd@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/is-path-cwd/-/is-path-cwd-1.0.0.tgz#d225ec23132e89edd38fda767472e62e65f1106d" - -is-path-in-cwd@^1.0.0: - version "1.0.1" - resolved "https://registry.yarnpkg.com/is-path-in-cwd/-/is-path-in-cwd-1.0.1.tgz#5ac48b345ef675339bd6c7a48a912110b241cf52" - dependencies: - is-path-inside "^1.0.0" - -is-path-inside@^1.0.0: - version "1.0.1" - resolved "https://registry.yarnpkg.com/is-path-inside/-/is-path-inside-1.0.1.tgz#8ef5b7de50437a3fdca6b4e865ef7aa55cb48036" - dependencies: - path-is-inside "^1.0.1" - -is-plain-obj@^1.0.0, is-plain-obj@^1.1.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/is-plain-obj/-/is-plain-obj-1.1.0.tgz#71a50c8429dfca773c92a390a4a03b39fcd51d3e" - -is-posix-bracket@^0.1.0: - version "0.1.1" - resolved "https://registry.yarnpkg.com/is-posix-bracket/-/is-posix-bracket-0.1.1.tgz#3334dc79774368e92f016e6fbc0a88f5cd6e6bc4" - -is-primitive@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/is-primitive/-/is-primitive-2.0.0.tgz#207bab91638499c07b2adf240a41a87210034575" - -is-redirect@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/is-redirect/-/is-redirect-1.0.0.tgz#1d03dded53bd8db0f30c26e4f95d36fc7c87dc24" - -is-regex@^1.0.4: - version "1.0.4" - resolved "https://registry.yarnpkg.com/is-regex/-/is-regex-1.0.4.tgz#5517489b547091b0930e095654ced25ee97e9491" - dependencies: - has "^1.0.1" - -is-relative-url@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/is-relative-url/-/is-relative-url-2.0.0.tgz#72902d7fe04b3d4792e7db15f9db84b7204c9cef" - dependencies: - is-absolute-url "^2.0.0" - -is-retry-allowed@^1.0.0, is-retry-allowed@^1.1.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/is-retry-allowed/-/is-retry-allowed-1.1.0.tgz#11a060568b67339444033d0125a61a20d564fb34" - -is-stream@^1.0.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-1.1.0.tgz#12d4a3dd4e68e0b79ceb8dbc84173ae80d91ca44" - -is-symbol@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/is-symbol/-/is-symbol-1.0.1.tgz#3cc59f00025194b6ab2e38dbae6689256b660572" - -is-typedarray@~1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/is-typedarray/-/is-typedarray-1.0.0.tgz#e479c80858df0c1b11ddda6940f96011fcda4a9a" - -is-utf8@^0.2.0: - version "0.2.1" - resolved "https://registry.yarnpkg.com/is-utf8/-/is-utf8-0.2.1.tgz#4b0da1442104d1b336340e80797e865cf39f7d72" - -is-whitespace-character@^1.0.0: - version "1.0.2" - resolved "https://registry.yarnpkg.com/is-whitespace-character/-/is-whitespace-character-1.0.2.tgz#ede53b4c6f6fb3874533751ec9280d01928d03ed" - -is-word-character@^1.0.0: - version "1.0.2" - resolved "https://registry.yarnpkg.com/is-word-character/-/is-word-character-1.0.2.tgz#46a5dac3f2a1840898b91e576cd40d493f3ae553" - -isarray@1.0.0, isarray@~1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11" - -isemail@^3.1.2: - version "3.1.3" - resolved "https://registry.yarnpkg.com/isemail/-/isemail-3.1.3.tgz#64f37fc113579ea12523165c3ebe3a71a56ce571" - dependencies: - punycode "2.x.x" - -isobject@^2.0.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/isobject/-/isobject-2.1.0.tgz#f065561096a3f1da2ef46272f815c840d87e0c89" - dependencies: - isarray "1.0.0" - -isstream@~0.1.2: - version "0.1.2" - resolved "https://registry.yarnpkg.com/isstream/-/isstream-0.1.2.tgz#47e63f7af55afa6f92e1500e690eb8b8529c099a" - -isurl@^1.0.0-alpha5: - version "1.0.0" - resolved "https://registry.yarnpkg.com/isurl/-/isurl-1.0.0.tgz#b27f4f49f3cdaa3ea44a0a5b7f3462e6edc39d67" - dependencies: - has-to-string-tag-x "^1.2.0" - is-object "^1.0.1" - -js-yaml@^3.12.0, js-yaml@^3.2.4, js-yaml@^3.6.1: - version "3.12.0" - resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-3.12.0.tgz#eaed656ec8344f10f527c6bfa1b6e2244de167d1" - dependencies: - argparse "^1.0.7" - esprima "^4.0.0" - -jsbn@~0.1.0: - version "0.1.1" - resolved "https://registry.yarnpkg.com/jsbn/-/jsbn-0.1.1.tgz#a5e654c2e5a2deb5f201d96cefbca80c0ef2f513" - -json-buffer@3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/json-buffer/-/json-buffer-3.0.0.tgz#5b1f397afc75d677bde8bcfc0e47e1f9a3d9a898" - -json-parse-better-errors@^1.0.1: - version "1.0.2" - resolved "https://registry.yarnpkg.com/json-parse-better-errors/-/json-parse-better-errors-1.0.2.tgz#bb867cfb3450e69107c131d1c514bab3dc8bcaa9" - -json-schema-traverse@^0.3.0: - version "0.3.1" - resolved "https://registry.yarnpkg.com/json-schema-traverse/-/json-schema-traverse-0.3.1.tgz#349a6d44c53a51de89b40805c5d5e59b417d3340" - -json-schema@0.2.3: - version "0.2.3" - resolved "https://registry.yarnpkg.com/json-schema/-/json-schema-0.2.3.tgz#b480c892e59a2f05954ce727bd3f2a4e882f9e13" - -json-stable-stringify@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/json-stable-stringify/-/json-stable-stringify-1.0.1.tgz#9a759d39c5f2ff503fd5300646ed445f88c4f9af" - dependencies: - jsonify "~0.0.0" - -json-stringify-safe@~5.0.1: - version "5.0.1" - resolved "https://registry.yarnpkg.com/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz#1296a2d58fd45f19a0f6ce01d65701e2c735b6eb" - -json5@^0.5.1: - version "0.5.1" - resolved "https://registry.yarnpkg.com/json5/-/json5-0.5.1.tgz#1eade7acc012034ad84e2396767ead9fa5495821" - -json5@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/json5/-/json5-1.0.1.tgz#779fb0018604fa854eacbf6252180d83543e3dbe" - dependencies: - minimist "^1.2.0" - -jsonify@~0.0.0: - version "0.0.0" - resolved "https://registry.yarnpkg.com/jsonify/-/jsonify-0.0.0.tgz#2c74b6ee41d93ca51b7b5aaee8f503631d252a73" - -jsprim@^1.2.2: - version "1.4.1" - resolved "https://registry.yarnpkg.com/jsprim/-/jsprim-1.4.1.tgz#313e66bc1e5cc06e438bc1b7499c2e5c56acb6a2" - dependencies: - assert-plus "1.0.0" - extsprintf "1.3.0" - json-schema "0.2.3" - verror "1.10.0" - -keyv@3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/keyv/-/keyv-3.0.0.tgz#44923ba39e68b12a7cec7df6c3268c031f2ef373" - dependencies: - json-buffer "3.0.0" - -kind-of@^3.0.2: - version "3.2.2" - resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-3.2.2.tgz#31ea21a734bab9bbb0f32466d893aea51e4a3c64" - dependencies: - is-buffer "^1.1.5" - -kind-of@^6.0.0: - version "6.0.2" - resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-6.0.2.tgz#01146b36a6218e64e58f3a8d66de5d7fc6f6d051" - -levn@~0.3.0: - version "0.3.0" - resolved "https://registry.yarnpkg.com/levn/-/levn-0.3.0.tgz#3b09924edf9f083c0490fdd4c0bc4421e04764ee" - dependencies: - prelude-ls "~1.1.2" - type-check "~0.3.2" - -link-check@^4.1.0: - version "4.4.4" - resolved "https://registry.yarnpkg.com/link-check/-/link-check-4.4.4.tgz#08dbb881b70c23f1c173889c3a34d682c2e68c1a" - dependencies: - is-relative-url "^2.0.0" - isemail "^3.1.2" - ms "^2.1.1" - request "^2.87.0" - -load-json-file@^1.0.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/load-json-file/-/load-json-file-1.1.0.tgz#956905708d58b4bab4c2261b04f59f31c99374c0" - dependencies: - graceful-fs "^4.1.2" - parse-json "^2.2.0" - pify "^2.0.0" - pinkie-promise "^2.0.0" - strip-bom "^2.0.0" - -load-json-file@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/load-json-file/-/load-json-file-4.0.0.tgz#2f5f45ab91e33216234fd53adab668eb4ec0993b" - dependencies: - graceful-fs "^4.1.2" - parse-json "^4.0.0" - pify "^3.0.0" - strip-bom "^3.0.0" - -load-plugin@^2.0.0: - version "2.2.2" - resolved "https://registry.yarnpkg.com/load-plugin/-/load-plugin-2.2.2.tgz#ebc7599491ff33e5077719fbe051d5725a9f7a89" - dependencies: - npm-prefix "^1.2.0" - resolve-from "^4.0.0" - -locate-path@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/locate-path/-/locate-path-2.0.0.tgz#2b568b265eec944c6d9c0de9c3dbbbca0354cd8e" - dependencies: - p-locate "^2.0.0" - path-exists "^3.0.0" - -lodash@^4.0.0, lodash@^4.17.4: - version "4.17.10" - resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.10.tgz#1b7793cf7259ea38fb3661d4d38b3260af8ae4e7" - -log-symbols@^1.0.2: - version "1.0.2" - resolved "https://registry.yarnpkg.com/log-symbols/-/log-symbols-1.0.2.tgz#376ff7b58ea3086a0f09facc74617eca501e1a18" - dependencies: - chalk "^1.0.0" - -longest-streak@^2.0.1: - version "2.0.2" - resolved "https://registry.yarnpkg.com/longest-streak/-/longest-streak-2.0.2.tgz#2421b6ba939a443bb9ffebf596585a50b4c38e2e" - -lowercase-keys@1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/lowercase-keys/-/lowercase-keys-1.0.0.tgz#4e3366b39e7f5457e35f1324bdf6f88d0bfc7306" - -lowercase-keys@^1.0.0: - version "1.0.1" - resolved "https://registry.yarnpkg.com/lowercase-keys/-/lowercase-keys-1.0.1.tgz#6f9e30b47084d971a7c820ff15a6c5167b74c26f" - -map-like@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/map-like/-/map-like-2.0.0.tgz#94496d49ad333c0dc3234b27adbbd1e8535953b4" - -markdown-escapes@^1.0.0: - version "1.0.2" - resolved "https://registry.yarnpkg.com/markdown-escapes/-/markdown-escapes-1.0.2.tgz#e639cbde7b99c841c0bacc8a07982873b46d2122" - -markdown-extensions@^1.1.0: - version "1.1.1" - resolved "https://registry.yarnpkg.com/markdown-extensions/-/markdown-extensions-1.1.1.tgz#fea03b539faeaee9b4ef02a3769b455b189f7fc3" - -markdown-table@^1.1.0: - version "1.1.2" - resolved "https://registry.yarnpkg.com/markdown-table/-/markdown-table-1.1.2.tgz#c78db948fa879903a41bce522e3b96f801c63786" - -math-random@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/math-random/-/math-random-1.0.1.tgz#8b3aac588b8a66e4975e3cdea67f7bb329601fac" - -md5@^2.2.1: - version "2.2.1" - resolved "https://registry.yarnpkg.com/md5/-/md5-2.2.1.tgz#53ab38d5fe3c8891ba465329ea23fac0540126f9" - dependencies: - charenc "~0.0.1" - crypt "~0.0.1" - is-buffer "~1.1.1" - -mdast-util-compact@^1.0.0: - version "1.0.1" - resolved "https://registry.yarnpkg.com/mdast-util-compact/-/mdast-util-compact-1.0.1.tgz#cdb5f84e2b6a2d3114df33bd05d9cb32e3c4083a" - dependencies: - unist-util-modify-children "^1.0.0" - unist-util-visit "^1.1.0" - -micromatch@^2.1.5: - version "2.3.11" - resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-2.3.11.tgz#86677c97d1720b363431d04d0d15293bd38c1565" - dependencies: - arr-diff "^2.0.0" - array-unique "^0.2.1" - braces "^1.8.2" - expand-brackets "^0.1.4" - extglob "^0.3.1" - filename-regex "^2.0.0" - is-extglob "^1.0.0" - is-glob "^2.0.1" - kind-of "^3.0.2" - normalize-path "^2.0.1" - object.omit "^2.0.0" - parse-glob "^3.0.4" - regex-cache "^0.4.2" - -mime-db@~1.35.0: - version "1.35.0" - resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.35.0.tgz#0569d657466491283709663ad379a99b90d9ab47" - -mime-types@^2.1.12, mime-types@~2.1.17: - version "2.1.19" - resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.19.tgz#71e464537a7ef81c15f2db9d97e913fc0ff606f0" - dependencies: - mime-db "~1.35.0" - -mimic-response@^1.0.0: - version "1.0.1" - resolved "https://registry.yarnpkg.com/mimic-response/-/mimic-response-1.0.1.tgz#4923538878eef42063cb8a3e3b0798781487ab1b" - -minimatch@^3.0.2, minimatch@^3.0.4: - version "3.0.4" - resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083" - dependencies: - brace-expansion "^1.1.7" - -minimist@0.0.8: - version "0.0.8" - resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.0.8.tgz#857fcabfc3397d2625b8228262e86aa7a011b05d" - -minimist@^1.2.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.0.tgz#a35008b20f41383eec1fb914f4cd5df79a264284" - -minipass@^2.2.1, minipass@^2.3.3: - version "2.3.3" - resolved "https://registry.yarnpkg.com/minipass/-/minipass-2.3.3.tgz#a7dcc8b7b833f5d368759cce544dccb55f50f233" - dependencies: - safe-buffer "^5.1.2" - yallist "^3.0.0" - -minizlib@^1.1.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/minizlib/-/minizlib-1.1.0.tgz#11e13658ce46bc3a70a267aac58359d1e0c29ceb" - dependencies: - minipass "^2.2.1" - -mkdirp@^0.5.0, mkdirp@^0.5.1: - version "0.5.1" - resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.1.tgz#30057438eac6cf7f8c4767f38648d6697d75c903" - dependencies: - minimist "0.0.8" - -ms@2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8" - -ms@^2.1.1: - version "2.1.1" - resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.1.tgz#30a5864eb3ebb0a66f2ebe6d727af06a09d86e0a" - -nan@^2.9.2: - version "2.10.0" - resolved "https://registry.yarnpkg.com/nan/-/nan-2.10.0.tgz#96d0cd610ebd58d4b4de9cc0c6828cda99c7548f" - -needle@^2.2.1: - version "2.2.1" - resolved "https://registry.yarnpkg.com/needle/-/needle-2.2.1.tgz#b5e325bd3aae8c2678902fa296f729455d1d3a7d" - dependencies: - debug "^2.1.2" - iconv-lite "^0.4.4" - sax "^1.2.4" - -nlcst-to-string@^2.0.0: - version "2.0.2" - resolved "https://registry.yarnpkg.com/nlcst-to-string/-/nlcst-to-string-2.0.2.tgz#7125af4d4d369850c697192a658f01f36af9937b" - -no-cliches@^0.1.0: - version "0.1.0" - resolved "https://registry.yarnpkg.com/no-cliches/-/no-cliches-0.1.0.tgz#f4eb81a551fecde813f8c611e35e64a5118dc38c" - -node-pre-gyp@^0.10.0: - version "0.10.3" - resolved "https://registry.yarnpkg.com/node-pre-gyp/-/node-pre-gyp-0.10.3.tgz#3070040716afdc778747b61b6887bf78880b80fc" - dependencies: - detect-libc "^1.0.2" - mkdirp "^0.5.1" - needle "^2.2.1" - nopt "^4.0.1" - npm-packlist "^1.1.6" - npmlog "^4.0.2" - rc "^1.2.7" - rimraf "^2.6.1" - semver "^5.3.0" - tar "^4" - -nopt@^4.0.1: - version "4.0.1" - resolved "https://registry.yarnpkg.com/nopt/-/nopt-4.0.1.tgz#d0d4685afd5415193c8c7505602d0d17cd64474d" - dependencies: - abbrev "1" - osenv "^0.1.4" - -normalize-package-data@^2.3.2: - version "2.4.0" - resolved "https://registry.yarnpkg.com/normalize-package-data/-/normalize-package-data-2.4.0.tgz#12f95a307d58352075a04907b84ac8be98ac012f" - dependencies: - hosted-git-info "^2.1.4" - is-builtin-module "^1.0.0" - semver "2 || 3 || 4 || 5" - validate-npm-package-license "^3.0.1" - -normalize-path@^2.0.0, normalize-path@^2.0.1: - version "2.1.1" - resolved "https://registry.yarnpkg.com/normalize-path/-/normalize-path-2.1.1.tgz#1ab28b556e198363a8c1a6f7e6fa20137fe6aed9" - dependencies: - remove-trailing-separator "^1.0.1" - -normalize-url@2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/normalize-url/-/normalize-url-2.0.1.tgz#835a9da1551fa26f70e92329069a23aa6574d7e6" - dependencies: - prepend-http "^2.0.0" - query-string "^5.0.1" - sort-keys "^2.0.0" - -npm-bundled@^1.0.1: - version "1.0.3" - resolved "https://registry.yarnpkg.com/npm-bundled/-/npm-bundled-1.0.3.tgz#7e71703d973af3370a9591bafe3a63aca0be2308" - -npm-packlist@^1.1.6: - version "1.1.11" - resolved "https://registry.yarnpkg.com/npm-packlist/-/npm-packlist-1.1.11.tgz#84e8c683cbe7867d34b1d357d893ce29e28a02de" - dependencies: - ignore-walk "^3.0.1" - npm-bundled "^1.0.1" - -npm-prefix@^1.2.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/npm-prefix/-/npm-prefix-1.2.0.tgz#e619455f7074ba54cc66d6d0d37dd9f1be6bcbc0" - dependencies: - rc "^1.1.0" - shellsubstitute "^1.1.0" - untildify "^2.1.0" - -npmlog@^4.0.2: - version "4.1.2" - resolved "https://registry.yarnpkg.com/npmlog/-/npmlog-4.1.2.tgz#08a7f2a8bf734604779a9efa4ad5cc717abb954b" - dependencies: - are-we-there-yet "~1.1.2" - console-control-strings "~1.1.0" - gauge "~2.7.3" - set-blocking "~2.0.0" - -number-is-nan@^1.0.0: - version "1.0.1" - resolved "https://registry.yarnpkg.com/number-is-nan/-/number-is-nan-1.0.1.tgz#097b602b53422a522c1afb8790318336941a011d" - -oauth-sign@~0.8.2: - version "0.8.2" - resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.8.2.tgz#46a6ab7f0aead8deae9ec0565780b7d4efeb9d43" - -object-assign@^4.0.1, object-assign@^4.1.0, object-assign@^4.1.1: - version "4.1.1" - resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863" - -object-keys@^1.0.11, object-keys@^1.0.12, object-keys@^1.0.8: - version "1.0.12" - resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-1.0.12.tgz#09c53855377575310cca62f55bb334abff7b3ed2" - -object.assign@^4.0.4: - version "4.1.0" - resolved "https://registry.yarnpkg.com/object.assign/-/object.assign-4.1.0.tgz#968bf1100d7956bb3ca086f006f846b3bc4008da" - dependencies: - define-properties "^1.1.2" - function-bind "^1.1.1" - has-symbols "^1.0.0" - object-keys "^1.0.11" - -object.omit@^2.0.0: - version "2.0.1" - resolved "https://registry.yarnpkg.com/object.omit/-/object.omit-2.0.1.tgz#1a9c744829f39dbb858c76ca3579ae2a54ebd1fa" - dependencies: - for-own "^0.1.4" - is-extendable "^0.1.1" - -once@^1.3.0: - version "1.4.0" - resolved "https://registry.yarnpkg.com/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1" - dependencies: - wrappy "1" - -optionator@^0.8.0, optionator@^0.8.1: - version "0.8.2" - resolved "https://registry.yarnpkg.com/optionator/-/optionator-0.8.2.tgz#364c5e409d3f4d6301d6c0b4c05bba50180aeb64" - dependencies: - deep-is "~0.1.3" - fast-levenshtein "~2.0.4" - levn "~0.3.0" - prelude-ls "~1.1.2" - type-check "~0.3.2" - wordwrap "~1.0.0" - -os-homedir@^1.0.0: - version "1.0.2" - resolved "https://registry.yarnpkg.com/os-homedir/-/os-homedir-1.0.2.tgz#ffbc4988336e0e833de0c168c7ef152121aa7fb3" - -os-tmpdir@^1.0.0: - version "1.0.2" - resolved "https://registry.yarnpkg.com/os-tmpdir/-/os-tmpdir-1.0.2.tgz#bbe67406c79aa85c5cfec766fe5734555dfa1274" - -osenv@^0.1.4: - version "0.1.5" - resolved "https://registry.yarnpkg.com/osenv/-/osenv-0.1.5.tgz#85cdfafaeb28e8677f416e287592b5f3f49ea410" - dependencies: - os-homedir "^1.0.0" - os-tmpdir "^1.0.0" - -p-any@^1.0.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/p-any/-/p-any-1.1.0.tgz#1d03835c7eed1e34b8e539c47b7b60d0d015d4e1" - dependencies: - p-some "^2.0.0" - -p-cancelable@^0.4.0: - version "0.4.1" - resolved "https://registry.yarnpkg.com/p-cancelable/-/p-cancelable-0.4.1.tgz#35f363d67d52081c8d9585e37bcceb7e0bbcb2a0" - -p-finally@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/p-finally/-/p-finally-1.0.0.tgz#3fbcfb15b899a44123b34b6dcc18b724336a2cae" - -p-is-promise@^1.1.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/p-is-promise/-/p-is-promise-1.1.0.tgz#9c9456989e9f6588017b0434d56097675c3da05e" - -p-limit@^1.1.0: - version "1.3.0" - resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-1.3.0.tgz#b86bd5f0c25690911c7590fcbfc2010d54b3ccb8" - dependencies: - p-try "^1.0.0" - -p-locate@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/p-locate/-/p-locate-2.0.0.tgz#20a0103b222a70c8fd39cc2e580680f3dde5ec43" - dependencies: - p-limit "^1.1.0" - -p-some@^2.0.0: - version "2.0.1" - resolved "https://registry.yarnpkg.com/p-some/-/p-some-2.0.1.tgz#65d87c8b154edbcf5221d167778b6d2e150f6f06" - dependencies: - aggregate-error "^1.0.0" - -p-timeout@^1.0.0: - version "1.2.1" - resolved "https://registry.yarnpkg.com/p-timeout/-/p-timeout-1.2.1.tgz#5eb3b353b7fce99f101a1038880bb054ebbea386" - dependencies: - p-finally "^1.0.0" - -p-timeout@^2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/p-timeout/-/p-timeout-2.0.1.tgz#d8dd1979595d2dc0139e1fe46b8b646cb3cdf038" - dependencies: - p-finally "^1.0.0" - -p-try@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/p-try/-/p-try-1.0.0.tgz#cbc79cdbaf8fd4228e13f621f2b1a237c1b207b3" - -parse-entities@^1.0.2, parse-entities@^1.1.0: - version "1.1.2" - resolved "https://registry.yarnpkg.com/parse-entities/-/parse-entities-1.1.2.tgz#9eaf719b29dc3bd62246b4332009072e01527777" - dependencies: - character-entities "^1.0.0" - character-entities-legacy "^1.0.0" - character-reference-invalid "^1.0.0" - is-alphanumerical "^1.0.0" - is-decimal "^1.0.0" - is-hexadecimal "^1.0.0" - -parse-glob@^3.0.4: - version "3.0.4" - resolved "https://registry.yarnpkg.com/parse-glob/-/parse-glob-3.0.4.tgz#b2c376cfb11f35513badd173ef0bb6e3a388391c" - dependencies: - glob-base "^0.3.0" - is-dotfile "^1.0.0" - is-extglob "^1.0.0" - is-glob "^2.0.0" - -parse-json@^2.2.0: - version "2.2.0" - resolved "https://registry.yarnpkg.com/parse-json/-/parse-json-2.2.0.tgz#f480f40434ef80741f8469099f8dea18f55a4dc9" - dependencies: - error-ex "^1.2.0" - -parse-json@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/parse-json/-/parse-json-4.0.0.tgz#be35f5425be1f7f6c747184f98a788cb99477ee0" - dependencies: - error-ex "^1.3.1" - json-parse-better-errors "^1.0.1" - -passive-voice@^0.1.0: - version "0.1.0" - resolved "https://registry.yarnpkg.com/passive-voice/-/passive-voice-0.1.0.tgz#16ff91ae40ba0e92c43e671763fdc842a70270b1" - -path-exists@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/path-exists/-/path-exists-3.0.0.tgz#ce0ebeaa5f78cb18925ea7d810d7b59b010fd515" - -path-is-absolute@^1.0.0: - version "1.0.1" - resolved "https://registry.yarnpkg.com/path-is-absolute/-/path-is-absolute-1.0.1.tgz#174b9268735534ffbc7ace6bf53a5a9e1b5c5f5f" - -path-is-inside@^1.0.1: - version "1.0.2" - resolved "https://registry.yarnpkg.com/path-is-inside/-/path-is-inside-1.0.2.tgz#365417dede44430d1c11af61027facf074bdfc53" - -path-to-glob-pattern@^1.0.2: - version "1.0.2" - resolved "https://registry.yarnpkg.com/path-to-glob-pattern/-/path-to-glob-pattern-1.0.2.tgz#473e6a3a292a9d13fbae3edccee72d3baba8c619" - -path-type@^1.0.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/path-type/-/path-type-1.1.0.tgz#59c44f7ee491da704da415da5a4070ba4f8fe441" - dependencies: - graceful-fs "^4.1.2" - pify "^2.0.0" - pinkie-promise "^2.0.0" - -path-type@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/path-type/-/path-type-3.0.0.tgz#cef31dc8e0a1a3bb0d105c0cd97cf3bf47f4e36f" - dependencies: - pify "^3.0.0" - -performance-now@^2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-2.1.0.tgz#6309f4e0e5fa913ec1c69307ae364b4b377c9e7b" - -pify@^2.0.0: - version "2.3.0" - resolved "https://registry.yarnpkg.com/pify/-/pify-2.3.0.tgz#ed141a6ac043a849ea588498e7dca8b15330e90c" - -pify@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/pify/-/pify-3.0.0.tgz#e5a4acd2c101fdf3d9a4d07f0dbc4db49dd28176" - -pinkie-promise@^2.0.0: - version "2.0.1" - resolved "https://registry.yarnpkg.com/pinkie-promise/-/pinkie-promise-2.0.1.tgz#2135d6dfa7a358c069ac9b178776288228450ffa" - dependencies: - pinkie "^2.0.0" - -pinkie@^2.0.0: - version "2.0.4" - resolved "https://registry.yarnpkg.com/pinkie/-/pinkie-2.0.4.tgz#72556b80cfa0d48a974e80e77248e80ed4f7f870" - -pluralize@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/pluralize/-/pluralize-2.0.0.tgz#72b726aa6fac1edeee42256c7d8dc256b335677f" - -prelude-ls@~1.1.2: - version "1.1.2" - resolved "https://registry.yarnpkg.com/prelude-ls/-/prelude-ls-1.1.2.tgz#21932a549f5e52ffd9a827f570e04be62a97da54" - -prepend-http@^1.0.1: - version "1.0.4" - resolved "https://registry.yarnpkg.com/prepend-http/-/prepend-http-1.0.4.tgz#d4f4562b0ce3696e41ac52d0e002e57a635dc6dc" - -prepend-http@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/prepend-http/-/prepend-http-2.0.0.tgz#e92434bfa5ea8c19f41cdfd401d741a3c819d897" - -preserve@^0.2.0: - version "0.2.0" - resolved "https://registry.yarnpkg.com/preserve/-/preserve-0.2.0.tgz#815ed1f6ebc65926f865b310c0713bcb3315ce4b" - -prettier@^1.13.7: - version "1.14.2" - resolved "https://registry.yarnpkg.com/prettier/-/prettier-1.14.2.tgz#0ac1c6e1a90baa22a62925f41963c841983282f9" - -process-nextick-args@~2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.0.tgz#a37d732f4271b4ab1ad070d35508e8290788ffaa" - -public-ip@^2.3.0: - version "2.4.0" - resolved "https://registry.yarnpkg.com/public-ip/-/public-ip-2.4.0.tgz#f00c028a15366d8c798e47efab6acd09a17666da" - dependencies: - dns-socket "^1.6.2" - got "^8.0.0" - is-ip "^2.0.0" - pify "^3.0.0" - -punycode@2.x.x: - version "2.1.1" - resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.1.1.tgz#b58b010ac40c22c5657616c8d2c2c02c7bf479ec" - -punycode@^1.4.1: - version "1.4.1" - resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.4.1.tgz#c0d5a63b2718800ad8e1eb0fa5269c84dd41845e" - -qs@~6.5.1: - version "6.5.2" - resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.2.tgz#cb3ae806e8740444584ef154ce8ee98d403f3e36" - -query-string@^5.0.1: - version "5.1.1" - resolved "https://registry.yarnpkg.com/query-string/-/query-string-5.1.1.tgz#a78c012b71c17e05f2e3fa2319dd330682efb3cb" - dependencies: - decode-uri-component "^0.2.0" - object-assign "^4.1.0" - strict-uri-encode "^1.0.0" - -randomatic@^3.0.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/randomatic/-/randomatic-3.1.0.tgz#36f2ca708e9e567f5ed2ec01949026d50aa10116" - dependencies: - is-number "^4.0.0" - kind-of "^6.0.0" - math-random "^1.0.1" - -rc-config-loader@^2.0.1: - version "2.0.2" - resolved "https://registry.yarnpkg.com/rc-config-loader/-/rc-config-loader-2.0.2.tgz#46eb2f98fb5b2aa7b1119d66c0554de5133f1bc1" - dependencies: - debug "^3.1.0" - js-yaml "^3.12.0" - json5 "^1.0.1" - object-assign "^4.1.0" - object-keys "^1.0.12" - path-exists "^3.0.0" - require-from-string "^2.0.2" - -rc@^1.1.0, rc@^1.2.7: - version "1.2.8" - resolved "https://registry.yarnpkg.com/rc/-/rc-1.2.8.tgz#cd924bf5200a075b83c188cd6b9e211b7fc0d3ed" - dependencies: - deep-extend "^0.6.0" - ini "~1.3.0" - minimist "^1.2.0" - strip-json-comments "~2.0.1" - -read-pkg-up@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/read-pkg-up/-/read-pkg-up-3.0.0.tgz#3ed496685dba0f8fe118d0691dc51f4a1ff96f07" - dependencies: - find-up "^2.0.0" - read-pkg "^3.0.0" - -read-pkg@^1.1.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/read-pkg/-/read-pkg-1.1.0.tgz#f5ffaa5ecd29cb31c0474bca7d756b6bb29e3f28" - dependencies: - load-json-file "^1.0.0" - normalize-package-data "^2.3.2" - path-type "^1.0.0" - -read-pkg@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/read-pkg/-/read-pkg-3.0.0.tgz#9cbc686978fee65d16c00e2b19c237fcf6e38389" - dependencies: - load-json-file "^4.0.0" - normalize-package-data "^2.3.2" - path-type "^3.0.0" - -readable-stream@^2.0.0, readable-stream@^2.0.2, readable-stream@^2.0.6, readable-stream@^2.2.2: - version "2.3.6" - resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.6.tgz#b11c27d88b8ff1fbe070643cf94b0c79ae1b0aaf" - dependencies: - core-util-is "~1.0.0" - inherits "~2.0.3" - isarray "~1.0.0" - process-nextick-args "~2.0.0" - safe-buffer "~5.1.1" - string_decoder "~1.1.1" - util-deprecate "~1.0.1" - -readdirp@^2.0.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/readdirp/-/readdirp-2.1.0.tgz#4ed0ad060df3073300c48440373f72d1cc642d78" - dependencies: - graceful-fs "^4.1.2" - minimatch "^3.0.2" - readable-stream "^2.0.2" - set-immediate-shim "^1.0.1" - -regex-cache@^0.4.2: - version "0.4.4" - resolved "https://registry.yarnpkg.com/regex-cache/-/regex-cache-0.4.4.tgz#75bdc58a2a1496cec48a12835bc54c8d562336dd" - dependencies: - is-equal-shallow "^0.1.3" - -remark-cli@^5.0.0: - version "5.0.0" - resolved "https://registry.yarnpkg.com/remark-cli/-/remark-cli-5.0.0.tgz#9feefd06474f3d0ff132df21b5334c546df12ab6" - dependencies: - markdown-extensions "^1.1.0" - remark "^9.0.0" - unified-args "^5.0.0" - -remark-frontmatter@^1.2.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/remark-frontmatter/-/remark-frontmatter-1.2.0.tgz#67905d178c0fe531ed12c57b98759f101fc2c1b5" - dependencies: - fault "^1.0.1" - xtend "^4.0.1" - -remark-lint-no-dead-urls@^0.3.0: - version "0.3.0" - resolved "https://registry.yarnpkg.com/remark-lint-no-dead-urls/-/remark-lint-no-dead-urls-0.3.0.tgz#b640ecbb4ccaf780afe28c8d13e79f5dc6769449" - dependencies: - is-online "^7.0.0" - is-relative-url "^2.0.0" - link-check "^4.1.0" - unified-lint-rule "^1.0.1" - unist-util-visit "^1.1.3" - -remark-lint-write-good@^1.0.3: - version "1.0.3" - resolved "https://registry.yarnpkg.com/remark-lint-write-good/-/remark-lint-write-good-1.0.3.tgz#daa4cf122212cfa06e437702ef7b43a12875bd5d" - dependencies: - nlcst-to-string "^2.0.0" - unified-lint-rule "^1.0.1" - unist-util-visit "^1.1.1" - write-good "^0.11.1" - -remark-parse@^5.0.0: - version "5.0.0" - resolved "https://registry.yarnpkg.com/remark-parse/-/remark-parse-5.0.0.tgz#4c077f9e499044d1d5c13f80d7a98cf7b9285d95" - dependencies: - collapse-white-space "^1.0.2" - is-alphabetical "^1.0.0" - is-decimal "^1.0.0" - is-whitespace-character "^1.0.0" - is-word-character "^1.0.0" - markdown-escapes "^1.0.0" - parse-entities "^1.1.0" - repeat-string "^1.5.4" - state-toggle "^1.0.0" - trim "0.0.1" - trim-trailing-lines "^1.0.0" - unherit "^1.0.4" - unist-util-remove-position "^1.0.0" - vfile-location "^2.0.0" - xtend "^4.0.1" - -remark-stringify@^5.0.0: - version "5.0.0" - resolved "https://registry.yarnpkg.com/remark-stringify/-/remark-stringify-5.0.0.tgz#336d3a4d4a6a3390d933eeba62e8de4bd280afba" - dependencies: - ccount "^1.0.0" - is-alphanumeric "^1.0.0" - is-decimal "^1.0.0" - is-whitespace-character "^1.0.0" - longest-streak "^2.0.1" - markdown-escapes "^1.0.0" - markdown-table "^1.1.0" - mdast-util-compact "^1.0.0" - parse-entities "^1.0.2" - repeat-string "^1.5.4" - state-toggle "^1.0.0" - stringify-entities "^1.0.1" - unherit "^1.0.4" - xtend "^4.0.1" - -remark@^9.0.0: - version "9.0.0" - resolved "https://registry.yarnpkg.com/remark/-/remark-9.0.0.tgz#c5cfa8ec535c73a67c4b0f12bfdbd3a67d8b2f60" - dependencies: - remark-parse "^5.0.0" - remark-stringify "^5.0.0" - unified "^6.0.0" - -remove-trailing-separator@^1.0.1: - version "1.1.0" - resolved "https://registry.yarnpkg.com/remove-trailing-separator/-/remove-trailing-separator-1.1.0.tgz#c24bce2a283adad5bc3f58e0d48249b92379d8ef" - -repeat-element@^1.1.2: - version "1.1.2" - resolved "https://registry.yarnpkg.com/repeat-element/-/repeat-element-1.1.2.tgz#ef089a178d1483baae4d93eb98b4f9e4e11d990a" - -repeat-string@^1.5.0, repeat-string@^1.5.2, repeat-string@^1.5.4: - version "1.6.1" - resolved "https://registry.yarnpkg.com/repeat-string/-/repeat-string-1.6.1.tgz#8dcae470e1c88abc2d600fff4a776286da75e637" - -replace-ext@1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/replace-ext/-/replace-ext-1.0.0.tgz#de63128373fcbf7c3ccfa4de5a480c45a67958eb" - -request@^2.87.0: - version "2.87.0" - resolved "https://registry.yarnpkg.com/request/-/request-2.87.0.tgz#32f00235cd08d482b4d0d68db93a829c0ed5756e" - dependencies: - aws-sign2 "~0.7.0" - aws4 "^1.6.0" - caseless "~0.12.0" - combined-stream "~1.0.5" - extend "~3.0.1" - forever-agent "~0.6.1" - form-data "~2.3.1" - har-validator "~5.0.3" - http-signature "~1.2.0" - is-typedarray "~1.0.0" - isstream "~0.1.2" - json-stringify-safe "~5.0.1" - mime-types "~2.1.17" - oauth-sign "~0.8.2" - performance-now "^2.1.0" - qs "~6.5.1" - safe-buffer "^5.1.1" - tough-cookie "~2.3.3" - tunnel-agent "^0.6.0" - uuid "^3.1.0" - -require-from-string@^2.0.2: - version "2.0.2" - resolved "https://registry.yarnpkg.com/require-from-string/-/require-from-string-2.0.2.tgz#89a7fdd938261267318eafe14f9c32e598c36909" - -resolve-from@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/resolve-from/-/resolve-from-4.0.0.tgz#4abcd852ad32dd7baabfe9b40e00a36db5f392e6" - -responselike@1.0.2: - version "1.0.2" - resolved "https://registry.yarnpkg.com/responselike/-/responselike-1.0.2.tgz#918720ef3b631c5642be068f15ade5a46f4ba1e7" - dependencies: - lowercase-keys "^1.0.0" - -rimraf@^2.2.8, rimraf@^2.6.1: - version "2.6.2" - resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.6.2.tgz#2ed8150d24a16ea8651e6d6ef0f47c4158ce7a36" - dependencies: - glob "^7.0.5" - -safe-buffer@^5.0.1, safe-buffer@^5.1.1, safe-buffer@^5.1.2, safe-buffer@~5.1.0, safe-buffer@~5.1.1: - version "5.1.2" - resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d" - -"safer-buffer@>= 2.1.2 < 3", safer-buffer@^2.0.2, safer-buffer@^2.1.0, safer-buffer@~2.1.0: - version "2.1.2" - resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" - -sax@^1.2.4: - version "1.2.4" - resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" - -"semver@2 || 3 || 4 || 5", semver@^5.3.0: - version "5.5.0" - resolved "https://registry.yarnpkg.com/semver/-/semver-5.5.0.tgz#dc4bbc7a6ca9d916dee5d43516f0092b58f7b8ab" - -set-blocking@~2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/set-blocking/-/set-blocking-2.0.0.tgz#045f9782d011ae9a6803ddd382b24392b3d890f7" - -set-immediate-shim@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/set-immediate-shim/-/set-immediate-shim-1.0.1.tgz#4b2b1b27eb808a9f8dcc481a58e5e56f599f3f61" - -shellsubstitute@^1.1.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/shellsubstitute/-/shellsubstitute-1.2.0.tgz#e4f702a50c518b0f6fe98451890d705af29b6b70" - -signal-exit@^3.0.0: - version "3.0.2" - resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.2.tgz#b5fdc08f1287ea1178628e415e25132b73646c6d" - -slice-ansi@0.0.4: - version "0.0.4" - resolved "https://registry.yarnpkg.com/slice-ansi/-/slice-ansi-0.0.4.tgz#edbf8903f66f7ce2f8eafd6ceed65e264c831b35" - -sliced@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/sliced/-/sliced-1.0.1.tgz#0b3a662b5d04c3177b1926bea82b03f837a2ef41" - -sort-keys@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/sort-keys/-/sort-keys-2.0.0.tgz#658535584861ec97d730d6cf41822e1f56684128" - dependencies: - is-plain-obj "^1.0.0" - -spdx-correct@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/spdx-correct/-/spdx-correct-3.0.0.tgz#05a5b4d7153a195bc92c3c425b69f3b2a9524c82" - dependencies: - spdx-expression-parse "^3.0.0" - spdx-license-ids "^3.0.0" - -spdx-exceptions@^2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/spdx-exceptions/-/spdx-exceptions-2.1.0.tgz#2c7ae61056c714a5b9b9b2b2af7d311ef5c78fe9" - -spdx-expression-parse@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/spdx-expression-parse/-/spdx-expression-parse-3.0.0.tgz#99e119b7a5da00e05491c9fa338b7904823b41d0" - dependencies: - spdx-exceptions "^2.1.0" - spdx-license-ids "^3.0.0" - -spdx-license-ids@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/spdx-license-ids/-/spdx-license-ids-3.0.0.tgz#7a7cd28470cc6d3a1cfe6d66886f6bc430d3ac87" - -split-lines@^1.1.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/split-lines/-/split-lines-1.1.0.tgz#3abba8f598614142f9db8d27ab6ab875662a1e09" - -sprintf-js@~1.0.2: - version "1.0.3" - resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c" - -sshpk@^1.7.0: - version "1.14.2" - resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.14.2.tgz#c6fc61648a3d9c4e764fd3fcdf4ea105e492ba98" - dependencies: - asn1 "~0.2.3" - assert-plus "^1.0.0" - dashdash "^1.12.0" - getpass "^0.1.1" - safer-buffer "^2.0.2" - optionalDependencies: - bcrypt-pbkdf "^1.0.0" - ecc-jsbn "~0.1.1" - jsbn "~0.1.0" - tweetnacl "~0.14.0" - -state-toggle@^1.0.0: - version "1.0.1" - resolved "https://registry.yarnpkg.com/state-toggle/-/state-toggle-1.0.1.tgz#c3cb0974f40a6a0f8e905b96789eb41afa1cde3a" - -strict-uri-encode@^1.0.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/strict-uri-encode/-/strict-uri-encode-1.1.0.tgz#279b225df1d582b1f54e65addd4352e18faa0713" - -string-width@^1.0.0, string-width@^1.0.1: - version "1.0.2" - resolved "https://registry.yarnpkg.com/string-width/-/string-width-1.0.2.tgz#118bdf5b8cdc51a2a7e70d211e07e2b0b9b107d3" - dependencies: - code-point-at "^1.0.0" - is-fullwidth-code-point "^1.0.0" - strip-ansi "^3.0.0" - -"string-width@^1.0.2 || 2", string-width@^2.0.0: - version "2.1.1" - resolved "https://registry.yarnpkg.com/string-width/-/string-width-2.1.1.tgz#ab93f27a8dc13d28cac815c462143a6d9012ae9e" - dependencies: - is-fullwidth-code-point "^2.0.0" - strip-ansi "^4.0.0" - -string.prototype.padstart@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/string.prototype.padstart/-/string.prototype.padstart-3.0.0.tgz#5bcfad39f4649bb2d031292e19bcf0b510d4b242" - dependencies: - define-properties "^1.1.2" - es-abstract "^1.4.3" - function-bind "^1.0.2" - -string_decoder@~1.1.1: - version "1.1.1" - resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.1.1.tgz#9cf1611ba62685d7030ae9e4ba34149c3af03fc8" - dependencies: - safe-buffer "~5.1.0" - -stringify-entities@^1.0.1: - version "1.3.2" - resolved "https://registry.yarnpkg.com/stringify-entities/-/stringify-entities-1.3.2.tgz#a98417e5471fd227b3e45d3db1861c11caf668f7" - dependencies: - character-entities-html4 "^1.0.0" - character-entities-legacy "^1.0.0" - is-alphanumerical "^1.0.0" - is-hexadecimal "^1.0.0" - -strip-ansi@^3.0.0, strip-ansi@^3.0.1: - version "3.0.1" - resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-3.0.1.tgz#6a385fb8853d952d5ff05d0e8aaf94278dc63dcf" - dependencies: - ansi-regex "^2.0.0" - -strip-ansi@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-4.0.0.tgz#a8479022eb1ac368a871389b635262c505ee368f" - dependencies: - ansi-regex "^3.0.0" - -strip-bom@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/strip-bom/-/strip-bom-2.0.0.tgz#6219a85616520491f35788bdbf1447a99c7e6b0e" - dependencies: - is-utf8 "^0.2.0" - -strip-bom@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/strip-bom/-/strip-bom-3.0.0.tgz#2334c18e9c759f7bdd56fdef7e9ae3d588e68ed3" - -strip-json-comments@~2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-2.0.1.tgz#3c531942e908c2697c0ec344858c286c7ca0a60a" - -structured-source@^3.0.2: - version "3.0.2" - resolved "https://registry.yarnpkg.com/structured-source/-/structured-source-3.0.2.tgz#dd802425e0f53dc4a6e7aca3752901a1ccda7af5" - dependencies: - boundary "^1.0.1" - -supports-color@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-2.0.0.tgz#535d045ce6b6363fa40117084629995e9df324c7" - -supports-color@^4.1.0: - version "4.5.0" - resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-4.5.0.tgz#be7a0de484dec5c5cddf8b3d59125044912f635b" - dependencies: - has-flag "^2.0.0" - -supports-color@^5.3.0: - version "5.4.0" - resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-5.4.0.tgz#1c6b337402c2137605efe19f10fec390f6faab54" - dependencies: - has-flag "^3.0.0" - -table@^3.7.8: - version "3.8.3" - resolved "https://registry.yarnpkg.com/table/-/table-3.8.3.tgz#2bbc542f0fda9861a755d3947fefd8b3f513855f" - dependencies: - ajv "^4.7.0" - ajv-keywords "^1.0.0" - chalk "^1.1.1" - lodash "^4.0.0" - slice-ansi "0.0.4" - string-width "^2.0.0" - -tar@^4: - version "4.4.6" - resolved "https://registry.yarnpkg.com/tar/-/tar-4.4.6.tgz#63110f09c00b4e60ac8bcfe1bf3c8660235fbc9b" - dependencies: - chownr "^1.0.1" - fs-minipass "^1.2.5" - minipass "^2.3.3" - minizlib "^1.1.0" - mkdirp "^0.5.0" - safe-buffer "^5.1.2" - yallist "^3.0.2" - -text-table@^0.2.0: - version "0.2.0" - resolved "https://registry.yarnpkg.com/text-table/-/text-table-0.2.0.tgz#7f5ee823ae805207c00af2df4a84ec3fcfa570b4" - -textlint-rule-helper@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/textlint-rule-helper/-/textlint-rule-helper-2.0.0.tgz#95cb4696c95c4258d2e3389e9e64b849f9721382" - dependencies: - unist-util-visit "^1.1.0" - -textlint-rule-stop-words@^1.0.3: - version "1.0.3" - resolved "https://registry.yarnpkg.com/textlint-rule-stop-words/-/textlint-rule-stop-words-1.0.3.tgz#fe2f40cbe5837331b2a09fdec57cc71758093bf0" - dependencies: - lodash "^4.17.4" - split-lines "^1.1.0" - textlint-rule-helper "^2.0.0" - -textlint@^10.2.1: - version "10.2.1" - resolved "https://registry.yarnpkg.com/textlint/-/textlint-10.2.1.tgz#ee22b7967d59cef7c74a04a5f4e8883134e5c79d" - dependencies: - "@textlint/ast-node-types" "^4.0.2" - "@textlint/ast-traverse" "^2.0.8" - "@textlint/feature-flag" "^3.0.4" - "@textlint/fixer-formatter" "^3.0.7" - "@textlint/kernel" "^2.0.9" - "@textlint/linter-formatter" "^3.0.7" - "@textlint/textlint-plugin-markdown" "^4.0.10" - "@textlint/textlint-plugin-text" "^3.0.10" - "@types/bluebird" "^3.5.18" - bluebird "^3.0.5" - debug "^2.1.0" - deep-equal "^1.0.1" - file-entry-cache "^2.0.0" - get-stdin "^5.0.1" - glob "^7.1.1" - interop-require "^1.0.0" - is-file "^1.0.0" - log-symbols "^1.0.2" - map-like "^2.0.0" - md5 "^2.2.1" - mkdirp "^0.5.0" - object-assign "^4.0.1" - optionator "^0.8.0" - path-to-glob-pattern "^1.0.2" - rc-config-loader "^2.0.1" - read-pkg "^1.1.0" - read-pkg-up "^3.0.0" - structured-source "^3.0.2" - try-resolve "^1.0.1" - unique-concat "^0.2.2" - -timed-out@^4.0.0, timed-out@^4.0.1: - version "4.0.1" - resolved "https://registry.yarnpkg.com/timed-out/-/timed-out-4.0.1.tgz#f32eacac5a175bea25d7fab565ab3ed8741ef56f" - -to-vfile@^2.0.0: - version "2.2.0" - resolved "https://registry.yarnpkg.com/to-vfile/-/to-vfile-2.2.0.tgz#342d1705e6df526d569b1fc8bfa29f1f36d6c416" - dependencies: - is-buffer "^1.1.4" - vfile "^2.0.0" - x-is-function "^1.0.4" - -too-wordy@^0.1.4: - version "0.1.4" - resolved "https://registry.yarnpkg.com/too-wordy/-/too-wordy-0.1.4.tgz#8e7b20a7b7a4d8fc3759f4e00c4929993d1b12f0" - -tough-cookie@~2.3.3: - version "2.3.4" - resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.3.4.tgz#ec60cee38ac675063ffc97a5c18970578ee83655" - dependencies: - punycode "^1.4.1" - -traverse@^0.6.6: - version "0.6.6" - resolved "https://registry.yarnpkg.com/traverse/-/traverse-0.6.6.tgz#cbdf560fd7b9af632502fed40f918c157ea97137" - -trim-trailing-lines@^1.0.0: - version "1.1.1" - resolved "https://registry.yarnpkg.com/trim-trailing-lines/-/trim-trailing-lines-1.1.1.tgz#e0ec0810fd3c3f1730516b45f49083caaf2774d9" - -trim@0.0.1: - version "0.0.1" - resolved "https://registry.yarnpkg.com/trim/-/trim-0.0.1.tgz#5858547f6b290757ee95cccc666fb50084c460dd" - -trough@^1.0.0: - version "1.0.3" - resolved "https://registry.yarnpkg.com/trough/-/trough-1.0.3.tgz#e29bd1614c6458d44869fc28b255ab7857ef7c24" - -try-resolve@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/try-resolve/-/try-resolve-1.0.1.tgz#cfde6fabd72d63e5797cfaab873abbe8e700e912" - -tunnel-agent@^0.6.0: - version "0.6.0" - resolved "https://registry.yarnpkg.com/tunnel-agent/-/tunnel-agent-0.6.0.tgz#27a5dea06b36b04a0a9966774b290868f0fc40fd" - dependencies: - safe-buffer "^5.0.1" - -tweetnacl@^0.14.3, tweetnacl@~0.14.0: - version "0.14.5" - resolved "https://registry.yarnpkg.com/tweetnacl/-/tweetnacl-0.14.5.tgz#5ae68177f192d4456269d108afa93ff8743f4f64" - -type-check@~0.3.2: - version "0.3.2" - resolved "https://registry.yarnpkg.com/type-check/-/type-check-0.3.2.tgz#5884cab512cf1d355e3fb784f30804b2b520db72" - dependencies: - prelude-ls "~1.1.2" - -typedarray@^0.0.6: - version "0.0.6" - resolved "https://registry.yarnpkg.com/typedarray/-/typedarray-0.0.6.tgz#867ac74e3864187b1d3d47d996a78ec5c8830777" - -unherit@^1.0.4: - version "1.1.1" - resolved "https://registry.yarnpkg.com/unherit/-/unherit-1.1.1.tgz#132748da3e88eab767e08fabfbb89c5e9d28628c" - dependencies: - inherits "^2.0.1" - xtend "^4.0.1" - -unified-args@^5.0.0: - version "5.1.0" - resolved "https://registry.yarnpkg.com/unified-args/-/unified-args-5.1.0.tgz#1889200e072998a662e6e84d817d6f4b5f448dd1" - dependencies: - camelcase "^4.0.0" - chalk "^2.0.0" - chokidar "^1.5.1" - json5 "^0.5.1" - minimist "^1.2.0" - text-table "^0.2.0" - unified-engine "^5.1.0" - -unified-engine@^5.1.0: - version "5.1.0" - resolved "https://registry.yarnpkg.com/unified-engine/-/unified-engine-5.1.0.tgz#30db83bcc76c821f773bb5a8a491aa0e2471e3d1" - dependencies: - concat-stream "^1.5.1" - debug "^3.1.0" - fault "^1.0.0" - fn-name "^2.0.1" - glob "^7.0.3" - ignore "^3.2.0" - is-empty "^1.0.0" - is-hidden "^1.0.1" - is-object "^1.0.1" - js-yaml "^3.6.1" - load-plugin "^2.0.0" - parse-json "^4.0.0" - to-vfile "^2.0.0" - trough "^1.0.0" - unist-util-inspect "^4.1.2" - vfile-reporter "^4.0.0" - vfile-statistics "^1.1.0" - x-is-function "^1.0.4" - x-is-string "^0.1.0" - xtend "^4.0.1" - -unified-lint-rule@^1.0.1: - version "1.0.3" - resolved "https://registry.yarnpkg.com/unified-lint-rule/-/unified-lint-rule-1.0.3.tgz#e302b0c4a7ac428c0980e049a500e59528001299" - dependencies: - wrapped "^1.0.1" - -unified@^6.0.0, unified@^6.1.6: - version "6.2.0" - resolved "https://registry.yarnpkg.com/unified/-/unified-6.2.0.tgz#7fbd630f719126d67d40c644b7e3f617035f6dba" - dependencies: - bail "^1.0.0" - extend "^3.0.0" - is-plain-obj "^1.1.0" - trough "^1.0.0" - vfile "^2.0.0" - x-is-string "^0.1.0" - -unique-concat@^0.2.2: - version "0.2.2" - resolved "https://registry.yarnpkg.com/unique-concat/-/unique-concat-0.2.2.tgz#9210f9bdcaacc5e1e3929490d7c019df96f18712" - -unist-util-inspect@^4.1.2: - version "4.1.3" - resolved "https://registry.yarnpkg.com/unist-util-inspect/-/unist-util-inspect-4.1.3.tgz#39470e6d77485db285966df78431219aa1287822" - dependencies: - is-empty "^1.0.0" - -unist-util-is@^2.1.2: - version "2.1.2" - resolved "https://registry.yarnpkg.com/unist-util-is/-/unist-util-is-2.1.2.tgz#1193fa8f2bfbbb82150633f3a8d2eb9a1c1d55db" - -unist-util-modify-children@^1.0.0: - version "1.1.2" - resolved "https://registry.yarnpkg.com/unist-util-modify-children/-/unist-util-modify-children-1.1.2.tgz#c7f1b91712554ee59c47a05b551ed3e052a4e2d1" - dependencies: - array-iterate "^1.0.0" - -unist-util-remove-position@^1.0.0: - version "1.1.2" - resolved "https://registry.yarnpkg.com/unist-util-remove-position/-/unist-util-remove-position-1.1.2.tgz#86b5dad104d0bbfbeb1db5f5c92f3570575c12cb" - dependencies: - unist-util-visit "^1.1.0" - -unist-util-stringify-position@^1.0.0, unist-util-stringify-position@^1.1.1: - version "1.1.2" - resolved "https://registry.yarnpkg.com/unist-util-stringify-position/-/unist-util-stringify-position-1.1.2.tgz#3f37fcf351279dcbca7480ab5889bb8a832ee1c6" - -unist-util-visit-parents@^2.0.0: - version "2.0.1" - resolved "https://registry.yarnpkg.com/unist-util-visit-parents/-/unist-util-visit-parents-2.0.1.tgz#63fffc8929027bee04bfef7d2cce474f71cb6217" - dependencies: - unist-util-is "^2.1.2" - -unist-util-visit@^1.1.0, unist-util-visit@^1.1.1, unist-util-visit@^1.1.3: - version "1.4.0" - resolved "https://registry.yarnpkg.com/unist-util-visit/-/unist-util-visit-1.4.0.tgz#1cb763647186dc26f5e1df5db6bd1e48b3cc2fb1" - dependencies: - unist-util-visit-parents "^2.0.0" - -untildify@^2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/untildify/-/untildify-2.1.0.tgz#17eb2807987f76952e9c0485fc311d06a826a2e0" - dependencies: - os-homedir "^1.0.0" - -unzip-response@^2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/unzip-response/-/unzip-response-2.0.1.tgz#d2f0f737d16b0615e72a6935ed04214572d56f97" - -url-parse-lax@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/url-parse-lax/-/url-parse-lax-1.0.0.tgz#7af8f303645e9bd79a272e7a14ac68bc0609da73" - dependencies: - prepend-http "^1.0.1" - -url-parse-lax@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/url-parse-lax/-/url-parse-lax-3.0.0.tgz#16b5cafc07dbe3676c1b1999177823d6503acb0c" - dependencies: - prepend-http "^2.0.0" - -url-to-options@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/url-to-options/-/url-to-options-1.0.1.tgz#1505a03a289a48cbd7a434efbaeec5055f5633a9" - -util-deprecate@~1.0.1: - version "1.0.2" - resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf" - -uuid@^3.1.0: - version "3.3.2" - resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.3.2.tgz#1b4af4955eb3077c501c23872fc6513811587131" - -validate-npm-package-license@^3.0.1: - version "3.0.4" - resolved "https://registry.yarnpkg.com/validate-npm-package-license/-/validate-npm-package-license-3.0.4.tgz#fc91f6b9c7ba15c857f4cb2c5defeec39d4f410a" - dependencies: - spdx-correct "^3.0.0" - spdx-expression-parse "^3.0.0" - -verror@1.10.0: - version "1.10.0" - resolved "https://registry.yarnpkg.com/verror/-/verror-1.10.0.tgz#3a105ca17053af55d6e270c1f8288682e18da400" - dependencies: - assert-plus "^1.0.0" - core-util-is "1.0.2" - extsprintf "^1.2.0" - -vfile-location@^2.0.0: - version "2.0.3" - resolved "https://registry.yarnpkg.com/vfile-location/-/vfile-location-2.0.3.tgz#083ba80e50968e8d420be49dd1ea9a992131df77" - -vfile-message@^1.0.0: - version "1.0.1" - resolved "https://registry.yarnpkg.com/vfile-message/-/vfile-message-1.0.1.tgz#51a2ccd8a6b97a7980bb34efb9ebde9632e93677" - dependencies: - unist-util-stringify-position "^1.1.1" - -vfile-reporter@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/vfile-reporter/-/vfile-reporter-4.0.0.tgz#ea6f0ae1342f4841573985e05f941736f27de9da" - dependencies: - repeat-string "^1.5.0" - string-width "^1.0.0" - supports-color "^4.1.0" - unist-util-stringify-position "^1.0.0" - vfile-statistics "^1.1.0" - -vfile-statistics@^1.1.0: - version "1.1.1" - resolved "https://registry.yarnpkg.com/vfile-statistics/-/vfile-statistics-1.1.1.tgz#a22fd4eb844c9eaddd781ad3b3246db88375e2e3" - -vfile@^2.0.0: - version "2.3.0" - resolved "https://registry.yarnpkg.com/vfile/-/vfile-2.3.0.tgz#e62d8e72b20e83c324bc6c67278ee272488bf84a" - dependencies: - is-buffer "^1.1.4" - replace-ext "1.0.0" - unist-util-stringify-position "^1.0.0" - vfile-message "^1.0.0" - -weasel-words@^0.1.1: - version "0.1.1" - resolved "https://registry.yarnpkg.com/weasel-words/-/weasel-words-0.1.1.tgz#7137946585c73fe44882013853bd000c5d687a4e" - -wide-align@^1.1.0: - version "1.1.3" - resolved "https://registry.yarnpkg.com/wide-align/-/wide-align-1.1.3.tgz#ae074e6bdc0c14a431e804e624549c633b000457" - dependencies: - string-width "^1.0.2 || 2" - -wordwrap@~1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/wordwrap/-/wordwrap-1.0.0.tgz#27584810891456a4171c8d0226441ade90cbcaeb" - -wrapped@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/wrapped/-/wrapped-1.0.1.tgz#c783d9d807b273e9b01e851680a938c87c907242" - dependencies: - co "3.1.0" - sliced "^1.0.1" - -wrappy@1: - version "1.0.2" - resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f" - -write-good@^0.11.1: - version "0.11.3" - resolved "https://registry.yarnpkg.com/write-good/-/write-good-0.11.3.tgz#8eeb5da9a8e155dafb1325d27eba33cb67d24d8c" - dependencies: - adverb-where "0.0.9" - e-prime "^0.10.2" - no-cliches "^0.1.0" - object.assign "^4.0.4" - passive-voice "^0.1.0" - too-wordy "^0.1.4" - weasel-words "^0.1.1" - -write@^0.2.1: - version "0.2.1" - resolved "https://registry.yarnpkg.com/write/-/write-0.2.1.tgz#5fc03828e264cea3fe91455476f7a3c566cb0757" - dependencies: - mkdirp "^0.5.1" - -x-is-function@^1.0.4: - version "1.0.4" - resolved "https://registry.yarnpkg.com/x-is-function/-/x-is-function-1.0.4.tgz#5d294dc3d268cbdd062580e0c5df77a391d1fa1e" - -x-is-string@^0.1.0: - version "0.1.0" - resolved "https://registry.yarnpkg.com/x-is-string/-/x-is-string-0.1.0.tgz#474b50865af3a49a9c4657f05acd145458f77d82" - -xml-escape@^1.0.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/xml-escape/-/xml-escape-1.1.0.tgz#3904c143fa8eb3a0030ec646d2902a2f1b706c44" - -xtend@^4.0.1: - version "4.0.1" - resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.1.tgz#a5c6d532be656e23db820efb943a1f04998d63af" - -yallist@^3.0.0, yallist@^3.0.2: - version "3.0.2" - resolved "https://registry.yarnpkg.com/yallist/-/yallist-3.0.2.tgz#8452b4bb7e83c7c188d8041c1a837c773d6d8bb9" diff --git a/evidence/pool.go b/evidence/pool.go index 0f3d482a..b5fdbdf1 100644 --- a/evidence/pool.go +++ b/evidence/pool.go @@ -57,10 +57,10 @@ func (evpool *EvidencePool) PriorityEvidence() []types.Evidence { return evpool.evidenceStore.PriorityEvidence() } -// PendingEvidence returns uncommitted evidence up to maxBytes. -// If maxBytes is -1, all evidence is returned. -func (evpool *EvidencePool) PendingEvidence(maxBytes int64) []types.Evidence { - return evpool.evidenceStore.PendingEvidence(maxBytes) +// PendingEvidence returns up to maxNum uncommitted evidence. +// If maxNum is -1, all evidence is returned. +func (evpool *EvidencePool) PendingEvidence(maxNum int64) []types.Evidence { + return evpool.evidenceStore.PendingEvidence(maxNum) } // State returns the current state of the evpool. @@ -127,7 +127,7 @@ func (evpool *EvidencePool) MarkEvidenceAsCommitted(height int64, evidence []typ } // remove committed evidence from the clist - maxAge := evpool.State().ConsensusParams.EvidenceParams.MaxAge + maxAge := evpool.State().ConsensusParams.Evidence.MaxAge evpool.removeEvidence(height, maxAge, blockEvidenceMap) } diff --git a/evidence/pool_test.go b/evidence/pool_test.go index c3ed569e..1f4f1a06 100644 --- a/evidence/pool_test.go +++ b/evidence/pool_test.go @@ -13,8 +13,6 @@ import ( tmtime "github.com/tendermint/tendermint/types/time" ) -var mockState = sm.State{} - func TestMain(m *testing.M) { types.RegisterMockEvidences(cdc) @@ -35,10 +33,10 @@ func initializeValidatorState(valAddr []byte, height int64) dbm.DB { LastBlockHeight: 0, LastBlockTime: tmtime.Now(), Validators: valSet, - NextValidators: valSet.CopyIncrementAccum(1), + NextValidators: valSet.CopyIncrementProposerPriority(1), LastHeightValidatorsChanged: 1, ConsensusParams: types.ConsensusParams{ - EvidenceParams: types.EvidenceParams{ + Evidence: types.EvidenceParams{ MaxAge: 1000000, }, }, diff --git a/evidence/reactor.go b/evidence/reactor.go index cfe47364..bbbab3e9 100644 --- a/evidence/reactor.go +++ b/evidence/reactor.go @@ -48,7 +48,7 @@ func (evR *EvidenceReactor) SetLogger(l log.Logger) { // It returns the list of channels for this reactor. func (evR *EvidenceReactor) GetChannels() []*p2p.ChannelDescriptor { return []*p2p.ChannelDescriptor{ - &p2p.ChannelDescriptor{ + { ID: EvidenceChannel, Priority: 5, }, @@ -74,6 +74,13 @@ func (evR *EvidenceReactor) Receive(chID byte, src p2p.Peer, msgBytes []byte) { evR.Switch.StopPeerForError(src, err) return } + + if err = msg.ValidateBasic(); err != nil { + evR.Logger.Error("Peer sent us invalid msg", "peer", src, "msg", msg, "err", err) + evR.Switch.StopPeerForError(src, err) + return + } + evR.Logger.Debug("Receive", "src", src, "chId", chID, "msg", msg) switch msg := msg.(type) { @@ -153,18 +160,21 @@ func (evR *EvidenceReactor) broadcastEvidenceRoutine(peer p2p.Peer) { // Returns the message to send the peer, or nil if the evidence is invalid for the peer. // If message is nil, return true if we should sleep and try again. func (evR EvidenceReactor) checkSendEvidenceMessage(peer p2p.Peer, ev types.Evidence) (msg EvidenceMessage, retry bool) { - // make sure the peer is up to date evHeight := ev.Height() peerState, ok := peer.Get(types.PeerStateKey).(PeerState) if !ok { - evR.Logger.Info("Found peer without PeerState", "peer", peer) + // Peer does not have a state yet. We set it in the consensus reactor, but + // when we add peer in Switch, the order we call reactors#AddPeer is + // different every time due to us using a map. Sometimes other reactors + // will be initialized before the consensus reactor. We should wait a few + // milliseconds and retry. return nil, true } // NOTE: We only send evidence to peers where // peerHeight - maxAge < evidenceHeight < peerHeight - maxAge := evR.evpool.State().ConsensusParams.EvidenceParams.MaxAge + maxAge := evR.evpool.State().ConsensusParams.Evidence.MaxAge peerHeight := peerState.GetHeight() if peerHeight < evHeight { // peer is behind. sleep while he catches up @@ -191,7 +201,9 @@ type PeerState interface { // Messages // EvidenceMessage is a message sent or received by the EvidenceReactor. -type EvidenceMessage interface{} +type EvidenceMessage interface { + ValidateBasic() error +} func RegisterEvidenceMessages(cdc *amino.Codec) { cdc.RegisterInterface((*EvidenceMessage)(nil), nil) @@ -209,11 +221,21 @@ func decodeMsg(bz []byte) (msg EvidenceMessage, err error) { //------------------------------------- -// EvidenceMessage contains a list of evidence. +// EvidenceListMessage contains a list of evidence. type EvidenceListMessage struct { Evidence []types.Evidence } +// ValidateBasic performs basic validation. +func (m *EvidenceListMessage) ValidateBasic() error { + for i, ev := range m.Evidence { + if err := ev.ValidateBasic(); err != nil { + return fmt.Errorf("Invalid evidence (#%d): %v", i, err) + } + } + return nil +} + // String returns a string representation of the EvidenceListMessage. func (m *EvidenceListMessage) String() string { return fmt.Sprintf("[EvidenceListMessage %v]", m.Evidence) diff --git a/evidence/reactor_test.go b/evidence/reactor_test.go index ea9657d2..1c4e731a 100644 --- a/evidence/reactor_test.go +++ b/evidence/reactor_test.go @@ -10,6 +10,7 @@ import ( "github.com/stretchr/testify/assert" cfg "github.com/tendermint/tendermint/config" + "github.com/tendermint/tendermint/crypto/secp256k1" dbm "github.com/tendermint/tendermint/libs/db" "github.com/tendermint/tendermint/libs/log" "github.com/tendermint/tendermint/p2p" @@ -164,6 +165,16 @@ func TestReactorSelectiveBroadcast(t *testing.T) { // make reactors from statedb reactors := makeAndConnectEvidenceReactors(config, []dbm.DB{stateDB1, stateDB2}) + + // set the peer height on each reactor + for _, r := range reactors { + for _, peer := range r.Switch.Peers().List() { + ps := peerState{height1} + peer.Set(types.PeerStateKey, ps) + } + } + + // update the first reactor peer's height to be very small peer := reactors[0].Switch.Peers().List()[0] ps := peerState{height2} peer.Set(types.PeerStateKey, ps) @@ -178,3 +189,30 @@ func TestReactorSelectiveBroadcast(t *testing.T) { peers := reactors[1].Switch.Peers().List() assert.Equal(t, 1, len(peers)) } +func TestEvidenceListMessageValidationBasic(t *testing.T) { + + testCases := []struct { + testName string + malleateEvListMsg func(*EvidenceListMessage) + expectErr bool + }{ + {"Good EvidenceListMessage", func(evList *EvidenceListMessage) {}, false}, + {"Invalid EvidenceListMessage", func(evList *EvidenceListMessage) { + evList.Evidence = append(evList.Evidence, + &types.DuplicateVoteEvidence{PubKey: secp256k1.GenPrivKey().PubKey()}) + }, true}, + } + for _, tc := range testCases { + t.Run(tc.testName, func(t *testing.T) { + evListMsg := &EvidenceListMessage{} + n := 3 + valAddr := []byte("myval") + evListMsg.Evidence = make([]types.Evidence, n) + for i := 0; i < n; i++ { + evListMsg.Evidence[i] = types.NewMockGoodEvidence(int64(i+1), 0, valAddr) + } + tc.malleateEvListMsg(evListMsg) + assert.Equal(t, tc.expectErr, evListMsg.ValidateBasic() != nil, "Validate Basic had an unexpected result") + }) + } +} diff --git a/evidence/store.go b/evidence/store.go index ccfd2d48..17b37aab 100644 --- a/evidence/store.go +++ b/evidence/store.go @@ -86,26 +86,26 @@ func (store *EvidenceStore) PriorityEvidence() (evidence []types.Evidence) { return l } -// PendingEvidence returns known uncommitted evidence up to maxBytes. -// If maxBytes is -1, all evidence is returned. -func (store *EvidenceStore) PendingEvidence(maxBytes int64) (evidence []types.Evidence) { - return store.listEvidence(baseKeyPending, maxBytes) +// PendingEvidence returns up to maxNum known, uncommitted evidence. +// If maxNum is -1, all evidence is returned. +func (store *EvidenceStore) PendingEvidence(maxNum int64) (evidence []types.Evidence) { + return store.listEvidence(baseKeyPending, maxNum) } -// listEvidence lists the evidence for the given prefix key up to maxBytes. +// listEvidence lists up to maxNum pieces of evidence for the given prefix key. // It is wrapped by PriorityEvidence and PendingEvidence for convenience. -// If maxBytes is -1, there's no cap on the size of returned evidence. -func (store *EvidenceStore) listEvidence(prefixKey string, maxBytes int64) (evidence []types.Evidence) { - var bytes int64 +// If maxNum is -1, there's no cap on the size of returned evidence. +func (store *EvidenceStore) listEvidence(prefixKey string, maxNum int64) (evidence []types.Evidence) { + var count int64 iter := dbm.IteratePrefix(store.db, []byte(prefixKey)) defer iter.Close() for ; iter.Valid(); iter.Next() { val := iter.Value() - if maxBytes > 0 && bytes+int64(len(val)) > maxBytes { + if count == maxNum { return evidence } - bytes += int64(len(val)) + count++ var ei EvidenceInfo err := cdc.UnmarshalBinaryBare(val, &ei) diff --git a/evidence/wire.go b/evidence/wire.go index 73ff33b2..6752bc3e 100644 --- a/evidence/wire.go +++ b/evidence/wire.go @@ -1,7 +1,7 @@ package evidence import ( - "github.com/tendermint/go-amino" + amino "github.com/tendermint/go-amino" cryptoAmino "github.com/tendermint/tendermint/crypto/encoding/amino" "github.com/tendermint/tendermint/types" ) @@ -13,3 +13,8 @@ func init() { cryptoAmino.RegisterAmino(cdc) types.RegisterEvidences(cdc) } + +// For testing purposes only +func RegisterMockEvidences() { + types.RegisterMockEvidences(cdc) +} diff --git a/libs/autofile/autofile.go b/libs/autofile/autofile.go index 6822545e..e428e26c 100644 --- a/libs/autofile/autofile.go +++ b/libs/autofile/autofile.go @@ -8,7 +8,6 @@ import ( "time" cmn "github.com/tendermint/tendermint/libs/common" - "github.com/tendermint/tendermint/libs/errors" ) /* AutoFile usage @@ -83,6 +82,8 @@ func OpenAutoFile(path string) (*AutoFile, error) { return af, nil } +// Close shuts down the closing goroutine, SIGHUP handler and closes the +// AutoFile. func (af *AutoFile) Close() error { af.closeTicker.Stop() close(af.closeTickerStopc) @@ -116,6 +117,10 @@ func (af *AutoFile) closeFile() (err error) { return file.Close() } +// Write writes len(b) bytes to the AutoFile. It returns the number of bytes +// written and an error, if any. Write returns a non-nil error when n != +// len(b). +// Opens AutoFile if needed. func (af *AutoFile) Write(b []byte) (n int, err error) { af.mtx.Lock() defer af.mtx.Unlock() @@ -130,6 +135,10 @@ func (af *AutoFile) Write(b []byte) (n int, err error) { return } +// Sync commits the current contents of the file to stable storage. Typically, +// this means flushing the file system's in-memory copy of recently written +// data to disk. +// Opens AutoFile if needed. func (af *AutoFile) Sync() error { af.mtx.Lock() defer af.mtx.Unlock() @@ -147,34 +156,33 @@ func (af *AutoFile) openFile() error { if err != nil { return err } - fileInfo, err := file.Stat() - if err != nil { - return err - } - if fileInfo.Mode() != autoFilePerms { - return errors.NewErrPermissionsChanged(file.Name(), fileInfo.Mode(), autoFilePerms) - } + // fileInfo, err := file.Stat() + // if err != nil { + // return err + // } + // if fileInfo.Mode() != autoFilePerms { + // return errors.NewErrPermissionsChanged(file.Name(), fileInfo.Mode(), autoFilePerms) + // } af.file = file return nil } +// Size returns the size of the AutoFile. It returns -1 and an error if fails +// get stats or open file. +// Opens AutoFile if needed. func (af *AutoFile) Size() (int64, error) { af.mtx.Lock() defer af.mtx.Unlock() if af.file == nil { - err := af.openFile() - if err != nil { - if err == os.ErrNotExist { - return 0, nil - } + if err := af.openFile(); err != nil { return -1, err } } + stat, err := af.file.Stat() if err != nil { return -1, err } return stat.Size(), nil - } diff --git a/libs/autofile/autofile_test.go b/libs/autofile/autofile_test.go index 5408d820..d9c90309 100644 --- a/libs/autofile/autofile_test.go +++ b/libs/autofile/autofile_test.go @@ -10,7 +10,6 @@ import ( "github.com/stretchr/testify/require" cmn "github.com/tendermint/tendermint/libs/common" - "github.com/tendermint/tendermint/libs/errors" ) func TestSIGHUP(t *testing.T) { @@ -58,29 +57,66 @@ func TestSIGHUP(t *testing.T) { } } -// Manually modify file permissions, close, and reopen using autofile: -// We expect the file permissions to be changed back to the intended perms. -func TestOpenAutoFilePerms(t *testing.T) { - file, err := ioutil.TempFile("", "permission_test") - require.NoError(t, err) - err = file.Close() - require.NoError(t, err) - name := file.Name() +// // Manually modify file permissions, close, and reopen using autofile: +// // We expect the file permissions to be changed back to the intended perms. +// func TestOpenAutoFilePerms(t *testing.T) { +// file, err := ioutil.TempFile("", "permission_test") +// require.NoError(t, err) +// err = file.Close() +// require.NoError(t, err) +// name := file.Name() - // open and change permissions - af, err := OpenAutoFile(name) +// // open and change permissions +// af, err := OpenAutoFile(name) +// require.NoError(t, err) +// err = af.file.Chmod(0755) +// require.NoError(t, err) +// err = af.Close() +// require.NoError(t, err) + +// // reopen and expect an ErrPermissionsChanged as Cause +// af, err = OpenAutoFile(name) +// require.Error(t, err) +// if e, ok := err.(*errors.ErrPermissionsChanged); ok { +// t.Logf("%v", e) +// } else { +// t.Errorf("unexpected error %v", e) +// } +// } + +func TestAutoFileSize(t *testing.T) { + // First, create an AutoFile writing to a tempfile dir + f, err := ioutil.TempFile("", "sighup_test") require.NoError(t, err) - err = af.file.Chmod(0755) + err = f.Close() require.NoError(t, err) + + // Here is the actual AutoFile. + af, err := OpenAutoFile(f.Name()) + require.NoError(t, err) + + // 1. Empty file + size, err := af.Size() + require.Zero(t, size) + require.NoError(t, err) + + // 2. Not empty file + data := []byte("Maniac\n") + _, err = af.Write(data) + require.NoError(t, err) + size, err = af.Size() + require.EqualValues(t, len(data), size) + require.NoError(t, err) + + // 3. Not existing file err = af.Close() require.NoError(t, err) + err = os.Remove(f.Name()) + require.NoError(t, err) + size, err = af.Size() + require.EqualValues(t, 0, size, "Expected a new file to be empty") + require.NoError(t, err) - // reopen and expect an ErrPermissionsChanged as Cause - af, err = OpenAutoFile(name) - require.Error(t, err) - if e, ok := err.(*errors.ErrPermissionsChanged); ok { - t.Logf("%v", e) - } else { - t.Errorf("unexpected error %v", e) - } + // Cleanup + _ = os.Remove(f.Name()) } diff --git a/libs/autofile/group.go b/libs/autofile/group.go index ea272b61..7e926946 100644 --- a/libs/autofile/group.go +++ b/libs/autofile/group.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "io" - "log" "os" "path" "path/filepath" @@ -68,6 +67,11 @@ type Group struct { minIndex int // Includes head maxIndex int // Includes head, where Head will move to + // close this when the processTicks routine is done. + // this ensures we can cleanup the dir after calling Stop + // and the routine won't be trying to access it anymore + doneProcessTicks chan struct{} + // TODO: When we start deleting files, we need to start tracking GroupReaders // and their dependencies. } @@ -91,6 +95,7 @@ func OpenGroup(headPath string, groupOptions ...func(*Group)) (g *Group, err err groupCheckDuration: defaultGroupCheckDuration, minIndex: 0, maxIndex: 0, + doneProcessTicks: make(chan struct{}), } for _, option := range groupOptions { @@ -141,6 +146,11 @@ func (g *Group) OnStop() { g.Flush() // flush any uncommitted data } +func (g *Group) Wait() { + // wait for processTicks routine to finish + <-g.doneProcessTicks +} + // Close closes the head file. The group must be stopped by this moment. func (g *Group) Close() { g.Flush() // flush any uncommitted data @@ -212,6 +222,7 @@ func (g *Group) Flush() error { } func (g *Group) processTicks() { + defer close(g.doneProcessTicks) for { select { case <-g.ticker.C: @@ -231,7 +242,8 @@ func (g *Group) checkHeadSizeLimit() { } size, err := g.Head.Size() if err != nil { - panic(err) + g.Logger.Error("Group's head may grow without bound", "head", g.Head.Path, "err", err) + return } if size >= limit { g.RotateFile() @@ -253,21 +265,21 @@ func (g *Group) checkTotalSizeLimit() { } if index == gInfo.MaxIndex { // Special degenerate case, just do nothing. - log.Println("WARNING: Group's head " + g.Head.Path + "may grow without bound") + g.Logger.Error("Group's head may grow without bound", "head", g.Head.Path) return } pathToRemove := filePathForIndex(g.Head.Path, index, gInfo.MaxIndex) - fileInfo, err := os.Stat(pathToRemove) + fInfo, err := os.Stat(pathToRemove) if err != nil { - log.Println("WARNING: Failed to fetch info for file @" + pathToRemove) + g.Logger.Error("Failed to fetch info for file", "file", pathToRemove) continue } err = os.Remove(pathToRemove) if err != nil { - log.Println(err) + g.Logger.Error("Failed to remove path", "path", pathToRemove) return } - totalSize -= fileInfo.Size() + totalSize -= fInfo.Size() } } diff --git a/libs/cli/flags/log_level_test.go b/libs/cli/flags/log_level_test.go index 1503ec28..c4c1707b 100644 --- a/libs/cli/flags/log_level_test.go +++ b/libs/cli/flags/log_level_test.go @@ -51,7 +51,7 @@ func TestParseLogLevel(t *testing.T) { buf.Reset() - logger.With("module", "wire").Debug("Kingpin") + logger.With("module", "mempool").With("module", "wire").Debug("Kingpin") if have := strings.TrimSpace(buf.String()); c.expectedLogLines[0] != have { t.Errorf("\nwant '%s'\nhave '%s'\nlevel '%s'", c.expectedLogLines[0], have, c.lvl) } diff --git a/libs/clist/clist_test.go b/libs/clist/clist_test.go index 4ded6177..13aca357 100644 --- a/libs/clist/clist_test.go +++ b/libs/clist/clist_test.go @@ -65,12 +65,13 @@ func TestSmall(t *testing.T) { } -/* -This test is quite hacky because it relies on SetFinalizer -which isn't guaranteed to run at all. -*/ -// nolint: megacheck +// This test is quite hacky because it relies on SetFinalizer +// which isn't guaranteed to run at all. +//nolint:unused,deadcode func _TestGCFifo(t *testing.T) { + if runtime.GOARCH != "amd64" { + t.Skipf("Skipping on non-amd64 machine") + } const numElements = 1000000 l := New() @@ -113,12 +114,13 @@ func _TestGCFifo(t *testing.T) { } } -/* -This test is quite hacky because it relies on SetFinalizer -which isn't guaranteed to run at all. -*/ -// nolint: megacheck +// This test is quite hacky because it relies on SetFinalizer +// which isn't guaranteed to run at all. +//nolint:unused,deadcode func _TestGCRandom(t *testing.T) { + if runtime.GOARCH != "amd64" { + t.Skipf("Skipping on non-amd64 machine") + } const numElements = 1000000 l := New() diff --git a/libs/common/colors.go b/libs/common/colors.go index 4837f97b..89dda2c9 100644 --- a/libs/common/colors.go +++ b/libs/common/colors.go @@ -43,7 +43,7 @@ func treat(s string, color string) string { } func treatAll(color string, args ...interface{}) string { - var parts []string + parts := make([]string, 0, len(args)) for _, arg := range args { parts = append(parts, treat(fmt.Sprintf("%v", arg), color)) } diff --git a/libs/common/errors.go b/libs/common/errors.go index 1dc909e8..10e40ebd 100644 --- a/libs/common/errors.go +++ b/libs/common/errors.go @@ -146,7 +146,7 @@ func (err *cmnError) Format(s fmt.State, verb rune) { s.Write([]byte("--= /Error =--\n")) } else { // Write msg. - s.Write([]byte(fmt.Sprintf("Error{%v}", err.data))) // TODO tick-esc? + s.Write([]byte(fmt.Sprintf("%v", err.data))) } } } diff --git a/libs/common/errors_test.go b/libs/common/errors_test.go index 52c78a76..b85936dd 100644 --- a/libs/common/errors_test.go +++ b/libs/common/errors_test.go @@ -24,7 +24,7 @@ func TestErrorPanic(t *testing.T) { var err = capturePanic() assert.Equal(t, pnk{"something"}, err.Data()) - assert.Equal(t, "Error{{something}}", fmt.Sprintf("%v", err)) + assert.Equal(t, "{something}", fmt.Sprintf("%v", err)) assert.Contains(t, fmt.Sprintf("%#v", err), "This is the message in ErrorWrap(r, message).") assert.Contains(t, fmt.Sprintf("%#v", err), "Stack Trace:\n 0") } @@ -34,7 +34,7 @@ func TestErrorWrapSomething(t *testing.T) { var err = ErrorWrap("something", "formatter%v%v", 0, 1) assert.Equal(t, "something", err.Data()) - assert.Equal(t, "Error{something}", fmt.Sprintf("%v", err)) + assert.Equal(t, "something", fmt.Sprintf("%v", err)) assert.Regexp(t, `formatter01\n`, fmt.Sprintf("%#v", err)) assert.Contains(t, fmt.Sprintf("%#v", err), "Stack Trace:\n 0") } @@ -46,7 +46,7 @@ func TestErrorWrapNothing(t *testing.T) { assert.Equal(t, FmtError{"formatter%v%v", []interface{}{0, 1}}, err.Data()) - assert.Equal(t, "Error{formatter01}", fmt.Sprintf("%v", err)) + assert.Equal(t, "formatter01", fmt.Sprintf("%v", err)) assert.Contains(t, fmt.Sprintf("%#v", err), `Data: common.FmtError{format:"formatter%v%v", args:[]interface {}{0, 1}}`) assert.Contains(t, fmt.Sprintf("%#v", err), "Stack Trace:\n 0") } @@ -58,7 +58,7 @@ func TestErrorNewError(t *testing.T) { assert.Equal(t, FmtError{"formatter%v%v", []interface{}{0, 1}}, err.Data()) - assert.Equal(t, "Error{formatter01}", fmt.Sprintf("%v", err)) + assert.Equal(t, "formatter01", fmt.Sprintf("%v", err)) assert.Contains(t, fmt.Sprintf("%#v", err), `Data: common.FmtError{format:"formatter%v%v", args:[]interface {}{0, 1}}`) assert.NotContains(t, fmt.Sprintf("%#v", err), "Stack Trace") } @@ -70,7 +70,7 @@ func TestErrorNewErrorWithStacktrace(t *testing.T) { assert.Equal(t, FmtError{"formatter%v%v", []interface{}{0, 1}}, err.Data()) - assert.Equal(t, "Error{formatter01}", fmt.Sprintf("%v", err)) + assert.Equal(t, "formatter01", fmt.Sprintf("%v", err)) assert.Contains(t, fmt.Sprintf("%#v", err), `Data: common.FmtError{format:"formatter%v%v", args:[]interface {}{0, 1}}`) assert.Contains(t, fmt.Sprintf("%#v", err), "Stack Trace:\n 0") } @@ -85,7 +85,7 @@ func TestErrorNewErrorWithTrace(t *testing.T) { assert.Equal(t, FmtError{"formatter%v%v", []interface{}{0, 1}}, err.Data()) - assert.Equal(t, "Error{formatter01}", fmt.Sprintf("%v", err)) + assert.Equal(t, "formatter01", fmt.Sprintf("%v", err)) assert.Contains(t, fmt.Sprintf("%#v", err), `Data: common.FmtError{format:"formatter%v%v", args:[]interface {}{0, 1}}`) dump := fmt.Sprintf("%#v", err) assert.NotContains(t, dump, "Stack Trace") diff --git a/libs/common/net.go b/libs/common/net.go index bdbe38f7..c7fff4cc 100644 --- a/libs/common/net.go +++ b/libs/common/net.go @@ -24,3 +24,20 @@ func ProtocolAndAddress(listenAddr string) (string, string) { } return protocol, address } + +// GetFreePort gets a free port from the operating system. +// Ripped from https://github.com/phayes/freeport. +// BSD-licensed. +func GetFreePort() (int, error) { + addr, err := net.ResolveTCPAddr("tcp", "localhost:0") + if err != nil { + return 0, err + } + + l, err := net.ListenTCP("tcp", addr) + if err != nil { + return 0, err + } + defer l.Close() + return l.Addr().(*net.TCPAddr).Port, nil +} diff --git a/libs/common/string.go b/libs/common/string.go index e125043d..ddf350b1 100644 --- a/libs/common/string.go +++ b/libs/common/string.go @@ -61,3 +61,16 @@ func ASCIITrim(s string) string { } return string(r) } + +// StringSliceEqual checks if string slices a and b are equal +func StringSliceEqual(a, b []string) bool { + if len(a) != len(b) { + return false + } + for i := 0; i < len(a); i++ { + if a[i] != b[i] { + return false + } + } + return true +} diff --git a/libs/common/string_test.go b/libs/common/string_test.go index 5e7ae98c..35b6fafc 100644 --- a/libs/common/string_test.go +++ b/libs/common/string_test.go @@ -3,6 +3,8 @@ package common import ( "testing" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/assert" ) @@ -35,3 +37,22 @@ func TestASCIITrim(t *testing.T) { assert.Equal(t, ASCIITrim(" a "), "a") assert.Panics(t, func() { ASCIITrim("\xC2\xA2") }) } + +func TestStringSliceEqual(t *testing.T) { + tests := []struct { + a []string + b []string + want bool + }{ + {[]string{"hello", "world"}, []string{"hello", "world"}, true}, + {[]string{"test"}, []string{"test"}, true}, + {[]string{"test1"}, []string{"test2"}, false}, + {[]string{"hello", "world."}, []string{"hello", "world!"}, false}, + {[]string{"only 1 word"}, []string{"two", "words!"}, false}, + {[]string{"two", "words!"}, []string{"only 1 word"}, false}, + } + for i, tt := range tests { + require.Equal(t, tt.want, StringSliceEqual(tt.a, tt.b), + "StringSliceEqual failed on test %d", i) + } +} diff --git a/libs/db/backend_test.go b/libs/db/backend_test.go index 2aebde1c..fb2a3d0b 100644 --- a/libs/db/backend_test.go +++ b/libs/db/backend_test.go @@ -180,13 +180,13 @@ func testDBIterator(t *testing.T, backend DBBackendType) { verifyIterator(t, db.ReverseIterator(nil, nil), []int64{9, 8, 7, 5, 4, 3, 2, 1, 0}, "reverse iterator") verifyIterator(t, db.Iterator(nil, int642Bytes(0)), []int64(nil), "forward iterator to 0") - verifyIterator(t, db.ReverseIterator(nil, int642Bytes(10)), []int64(nil), "reverse iterator 10") + verifyIterator(t, db.ReverseIterator(int642Bytes(10), nil), []int64(nil), "reverse iterator from 10 (ex)") verifyIterator(t, db.Iterator(int642Bytes(0), nil), []int64{0, 1, 2, 3, 4, 5, 7, 8, 9}, "forward iterator from 0") verifyIterator(t, db.Iterator(int642Bytes(1), nil), []int64{1, 2, 3, 4, 5, 7, 8, 9}, "forward iterator from 1") - verifyIterator(t, db.ReverseIterator(int642Bytes(10), nil), []int64{9, 8, 7, 5, 4, 3, 2, 1, 0}, "reverse iterator from 10") - verifyIterator(t, db.ReverseIterator(int642Bytes(9), nil), []int64{9, 8, 7, 5, 4, 3, 2, 1, 0}, "reverse iterator from 9") - verifyIterator(t, db.ReverseIterator(int642Bytes(8), nil), []int64{8, 7, 5, 4, 3, 2, 1, 0}, "reverse iterator from 8") + verifyIterator(t, db.ReverseIterator(nil, int642Bytes(10)), []int64{9, 8, 7, 5, 4, 3, 2, 1, 0}, "reverse iterator from 10 (ex)") + verifyIterator(t, db.ReverseIterator(nil, int642Bytes(9)), []int64{8, 7, 5, 4, 3, 2, 1, 0}, "reverse iterator from 9 (ex)") + verifyIterator(t, db.ReverseIterator(nil, int642Bytes(8)), []int64{7, 5, 4, 3, 2, 1, 0}, "reverse iterator from 8 (ex)") verifyIterator(t, db.Iterator(int642Bytes(5), int642Bytes(6)), []int64{5}, "forward iterator from 5 to 6") verifyIterator(t, db.Iterator(int642Bytes(5), int642Bytes(7)), []int64{5}, "forward iterator from 5 to 7") @@ -195,20 +195,20 @@ func testDBIterator(t *testing.T, backend DBBackendType) { verifyIterator(t, db.Iterator(int642Bytes(6), int642Bytes(8)), []int64{7}, "forward iterator from 6 to 8") verifyIterator(t, db.Iterator(int642Bytes(7), int642Bytes(8)), []int64{7}, "forward iterator from 7 to 8") - verifyIterator(t, db.ReverseIterator(int642Bytes(5), int642Bytes(4)), []int64{5}, "reverse iterator from 5 to 4") - verifyIterator(t, db.ReverseIterator(int642Bytes(6), int642Bytes(4)), []int64{5}, "reverse iterator from 6 to 4") - verifyIterator(t, db.ReverseIterator(int642Bytes(7), int642Bytes(4)), []int64{7, 5}, "reverse iterator from 7 to 4") - verifyIterator(t, db.ReverseIterator(int642Bytes(6), int642Bytes(5)), []int64(nil), "reverse iterator from 6 to 5") - verifyIterator(t, db.ReverseIterator(int642Bytes(7), int642Bytes(5)), []int64{7}, "reverse iterator from 7 to 5") - verifyIterator(t, db.ReverseIterator(int642Bytes(7), int642Bytes(6)), []int64{7}, "reverse iterator from 7 to 6") + verifyIterator(t, db.ReverseIterator(int642Bytes(4), int642Bytes(5)), []int64{4}, "reverse iterator from 5 (ex) to 4") + verifyIterator(t, db.ReverseIterator(int642Bytes(4), int642Bytes(6)), []int64{5, 4}, "reverse iterator from 6 (ex) to 4") + verifyIterator(t, db.ReverseIterator(int642Bytes(4), int642Bytes(7)), []int64{5, 4}, "reverse iterator from 7 (ex) to 4") + verifyIterator(t, db.ReverseIterator(int642Bytes(5), int642Bytes(6)), []int64{5}, "reverse iterator from 6 (ex) to 5") + verifyIterator(t, db.ReverseIterator(int642Bytes(5), int642Bytes(7)), []int64{5}, "reverse iterator from 7 (ex) to 5") + verifyIterator(t, db.ReverseIterator(int642Bytes(6), int642Bytes(7)), []int64(nil), "reverse iterator from 7 (ex) to 6") verifyIterator(t, db.Iterator(int642Bytes(0), int642Bytes(1)), []int64{0}, "forward iterator from 0 to 1") - verifyIterator(t, db.ReverseIterator(int642Bytes(9), int642Bytes(8)), []int64{9}, "reverse iterator from 9 to 8") + verifyIterator(t, db.ReverseIterator(int642Bytes(8), int642Bytes(9)), []int64{8}, "reverse iterator from 9 (ex) to 8") verifyIterator(t, db.Iterator(int642Bytes(2), int642Bytes(4)), []int64{2, 3}, "forward iterator from 2 to 4") verifyIterator(t, db.Iterator(int642Bytes(4), int642Bytes(2)), []int64(nil), "forward iterator from 4 to 2") - verifyIterator(t, db.ReverseIterator(int642Bytes(4), int642Bytes(2)), []int64{4, 3}, "reverse iterator from 4 to 2") - verifyIterator(t, db.ReverseIterator(int642Bytes(2), int642Bytes(4)), []int64(nil), "reverse iterator from 2 to 4") + verifyIterator(t, db.ReverseIterator(int642Bytes(2), int642Bytes(4)), []int64{3, 2}, "reverse iterator from 4 (ex) to 2") + verifyIterator(t, db.ReverseIterator(int642Bytes(4), int642Bytes(2)), []int64(nil), "reverse iterator from 2 (ex) to 4") } diff --git a/libs/db/c_level_db.go b/libs/db/c_level_db.go index 30746126..7f74b2a7 100644 --- a/libs/db/c_level_db.go +++ b/libs/db/c_level_db.go @@ -205,13 +205,13 @@ type cLevelDBIterator struct { func newCLevelDBIterator(source *levigo.Iterator, start, end []byte, isReverse bool) *cLevelDBIterator { if isReverse { - if start == nil { + if end == nil { source.SeekToLast() } else { - source.Seek(start) + source.Seek(end) if source.Valid() { - soakey := source.Key() // start or after key - if bytes.Compare(start, soakey) < 0 { + eoakey := source.Key() // end or after key + if bytes.Compare(end, eoakey) <= 0 { source.Prev() } } else { @@ -255,10 +255,11 @@ func (itr cLevelDBIterator) Valid() bool { } // If key is end or past it, invalid. + var start = itr.start var end = itr.end var key = itr.source.Key() if itr.isReverse { - if end != nil && bytes.Compare(key, end) <= 0 { + if start != nil && bytes.Compare(key, start) < 0 { itr.isInvalid = true return false } diff --git a/libs/db/fsdb.go b/libs/db/fsdb.go index 92c059d4..2d82e774 100644 --- a/libs/db/fsdb.go +++ b/libs/db/fsdb.go @@ -12,7 +12,6 @@ import ( "github.com/pkg/errors" cmn "github.com/tendermint/tendermint/libs/common" - tmerrors "github.com/tendermint/tendermint/libs/errors" ) const ( @@ -162,7 +161,7 @@ func (db *FSDB) MakeIterator(start, end []byte, isReversed bool) Iterator { // We need a copy of all of the keys. // Not the best, but probably not a bottleneck depending. - keys, err := list(db.dir, start, end, isReversed) + keys, err := list(db.dir, start, end) if err != nil { panic(errors.Wrapf(err, "Listing keys in %s", db.dir)) } @@ -207,13 +206,13 @@ func write(path string, d []byte) error { return err } defer f.Close() - fInfo, err := f.Stat() - if err != nil { - return err - } - if fInfo.Mode() != keyPerm { - return tmerrors.NewErrPermissionsChanged(f.Name(), keyPerm, fInfo.Mode()) - } + // fInfo, err := f.Stat() + // if err != nil { + // return err + // } + // if fInfo.Mode() != keyPerm { + // return tmerrors.NewErrPermissionsChanged(f.Name(), keyPerm, fInfo.Mode()) + // } _, err = f.Write(d) if err != nil { return err @@ -230,7 +229,7 @@ func remove(path string) error { // List keys in a directory, stripping of escape sequences and dir portions. // CONTRACT: returns os errors directly without wrapping. -func list(dirPath string, start, end []byte, isReversed bool) ([]string, error) { +func list(dirPath string, start, end []byte) ([]string, error) { dir, err := os.Open(dirPath) if err != nil { return nil, err @@ -248,7 +247,7 @@ func list(dirPath string, start, end []byte, isReversed bool) ([]string, error) return nil, fmt.Errorf("Failed to unescape %s while listing", name) } key := unescapeKey([]byte(n)) - if IsKeyInDomain(key, start, end, isReversed) { + if IsKeyInDomain(key, start, end) { keys = append(keys, string(key)) } } diff --git a/libs/db/go_level_db.go b/libs/db/go_level_db.go index 8a488792..fd487a4d 100644 --- a/libs/db/go_level_db.go +++ b/libs/db/go_level_db.go @@ -213,13 +213,13 @@ var _ Iterator = (*goLevelDBIterator)(nil) func newGoLevelDBIterator(source iterator.Iterator, start, end []byte, isReverse bool) *goLevelDBIterator { if isReverse { - if start == nil { + if end == nil { source.Last() } else { - valid := source.Seek(start) + valid := source.Seek(end) if valid { - soakey := source.Key() // start or after key - if bytes.Compare(start, soakey) < 0 { + eoakey := source.Key() // end or after key + if bytes.Compare(end, eoakey) <= 0 { source.Prev() } } else { @@ -265,11 +265,12 @@ func (itr *goLevelDBIterator) Valid() bool { } // If key is end or past it, invalid. + var start = itr.start var end = itr.end var key = itr.source.Key() if itr.isReverse { - if end != nil && bytes.Compare(key, end) <= 0 { + if start != nil && bytes.Compare(key, start) < 0 { itr.isInvalid = true return false } diff --git a/libs/db/mem_db.go b/libs/db/mem_db.go index 58012301..ff516bc7 100644 --- a/libs/db/mem_db.go +++ b/libs/db/mem_db.go @@ -237,7 +237,7 @@ func (itr *memDBIterator) assertIsValid() { func (db *MemDB) getSortedKeys(start, end []byte, reverse bool) []string { keys := []string{} for key := range db.db { - inDomain := IsKeyInDomain([]byte(key), start, end, reverse) + inDomain := IsKeyInDomain([]byte(key), start, end) if inDomain { keys = append(keys, key) } diff --git a/libs/db/prefix_db.go b/libs/db/prefix_db.go index 9dc4ee97..40d72560 100644 --- a/libs/db/prefix_db.go +++ b/libs/db/prefix_db.go @@ -131,27 +131,13 @@ func (pdb *prefixDB) ReverseIterator(start, end []byte) Iterator { defer pdb.mtx.Unlock() var pstart, pend []byte - if start == nil { - // This may cause the underlying iterator to start with - // an item which doesn't start with prefix. We will skip - // that item later in this function. See 'skipOne'. - pstart = cpIncr(pdb.prefix) - } else { - pstart = append(cp(pdb.prefix), start...) - } + pstart = append(cp(pdb.prefix), start...) if end == nil { - // This may cause the underlying iterator to end with an - // item which doesn't start with prefix. The - // prefixIterator will terminate iteration - // automatically upon detecting this. - pend = cpDecr(pdb.prefix) + pend = cpIncr(pdb.prefix) } else { pend = append(cp(pdb.prefix), end...) } ritr := pdb.db.ReverseIterator(pstart, pend) - if start == nil { - skipOne(ritr, cpIncr(pdb.prefix)) - } return newPrefixIterator( pdb.prefix, start, @@ -310,7 +296,6 @@ func (itr *prefixIterator) Next() { } itr.source.Next() if !itr.source.Valid() || !bytes.HasPrefix(itr.source.Key(), itr.prefix) { - itr.source.Close() itr.valid = false return } @@ -345,13 +330,3 @@ func stripPrefix(key []byte, prefix []byte) (stripped []byte) { } return key[len(prefix):] } - -// If the first iterator item is skipKey, then -// skip it. -func skipOne(itr Iterator, skipKey []byte) { - if itr.Valid() { - if bytes.Equal(itr.Key(), skipKey) { - itr.Next() - } - } -} diff --git a/libs/db/prefix_db_test.go b/libs/db/prefix_db_test.go index 60809f15..e3e37c7d 100644 --- a/libs/db/prefix_db_test.go +++ b/libs/db/prefix_db_test.go @@ -113,8 +113,46 @@ func TestPrefixDBReverseIterator2(t *testing.T) { db := mockDBWithStuff() pdb := NewPrefixDB(db, bz("key")) + itr := pdb.ReverseIterator(bz(""), nil) + checkDomain(t, itr, bz(""), nil) + checkItem(t, itr, bz("3"), bz("value3")) + checkNext(t, itr, true) + checkItem(t, itr, bz("2"), bz("value2")) + checkNext(t, itr, true) + checkItem(t, itr, bz("1"), bz("value1")) + checkNext(t, itr, true) + checkItem(t, itr, bz(""), bz("value")) + checkNext(t, itr, false) + checkInvalid(t, itr) + itr.Close() +} + +func TestPrefixDBReverseIterator3(t *testing.T) { + db := mockDBWithStuff() + pdb := NewPrefixDB(db, bz("key")) + itr := pdb.ReverseIterator(nil, bz("")) checkDomain(t, itr, nil, bz("")) + checkInvalid(t, itr) + itr.Close() +} + +func TestPrefixDBReverseIterator4(t *testing.T) { + db := mockDBWithStuff() + pdb := NewPrefixDB(db, bz("key")) + + itr := pdb.ReverseIterator(bz(""), bz("")) + checkDomain(t, itr, bz(""), bz("")) + checkInvalid(t, itr) + itr.Close() +} + +func TestPrefixDBReverseIterator5(t *testing.T) { + db := mockDBWithStuff() + pdb := NewPrefixDB(db, bz("key")) + + itr := pdb.ReverseIterator(bz("1"), nil) + checkDomain(t, itr, bz("1"), nil) checkItem(t, itr, bz("3"), bz("value3")) checkNext(t, itr, true) checkItem(t, itr, bz("2"), bz("value2")) @@ -125,23 +163,30 @@ func TestPrefixDBReverseIterator2(t *testing.T) { itr.Close() } -func TestPrefixDBReverseIterator3(t *testing.T) { +func TestPrefixDBReverseIterator6(t *testing.T) { db := mockDBWithStuff() pdb := NewPrefixDB(db, bz("key")) - itr := pdb.ReverseIterator(bz(""), nil) - checkDomain(t, itr, bz(""), nil) - checkItem(t, itr, bz(""), bz("value")) + itr := pdb.ReverseIterator(bz("2"), nil) + checkDomain(t, itr, bz("2"), nil) + checkItem(t, itr, bz("3"), bz("value3")) + checkNext(t, itr, true) + checkItem(t, itr, bz("2"), bz("value2")) checkNext(t, itr, false) checkInvalid(t, itr) itr.Close() } -func TestPrefixDBReverseIterator4(t *testing.T) { +func TestPrefixDBReverseIterator7(t *testing.T) { db := mockDBWithStuff() pdb := NewPrefixDB(db, bz("key")) - itr := pdb.ReverseIterator(bz(""), bz("")) + itr := pdb.ReverseIterator(nil, bz("2")) + checkDomain(t, itr, nil, bz("2")) + checkItem(t, itr, bz("1"), bz("value1")) + checkNext(t, itr, true) + checkItem(t, itr, bz(""), bz("value")) + checkNext(t, itr, false) checkInvalid(t, itr) itr.Close() } diff --git a/libs/db/types.go b/libs/db/types.go index ad78859a..9b9c6d0b 100644 --- a/libs/db/types.go +++ b/libs/db/types.go @@ -34,9 +34,9 @@ type DB interface { Iterator(start, end []byte) Iterator // Iterate over a domain of keys in descending order. End is exclusive. - // Start must be greater than end, or the Iterator is invalid. - // If start is nil, iterates from the last/greatest item (inclusive). - // If end is nil, iterates up to the first/least item (inclusive). + // Start must be less than end, or the Iterator is invalid. + // If start is nil, iterates up to the first/least item (inclusive). + // If end is nil, iterates from the last/greatest item (inclusive). // CONTRACT: No writes may happen within a domain while an iterator exists over it. // CONTRACT: start, end readonly []byte ReverseIterator(start, end []byte) Iterator diff --git a/libs/db/util.go b/libs/db/util.go index 51277ac4..e927c354 100644 --- a/libs/db/util.go +++ b/libs/db/util.go @@ -33,46 +33,13 @@ func cpIncr(bz []byte) (ret []byte) { return nil } -// Returns a slice of the same length (big endian) -// except decremented by one. -// Returns nil on underflow (e.g. if bz bytes are all 0x00) -// CONTRACT: len(bz) > 0 -func cpDecr(bz []byte) (ret []byte) { - if len(bz) == 0 { - panic("cpDecr expects non-zero bz length") - } - ret = cp(bz) - for i := len(bz) - 1; i >= 0; i-- { - if ret[i] > byte(0x00) { - ret[i]-- - return - } - ret[i] = byte(0xFF) - if i == 0 { - // Underflow - return nil - } - } - return nil -} - // See DB interface documentation for more information. -func IsKeyInDomain(key, start, end []byte, isReverse bool) bool { - if !isReverse { - if bytes.Compare(key, start) < 0 { - return false - } - if end != nil && bytes.Compare(end, key) <= 0 { - return false - } - return true - } else { - if start != nil && bytes.Compare(start, key) < 0 { - return false - } - if end != nil && bytes.Compare(key, end) <= 0 { - return false - } - return true +func IsKeyInDomain(key, start, end []byte) bool { + if bytes.Compare(key, start) < 0 { + return false } + if end != nil && bytes.Compare(end, key) <= 0 { + return false + } + return true } diff --git a/libs/errors/errors.go b/libs/errors/errors.go index ae5d9439..a0338278 100644 --- a/libs/errors/errors.go +++ b/libs/errors/errors.go @@ -1,26 +1,21 @@ // Package errors contains errors that are thrown across packages. package errors -import ( - "fmt" - "os" -) +// // ErrPermissionsChanged occurs if the file permission have changed since the file was created. +// type ErrPermissionsChanged struct { +// name string +// got, want os.FileMode +// } -// ErrPermissionsChanged occurs if the file permission have changed since the file was created. -type ErrPermissionsChanged struct { - name string - got, want os.FileMode -} +// func NewErrPermissionsChanged(name string, got, want os.FileMode) *ErrPermissionsChanged { +// return &ErrPermissionsChanged{name: name, got: got, want: want} +// } -func NewErrPermissionsChanged(name string, got, want os.FileMode) *ErrPermissionsChanged { - return &ErrPermissionsChanged{name: name, got: got, want: want} -} - -func (e ErrPermissionsChanged) Error() string { - return fmt.Sprintf( - "file: [%v]\nexpected file permissions: %v, got: %v", - e.name, - e.want, - e.got, - ) -} +// func (e ErrPermissionsChanged) Error() string { +// return fmt.Sprintf( +// "file: [%v]\nexpected file permissions: %v, got: %v", +// e.name, +// e.want, +// e.got, +// ) +// } diff --git a/libs/events/events.go b/libs/events/events.go index fb90bbea..34333a06 100644 --- a/libs/events/events.go +++ b/libs/events/events.go @@ -188,7 +188,7 @@ func (cell *eventCell) RemoveListener(listenerID string) int { func (cell *eventCell) FireEvent(data EventData) { cell.mtx.RLock() - var eventCallbacks []EventCallback + eventCallbacks := make([]EventCallback, 0, len(cell.listeners)) for _, cb := range cell.listeners { eventCallbacks = append(eventCallbacks, cb) } diff --git a/libs/fail/fail.go b/libs/fail/fail.go new file mode 100644 index 00000000..d7912af5 --- /dev/null +++ b/libs/fail/fail.go @@ -0,0 +1,79 @@ +package fail + +import ( + "fmt" + "math/rand" + "os" + "strconv" +) + +var callIndexToFail int + +func init() { + callIndexToFailS := os.Getenv("FAIL_TEST_INDEX") + + if callIndexToFailS == "" { + callIndexToFail = -1 + } else { + var err error + callIndexToFail, err = strconv.Atoi(callIndexToFailS) + if err != nil { + callIndexToFail = -1 + } + } +} + +// Fail when FAIL_TEST_INDEX == callIndex +var ( + callIndex int //indexes Fail calls + + callRandIndex int // indexes a run of FailRand calls + callRandIndexToFail = -1 // the callRandIndex to fail on in FailRand +) + +func Fail() { + if callIndexToFail < 0 { + return + } + + if callIndex == callIndexToFail { + Exit() + } + + callIndex += 1 +} + +// FailRand should be called n successive times. +// It will fail on a random one of those calls +// n must be greater than 0 +func FailRand(n int) { + if callIndexToFail < 0 { + return + } + + if callRandIndexToFail < 0 { + // first call in the loop, pick a random index to fail at + callRandIndexToFail = rand.Intn(n) + callRandIndex = 0 + } + + if callIndex == callIndexToFail { + if callRandIndex == callRandIndexToFail { + Exit() + } + } + + callRandIndex += 1 + + if callRandIndex == n { + callIndex += 1 + } +} + +func Exit() { + fmt.Printf("*** fail-test %d ***\n", callIndex) + os.Exit(1) + // proc, _ := os.FindProcess(os.Getpid()) + // proc.Signal(os.Interrupt) + // panic(fmt.Sprintf("*** fail-test %d ***", callIndex)) +} diff --git a/libs/flowrate/io_test.go b/libs/flowrate/io_test.go index c84029d5..ab2c7121 100644 --- a/libs/flowrate/io_test.go +++ b/libs/flowrate/io_test.go @@ -81,12 +81,12 @@ func TestReader(t *testing.T) { // Active, Start, Duration, Idle, Bytes, Samples, InstRate, CurRate, AvgRate, PeakRate, BytesRem, TimeRem, Progress want := []Status{ - Status{true, start, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, - Status{true, start, _100ms, 0, 10, 1, 100, 100, 100, 100, 0, 0, 0}, - Status{true, start, _200ms, _100ms, 20, 2, 100, 100, 100, 100, 0, 0, 0}, - Status{true, start, _300ms, _200ms, 20, 3, 0, 90, 67, 100, 0, 0, 0}, - Status{false, start, _300ms, 0, 20, 3, 0, 0, 67, 100, 0, 0, 0}, - Status{false, start, _300ms, 0, 20, 3, 0, 0, 67, 100, 0, 0, 0}, + {true, start, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, + {true, start, _100ms, 0, 10, 1, 100, 100, 100, 100, 0, 0, 0}, + {true, start, _200ms, _100ms, 20, 2, 100, 100, 100, 100, 0, 0, 0}, + {true, start, _300ms, _200ms, 20, 3, 0, 90, 67, 100, 0, 0, 0}, + {false, start, _300ms, 0, 20, 3, 0, 0, 67, 100, 0, 0, 0}, + {false, start, _300ms, 0, 20, 3, 0, 0, 67, 100, 0, 0, 0}, } for i, s := range status { if !statusesAreEqual(&s, &want[i]) { @@ -139,8 +139,8 @@ func TestWriter(t *testing.T) { // Active, Start, Duration, Idle, Bytes, Samples, InstRate, CurRate, AvgRate, PeakRate, BytesRem, TimeRem, Progress want := []Status{ - Status{true, start, _400ms, 0, 80, 4, 200, 200, 200, 200, 20, _100ms, 80000}, - Status{true, start, _500ms, _100ms, 100, 5, 200, 200, 200, 200, 0, 0, 100000}, + {true, start, _400ms, 0, 80, 4, 200, 200, 200, 200, 20, _100ms, 80000}, + {true, start, _500ms, _100ms, 100, 5, 200, 200, 200, 200, 0, 0, 100000}, } for i, s := range status { if !statusesAreEqual(&s, &want[i]) { diff --git a/libs/log/filter.go b/libs/log/filter.go index 768c09b8..b71447ed 100644 --- a/libs/log/filter.go +++ b/libs/log/filter.go @@ -11,9 +11,10 @@ const ( ) type filter struct { - next Logger - allowed level // XOR'd levels for default case - allowedKeyvals map[keyval]level // When key-value match, use this level + next Logger + allowed level // XOR'd levels for default case + initiallyAllowed level // XOR'd levels for initial case + allowedKeyvals map[keyval]level // When key-value match, use this level } type keyval struct { @@ -33,6 +34,7 @@ func NewFilter(next Logger, options ...Option) Logger { for _, option := range options { option(l) } + l.initiallyAllowed = l.allowed return l } @@ -76,14 +78,45 @@ func (l *filter) Error(msg string, keyvals ...interface{}) { // logger = log.NewFilter(logger, log.AllowError(), log.AllowInfoWith("module", "crypto"), log.AllowNoneWith("user", "Sam")) // logger.With("user", "Sam").With("module", "crypto").Info("Hello") # produces "I... Hello module=crypto user=Sam" func (l *filter) With(keyvals ...interface{}) Logger { + keyInAllowedKeyvals := false + for i := len(keyvals) - 2; i >= 0; i -= 2 { for kv, allowed := range l.allowedKeyvals { - if keyvals[i] == kv.key && keyvals[i+1] == kv.value { - return &filter{next: l.next.With(keyvals...), allowed: allowed, allowedKeyvals: l.allowedKeyvals} + if keyvals[i] == kv.key { + keyInAllowedKeyvals = true + // Example: + // logger = log.NewFilter(logger, log.AllowError(), log.AllowInfoWith("module", "crypto")) + // logger.With("module", "crypto") + if keyvals[i+1] == kv.value { + return &filter{ + next: l.next.With(keyvals...), + allowed: allowed, // set the desired level + allowedKeyvals: l.allowedKeyvals, + initiallyAllowed: l.initiallyAllowed, + } + } } } } - return &filter{next: l.next.With(keyvals...), allowed: l.allowed, allowedKeyvals: l.allowedKeyvals} + + // Example: + // logger = log.NewFilter(logger, log.AllowError(), log.AllowInfoWith("module", "crypto")) + // logger.With("module", "main") + if keyInAllowedKeyvals { + return &filter{ + next: l.next.With(keyvals...), + allowed: l.initiallyAllowed, // return back to initially allowed + allowedKeyvals: l.allowedKeyvals, + initiallyAllowed: l.initiallyAllowed, + } + } + + return &filter{ + next: l.next.With(keyvals...), + allowed: l.allowed, // simply continue with the current level + allowedKeyvals: l.allowedKeyvals, + initiallyAllowed: l.initiallyAllowed, + } } //-------------------------------------------------------------------------------- diff --git a/libs/log/testing_logger.go b/libs/log/testing_logger.go index 81482bef..8914bd81 100644 --- a/libs/log/testing_logger.go +++ b/libs/log/testing_logger.go @@ -1,6 +1,7 @@ package log import ( + "io" "os" "testing" @@ -19,12 +20,22 @@ var ( // inside a test (not in the init func) because // verbose flag only set at the time of testing. func TestingLogger() Logger { + return TestingLoggerWithOutput(os.Stdout) +} + +// TestingLoggerWOutput returns a TMLogger which writes to (w io.Writer) if testing being run +// with the verbose (-v) flag, NopLogger otherwise. +// +// Note that the call to TestingLoggerWithOutput(w io.Writer) must be made +// inside a test (not in the init func) because +// verbose flag only set at the time of testing. +func TestingLoggerWithOutput(w io.Writer) Logger { if _testingLogger != nil { return _testingLogger } if testing.Verbose() { - _testingLogger = NewTMLogger(NewSyncWriter(os.Stdout)) + _testingLogger = NewTMLogger(NewSyncWriter(w)) } else { _testingLogger = NewNopLogger() } diff --git a/libs/log/tmfmt_logger.go b/libs/log/tmfmt_logger.go index de155fef..d841263e 100644 --- a/libs/log/tmfmt_logger.go +++ b/libs/log/tmfmt_logger.go @@ -84,13 +84,13 @@ func (l tmfmtLogger) Log(keyvals ...interface{}) error { // Form a custom Tendermint line // // Example: - // D[05-02|11:06:44.322] Stopping AddrBook (ignoring: already stopped) + // D[2016-05-02|11:06:44.322] Stopping AddrBook (ignoring: already stopped) // // Description: // D - first character of the level, uppercase (ASCII only) - // [05-02|11:06:44.322] - our time format (see https://golang.org/src/time/format.go) + // [2016-05-02|11:06:44.322] - our time format (see https://golang.org/src/time/format.go) // Stopping ... - message - enc.buf.WriteString(fmt.Sprintf("%c[%s] %-44s ", lvl[0]-32, time.Now().Format("01-02|15:04:05.000"), msg)) + enc.buf.WriteString(fmt.Sprintf("%c[%s] %-44s ", lvl[0]-32, time.Now().Format("2006-01-02|15:04:05.000"), msg)) if module != unknown { enc.buf.WriteString("module=" + module + " ") diff --git a/libs/pubsub/pubsub.go b/libs/pubsub/pubsub.go index c104439f..a39c8c73 100644 --- a/libs/pubsub/pubsub.go +++ b/libs/pubsub/pubsub.go @@ -9,6 +9,45 @@ // When some message is published, we match it with all queries. If there is a // match, this message will be pushed to all clients, subscribed to that query. // See query subpackage for our implementation. +// +// Due to the blocking send implementation, a single subscriber can freeze an +// entire server by not reading messages before it unsubscribes. To avoid such +// scenario, subscribers must either: +// +// a) make sure they continue to read from the out channel until +// Unsubscribe(All) is called +// +// s.Subscribe(ctx, sub, qry, out) +// go func() { +// for msg := range out { +// // handle msg +// // will exit automatically when out is closed by Unsubscribe(All) +// } +// }() +// s.UnsubscribeAll(ctx, sub) +// +// b) drain the out channel before calling Unsubscribe(All) +// +// s.Subscribe(ctx, sub, qry, out) +// defer func() { +// // drain out to make sure we don't block +// LOOP: +// for { +// select { +// case <-out: +// default: +// break LOOP +// } +// } +// s.UnsubscribeAll(ctx, sub) +// }() +// for msg := range out { +// // handle msg +// if err != nil { +// return err +// } +// } +// package pubsub import ( @@ -62,7 +101,7 @@ type Server struct { cmdsCap int mtx sync.RWMutex - subscriptions map[string]map[string]Query // subscriber -> query (string) -> Query + subscriptions map[string]map[string]struct{} // subscriber -> query (string) -> empty struct } // Option sets a parameter for the server. @@ -104,7 +143,7 @@ func (ts tagMap) Len() int { // provided, the resulting server's queue is unbuffered. func NewServer(options ...Option) *Server { s := &Server{ - subscriptions: make(map[string]map[string]Query), + subscriptions: make(map[string]map[string]struct{}), } s.BaseService = *cmn.NewBaseService(nil, "PubSub", s) @@ -154,11 +193,9 @@ func (s *Server) Subscribe(ctx context.Context, clientID string, query Query, ou case s.cmds <- cmd{op: sub, clientID: clientID, query: query, ch: out}: s.mtx.Lock() if _, ok = s.subscriptions[clientID]; !ok { - s.subscriptions[clientID] = make(map[string]Query) + s.subscriptions[clientID] = make(map[string]struct{}) } - // preserve original query - // see Unsubscribe - s.subscriptions[clientID][query.String()] = query + s.subscriptions[clientID][query.String()] = struct{}{} s.mtx.Unlock() return nil case <-ctx.Done(): @@ -172,22 +209,23 @@ func (s *Server) Subscribe(ctx context.Context, clientID string, query Query, ou // returned to the caller if the context is canceled or if subscription does // not exist. func (s *Server) Unsubscribe(ctx context.Context, clientID string, query Query) error { - var origQuery Query s.mtx.RLock() clientSubscriptions, ok := s.subscriptions[clientID] if ok { - origQuery, ok = clientSubscriptions[query.String()] + _, ok = clientSubscriptions[query.String()] } s.mtx.RUnlock() if !ok { return ErrSubscriptionNotFound } - // original query is used here because we're using pointers as map keys select { - case s.cmds <- cmd{op: unsub, clientID: clientID, query: origQuery}: + case s.cmds <- cmd{op: unsub, clientID: clientID, query: query}: s.mtx.Lock() delete(clientSubscriptions, query.String()) + if len(clientSubscriptions) == 0 { + delete(s.subscriptions, clientID) + } s.mtx.Unlock() return nil case <-ctx.Done(): @@ -247,17 +285,27 @@ func (s *Server) OnStop() { // NOTE: not goroutine safe type state struct { - // query -> client -> ch - queries map[Query]map[string]chan<- interface{} - // client -> query -> struct{} - clients map[string]map[Query]struct{} + // query string -> client -> ch + queryToChanMap map[string]map[string]chan<- interface{} + // client -> query string -> struct{} + clientToQueryMap map[string]map[string]struct{} + // query string -> queryPlusRefCount + queries map[string]*queryPlusRefCount +} + +// queryPlusRefCount holds a pointer to a query and reference counter. When +// refCount is zero, query will be removed. +type queryPlusRefCount struct { + q Query + refCount int } // OnStart implements Service.OnStart by starting the server. func (s *Server) OnStart() error { go s.loop(state{ - queries: make(map[Query]map[string]chan<- interface{}), - clients: make(map[string]map[Query]struct{}), + queryToChanMap: make(map[string]map[string]chan<- interface{}), + clientToQueryMap: make(map[string]map[string]struct{}), + queries: make(map[string]*queryPlusRefCount), }) return nil } @@ -278,7 +326,7 @@ loop: state.removeAll(cmd.clientID) } case shutdown: - for clientID := range state.clients { + for clientID := range state.clientToQueryMap { state.removeAll(clientID) } break loop @@ -291,66 +339,99 @@ loop: } func (state *state) add(clientID string, q Query, ch chan<- interface{}) { + qStr := q.String() // initialize clientToChannelMap per query if needed - if _, ok := state.queries[q]; !ok { - state.queries[q] = make(map[string]chan<- interface{}) + if _, ok := state.queryToChanMap[qStr]; !ok { + state.queryToChanMap[qStr] = make(map[string]chan<- interface{}) } // create subscription - state.queries[q][clientID] = ch + state.queryToChanMap[qStr][clientID] = ch + + // initialize queries if needed + if _, ok := state.queries[qStr]; !ok { + state.queries[qStr] = &queryPlusRefCount{q: q, refCount: 0} + } + // increment reference counter + state.queries[qStr].refCount++ // add client if needed - if _, ok := state.clients[clientID]; !ok { - state.clients[clientID] = make(map[Query]struct{}) + if _, ok := state.clientToQueryMap[clientID]; !ok { + state.clientToQueryMap[clientID] = make(map[string]struct{}) } - state.clients[clientID][q] = struct{}{} + state.clientToQueryMap[clientID][qStr] = struct{}{} } func (state *state) remove(clientID string, q Query) { - clientToChannelMap, ok := state.queries[q] + qStr := q.String() + + clientToChannelMap, ok := state.queryToChanMap[qStr] if !ok { return } ch, ok := clientToChannelMap[clientID] - if ok { - close(ch) - - delete(state.clients[clientID], q) - - // if it not subscribed to anything else, remove the client - if len(state.clients[clientID]) == 0 { - delete(state.clients, clientID) - } - - delete(state.queries[q], clientID) - if len(state.queries[q]) == 0 { - delete(state.queries, q) - } - } -} - -func (state *state) removeAll(clientID string) { - queryMap, ok := state.clients[clientID] if !ok { return } - for q := range queryMap { - ch := state.queries[q][clientID] + close(ch) + + // remove the query from client map. + // if client is not subscribed to anything else, remove it. + delete(state.clientToQueryMap[clientID], qStr) + if len(state.clientToQueryMap[clientID]) == 0 { + delete(state.clientToQueryMap, clientID) + } + + // remove the client from query map. + // if query has no other clients subscribed, remove it. + delete(state.queryToChanMap[qStr], clientID) + if len(state.queryToChanMap[qStr]) == 0 { + delete(state.queryToChanMap, qStr) + } + + // decrease ref counter in queries + state.queries[qStr].refCount-- + // remove the query if nobody else is using it + if state.queries[qStr].refCount == 0 { + delete(state.queries, qStr) + } +} + +func (state *state) removeAll(clientID string) { + queryMap, ok := state.clientToQueryMap[clientID] + if !ok { + return + } + + for qStr := range queryMap { + ch := state.queryToChanMap[qStr][clientID] close(ch) - delete(state.queries[q], clientID) - if len(state.queries[q]) == 0 { - delete(state.queries, q) + // remove the client from query map. + // if query has no other clients subscribed, remove it. + delete(state.queryToChanMap[qStr], clientID) + if len(state.queryToChanMap[qStr]) == 0 { + delete(state.queryToChanMap, qStr) + } + + // decrease ref counter in queries + state.queries[qStr].refCount-- + // remove the query if nobody else is using it + if state.queries[qStr].refCount == 0 { + delete(state.queries, qStr) } } - delete(state.clients, clientID) + + // remove the client. + delete(state.clientToQueryMap, clientID) } func (state *state) send(msg interface{}, tags TagMap) { - for q, clientToChannelMap := range state.queries { + for qStr, clientToChannelMap := range state.queryToChanMap { + q := state.queries[qStr].q if q.Matches(tags) { for _, ch := range clientToChannelMap { ch <- msg diff --git a/libs/pubsub/pubsub_test.go b/libs/pubsub/pubsub_test.go index 5e9931e4..bb660d9e 100644 --- a/libs/pubsub/pubsub_test.go +++ b/libs/pubsub/pubsub_test.go @@ -115,6 +115,25 @@ func TestUnsubscribe(t *testing.T) { assert.False(t, ok) } +func TestClientUnsubscribesTwice(t *testing.T) { + s := pubsub.NewServer() + s.SetLogger(log.TestingLogger()) + s.Start() + defer s.Stop() + + ctx := context.Background() + ch := make(chan interface{}) + err := s.Subscribe(ctx, clientID, query.MustParse("tm.events.type='NewBlock'"), ch) + require.NoError(t, err) + err = s.Unsubscribe(ctx, clientID, query.MustParse("tm.events.type='NewBlock'")) + require.NoError(t, err) + + err = s.Unsubscribe(ctx, clientID, query.MustParse("tm.events.type='NewBlock'")) + assert.Equal(t, pubsub.ErrSubscriptionNotFound, err) + err = s.UnsubscribeAll(ctx, clientID) + assert.Equal(t, pubsub.ErrSubscriptionNotFound, err) +} + func TestResubscribe(t *testing.T) { s := pubsub.NewServer() s.SetLogger(log.TestingLogger()) diff --git a/libs/pubsub/query/query_test.go b/libs/pubsub/query/query_test.go index f0d94099..d1810f46 100644 --- a/libs/pubsub/query/query_test.go +++ b/libs/pubsub/query/query_test.go @@ -73,9 +73,9 @@ func TestConditions(t *testing.T) { s string conditions []query.Condition }{ - {s: "tm.events.type='NewBlock'", conditions: []query.Condition{query.Condition{Tag: "tm.events.type", Op: query.OpEqual, Operand: "NewBlock"}}}, - {s: "tx.gas > 7 AND tx.gas < 9", conditions: []query.Condition{query.Condition{Tag: "tx.gas", Op: query.OpGreater, Operand: int64(7)}, query.Condition{Tag: "tx.gas", Op: query.OpLess, Operand: int64(9)}}}, - {s: "tx.time >= TIME 2013-05-03T14:45:00Z", conditions: []query.Condition{query.Condition{Tag: "tx.time", Op: query.OpGreaterEqual, Operand: txTime}}}, + {s: "tm.events.type='NewBlock'", conditions: []query.Condition{{Tag: "tm.events.type", Op: query.OpEqual, Operand: "NewBlock"}}}, + {s: "tx.gas > 7 AND tx.gas < 9", conditions: []query.Condition{{Tag: "tx.gas", Op: query.OpGreater, Operand: int64(7)}, {Tag: "tx.gas", Op: query.OpLess, Operand: int64(9)}}}, + {s: "tx.time >= TIME 2013-05-03T14:45:00Z", conditions: []query.Condition{{Tag: "tx.time", Op: query.OpGreaterEqual, Operand: txTime}}}, } for _, tc := range testCases { diff --git a/libs/test.sh b/libs/test.sh index ecf17fc4..64898b0d 100755 --- a/libs/test.sh +++ b/libs/test.sh @@ -2,7 +2,7 @@ set -e # run the linter -# make metalinter_test +# make lint # setup certs make gen_certs diff --git a/lite/base_verifier.go b/lite/base_verifier.go index fcde01c0..9eb880bb 100644 --- a/lite/base_verifier.go +++ b/lite/base_verifier.go @@ -35,34 +35,40 @@ func NewBaseVerifier(chainID string, height int64, valset *types.ValidatorSet) * } // Implements Verifier. -func (bc *BaseVerifier) ChainID() string { - return bc.chainID +func (bv *BaseVerifier) ChainID() string { + return bv.chainID } // Implements Verifier. -func (bc *BaseVerifier) Verify(signedHeader types.SignedHeader) error { +func (bv *BaseVerifier) Verify(signedHeader types.SignedHeader) error { - // We can't verify commits older than bc.height. - if signedHeader.Height < bc.height { + // We can't verify commits for a different chain. + if signedHeader.ChainID != bv.chainID { + return cmn.NewError("BaseVerifier chainID is %v, cannot verify chainID %v", + bv.chainID, signedHeader.ChainID) + } + + // We can't verify commits older than bv.height. + if signedHeader.Height < bv.height { return cmn.NewError("BaseVerifier height is %v, cannot verify height %v", - bc.height, signedHeader.Height) + bv.height, signedHeader.Height) } // We can't verify with the wrong validator set. if !bytes.Equal(signedHeader.ValidatorsHash, - bc.valset.Hash()) { - return lerr.ErrUnexpectedValidators(signedHeader.ValidatorsHash, bc.valset.Hash()) + bv.valset.Hash()) { + return lerr.ErrUnexpectedValidators(signedHeader.ValidatorsHash, bv.valset.Hash()) } // Do basic sanity checks. - err := signedHeader.ValidateBasic(bc.chainID) + err := signedHeader.ValidateBasic(bv.chainID) if err != nil { return cmn.ErrorWrap(err, "in verify") } // Check commit signatures. - err = bc.valset.VerifyCommit( - bc.chainID, signedHeader.Commit.BlockID, + err = bv.valset.VerifyCommit( + bv.chainID, signedHeader.Commit.BlockID, signedHeader.Height, signedHeader.Commit) if err != nil { return cmn.ErrorWrap(err, "in verify") diff --git a/lite/commit.go b/lite/commit.go index 25efb8dc..6cd35417 100644 --- a/lite/commit.go +++ b/lite/commit.go @@ -8,7 +8,7 @@ import ( "github.com/tendermint/tendermint/types" ) -// FullCommit is a signed header (the block header and a commit that signs it), +// FullCommit contains a SignedHeader (the block header and a commit that signs it), // the validator set which signed the commit, and the next validator set. The // next validator set (which is proven from the block header) allows us to // revert to block-by-block updating of lite Verifier's latest validator set, diff --git a/lite/dbprovider.go b/lite/dbprovider.go index cab695b4..ef1b2a59 100644 --- a/lite/dbprovider.go +++ b/lite/dbprovider.go @@ -13,6 +13,9 @@ import ( "github.com/tendermint/tendermint/types" ) +var _ PersistentProvider = (*DBProvider)(nil) + +// DBProvider stores commits and validator sets in a DB. type DBProvider struct { logger log.Logger label string @@ -56,7 +59,7 @@ func (dbp *DBProvider) SaveFullCommit(fc FullCommit) error { // We might be overwriting what we already have, but // it makes the logic easier for now. vsKey := validatorSetKey(fc.ChainID(), fc.Height()) - vsBz, err := dbp.cdc.MarshalBinary(fc.Validators) + vsBz, err := dbp.cdc.MarshalBinaryLengthPrefixed(fc.Validators) if err != nil { return err } @@ -64,7 +67,7 @@ func (dbp *DBProvider) SaveFullCommit(fc FullCommit) error { // Save the fc.NextValidators. nvsKey := validatorSetKey(fc.ChainID(), fc.Height()+1) - nvsBz, err := dbp.cdc.MarshalBinary(fc.NextValidators) + nvsBz, err := dbp.cdc.MarshalBinaryLengthPrefixed(fc.NextValidators) if err != nil { return err } @@ -72,7 +75,7 @@ func (dbp *DBProvider) SaveFullCommit(fc FullCommit) error { // Save the fc.SignedHeader shKey := signedHeaderKey(fc.ChainID(), fc.Height()) - shBz, err := dbp.cdc.MarshalBinary(fc.SignedHeader) + shBz, err := dbp.cdc.MarshalBinaryLengthPrefixed(fc.SignedHeader) if err != nil { return err } @@ -105,8 +108,8 @@ func (dbp *DBProvider) LatestFullCommit(chainID string, minHeight, maxHeight int } itr := dbp.db.ReverseIterator( - signedHeaderKey(chainID, maxHeight), - signedHeaderKey(chainID, minHeight-1), + signedHeaderKey(chainID, minHeight), + append(signedHeaderKey(chainID, maxHeight), byte(0x00)), ) defer itr.Close() @@ -121,7 +124,7 @@ func (dbp *DBProvider) LatestFullCommit(chainID string, minHeight, maxHeight int // Found the latest full commit signed header. shBz := itr.Value() sh := types.SignedHeader{} - err := dbp.cdc.UnmarshalBinary(shBz, &sh) + err := dbp.cdc.UnmarshalBinaryLengthPrefixed(shBz, &sh) if err != nil { return FullCommit{}, err } else { @@ -150,7 +153,7 @@ func (dbp *DBProvider) getValidatorSet(chainID string, height int64) (valset *ty err = lerr.ErrUnknownValidators(chainID, height) return } - err = dbp.cdc.UnmarshalBinary(vsBz, &valset) + err = dbp.cdc.UnmarshalBinaryLengthPrefixed(vsBz, &valset) if err != nil { return } @@ -190,8 +193,8 @@ func (dbp *DBProvider) deleteAfterN(chainID string, after int) error { dbp.logger.Info("DBProvider.deleteAfterN()...", "chainID", chainID, "after", after) itr := dbp.db.ReverseIterator( - signedHeaderKey(chainID, 1<<63-1), - signedHeaderKey(chainID, 0), + signedHeaderKey(chainID, 1), + append(signedHeaderKey(chainID, 1<<63-1), byte(0x00)), ) defer itr.Close() diff --git a/lite/doc.go b/lite/doc.go index 00dcce68..c02b5021 100644 --- a/lite/doc.go +++ b/lite/doc.go @@ -15,9 +15,7 @@ for you, so you can just build nice applications. We design for clients who have no strong trust relationship with any Tendermint node, just the blockchain and validator set as a whole. -# Data structures - -## SignedHeader +SignedHeader SignedHeader is a block header along with a commit -- enough validator precommit-vote signatures to prove its validity (> 2/3 of the voting power) @@ -42,7 +40,7 @@ The FullCommit is also declared in this package as a convenience structure, which includes the SignedHeader along with the full current and next ValidatorSets. -## Verifier +Verifier A Verifier validates a new SignedHeader given the currently known state. There are two different types of Verifiers provided. @@ -53,42 +51,35 @@ SignedHeader, and that the SignedHeader was to be signed by the exact given validator set, and that the height of the commit is at least height (or greater). -SignedHeader.Commit may be signed by a different validator set, it can get -verified with a BaseVerifier as long as sufficient signatures from the -previous validator set are present in the commit. - DynamicVerifier - this Verifier implements an auto-update and persistence strategy to verify any SignedHeader of the blockchain. -## Provider and PersistentProvider +Provider and PersistentProvider A Provider allows us to store and retrieve the FullCommits. -```go -type Provider interface { - // LatestFullCommit returns the latest commit with - // minHeight <= height <= maxHeight. - // If maxHeight is zero, returns the latest where - // minHeight <= height. - LatestFullCommit(chainID string, minHeight, maxHeight int64) (FullCommit, error) -} -``` + type Provider interface { + // LatestFullCommit returns the latest commit with + // minHeight <= height <= maxHeight. + // If maxHeight is zero, returns the latest where + // minHeight <= height. + LatestFullCommit(chainID string, minHeight, maxHeight int64) (FullCommit, error) + } * client.NewHTTPProvider - query Tendermint rpc. A PersistentProvider is a Provider that also allows for saving state. This is used by the DynamicVerifier for persistence. -```go -type PersistentProvider interface { - Provider + type PersistentProvider interface { + Provider - // SaveFullCommit saves a FullCommit (without verification). - SaveFullCommit(fc FullCommit) error -} -``` + // SaveFullCommit saves a FullCommit (without verification). + SaveFullCommit(fc FullCommit) error + } * DBProvider - persistence provider for use with any libs/DB. + * MultiProvider - combine multiple providers. The suggested use for local light clients is client.NewHTTPProvider(...) for @@ -97,7 +88,7 @@ dbm.NewMemDB()), NewDBProvider("label", db.NewFileDB(...))) to store confirmed full commits (Trusted) -# How We Track Validators +How We Track Validators Unless you want to blindly trust the node you talk with, you need to trace every response back to a hash in a block header and validate the commit @@ -121,7 +112,7 @@ If we cannot update directly from H -> H' because there was too much change to the validator set, then we can look for some Hm (H < Hm < H') with a validator set Vm. Then we try to update H -> Hm and then Hm -> H' in two steps. If one of these steps doesn't work, then we continue bisecting, until we eventually -have to externally validate the valdiator set changes at every block. +have to externally validate the validator set changes at every block. Since we never trust any server in this protocol, only the signatures themselves, it doesn't matter if the seed comes from a (possibly malicious) diff --git a/lite/dynamic_verifier.go b/lite/dynamic_verifier.go index 6a772091..8b69d2d7 100644 --- a/lite/dynamic_verifier.go +++ b/lite/dynamic_verifier.go @@ -18,12 +18,17 @@ var _ Verifier = (*DynamicVerifier)(nil) // "source" provider to obtain the needed FullCommits to securely sync with // validator set changes. It stores properly validated data on the // "trusted" local system. +// TODO: make this single threaded and create a new +// ConcurrentDynamicVerifier that wraps it with concurrency. +// see https://github.com/tendermint/tendermint/issues/3170 type DynamicVerifier struct { - logger log.Logger chainID string - // These are only properly validated data, from local system. + logger log.Logger + + // Already validated, stored locally trusted PersistentProvider - // This is a source of new info, like a node rpc, or other import method. + + // New info, like a node rpc, or other import method. source Provider // pending map to synchronize concurrent verification requests @@ -35,8 +40,8 @@ type DynamicVerifier struct { // trusted provider to store validated data and the source provider to // obtain missing data (e.g. FullCommits). // -// The trusted provider should a CacheProvider, MemProvider or -// files.Provider. The source provider should be a client.HTTPProvider. +// The trusted provider should be a DBProvider. +// The source provider should be a client.HTTPProvider. func NewDynamicVerifier(chainID string, trusted PersistentProvider, source Provider) *DynamicVerifier { return &DynamicVerifier{ logger: log.NewNopLogger(), @@ -47,68 +52,71 @@ func NewDynamicVerifier(chainID string, trusted PersistentProvider, source Provi } } -func (ic *DynamicVerifier) SetLogger(logger log.Logger) { +func (dv *DynamicVerifier) SetLogger(logger log.Logger) { logger = logger.With("module", "lite") - ic.logger = logger - ic.trusted.SetLogger(logger) - ic.source.SetLogger(logger) + dv.logger = logger + dv.trusted.SetLogger(logger) + dv.source.SetLogger(logger) } // Implements Verifier. -func (ic *DynamicVerifier) ChainID() string { - return ic.chainID +func (dv *DynamicVerifier) ChainID() string { + return dv.chainID } // Implements Verifier. // // If the validators have changed since the last known time, it looks to -// ic.trusted and ic.source to prove the new validators. On success, it will -// try to store the SignedHeader in ic.trusted if the next +// dv.trusted and dv.source to prove the new validators. On success, it will +// try to store the SignedHeader in dv.trusted if the next // validator can be sourced. -func (ic *DynamicVerifier) Verify(shdr types.SignedHeader) error { +func (dv *DynamicVerifier) Verify(shdr types.SignedHeader) error { // Performs synchronization for multi-threads verification at the same height. - ic.mtx.Lock() - if pending := ic.pendingVerifications[shdr.Height]; pending != nil { - ic.mtx.Unlock() + dv.mtx.Lock() + if pending := dv.pendingVerifications[shdr.Height]; pending != nil { + dv.mtx.Unlock() <-pending // pending is chan struct{} } else { pending := make(chan struct{}) - ic.pendingVerifications[shdr.Height] = pending + dv.pendingVerifications[shdr.Height] = pending defer func() { close(pending) - ic.mtx.Lock() - delete(ic.pendingVerifications, shdr.Height) - ic.mtx.Unlock() + dv.mtx.Lock() + delete(dv.pendingVerifications, shdr.Height) + dv.mtx.Unlock() }() - ic.mtx.Unlock() + dv.mtx.Unlock() } + //Get the exact trusted commit for h, and if it is - // equal to shdr, then don't even verify it, - // and just return nil. - trustedFCSameHeight, err := ic.trusted.LatestFullCommit(ic.chainID, shdr.Height, shdr.Height) + // equal to shdr, then it's already trusted, so + // just return nil. + trustedFCSameHeight, err := dv.trusted.LatestFullCommit(dv.chainID, shdr.Height, shdr.Height) if err == nil { // If loading trust commit successfully, and trust commit equal to shdr, then don't verify it, // just return nil. if bytes.Equal(trustedFCSameHeight.SignedHeader.Hash(), shdr.Hash()) { - ic.logger.Info(fmt.Sprintf("Load full commit at height %d from cache, there is not need to verify.", shdr.Height)) + dv.logger.Info(fmt.Sprintf("Load full commit at height %d from cache, there is not need to verify.", shdr.Height)) return nil } } else if !lerr.IsErrCommitNotFound(err) { // Return error if it is not CommitNotFound error - ic.logger.Info(fmt.Sprintf("Encountered unknown error in loading full commit at height %d.", shdr.Height)) + dv.logger.Info(fmt.Sprintf("Encountered unknown error in loading full commit at height %d.", shdr.Height)) return err } // Get the latest known full commit <= h-1 from our trusted providers. // The full commit at h-1 contains the valset to sign for h. - h := shdr.Height - 1 - trustedFC, err := ic.trusted.LatestFullCommit(ic.chainID, 1, h) + prevHeight := shdr.Height - 1 + trustedFC, err := dv.trusted.LatestFullCommit(dv.chainID, 1, prevHeight) if err != nil { return err } - if trustedFC.Height() == h { + // sync up to the prevHeight and assert our latest NextValidatorSet + // is the ValidatorSet for the SignedHeader + if trustedFC.Height() == prevHeight { // Return error if valset doesn't match. if !bytes.Equal( trustedFC.NextValidators.Hash(), @@ -118,11 +126,12 @@ func (ic *DynamicVerifier) Verify(shdr types.SignedHeader) error { shdr.Header.ValidatorsHash) } } else { - // If valset doesn't match... - if !bytes.Equal(trustedFC.NextValidators.Hash(), + // If valset doesn't match, try to update + if !bytes.Equal( + trustedFC.NextValidators.Hash(), shdr.Header.ValidatorsHash) { // ... update. - trustedFC, err = ic.updateToHeight(h) + trustedFC, err = dv.updateToHeight(prevHeight) if err != nil { return err } @@ -137,14 +146,21 @@ func (ic *DynamicVerifier) Verify(shdr types.SignedHeader) error { } // Verify the signed header using the matching valset. - cert := NewBaseVerifier(ic.chainID, trustedFC.Height()+1, trustedFC.NextValidators) + cert := NewBaseVerifier(dv.chainID, trustedFC.Height()+1, trustedFC.NextValidators) err = cert.Verify(shdr) if err != nil { return err } + // By now, the SignedHeader is fully validated and we're synced up to + // SignedHeader.Height - 1. To sync to SignedHeader.Height, we need + // the validator set at SignedHeader.Height + 1 so we can verify the + // SignedHeader.NextValidatorSet. + // TODO: is the ValidateFull below mostly redundant with the BaseVerifier.Verify above? + // See https://github.com/tendermint/tendermint/issues/3174. + // Get the next validator set. - nextValset, err := ic.source.ValidatorSet(ic.chainID, shdr.Height+1) + nextValset, err := dv.source.ValidatorSet(dv.chainID, shdr.Height+1) if lerr.IsErrUnknownValidators(err) { // Ignore this error. return nil @@ -160,31 +176,31 @@ func (ic *DynamicVerifier) Verify(shdr types.SignedHeader) error { } // Validate the full commit. This checks the cryptographic // signatures of Commit against Validators. - if err := nfc.ValidateFull(ic.chainID); err != nil { + if err := nfc.ValidateFull(dv.chainID); err != nil { return err } // Trust it. - return ic.trusted.SaveFullCommit(nfc) + return dv.trusted.SaveFullCommit(nfc) } // verifyAndSave will verify if this is a valid source full commit given the -// best match trusted full commit, and if good, persist to ic.trusted. +// best match trusted full commit, and if good, persist to dv.trusted. // Returns ErrTooMuchChange when >2/3 of trustedFC did not sign sourceFC. // Panics if trustedFC.Height() >= sourceFC.Height(). -func (ic *DynamicVerifier) verifyAndSave(trustedFC, sourceFC FullCommit) error { +func (dv *DynamicVerifier) verifyAndSave(trustedFC, sourceFC FullCommit) error { if trustedFC.Height() >= sourceFC.Height() { panic("should not happen") } err := trustedFC.NextValidators.VerifyFutureCommit( sourceFC.Validators, - ic.chainID, sourceFC.SignedHeader.Commit.BlockID, + dv.chainID, sourceFC.SignedHeader.Commit.BlockID, sourceFC.SignedHeader.Height, sourceFC.SignedHeader.Commit, ) if err != nil { return err } - return ic.trusted.SaveFullCommit(sourceFC) + return dv.trusted.SaveFullCommit(sourceFC) } // updateToHeight will use divide-and-conquer to find a path to h. @@ -192,29 +208,30 @@ func (ic *DynamicVerifier) verifyAndSave(trustedFC, sourceFC FullCommit) error { // for height h, using repeated applications of bisection if necessary. // // Returns ErrCommitNotFound if source provider doesn't have the commit for h. -func (ic *DynamicVerifier) updateToHeight(h int64) (FullCommit, error) { +func (dv *DynamicVerifier) updateToHeight(h int64) (FullCommit, error) { // Fetch latest full commit from source. - sourceFC, err := ic.source.LatestFullCommit(ic.chainID, h, h) + sourceFC, err := dv.source.LatestFullCommit(dv.chainID, h, h) if err != nil { return FullCommit{}, err } - // Validate the full commit. This checks the cryptographic - // signatures of Commit against Validators. - if err := sourceFC.ValidateFull(ic.chainID); err != nil { - return FullCommit{}, err - } - // If sourceFC.Height() != h, we can't do it. if sourceFC.Height() != h { return FullCommit{}, lerr.ErrCommitNotFound() } + // Validate the full commit. This checks the cryptographic + // signatures of Commit against Validators. + if err := sourceFC.ValidateFull(dv.chainID); err != nil { + return FullCommit{}, err + } + + // Verify latest FullCommit against trusted FullCommits FOR_LOOP: for { // Fetch latest full commit from trusted. - trustedFC, err := ic.trusted.LatestFullCommit(ic.chainID, 1, h) + trustedFC, err := dv.trusted.LatestFullCommit(dv.chainID, 1, h) if err != nil { return FullCommit{}, err } @@ -224,21 +241,21 @@ FOR_LOOP: } // Try to update to full commit with checks. - err = ic.verifyAndSave(trustedFC, sourceFC) + err = dv.verifyAndSave(trustedFC, sourceFC) if err == nil { // All good! return sourceFC, nil } // Handle special case when err is ErrTooMuchChange. - if lerr.IsErrTooMuchChange(err) { + if types.IsErrTooMuchChange(err) { // Divide and conquer. start, end := trustedFC.Height(), sourceFC.Height() if !(start < end) { panic("should not happen") } mid := (start + end) / 2 - _, err = ic.updateToHeight(mid) + _, err = dv.updateToHeight(mid) if err != nil { return FullCommit{}, err } @@ -249,8 +266,8 @@ FOR_LOOP: } } -func (ic *DynamicVerifier) LastTrustedHeight() int64 { - fc, err := ic.trusted.LatestFullCommit(ic.chainID, 1, 1<<63-1) +func (dv *DynamicVerifier) LastTrustedHeight() int64 { + fc, err := dv.trusted.LatestFullCommit(dv.chainID, 1, 1<<63-1) if err != nil { panic("should not happen") } diff --git a/lite/dynamic_verifier_test.go b/lite/dynamic_verifier_test.go index 9ff8ed81..386de513 100644 --- a/lite/dynamic_verifier_test.go +++ b/lite/dynamic_verifier_test.go @@ -10,6 +10,7 @@ import ( dbm "github.com/tendermint/tendermint/libs/db" log "github.com/tendermint/tendermint/libs/log" + "github.com/tendermint/tendermint/types" ) func TestInquirerValidPath(t *testing.T) { @@ -70,6 +71,70 @@ func TestInquirerValidPath(t *testing.T) { assert.Nil(err, "%+v", err) } +func TestDynamicVerify(t *testing.T) { + trust := NewDBProvider("trust", dbm.NewMemDB()) + source := NewDBProvider("source", dbm.NewMemDB()) + + // 10 commits with one valset, 1 to change, + // 10 commits with the next one + n1, n2 := 10, 10 + nCommits := n1 + n2 + 1 + maxHeight := int64(nCommits) + fcz := make([]FullCommit, nCommits) + + // gen the 2 val sets + chainID := "dynamic-verifier" + power := int64(10) + keys1 := genPrivKeys(5) + vals1 := keys1.ToValidators(power, 0) + keys2 := genPrivKeys(5) + vals2 := keys2.ToValidators(power, 0) + + // make some commits with the first + for i := 0; i < n1; i++ { + fcz[i] = makeFullCommit(int64(i), keys1, vals1, vals1, chainID) + } + + // update the val set + fcz[n1] = makeFullCommit(int64(n1), keys1, vals1, vals2, chainID) + + // make some commits with the new one + for i := n1 + 1; i < nCommits; i++ { + fcz[i] = makeFullCommit(int64(i), keys2, vals2, vals2, chainID) + } + + // Save everything in the source + for _, fc := range fcz { + source.SaveFullCommit(fc) + } + + // Initialize a Verifier with the initial state. + err := trust.SaveFullCommit(fcz[0]) + require.Nil(t, err) + ver := NewDynamicVerifier(chainID, trust, source) + ver.SetLogger(log.TestingLogger()) + + // fetch the latest from the source + latestFC, err := source.LatestFullCommit(chainID, 1, maxHeight) + require.NoError(t, err) + + // try to update to the latest + err = ver.Verify(latestFC.SignedHeader) + require.NoError(t, err) + +} + +func makeFullCommit(height int64, keys privKeys, vals, nextVals *types.ValidatorSet, chainID string) FullCommit { + height += 1 + consHash := []byte("special-params") + appHash := []byte(fmt.Sprintf("h=%d", height)) + resHash := []byte(fmt.Sprintf("res=%d", height)) + return keys.GenFullCommit( + chainID, height, nil, + vals, nextVals, + appHash, consHash, resHash, 0, len(keys)) +} + func TestInquirerVerifyHistorical(t *testing.T) { assert, require := assert.New(t), require.New(t) trust := NewDBProvider("trust", dbm.NewMemDB()) diff --git a/lite/errors/errors.go b/lite/errors/errors.go index 59b6380d..75442c72 100644 --- a/lite/errors/errors.go +++ b/lite/errors/errors.go @@ -25,12 +25,6 @@ func (e errUnexpectedValidators) Error() string { e.got, e.want) } -type errTooMuchChange struct{} - -func (e errTooMuchChange) Error() string { - return "Insufficient signatures to validate due to valset changes" -} - type errUnknownValidators struct { chainID string height int64 @@ -85,22 +79,6 @@ func IsErrUnexpectedValidators(err error) bool { return false } -//----------------- -// ErrTooMuchChange - -// ErrTooMuchChange indicates that the underlying validator set was changed by >1/3. -func ErrTooMuchChange() error { - return cmn.ErrorWrap(errTooMuchChange{}, "") -} - -func IsErrTooMuchChange(err error) bool { - if err_, ok := err.(cmn.Error); ok { - _, ok := err_.Data().(errTooMuchChange) - return ok - } - return false -} - //----------------- // ErrUnknownValidators diff --git a/lite/helpers.go b/lite/helpers.go index 16d22e70..6b18b351 100644 --- a/lite/helpers.go +++ b/lite/helpers.go @@ -70,7 +70,7 @@ func (pkz privKeys) ToValidators(init, inc int64) *types.ValidatorSet { // signHeader properly signs the header with all keys from first to last exclusive. func (pkz privKeys) signHeader(header *types.Header, first, last int) *types.Commit { - votes := make([]*types.Vote, len(pkz)) + commitSigs := make([]*types.CommitSig, len(pkz)) // We need this list to keep the ordering. vset := pkz.ToValidators(1, 0) @@ -78,12 +78,12 @@ func (pkz privKeys) signHeader(header *types.Header, first, last int) *types.Com // Fill in the votes we want. for i := first; i < last && i < len(pkz); i++ { vote := makeVote(header, vset, pkz[i]) - votes[vote.ValidatorIndex] = vote + commitSigs[vote.ValidatorIndex] = vote.CommitSig() } res := &types.Commit{ BlockID: types.BlockID{Hash: header.Hash()}, - Precommits: votes, + Precommits: commitSigs, } return res } @@ -97,7 +97,7 @@ func makeVote(header *types.Header, valset *types.ValidatorSet, key crypto.PrivK Height: header.Height, Round: 1, Timestamp: tmtime.Now(), - Type: types.VoteTypePrecommit, + Type: types.PrecommitType, BlockID: types.BlockID{Hash: header.Hash()}, } // Sign it diff --git a/lite/multiprovider.go b/lite/multiprovider.go index 734d042c..a05e19b1 100644 --- a/lite/multiprovider.go +++ b/lite/multiprovider.go @@ -6,6 +6,8 @@ import ( "github.com/tendermint/tendermint/types" ) +var _ PersistentProvider = (*multiProvider)(nil) + // multiProvider allows you to place one or more caches in front of a source // Provider. It runs through them in order until a match is found. type multiProvider struct { diff --git a/lite/provider.go b/lite/provider.go index 97e06a06..ebab1626 100644 --- a/lite/provider.go +++ b/lite/provider.go @@ -1,7 +1,7 @@ package lite import ( - log "github.com/tendermint/tendermint/libs/log" + "github.com/tendermint/tendermint/libs/log" "github.com/tendermint/tendermint/types" ) diff --git a/lite/proxy/proxy.go b/lite/proxy/proxy.go index ffd9db1d..d7ffb27d 100644 --- a/lite/proxy/proxy.go +++ b/lite/proxy/proxy.go @@ -9,7 +9,7 @@ import ( rpcclient "github.com/tendermint/tendermint/rpc/client" "github.com/tendermint/tendermint/rpc/core" ctypes "github.com/tendermint/tendermint/rpc/core/types" - rpc "github.com/tendermint/tendermint/rpc/lib/server" + rpcserver "github.com/tendermint/tendermint/rpc/lib/server" ) const ( @@ -19,6 +19,7 @@ const ( // StartProxy will start the websocket manager on the client, // set up the rpc routes to proxy via the given client, // and start up an http/rpc server on the location given by bind (eg. :1234) +// NOTE: This function blocks - you may want to call it in a go-routine. func StartProxy(c rpcclient.Client, listenAddr string, logger log.Logger, maxOpenConnections int) error { err := c.Start() if err != nil { @@ -31,47 +32,49 @@ func StartProxy(c rpcclient.Client, listenAddr string, logger log.Logger, maxOpe // build the handler... mux := http.NewServeMux() - rpc.RegisterRPCFuncs(mux, r, cdc, logger) + rpcserver.RegisterRPCFuncs(mux, r, cdc, logger) - wm := rpc.NewWebsocketManager(r, cdc, rpc.EventSubscriber(c)) + wm := rpcserver.NewWebsocketManager(r, cdc, rpcserver.EventSubscriber(c)) wm.SetLogger(logger) core.SetLogger(logger) mux.HandleFunc(wsEndpoint, wm.WebsocketHandler) - _, err = rpc.StartHTTPServer(listenAddr, mux, logger, rpc.Config{MaxOpenConnections: maxOpenConnections}) - - return err + l, err := rpcserver.Listen(listenAddr, rpcserver.Config{MaxOpenConnections: maxOpenConnections}) + if err != nil { + return err + } + return rpcserver.StartHTTPServer(l, mux, logger) } // RPCRoutes just routes everything to the given client, as if it were // a tendermint fullnode. // // if we want security, the client must implement it as a secure client -func RPCRoutes(c rpcclient.Client) map[string]*rpc.RPCFunc { +func RPCRoutes(c rpcclient.Client) map[string]*rpcserver.RPCFunc { - return map[string]*rpc.RPCFunc{ + return map[string]*rpcserver.RPCFunc{ // Subscribe/unsubscribe are reserved for websocket events. // We can just use the core tendermint impl, which uses the // EventSwitch we registered in NewWebsocketManager above - "subscribe": rpc.NewWSRPCFunc(core.Subscribe, "query"), - "unsubscribe": rpc.NewWSRPCFunc(core.Unsubscribe, "query"), + "subscribe": rpcserver.NewWSRPCFunc(core.Subscribe, "query"), + "unsubscribe": rpcserver.NewWSRPCFunc(core.Unsubscribe, "query"), // info API - "status": rpc.NewRPCFunc(c.Status, ""), - "blockchain": rpc.NewRPCFunc(c.BlockchainInfo, "minHeight,maxHeight"), - "genesis": rpc.NewRPCFunc(c.Genesis, ""), - "block": rpc.NewRPCFunc(c.Block, "height"), - "commit": rpc.NewRPCFunc(c.Commit, "height"), - "tx": rpc.NewRPCFunc(c.Tx, "hash,prove"), - "validators": rpc.NewRPCFunc(c.Validators, ""), + "status": rpcserver.NewRPCFunc(c.Status, ""), + "blockchain": rpcserver.NewRPCFunc(c.BlockchainInfo, "minHeight,maxHeight"), + "genesis": rpcserver.NewRPCFunc(c.Genesis, ""), + "block": rpcserver.NewRPCFunc(c.Block, "height"), + "commit": rpcserver.NewRPCFunc(c.Commit, "height"), + "tx": rpcserver.NewRPCFunc(c.Tx, "hash,prove"), + "validators": rpcserver.NewRPCFunc(c.Validators, ""), // broadcast API - "broadcast_tx_commit": rpc.NewRPCFunc(c.BroadcastTxCommit, "tx"), - "broadcast_tx_sync": rpc.NewRPCFunc(c.BroadcastTxSync, "tx"), - "broadcast_tx_async": rpc.NewRPCFunc(c.BroadcastTxAsync, "tx"), + "broadcast_tx_commit": rpcserver.NewRPCFunc(c.BroadcastTxCommit, "tx"), + "broadcast_tx_sync": rpcserver.NewRPCFunc(c.BroadcastTxSync, "tx"), + "broadcast_tx_async": rpcserver.NewRPCFunc(c.BroadcastTxAsync, "tx"), // abci API - "abci_query": rpc.NewRPCFunc(c.ABCIQuery, "path,data,prove"), - "abci_info": rpc.NewRPCFunc(c.ABCIInfo, ""), + "abci_query": rpcserver.NewRPCFunc(c.ABCIQuery, "path,data,prove"), + "abci_info": rpcserver.NewRPCFunc(c.ABCIInfo, ""), } } diff --git a/lite/proxy/query_test.go b/lite/proxy/query_test.go index 0e30d755..9547f771 100644 --- a/lite/proxy/query_test.go +++ b/lite/proxy/query_test.go @@ -10,6 +10,7 @@ import ( "github.com/stretchr/testify/require" "github.com/tendermint/tendermint/abci/example/kvstore" + "github.com/tendermint/tendermint/crypto/merkle" "github.com/tendermint/tendermint/lite" certclient "github.com/tendermint/tendermint/lite/client" nm "github.com/tendermint/tendermint/node" @@ -20,6 +21,7 @@ import ( var node *nm.Node var chainID = "tendermint_test" // TODO use from config. +//nolint:unused var waitForEventTimeout = 5 * time.Second // TODO fix tests!! @@ -41,6 +43,7 @@ func kvstoreTx(k, v []byte) []byte { // TODO: enable it after general proof format has been adapted // in abci/examples/kvstore.go +//nolint:unused,deadcode func _TestAppProofs(t *testing.T) { assert, require := assert.New(t), require.New(t) @@ -143,12 +146,13 @@ func TestTxProofs(t *testing.T) { require.NotNil(err) require.Contains(err.Error(), "not found") - // Now let's check with the real tx hash. + // Now let's check with the real tx root hash. key = types.Tx(tx).Hash() res, err = cl.Tx(key, true) require.NoError(err, "%#v", err) require.NotNil(res) - err = res.Proof.Validate(key) + keyHash := merkle.SimpleHashFromByteSlices([][]byte{key}) + err = res.Proof.Validate(keyHash) assert.NoError(err, "%#v", err) commit, err := GetCertifiedCommit(br.Height, cl, cert) diff --git a/mempool/mempool.go b/mempool/mempool.go index 65cd5535..8550f2f8 100644 --- a/mempool/mempool.go +++ b/mempool/mempool.go @@ -11,7 +11,6 @@ import ( "github.com/pkg/errors" - amino "github.com/tendermint/go-amino" abci "github.com/tendermint/tendermint/abci/types" cfg "github.com/tendermint/tendermint/config" auto "github.com/tendermint/tendermint/libs/autofile" @@ -25,12 +24,12 @@ import ( // PreCheckFunc is an optional filter executed before CheckTx and rejects // transaction if false is returned. An example would be to ensure that a // transaction doesn't exceeded the block size. -type PreCheckFunc func(types.Tx) bool +type PreCheckFunc func(types.Tx) error // PostCheckFunc is an optional filter executed after CheckTx and rejects // transaction if false is returned. An example would be to ensure a // transaction doesn't require more gas than available for the block. -type PostCheckFunc func(types.Tx, *abci.ResponseCheckTx) bool +type PostCheckFunc func(types.Tx, *abci.ResponseCheckTx) error /* @@ -66,26 +65,61 @@ var ( // ErrMempoolIsFull means Tendermint & an application can't handle that much load ErrMempoolIsFull = errors.New("Mempool is full") + + // ErrTxTooLarge means the tx is too big to be sent in a message to other peers + ErrTxTooLarge = fmt.Errorf("Tx too large. Max size is %d", maxTxSize) ) +// ErrPreCheck is returned when tx is too big +type ErrPreCheck struct { + Reason error +} + +func (e ErrPreCheck) Error() string { + return e.Reason.Error() +} + +// IsPreCheckError returns true if err is due to pre check failure. +func IsPreCheckError(err error) bool { + _, ok := err.(ErrPreCheck) + return ok +} + // PreCheckAminoMaxBytes checks that the size of the transaction plus the amino // overhead is smaller or equal to the expected maxBytes. func PreCheckAminoMaxBytes(maxBytes int64) PreCheckFunc { - return func(tx types.Tx) bool { + return func(tx types.Tx) error { // We have to account for the amino overhead in the tx size as well - aminoOverhead := amino.UvarintSize(uint64(len(tx))) - return int64(len(tx)+aminoOverhead) <= maxBytes + // NOTE: fieldNum = 1 as types.Block.Data contains Txs []Tx as first field. + // If this field order ever changes this needs to updated here accordingly. + // NOTE: if some []Tx are encoded without a parenting struct, the + // fieldNum is also equal to 1. + aminoOverhead := types.ComputeAminoOverhead(tx, 1) + txSize := int64(len(tx)) + aminoOverhead + if txSize > maxBytes { + return fmt.Errorf("Tx size (including amino overhead) is too big: %d, max: %d", + txSize, maxBytes) + } + return nil } } // PostCheckMaxGas checks that the wanted gas is smaller or equal to the passed -// maxGas. Returns true if maxGas is -1. +// maxGas. Returns nil if maxGas is -1. func PostCheckMaxGas(maxGas int64) PostCheckFunc { - return func(tx types.Tx, res *abci.ResponseCheckTx) bool { + return func(tx types.Tx, res *abci.ResponseCheckTx) error { if maxGas == -1 { - return true + return nil } - return res.GasWanted <= maxGas + if res.GasWanted < 0 { + return fmt.Errorf("gas wanted %d is negative", + res.GasWanted) + } + if res.GasWanted > maxGas { + return fmt.Errorf("gas wanted %d is greater than max gas %d", + res.GasWanted, maxGas) + } + return nil } } @@ -104,7 +138,6 @@ type Mempool struct { proxyMtx sync.Mutex proxyAppConn proxy.AppConnMempool txs *clist.CList // concurrent linked-list of good txs - counter int64 // simple incrementing counter height int64 // the last block Update()'d to rechecking int32 // for re-checking filtered txs on Update() recheckCursor *clist.CElement // next expected response @@ -140,7 +173,6 @@ func NewMempool( config: config, proxyAppConn: proxyAppConn, txs: clist.New(), - counter: 0, height: height, rechecking: 0, recheckCursor: nil, @@ -189,39 +221,33 @@ func WithMetrics(metrics *Metrics) MempoolOption { return func(mem *Mempool) { mem.metrics = metrics } } +// InitWAL creates a directory for the WAL file and opens a file itself. +// +// *panics* if can't create directory or open file. +// *not thread safe* +func (mem *Mempool) InitWAL() { + walDir := mem.config.WalDir() + err := cmn.EnsureDir(walDir, 0700) + if err != nil { + panic(errors.Wrap(err, "Error ensuring Mempool WAL dir")) + } + af, err := auto.OpenAutoFile(walDir + "/wal") + if err != nil { + panic(errors.Wrap(err, "Error opening Mempool WAL file")) + } + mem.wal = af +} + // CloseWAL closes and discards the underlying WAL file. // Any further writes will not be relayed to disk. -func (mem *Mempool) CloseWAL() bool { - if mem == nil { - return false - } - +func (mem *Mempool) CloseWAL() { mem.proxyMtx.Lock() defer mem.proxyMtx.Unlock() - if mem.wal == nil { - return false - } - if err := mem.wal.Close(); err != nil && mem.logger != nil { - mem.logger.Error("Mempool.CloseWAL", "err", err) + if err := mem.wal.Close(); err != nil { + mem.logger.Error("Error closing WAL", "err", err) } mem.wal = nil - return true -} - -func (mem *Mempool) InitWAL() { - walDir := mem.config.WalDir() - if walDir != "" { - err := cmn.EnsureDir(walDir, 0700) - if err != nil { - cmn.PanicSanity(errors.Wrap(err, "Error ensuring Mempool wal dir")) - } - af, err := auto.OpenAutoFile(walDir + "/wal") - if err != nil { - cmn.PanicSanity(errors.Wrap(err, "Error opening Mempool wal file")) - } - mem.wal = af - } } // Lock locks the mempool. The consensus must be able to hold lock to safely update. @@ -279,14 +305,24 @@ func (mem *Mempool) TxsWaitChan() <-chan struct{} { // CONTRACT: Either cb will get called, or err returned. func (mem *Mempool) CheckTx(tx types.Tx, cb func(*abci.Response)) (err error) { mem.proxyMtx.Lock() + // use defer to unlock mutex because application (*local client*) might panic defer mem.proxyMtx.Unlock() if mem.Size() >= mem.config.Size { return ErrMempoolIsFull } - if mem.preCheck != nil && !mem.preCheck(tx) { - return + // The size of the corresponding amino-encoded TxMessage + // can't be larger than the maxMsgSize, otherwise we can't + // relay it to peers. + if len(tx) > maxTxSize { + return ErrTxTooLarge + } + + if mem.preCheck != nil { + if err := mem.preCheck(tx); err != nil { + return ErrPreCheck{err} + } } // CACHE @@ -336,22 +372,28 @@ func (mem *Mempool) resCbNormal(req *abci.Request, res *abci.Response) { switch r := res.Value.(type) { case *abci.Response_CheckTx: tx := req.GetCheckTx().Tx - if (r.CheckTx.Code == abci.CodeTypeOK) && - mem.isPostCheckPass(tx, r.CheckTx) { - mem.counter++ + var postCheckErr error + if mem.postCheck != nil { + postCheckErr = mem.postCheck(tx, r.CheckTx) + } + if (r.CheckTx.Code == abci.CodeTypeOK) && postCheckErr == nil { memTx := &mempoolTx{ - counter: mem.counter, height: mem.height, gasWanted: r.CheckTx.GasWanted, tx: tx, } mem.txs.PushBack(memTx) - mem.logger.Info("Added good transaction", "tx", TxID(tx), "res", r, "total", mem.Size()) + mem.logger.Info("Added good transaction", + "tx", TxID(tx), + "res", r, + "height", memTx.height, + "total", mem.Size(), + ) mem.metrics.TxSizeBytes.Observe(float64(len(tx))) mem.notifyTxsAvailable() } else { // ignore bad transaction - mem.logger.Info("Rejected bad transaction", "tx", TxID(tx), "res", r) + mem.logger.Info("Rejected bad transaction", "tx", TxID(tx), "res", r, "err", postCheckErr) mem.metrics.FailedTxs.Add(1) // remove from cache (it might be good later) mem.cache.Remove(tx) @@ -364,25 +406,28 @@ func (mem *Mempool) resCbNormal(req *abci.Request, res *abci.Response) { func (mem *Mempool) resCbRecheck(req *abci.Request, res *abci.Response) { switch r := res.Value.(type) { case *abci.Response_CheckTx: + tx := req.GetCheckTx().Tx memTx := mem.recheckCursor.Value.(*mempoolTx) - if !bytes.Equal(req.GetCheckTx().Tx, memTx.tx) { - cmn.PanicSanity( - fmt.Sprintf( - "Unexpected tx response from proxy during recheck\nExpected %X, got %X", - r.CheckTx.Data, - memTx.tx, - ), - ) + if !bytes.Equal(tx, memTx.tx) { + panic(fmt.Sprintf( + "Unexpected tx response from proxy during recheck\nExpected %X, got %X", + memTx.tx, + tx)) } - if (r.CheckTx.Code == abci.CodeTypeOK) && mem.isPostCheckPass(memTx.tx, r.CheckTx) { + var postCheckErr error + if mem.postCheck != nil { + postCheckErr = mem.postCheck(tx, r.CheckTx) + } + if (r.CheckTx.Code == abci.CodeTypeOK) && postCheckErr == nil { // Good, nothing to do. } else { // Tx became invalidated due to newly committed block. + mem.logger.Info("Tx is no longer valid", "tx", TxID(tx), "res", r, "err", postCheckErr) mem.txs.Remove(mem.recheckCursor) mem.recheckCursor.DetachPrev() // remove from cache (it might be good later) - mem.cache.Remove(req.GetCheckTx().Tx) + mem.cache.Remove(tx) } if mem.recheckCursor == mem.recheckEnd { mem.recheckCursor = nil @@ -447,16 +492,20 @@ func (mem *Mempool) ReapMaxBytesMaxGas(maxBytes, maxGas int64) types.Txs { for e := mem.txs.Front(); e != nil; e = e.Next() { memTx := e.Value.(*mempoolTx) // Check total size requirement - aminoOverhead := int64(amino.UvarintSize(uint64(len(memTx.tx)))) + aminoOverhead := types.ComputeAminoOverhead(memTx.tx, 1) if maxBytes > -1 && totalBytes+int64(len(memTx.tx))+aminoOverhead > maxBytes { return txs } totalBytes += int64(len(memTx.tx)) + aminoOverhead - // Check total gas requirement - if maxGas > -1 && totalGas+memTx.gasWanted > maxGas { + // Check total gas requirement. + // If maxGas is negative, skip this check. + // Since newTotalGas < masGas, which + // must be non-negative, it follows that this won't overflow. + newTotalGas := totalGas + memTx.gasWanted + if maxGas > -1 && newTotalGas > maxGas { return txs } - totalGas += memTx.gasWanted + totalGas = newTotalGas txs = append(txs, memTx.tx) } return txs @@ -495,12 +544,6 @@ func (mem *Mempool) Update( preCheck PreCheckFunc, postCheck PostCheckFunc, ) error { - // First, create a lookup map of txns in new txs. - txsMap := make(map[string]struct{}, len(txs)) - for _, tx := range txs { - txsMap[string(tx)] = struct{}{} - } - // Set height mem.height = height mem.notifiedTxsAvailable = false @@ -512,15 +555,26 @@ func (mem *Mempool) Update( mem.postCheck = postCheck } - // Remove transactions that are already in txs. - goodTxs := mem.filterTxs(txsMap) - // Recheck mempool txs if any txs were committed in the block - if mem.config.Recheck && len(goodTxs) > 0 { - mem.logger.Info("Recheck txs", "numtxs", len(goodTxs), "height", height) - mem.recheckTxs(goodTxs) - // At this point, mem.txs are being rechecked. - // mem.recheckCursor re-scans mem.txs and possibly removes some txs. - // Before mem.Reap(), we should wait for mem.recheckCursor to be nil. + // Add committed transactions to cache (if missing). + for _, tx := range txs { + _ = mem.cache.Push(tx) + } + + // Remove committed transactions. + txsLeft := mem.removeTxs(txs) + + // Either recheck non-committed txs to see if they became invalid + // or just notify there're some txs left. + if len(txsLeft) > 0 { + if mem.config.Recheck { + mem.logger.Info("Recheck txs", "numtxs", len(txsLeft), "height", height) + mem.recheckTxs(txsLeft) + // At this point, mem.txs are being rechecked. + // mem.recheckCursor re-scans mem.txs and possibly removes some txs. + // Before mem.Reap(), we should wait for mem.recheckCursor to be nil. + } else { + mem.notifyTxsAvailable() + } } // Update metrics @@ -529,12 +583,18 @@ func (mem *Mempool) Update( return nil } -func (mem *Mempool) filterTxs(blockTxsMap map[string]struct{}) []types.Tx { - goodTxs := make([]types.Tx, 0, mem.txs.Len()) +func (mem *Mempool) removeTxs(txs types.Txs) []types.Tx { + // Build a map for faster lookups. + txsMap := make(map[string]struct{}, len(txs)) + for _, tx := range txs { + txsMap[string(tx)] = struct{}{} + } + + txsLeft := make([]types.Tx, 0, mem.txs.Len()) for e := mem.txs.Front(); e != nil; e = e.Next() { memTx := e.Value.(*mempoolTx) - // Remove the tx if it's alredy in a block. - if _, ok := blockTxsMap[string(memTx.tx)]; ok { + // Remove the tx if it's already in a block. + if _, ok := txsMap[string(memTx.tx)]; ok { // remove from clist mem.txs.Remove(e) e.DetachPrev() @@ -542,15 +602,14 @@ func (mem *Mempool) filterTxs(blockTxsMap map[string]struct{}) []types.Tx { // NOTE: we don't remove committed txs from the cache. continue } - // Good tx! - goodTxs = append(goodTxs, memTx.tx) + txsLeft = append(txsLeft, memTx.tx) } - return goodTxs + return txsLeft } -// NOTE: pass in goodTxs because mem.txs can mutate concurrently. -func (mem *Mempool) recheckTxs(goodTxs []types.Tx) { - if len(goodTxs) == 0 { +// NOTE: pass in txs because mem.txs can mutate concurrently. +func (mem *Mempool) recheckTxs(txs []types.Tx) { + if len(txs) == 0 { return } atomic.StoreInt32(&mem.rechecking, 1) @@ -559,21 +618,16 @@ func (mem *Mempool) recheckTxs(goodTxs []types.Tx) { // Push txs to proxyAppConn // NOTE: resCb() may be called concurrently. - for _, tx := range goodTxs { + for _, tx := range txs { mem.proxyAppConn.CheckTxAsync(tx) } mem.proxyAppConn.FlushAsync() } -func (mem *Mempool) isPostCheckPass(tx types.Tx, r *abci.ResponseCheckTx) bool { - return mem.postCheck == nil || mem.postCheck(tx, r) -} - //-------------------------------------------------------------------------------- // mempoolTx is a transaction that successfully ran type mempoolTx struct { - counter int64 // a simple incrementing counter height int64 // height that this tx had been validated in gasWanted int64 // amount of gas this tx states it will require tx types.Tx // @@ -629,7 +683,7 @@ func (cache *mapTxCache) Push(tx types.Tx) bool { // Use the tx hash in the cache txHash := sha256.Sum256(tx) if moved, exists := cache.map_[txHash]; exists { - cache.list.MoveToFront(moved) + cache.list.MoveToBack(moved) return false } diff --git a/mempool/mempool_test.go b/mempool/mempool_test.go index 5aabd00e..9d21e734 100644 --- a/mempool/mempool_test.go +++ b/mempool/mempool_test.go @@ -19,6 +19,7 @@ import ( "github.com/tendermint/tendermint/abci/example/kvstore" abci "github.com/tendermint/tendermint/abci/types" cfg "github.com/tendermint/tendermint/config" + cmn "github.com/tendermint/tendermint/libs/common" "github.com/tendermint/tendermint/libs/log" "github.com/tendermint/tendermint/proxy" "github.com/tendermint/tendermint/types" @@ -66,7 +67,13 @@ func checkTxs(t *testing.T, mempool *Mempool, count int) types.Txs { t.Error(err) } if err := mempool.CheckTx(txBytes, nil); err != nil { - t.Fatalf("Error after CheckTx: %v", err) + // Skip invalid txs. + // TestMempoolFilters will fail otherwise. It asserts a number of txs + // returned. + if IsPreCheckError(err) { + continue + } + t.Fatalf("CheckTx failed: %v while checking #%d tx", err, i) } } return txs @@ -102,11 +109,11 @@ func TestReapMaxBytesMaxGas(t *testing.T) { {20, 0, -1, 0}, {20, 0, 10, 0}, {20, 10, 10, 0}, - {20, 21, 10, 1}, - {20, 210, -1, 10}, - {20, 210, 5, 5}, - {20, 210, 10, 10}, - {20, 210, 15, 10}, + {20, 22, 10, 1}, + {20, 220, -1, 10}, + {20, 220, 5, 5}, + {20, 220, 10, 10}, + {20, 220, 15, 10}, {20, 20000, -1, 20}, {20, 20000, 5, 5}, {20, 20000, 30, 20}, @@ -126,47 +133,29 @@ func TestMempoolFilters(t *testing.T) { mempool := newMempoolWithApp(cc) emptyTxArr := []types.Tx{[]byte{}} - nopPreFilter := func(tx types.Tx) bool { return true } - nopPostFilter := func(tx types.Tx, res *abci.ResponseCheckTx) bool { return true } - - // This is the same filter we expect to be used within node/node.go and state/execution.go - nBytePreFilter := func(n int) func(tx types.Tx) bool { - return func(tx types.Tx) bool { - // We have to account for the amino overhead in the tx size as well - aminoOverhead := amino.UvarintSize(uint64(len(tx))) - return (len(tx) + aminoOverhead) <= n - } - } - - nGasPostFilter := func(n int64) func(tx types.Tx, res *abci.ResponseCheckTx) bool { - return func(tx types.Tx, res *abci.ResponseCheckTx) bool { - if n == -1 { - return true - } - return res.GasWanted <= n - } - } + nopPreFilter := func(tx types.Tx) error { return nil } + nopPostFilter := func(tx types.Tx, res *abci.ResponseCheckTx) error { return nil } // each table driven test creates numTxsToCreate txs with checkTx, and at the end clears all remaining txs. // each tx has 20 bytes + amino overhead = 21 bytes, 1 gas tests := []struct { numTxsToCreate int - preFilter func(tx types.Tx) bool - postFilter func(tx types.Tx, res *abci.ResponseCheckTx) bool + preFilter PreCheckFunc + postFilter PostCheckFunc expectedNumTxs int }{ {10, nopPreFilter, nopPostFilter, 10}, - {10, nBytePreFilter(10), nopPostFilter, 0}, - {10, nBytePreFilter(20), nopPostFilter, 0}, - {10, nBytePreFilter(21), nopPostFilter, 10}, - {10, nopPreFilter, nGasPostFilter(-1), 10}, - {10, nopPreFilter, nGasPostFilter(0), 0}, - {10, nopPreFilter, nGasPostFilter(1), 10}, - {10, nopPreFilter, nGasPostFilter(3000), 10}, - {10, nBytePreFilter(10), nGasPostFilter(20), 0}, - {10, nBytePreFilter(30), nGasPostFilter(20), 10}, - {10, nBytePreFilter(21), nGasPostFilter(1), 10}, - {10, nBytePreFilter(21), nGasPostFilter(0), 0}, + {10, PreCheckAminoMaxBytes(10), nopPostFilter, 0}, + {10, PreCheckAminoMaxBytes(20), nopPostFilter, 0}, + {10, PreCheckAminoMaxBytes(22), nopPostFilter, 10}, + {10, nopPreFilter, PostCheckMaxGas(-1), 10}, + {10, nopPreFilter, PostCheckMaxGas(0), 0}, + {10, nopPreFilter, PostCheckMaxGas(1), 10}, + {10, nopPreFilter, PostCheckMaxGas(3000), 10}, + {10, PreCheckAminoMaxBytes(10), PostCheckMaxGas(20), 0}, + {10, PreCheckAminoMaxBytes(30), PostCheckMaxGas(20), 10}, + {10, PreCheckAminoMaxBytes(22), PostCheckMaxGas(1), 10}, + {10, PreCheckAminoMaxBytes(22), PostCheckMaxGas(0), 0}, } for tcIndex, tt := range tests { mempool.Update(1, emptyTxArr, tt.preFilter, tt.postFilter) @@ -176,6 +165,17 @@ func TestMempoolFilters(t *testing.T) { } } +func TestMempoolUpdateAddsTxsToCache(t *testing.T) { + app := kvstore.NewKVStoreApplication() + cc := proxy.NewLocalClientCreator(app) + mempool := newMempoolWithApp(cc) + mempool.Update(1, []types.Tx{[]byte{0x01}}, nil, nil) + err := mempool.CheckTx([]byte{0x01}, nil) + if assert.Error(t, err) { + assert.Equal(t, ErrTxInCache, err) + } +} + func TestTxsAvailable(t *testing.T) { app := kvstore.NewKVStoreApplication() cc := proxy.NewLocalClientCreator(app) @@ -385,20 +385,71 @@ func TestMempoolCloseWAL(t *testing.T) { // 7. Invoke CloseWAL() and ensure it discards the // WAL thus any other write won't go through. - require.True(t, mempool.CloseWAL(), "CloseWAL should CloseWAL") + mempool.CloseWAL() mempool.CheckTx(types.Tx([]byte("bar")), nil) sum2 := checksumFile(walFilepath, t) require.Equal(t, sum1, sum2, "expected no change to the WAL after invoking CloseWAL() since it was discarded") - // 8. Second CloseWAL should do nothing - require.False(t, mempool.CloseWAL(), "CloseWAL should CloseWAL") - - // 9. Sanity check to ensure that the WAL file still exists + // 8. Sanity check to ensure that the WAL file still exists m3, err := filepath.Glob(filepath.Join(rootDir, "*")) require.Nil(t, err, "successful globbing expected") require.Equal(t, 1, len(m3), "expecting the wal match in") } +// Size of the amino encoded TxMessage is the length of the +// encoded byte array, plus 1 for the struct field, plus 4 +// for the amino prefix. +func txMessageSize(tx types.Tx) int { + return amino.ByteSliceSize(tx) + 1 + 4 +} + +func TestMempoolMaxMsgSize(t *testing.T) { + app := kvstore.NewKVStoreApplication() + cc := proxy.NewLocalClientCreator(app) + mempl := newMempoolWithApp(cc) + + testCases := []struct { + len int + err bool + }{ + // check small txs. no error + {10, false}, + {1000, false}, + {1000000, false}, + + // check around maxTxSize + // changes from no error to error + {maxTxSize - 2, false}, + {maxTxSize - 1, false}, + {maxTxSize, false}, + {maxTxSize + 1, true}, + {maxTxSize + 2, true}, + + // check around maxMsgSize. all error + {maxMsgSize - 1, true}, + {maxMsgSize, true}, + {maxMsgSize + 1, true}, + } + + for i, testCase := range testCases { + caseString := fmt.Sprintf("case %d, len %d", i, testCase.len) + + tx := cmn.RandBytes(testCase.len) + err := mempl.CheckTx(tx, nil) + msg := &TxMessage{tx} + encoded := cdc.MustMarshalBinaryBare(msg) + require.Equal(t, len(encoded), txMessageSize(tx), caseString) + if !testCase.err { + require.True(t, len(encoded) <= maxMsgSize, caseString) + require.NoError(t, err, caseString) + } else { + require.True(t, len(encoded) > maxMsgSize, caseString) + require.Equal(t, err, ErrTxTooLarge, caseString) + } + } + +} + func checksumIt(data []byte) string { h := md5.New() h.Write(data) diff --git a/mempool/metrics.go b/mempool/metrics.go index 3418f1ef..5e4eaf5e 100644 --- a/mempool/metrics.go +++ b/mempool/metrics.go @@ -7,7 +7,11 @@ import ( stdprometheus "github.com/prometheus/client_golang/prometheus" ) -const MetricsSubsytem = "mempool" +const ( + // MetricsSubsystem is a subsystem shared by all metrics exposed by this + // package. + MetricsSubsystem = "mempool" +) // Metrics contains metrics exposed by this package. // see MetricsProvider for descriptions. @@ -23,33 +27,39 @@ type Metrics struct { } // PrometheusMetrics returns Metrics build using Prometheus client library. -func PrometheusMetrics(namespace string) *Metrics { +// Optionally, labels can be provided along with their values ("foo", +// "fooValue"). +func PrometheusMetrics(namespace string, labelsAndValues ...string) *Metrics { + labels := []string{} + for i := 0; i < len(labelsAndValues); i += 2 { + labels = append(labels, labelsAndValues[i]) + } return &Metrics{ Size: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ Namespace: namespace, - Subsystem: MetricsSubsytem, + Subsystem: MetricsSubsystem, Name: "size", Help: "Size of the mempool (number of uncommitted transactions).", - }, []string{}), + }, labels).With(labelsAndValues...), TxSizeBytes: prometheus.NewHistogramFrom(stdprometheus.HistogramOpts{ Namespace: namespace, - Subsystem: MetricsSubsytem, + Subsystem: MetricsSubsystem, Name: "tx_size_bytes", Help: "Transaction sizes in bytes.", Buckets: stdprometheus.ExponentialBuckets(1, 3, 17), - }, []string{}), + }, labels).With(labelsAndValues...), FailedTxs: prometheus.NewCounterFrom(stdprometheus.CounterOpts{ Namespace: namespace, - Subsystem: MetricsSubsytem, + Subsystem: MetricsSubsystem, Name: "failed_txs", Help: "Number of failed transactions.", - }, []string{}), + }, labels).With(labelsAndValues...), RecheckTimes: prometheus.NewCounterFrom(stdprometheus.CounterOpts{ Namespace: namespace, - Subsystem: MetricsSubsytem, + Subsystem: MetricsSubsystem, Name: "recheck_times", Help: "Number of times transactions are rechecked in the mempool.", - }, []string{}), + }, labels).With(labelsAndValues...), } } diff --git a/mempool/reactor.go b/mempool/reactor.go index 96988be7..ff87f050 100644 --- a/mempool/reactor.go +++ b/mempool/reactor.go @@ -6,7 +6,6 @@ import ( "time" amino "github.com/tendermint/go-amino" - abci "github.com/tendermint/tendermint/abci/types" "github.com/tendermint/tendermint/libs/clist" "github.com/tendermint/tendermint/libs/log" @@ -18,8 +17,10 @@ import ( const ( MempoolChannel = byte(0x30) - maxMsgSize = 1048576 // 1MB TODO make it configurable - peerCatchupSleepIntervalMS = 100 // If peer is behind, sleep this amount + maxMsgSize = 1048576 // 1MB TODO make it configurable + maxTxSize = maxMsgSize - 8 // account for amino overhead of TxMessage + + peerCatchupSleepIntervalMS = 100 // If peer is behind, sleep this amount ) // MempoolReactor handles mempool tx broadcasting amongst peers. @@ -98,11 +99,6 @@ func (memR *MempoolReactor) Receive(chID byte, src p2p.Peer, msgBytes []byte) { } } -// BroadcastTx is an alias for Mempool.CheckTx. Broadcasting itself happens in peer routines. -func (memR *MempoolReactor) BroadcastTx(tx types.Tx, cb func(*abci.Response)) error { - return memR.Mempool.CheckTx(tx, cb) -} - // PeerState describes the state of a peer. type PeerState interface { GetHeight() int64 @@ -133,16 +129,23 @@ func (memR *MempoolReactor) broadcastTxRoutine(peer p2p.Peer) { } memTx := next.Value.(*mempoolTx) + // make sure the peer is up to date - height := memTx.Height() - if peerState_i := peer.Get(types.PeerStateKey); peerState_i != nil { - peerState := peerState_i.(PeerState) - peerHeight := peerState.GetHeight() - if peerHeight < height-1 { // Allow for a lag of 1 block - time.Sleep(peerCatchupSleepIntervalMS * time.Millisecond) - continue - } + peerState, ok := peer.Get(types.PeerStateKey).(PeerState) + if !ok { + // Peer does not have a state yet. We set it in the consensus reactor, but + // when we add peer in Switch, the order we call reactors#AddPeer is + // different every time due to us using a map. Sometimes other reactors + // will be initialized before the consensus reactor. We should wait a few + // milliseconds and retry. + time.Sleep(peerCatchupSleepIntervalMS * time.Millisecond) + continue } + if peerState.GetHeight() < memTx.Height()-1 { // Allow for a lag of 1 block + time.Sleep(peerCatchupSleepIntervalMS * time.Millisecond) + continue + } + // send memTx msg := &TxMessage{Tx: memTx.tx} success := peer.Send(MempoolChannel, cdc.MustMarshalBinaryBare(msg)) diff --git a/mempool/reactor_test.go b/mempool/reactor_test.go index 8ac400b0..ad9ad8b4 100644 --- a/mempool/reactor_test.go +++ b/mempool/reactor_test.go @@ -21,6 +21,14 @@ import ( "github.com/tendermint/tendermint/types" ) +type peerState struct { + height int64 +} + +func (ps peerState) GetHeight() int64 { + return ps.height +} + // mempoolLogger is a TestingLogger which uses a different // color for each validator ("validator" key must exist). func mempoolLogger() log.Logger { @@ -107,6 +115,11 @@ func TestReactorBroadcastTxMessage(t *testing.T) { r.Stop() } }() + for _, r := range reactors { + for _, peer := range r.Switch.Peers().List() { + peer.Set(types.PeerStateKey, peerState{1}) + } + } // send a bunch of txs to the first reactor's mempool // and wait for them all to be received in the others diff --git a/mempool/wire.go b/mempool/wire.go index ed089726..9224af41 100644 --- a/mempool/wire.go +++ b/mempool/wire.go @@ -1,7 +1,7 @@ package mempool import ( - "github.com/tendermint/go-amino" + amino "github.com/tendermint/go-amino" ) var cdc = amino.NewCodec() diff --git a/node/node.go b/node/node.go index d1ab0f86..1b731981 100644 --- a/node/node.go +++ b/node/node.go @@ -3,18 +3,20 @@ package node import ( "bytes" "context" - "errors" "fmt" "net" "net/http" _ "net/http/pprof" + "os" "strings" "time" + "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" - "github.com/tendermint/go-amino" + "github.com/rs/cors" + amino "github.com/tendermint/go-amino" abci "github.com/tendermint/tendermint/abci/types" bc "github.com/tendermint/tendermint/blockchain" cfg "github.com/tendermint/tendermint/config" @@ -32,8 +34,7 @@ import ( rpccore "github.com/tendermint/tendermint/rpc/core" ctypes "github.com/tendermint/tendermint/rpc/core/types" grpccore "github.com/tendermint/tendermint/rpc/grpc" - "github.com/tendermint/tendermint/rpc/lib" - "github.com/tendermint/tendermint/rpc/lib/server" + rpcserver "github.com/tendermint/tendermint/rpc/lib/server" sm "github.com/tendermint/tendermint/state" "github.com/tendermint/tendermint/state/txindex" "github.com/tendermint/tendermint/state/txindex/kv" @@ -86,8 +87,26 @@ func DefaultNewNode(config *cfg.Config, logger log.Logger) (*Node, error) { if err != nil { return nil, err } + + // Convert old PrivValidator if it exists. + oldPrivVal := config.OldPrivValidatorFile() + newPrivValKey := config.PrivValidatorKeyFile() + newPrivValState := config.PrivValidatorStateFile() + if _, err := os.Stat(oldPrivVal); !os.IsNotExist(err) { + oldPV, err := privval.LoadOldFilePV(oldPrivVal) + if err != nil { + return nil, fmt.Errorf("Error reading OldPrivValidator from %v: %v\n", oldPrivVal, err) + } + logger.Info("Upgrading PrivValidator file", + "old", oldPrivVal, + "newKey", newPrivValKey, + "newState", newPrivValState, + ) + oldPV.Upgrade(newPrivValKey, newPrivValState) + } + return NewNode(config, - privval.LoadOrGenFilePV(config.PrivValidatorFile()), + privval.LoadOrGenFilePV(newPrivValKey, newPrivValState), nodeKey, proxy.DefaultClientCreator(config.ProxyApp, config.ABCI, config.DBDir()), DefaultGenesisDocProviderFunc(config), @@ -98,15 +117,17 @@ func DefaultNewNode(config *cfg.Config, logger log.Logger) (*Node, error) { } // MetricsProvider returns a consensus, p2p and mempool Metrics. -type MetricsProvider func() (*cs.Metrics, *p2p.Metrics, *mempl.Metrics, *sm.Metrics) +type MetricsProvider func(chainID string) (*cs.Metrics, *p2p.Metrics, *mempl.Metrics, *sm.Metrics) // DefaultMetricsProvider returns Metrics build using Prometheus client library // if Prometheus is enabled. Otherwise, it returns no-op Metrics. func DefaultMetricsProvider(config *cfg.InstrumentationConfig) MetricsProvider { - return func() (*cs.Metrics, *p2p.Metrics, *mempl.Metrics, *sm.Metrics) { + return func(chainID string) (*cs.Metrics, *p2p.Metrics, *mempl.Metrics, *sm.Metrics) { if config.Prometheus { - return cs.PrometheusMetrics(config.Namespace), p2p.PrometheusMetrics(config.Namespace), - mempl.PrometheusMetrics(config.Namespace), sm.PrometheusMetrics(config.Namespace) + return cs.PrometheusMetrics(config.Namespace, "chain_id", chainID), + p2p.PrometheusMetrics(config.Namespace, "chain_id", chainID), + mempl.PrometheusMetrics(config.Namespace, "chain_id", chainID), + sm.PrometheusMetrics(config.Namespace, "chain_id", chainID) } return cs.NopMetrics(), p2p.NopMetrics(), mempl.NopMetrics(), sm.NopMetrics() } @@ -196,37 +217,83 @@ func NewNode(config *cfg.Config, return nil, fmt.Errorf("Error starting proxy app connections: %v", err) } - // Create the handshaker, which calls RequestInfo and replays any blocks - // as necessary to sync tendermint with the app. + // EventBus and IndexerService must be started before the handshake because + // we might need to index the txs of the replayed block as this might not have happened + // when the node stopped last time (i.e. the node stopped after it saved the block + // but before it indexed the txs, or, endblocker panicked) + eventBus := types.NewEventBus() + eventBus.SetLogger(logger.With("module", "events")) + + err = eventBus.Start() + if err != nil { + return nil, err + } + + // Transaction indexing + var txIndexer txindex.TxIndexer + switch config.TxIndex.Indexer { + case "kv": + store, err := dbProvider(&DBContext{"tx_index", config}) + if err != nil { + return nil, err + } + if config.TxIndex.IndexTags != "" { + txIndexer = kv.NewTxIndex(store, kv.IndexTags(splitAndTrimEmpty(config.TxIndex.IndexTags, ",", " "))) + } else if config.TxIndex.IndexAllTags { + txIndexer = kv.NewTxIndex(store, kv.IndexAllTags()) + } else { + txIndexer = kv.NewTxIndex(store) + } + default: + txIndexer = &null.TxIndex{} + } + + indexerService := txindex.NewIndexerService(txIndexer, eventBus) + indexerService.SetLogger(logger.With("module", "txindex")) + + err = indexerService.Start() + if err != nil { + return nil, err + } + + // Create the handshaker, which calls RequestInfo, sets the AppVersion on the state, + // and replays any blocks as necessary to sync tendermint with the app. consensusLogger := logger.With("module", "consensus") handshaker := cs.NewHandshaker(stateDB, state, blockStore, genDoc) handshaker.SetLogger(consensusLogger) + handshaker.SetEventBus(eventBus) if err := handshaker.Handshake(proxyApp); err != nil { return nil, fmt.Errorf("Error during handshake: %v", err) } - // reload the state (it may have been updated by the handshake) + // Reload the state. It will have the Version.Consensus.App set by the + // Handshake, and may have other modifications as well (ie. depending on + // what happened during block replay). state = sm.LoadState(stateDB) - // If an address is provided, listen on the socket for a - // connection from an external signing process. - if config.PrivValidatorListenAddr != "" { - var ( - // TODO: persist this key so external signer - // can actually authenticate us - privKey = ed25519.GenPrivKey() - pvsc = privval.NewSocketPV( - logger.With("module", "privval"), - config.PrivValidatorListenAddr, - privKey, - ) + // Log the version info. + logger.Info("Version info", + "software", version.TMCoreSemVer, + "block", version.BlockProtocol, + "p2p", version.P2PProtocol, + ) + + // If the state and software differ in block version, at least log it. + if state.Version.Consensus.Block != version.BlockProtocol { + logger.Info("Software and state have different block protocols", + "software", version.BlockProtocol, + "state", state.Version.Consensus.Block, ) + } - if err := pvsc.Start(); err != nil { - return nil, fmt.Errorf("Error starting private validator client: %v", err) + if config.PrivValidatorListenAddr != "" { + // If an address is provided, listen on the socket for a connection from an + // external signing process. + // FIXME: we should start services inside OnStart + privValidator, err = createAndStartPrivValidatorSocketClient(config.PrivValidatorListenAddr, logger) + if err != nil { + return nil, errors.Wrap(err, "Error with private validator socket client") } - - privValidator = pvsc } // Decide whether to fast-sync or not @@ -234,19 +301,22 @@ func NewNode(config *cfg.Config, fastSync := config.FastSync if state.Validators.Size() == 1 { addr, _ := state.Validators.GetByIndex(0) - if bytes.Equal(privValidator.GetAddress(), addr) { + privValAddr := privValidator.GetPubKey().Address() + if bytes.Equal(privValAddr, addr) { fastSync = false } } + pubKey := privValidator.GetPubKey() + addr := pubKey.Address() // Log whether this node is a validator or an observer - if state.Validators.HasAddress(privValidator.GetAddress()) { - consensusLogger.Info("This node is a validator", "addr", privValidator.GetAddress(), "pubKey", privValidator.GetPubKey()) + if state.Validators.HasAddress(addr) { + consensusLogger.Info("This node is a validator", "addr", addr, "pubKey", pubKey) } else { - consensusLogger.Info("This node is not a validator", "addr", privValidator.GetAddress(), "pubKey", privValidator.GetPubKey()) + consensusLogger.Info("This node is not a validator", "addr", addr, "pubKey", pubKey) } - csMetrics, p2pMetrics, memplMetrics, smMetrics := metricsProvider() + csMetrics, p2pMetrics, memplMetrics, smMetrics := metricsProvider(genDoc.ChainID) // Make MempoolReactor mempool := mempl.NewMempool( @@ -254,21 +324,14 @@ func NewNode(config *cfg.Config, proxyApp.Mempool(), state.LastBlockHeight, mempl.WithMetrics(memplMetrics), - mempl.WithPreCheck( - mempl.PreCheckAminoMaxBytes( - types.MaxDataBytesUnknownEvidence( - state.ConsensusParams.BlockSize.MaxBytes, - state.Validators.Size(), - ), - ), - ), - mempl.WithPostCheck( - mempl.PostCheckMaxGas(state.ConsensusParams.BlockSize.MaxGas), - ), + mempl.WithPreCheck(sm.TxPreCheck(state)), + mempl.WithPostCheck(sm.TxPostCheck(state)), ) mempoolLogger := logger.With("module", "mempool") mempool.SetLogger(mempoolLogger) - mempool.InitWAL() // no need to have the mempool wal during tests + if config.Mempool.WalEnabled() { + mempool.InitWAL() // no need to have the mempool wal during tests + } mempoolReactor := mempl.NewMempoolReactor(config.Mempool, mempool) mempoolReactor.SetLogger(mempoolLogger) @@ -320,43 +383,30 @@ func NewNode(config *cfg.Config, consensusReactor := cs.NewConsensusReactor(consensusState, fastSync, cs.ReactorMetrics(csMetrics)) consensusReactor.SetLogger(consensusLogger) - eventBus := types.NewEventBus() - eventBus.SetLogger(logger.With("module", "events")) - // services which will be publishing and/or subscribing for messages (events) // consensusReactor will set it on consensusState and blockExecutor consensusReactor.SetEventBus(eventBus) - // Transaction indexing - var txIndexer txindex.TxIndexer - switch config.TxIndex.Indexer { - case "kv": - store, err := dbProvider(&DBContext{"tx_index", config}) - if err != nil { - return nil, err - } - if config.TxIndex.IndexTags != "" { - txIndexer = kv.NewTxIndex(store, kv.IndexTags(splitAndTrimEmpty(config.TxIndex.IndexTags, ",", " "))) - } else if config.TxIndex.IndexAllTags { - txIndexer = kv.NewTxIndex(store, kv.IndexAllTags()) - } else { - txIndexer = kv.NewTxIndex(store) - } - default: - txIndexer = &null.TxIndex{} - } - - indexerService := txindex.NewIndexerService(txIndexer, eventBus) - indexerService.SetLogger(logger.With("module", "txindex")) - - var ( - p2pLogger = logger.With("module", "p2p") - nodeInfo = makeNodeInfo(config, nodeKey.ID(), txIndexer, genDoc.ChainID) + p2pLogger := logger.With("module", "p2p") + nodeInfo, err := makeNodeInfo( + config, + nodeKey.ID(), + txIndexer, + genDoc.ChainID, + p2p.NewProtocolVersion( + version.P2PProtocol, // global + state.Version.Consensus.Block, + state.Version.Consensus.App, + ), ) + if err != nil { + return nil, err + } // Setup Transport. var ( - transport = p2p.NewMultiplexTransport(nodeInfo, *nodeKey) + mConnConfig = p2p.MConnConfig(config.P2P) + transport = p2p.NewMultiplexTransport(nodeInfo, *nodeKey, mConnConfig) connFilters = []p2p.ConnFilterFunc{} peerFilters = []p2p.PeerFilterFunc{} ) @@ -449,7 +499,7 @@ func NewNode(config *cfg.Config, Seeds: splitAndTrimEmpty(config.P2P.Seeds, ",", " "), SeedMode: config.P2P.SeedMode, }) - pexReactor.SetLogger(p2pLogger) + pexReactor.SetLogger(logger.With("module", "pex")) sw.AddReactor("PEX", pexReactor) } @@ -499,11 +549,6 @@ func (n *Node) OnStart() error { time.Sleep(genTime.Sub(now)) } - err := n.eventBus.Start() - if err != nil { - return err - } - // Add private IDs to addrbook to block those peers being added n.addrBook.AddPrivateIDs(splitAndTrimEmpty(n.config.P2P.PrivatePeerIDs, ",", " ")) @@ -547,8 +592,7 @@ func (n *Node) OnStart() error { } } - // start tx indexer - return n.indexerService.Start() + return nil } // OnStop stops the Node. It implements cmn.Service. @@ -565,6 +609,11 @@ func (n *Node) OnStop() { // TODO: gracefully disconnect from peers. n.sw.Stop() + // stop mempool WAL + if n.config.Mempool.WalEnabled() { + n.mempoolReactor.Mempool.CloseWAL() + } + if err := n.transport.Close(); err != nil { n.Logger.Error("Error closing transport", "err", err) } @@ -579,10 +628,8 @@ func (n *Node) OnStop() { } } - if pvsc, ok := n.privValidator.(*privval.SocketPV); ok { - if err := pvsc.Stop(); err != nil { - n.Logger.Error("Error stopping priv validator socket client", "err", err) - } + if pvsc, ok := n.privValidator.(cmn.Service); ok { + pvsc.Stop() } if n.prometheusSrv != nil { @@ -603,7 +650,8 @@ func (n *Node) ConfigureRPC() { rpccore.SetEvidencePool(n.evidencePool) rpccore.SetP2PPeers(n.sw) rpccore.SetP2PTransport(n) - rpccore.SetPubKey(n.privValidator.GetPubKey()) + pubKey := n.privValidator.GetPubKey() + rpccore.SetPubKey(pubKey) rpccore.SetGenesisDoc(n.genesisDoc) rpccore.SetAddrBook(n.addrBook) rpccore.SetProxyAppQuery(n.proxyApp.Query()) @@ -632,30 +680,42 @@ func (n *Node) startRPC() ([]net.Listener, error) { wm.SetLogger(rpcLogger.With("protocol", "websocket")) mux.HandleFunc("/websocket", wm.WebsocketHandler) rpcserver.RegisterRPCFuncs(mux, rpccore.Routes, coreCodec, rpcLogger) - listener, err := rpcserver.StartHTTPServer( + + listener, err := rpcserver.Listen( listenAddr, - mux, - rpcLogger, rpcserver.Config{MaxOpenConnections: n.config.RPC.MaxOpenConnections}, ) if err != nil { return nil, err } + + var rootHandler http.Handler = mux + if n.config.RPC.IsCorsEnabled() { + corsMiddleware := cors.New(cors.Options{ + AllowedOrigins: n.config.RPC.CORSAllowedOrigins, + AllowedMethods: n.config.RPC.CORSAllowedMethods, + AllowedHeaders: n.config.RPC.CORSAllowedHeaders, + }) + rootHandler = corsMiddleware.Handler(mux) + } + + go rpcserver.StartHTTPServer( + listener, + rootHandler, + rpcLogger, + ) listeners[i] = listener } // we expose a simplified api over grpc for convenience to app devs grpcListenAddr := n.config.RPC.GRPCListenAddress if grpcListenAddr != "" { - listener, err := grpccore.StartGRPCServer( - grpcListenAddr, - grpccore.Config{ - MaxOpenConnections: n.config.RPC.GRPCMaxOpenConnections, - }, - ) + listener, err := rpcserver.Listen( + grpcListenAddr, rpcserver.Config{MaxOpenConnections: n.config.RPC.GRPCMaxOpenConnections}) if err != nil { return nil, err } + go grpccore.StartGRPCServer(listener) listeners = append(listeners, listener) } @@ -756,15 +816,17 @@ func makeNodeInfo( nodeID p2p.ID, txIndexer txindex.TxIndexer, chainID string, -) p2p.NodeInfo { + protocolVersion p2p.ProtocolVersion, +) (p2p.NodeInfo, error) { txIndexerStatus := "on" if _, ok := txIndexer.(*null.TxIndex); ok { txIndexerStatus = "off" } - nodeInfo := p2p.NodeInfo{ - ID: nodeID, - Network: chainID, - Version: version.Version, + nodeInfo := p2p.DefaultNodeInfo{ + ProtocolVersion: protocolVersion, + ID_: nodeID, + Network: chainID, + Version: version.TMCoreSemVer, Channels: []byte{ bc.BlockchainChannel, cs.StateChannel, cs.DataChannel, cs.VoteChannel, cs.VoteSetBitsChannel, @@ -772,13 +834,9 @@ func makeNodeInfo( evidence.EvidenceChannel, }, Moniker: config.Moniker, - Other: p2p.NodeInfoOther{ - AminoVersion: amino.Version, - P2PVersion: p2p.Version, - ConsensusVersion: cs.Version, - RPCVersion: fmt.Sprintf("%v/%v", rpc.Version, rpccore.Version), - TxIndex: txIndexerStatus, - RPCAddress: config.RPC.ListenAddress, + Other: p2p.DefaultNodeInfoOther{ + TxIndex: txIndexerStatus, + RPCAddress: config.RPC.ListenAddress, }, } @@ -794,7 +852,8 @@ func makeNodeInfo( nodeInfo.ListenAddr = lAddr - return nodeInfo + err := nodeInfo.Validate() + return nodeInfo, err } //------------------------------------------------------------------------------ @@ -826,6 +885,39 @@ func saveGenesisDoc(db dbm.DB, genDoc *types.GenesisDoc) { db.SetSync(genesisDocKey, bytes) } +func createAndStartPrivValidatorSocketClient( + listenAddr string, + logger log.Logger, +) (types.PrivValidator, error) { + var listener net.Listener + + protocol, address := cmn.ProtocolAndAddress(listenAddr) + ln, err := net.Listen(protocol, address) + if err != nil { + return nil, err + } + switch protocol { + case "unix": + listener = privval.NewUnixListener(ln) + case "tcp": + // TODO: persist this key so external signer + // can actually authenticate us + listener = privval.NewTCPListener(ln, ed25519.GenPrivKey()) + default: + return nil, fmt.Errorf( + "Wrong listen address: expected either 'tcp' or 'unix' protocols, got %s", + protocol, + ) + } + + pvsc := privval.NewSocketVal(logger.With("module", "privval"), listener) + if err := pvsc.Start(); err != nil { + return nil, errors.Wrap(err, "failed to start private validator") + } + + return pvsc, nil +} + // splitAndTrimEmpty slices s into all subslices separated by sep and returns a // slice of the string s with all leading and trailing Unicode code points // contained in cutset removed. If sep is empty, SplitAndTrim splits after each diff --git a/node/node_test.go b/node/node_test.go index f4c1f6a1..3218c832 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -3,19 +3,30 @@ package node import ( "context" "fmt" + "net" "os" "syscall" "testing" "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" - "github.com/tendermint/tendermint/libs/log" - + "github.com/tendermint/tendermint/abci/example/kvstore" cfg "github.com/tendermint/tendermint/config" + "github.com/tendermint/tendermint/crypto/ed25519" + "github.com/tendermint/tendermint/evidence" + cmn "github.com/tendermint/tendermint/libs/common" + dbm "github.com/tendermint/tendermint/libs/db" + "github.com/tendermint/tendermint/libs/log" + mempl "github.com/tendermint/tendermint/mempool" + "github.com/tendermint/tendermint/p2p" + "github.com/tendermint/tendermint/privval" + "github.com/tendermint/tendermint/proxy" + sm "github.com/tendermint/tendermint/state" "github.com/tendermint/tendermint/types" - tmtime "github.com/tendermint/tendermint/types/time" + "github.com/tendermint/tendermint/version" ) func TestNodeStartStop(t *testing.T) { @@ -23,17 +34,16 @@ func TestNodeStartStop(t *testing.T) { // create & start node n, err := DefaultNewNode(config, log.TestingLogger()) - assert.NoError(t, err, "expected no err on DefaultNewNode") - err1 := n.Start() - if err1 != nil { - t.Error(err1) - } + require.NoError(t, err) + err = n.Start() + require.NoError(t, err) + t.Logf("Started node %v", n.sw.NodeInfo()) // wait for the node to produce a block blockCh := make(chan interface{}) err = n.EventBus().Subscribe(context.Background(), "node_test", types.EventQueryNewBlock, blockCh) - assert.NoError(t, err) + require.NoError(t, err) select { case <-blockCh: case <-time.After(10 * time.Second): @@ -78,16 +88,217 @@ func TestSplitAndTrimEmpty(t *testing.T) { } } -func TestNodeDelayedStop(t *testing.T) { - config := cfg.ResetTestRoot("node_delayed_node_test") +func TestNodeDelayedStart(t *testing.T) { + config := cfg.ResetTestRoot("node_delayed_start_test") now := tmtime.Now() // create & start node n, err := DefaultNewNode(config, log.TestingLogger()) - n.GenesisDoc().GenesisTime = now.Add(5 * time.Second) - assert.NoError(t, err) + n.GenesisDoc().GenesisTime = now.Add(2 * time.Second) + require.NoError(t, err) n.Start() startTime := tmtime.Now() assert.Equal(t, true, startTime.After(n.GenesisDoc().GenesisTime)) } + +func TestNodeSetAppVersion(t *testing.T) { + config := cfg.ResetTestRoot("node_app_version_test") + + // create & start node + n, err := DefaultNewNode(config, log.TestingLogger()) + require.NoError(t, err) + + // default config uses the kvstore app + var appVersion version.Protocol = kvstore.ProtocolVersion + + // check version is set in state + state := sm.LoadState(n.stateDB) + assert.Equal(t, state.Version.Consensus.App, appVersion) + + // check version is set in node info + assert.Equal(t, n.nodeInfo.(p2p.DefaultNodeInfo).ProtocolVersion.App, appVersion) +} + +func TestNodeSetPrivValTCP(t *testing.T) { + addr := "tcp://" + testFreeAddr(t) + + config := cfg.ResetTestRoot("node_priv_val_tcp_test") + config.BaseConfig.PrivValidatorListenAddr = addr + + dialer := privval.DialTCPFn(addr, 100*time.Millisecond, ed25519.GenPrivKey()) + pvsc := privval.NewRemoteSigner( + log.TestingLogger(), + config.ChainID(), + types.NewMockPV(), + dialer, + ) + privval.RemoteSignerConnDeadline(100 * time.Millisecond)(pvsc) + + go func() { + err := pvsc.Start() + if err != nil { + panic(err) + } + }() + defer pvsc.Stop() + + n, err := DefaultNewNode(config, log.TestingLogger()) + require.NoError(t, err) + assert.IsType(t, &privval.SocketVal{}, n.PrivValidator()) +} + +// address without a protocol must result in error +func TestPrivValidatorListenAddrNoProtocol(t *testing.T) { + addrNoPrefix := testFreeAddr(t) + + config := cfg.ResetTestRoot("node_priv_val_tcp_test") + config.BaseConfig.PrivValidatorListenAddr = addrNoPrefix + + _, err := DefaultNewNode(config, log.TestingLogger()) + assert.Error(t, err) +} + +func TestNodeSetPrivValIPC(t *testing.T) { + tmpfile := "/tmp/kms." + cmn.RandStr(6) + ".sock" + defer os.Remove(tmpfile) // clean up + + config := cfg.ResetTestRoot("node_priv_val_tcp_test") + config.BaseConfig.PrivValidatorListenAddr = "unix://" + tmpfile + + dialer := privval.DialUnixFn(tmpfile) + pvsc := privval.NewRemoteSigner( + log.TestingLogger(), + config.ChainID(), + types.NewMockPV(), + dialer, + ) + privval.RemoteSignerConnDeadline(100 * time.Millisecond)(pvsc) + + go func() { + err := pvsc.Start() + require.NoError(t, err) + }() + defer pvsc.Stop() + + n, err := DefaultNewNode(config, log.TestingLogger()) + require.NoError(t, err) + assert.IsType(t, &privval.SocketVal{}, n.PrivValidator()) + +} + +// testFreeAddr claims a free port so we don't block on listener being ready. +func testFreeAddr(t *testing.T) string { + ln, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + defer ln.Close() + + return fmt.Sprintf("127.0.0.1:%d", ln.Addr().(*net.TCPAddr).Port) +} + +// create a proposal block using real and full +// mempool and evidence pool and validate it. +func TestCreateProposalBlock(t *testing.T) { + config := cfg.ResetTestRoot("node_create_proposal") + cc := proxy.NewLocalClientCreator(kvstore.NewKVStoreApplication()) + proxyApp := proxy.NewAppConns(cc) + err := proxyApp.Start() + require.Nil(t, err) + defer proxyApp.Stop() + + logger := log.TestingLogger() + + var height int64 = 1 + state, stateDB := state(1, height) + maxBytes := 16384 + state.ConsensusParams.BlockSize.MaxBytes = int64(maxBytes) + proposerAddr, _ := state.Validators.GetByIndex(0) + + // Make Mempool + memplMetrics := mempl.PrometheusMetrics("node_test") + mempool := mempl.NewMempool( + config.Mempool, + proxyApp.Mempool(), + state.LastBlockHeight, + mempl.WithMetrics(memplMetrics), + mempl.WithPreCheck(sm.TxPreCheck(state)), + mempl.WithPostCheck(sm.TxPostCheck(state)), + ) + mempool.SetLogger(logger) + + // Make EvidencePool + types.RegisterMockEvidencesGlobal() + evidence.RegisterMockEvidences() + evidenceDB := dbm.NewMemDB() + evidenceStore := evidence.NewEvidenceStore(evidenceDB) + evidencePool := evidence.NewEvidencePool(stateDB, evidenceStore) + evidencePool.SetLogger(logger) + + // fill the evidence pool with more evidence + // than can fit in a block + minEvSize := 12 + numEv := (maxBytes / types.MaxEvidenceBytesDenominator) / minEvSize + for i := 0; i < numEv; i++ { + ev := types.NewMockRandomGoodEvidence(1, proposerAddr, cmn.RandBytes(minEvSize)) + err := evidencePool.AddEvidence(ev) + assert.NoError(t, err) + } + + // fill the mempool with more txs + // than can fit in a block + txLength := 1000 + for i := 0; i < maxBytes/txLength; i++ { + tx := cmn.RandBytes(txLength) + err := mempool.CheckTx(tx, nil) + assert.NoError(t, err) + } + + blockExec := sm.NewBlockExecutor( + stateDB, + logger, + proxyApp.Consensus(), + mempool, + evidencePool, + ) + + commit := &types.Commit{} + block, _ := blockExec.CreateProposalBlock( + height, + state, commit, + proposerAddr, + ) + + err = blockExec.ValidateBlock(state, block) + assert.NoError(t, err) + +} + +func state(nVals int, height int64) (sm.State, dbm.DB) { + vals := make([]types.GenesisValidator, nVals) + for i := 0; i < nVals; i++ { + secret := []byte(fmt.Sprintf("test%d", i)) + pk := ed25519.GenPrivKeyFromSecret(secret) + vals[i] = types.GenesisValidator{ + pk.PubKey().Address(), + pk.PubKey(), + 1000, + fmt.Sprintf("test%d", i), + } + } + s, _ := sm.MakeGenesisState(&types.GenesisDoc{ + ChainID: "test-chain", + Validators: vals, + AppHash: nil, + }) + + // save validators to db for 2 heights + stateDB := dbm.NewMemDB() + sm.SaveState(stateDB, s) + + for i := 1; i < int(height); i++ { + s.LastBlockHeight++ + s.LastValidators = s.Validators.Copy() + sm.SaveState(stateDB, s) + } + return s, stateDB +} diff --git a/p2p/README.md b/p2p/README.md index 819a5056..df4c2ae0 100644 --- a/p2p/README.md +++ b/p2p/README.md @@ -4,8 +4,8 @@ The p2p package provides an abstraction around peer-to-peer communication. Docs: -- [Connection](https://github.com/tendermint/tendermint/blob/master/docs/spec/docs/spec/p2p/connection.md) for details on how connections and multiplexing work -- [Peer](https://github.com/tendermint/tendermint/blob/master/docs/spec/docs/spec/p2p/peer.md) for details on peer ID, handshakes, and peer exchange -- [Node](https://github.com/tendermint/tendermint/blob/master/docs/spec/docs/spec/p2p/node.md) for details about different types of nodes and how they should work -- [Pex](https://github.com/tendermint/tendermint/blob/master/docs/spec/docs/spec/reactors/pex/pex.md) for details on peer discovery and exchange -- [Config](https://github.com/tendermint/tendermint/blob/master/docs/spec/docs/spec/p2p/config.md) for details on some config option +- [Connection](https://github.com/tendermint/tendermint/blob/master/docs/spec/p2p/connection.md) for details on how connections and multiplexing work +- [Peer](https://github.com/tendermint/tendermint/blob/master/docs/spec/p2p/peer.md) for details on peer ID, handshakes, and peer exchange +- [Node](https://github.com/tendermint/tendermint/blob/master/docs/spec/p2p/node.md) for details about different types of nodes and how they should work +- [Pex](https://github.com/tendermint/tendermint/blob/master/docs/spec/reactors/pex/pex.md) for details on peer discovery and exchange +- [Config](https://github.com/tendermint/tendermint/blob/master/docs/spec/p2p/config.md) for details on some config option diff --git a/p2p/conn/connection.go b/p2p/conn/connection.go index 0e33adab..c1e90ab7 100644 --- a/p2p/conn/connection.go +++ b/p2p/conn/connection.go @@ -8,6 +8,7 @@ import ( "math" "net" "reflect" + "sync" "sync/atomic" "time" @@ -84,7 +85,15 @@ type MConnection struct { errored uint32 config MConnConfig - quit chan struct{} + // Closing quitSendRoutine will cause the sendRoutine to eventually quit. + // doneSendRoutine is closed when the sendRoutine actually quits. + quitSendRoutine chan struct{} + doneSendRoutine chan struct{} + + // used to ensure FlushStop and OnStop + // are safe to call concurrently. + stopMtx sync.Mutex + flushTimer *cmn.ThrottleTimer // flush writes as necessary but throttled. pingTimer *cmn.RepeatTimer // send pings periodically @@ -156,6 +165,7 @@ func NewMConnectionWithConfig(conn net.Conn, chDescs []*ChannelDescriptor, onRec onReceive: onReceive, onError: onError, config: config, + created: time.Now(), } // Create channels @@ -190,25 +200,85 @@ func (c *MConnection) OnStart() error { if err := c.BaseService.OnStart(); err != nil { return err } - c.quit = make(chan struct{}) c.flushTimer = cmn.NewThrottleTimer("flush", c.config.FlushThrottle) c.pingTimer = cmn.NewRepeatTimer("ping", c.config.PingInterval) c.pongTimeoutCh = make(chan bool, 1) c.chStatsTimer = cmn.NewRepeatTimer("chStats", updateStats) + c.quitSendRoutine = make(chan struct{}) + c.doneSendRoutine = make(chan struct{}) go c.sendRoutine() go c.recvRoutine() return nil } -// OnStop implements BaseService -func (c *MConnection) OnStop() { +// stopServices stops the BaseService and timers and closes the quitSendRoutine. +// if the quitSendRoutine was already closed, it returns true, otherwise it returns false. +// It uses the stopMtx to ensure only one of FlushStop and OnStop can do this at a time. +func (c *MConnection) stopServices() (alreadyStopped bool) { + c.stopMtx.Lock() + defer c.stopMtx.Unlock() + + select { + case <-c.quitSendRoutine: + // already quit via FlushStop or OnStop + return true + default: + } + c.BaseService.OnStop() c.flushTimer.Stop() c.pingTimer.Stop() c.chStatsTimer.Stop() - if c.quit != nil { - close(c.quit) + + close(c.quitSendRoutine) + return false +} + +// FlushStop replicates the logic of OnStop. +// It additionally ensures that all successful +// .Send() calls will get flushed before closing +// the connection. +func (c *MConnection) FlushStop() { + if c.stopServices() { + return } + + // this block is unique to FlushStop + { + // wait until the sendRoutine exits + // so we dont race on calling sendSomePacketMsgs + <-c.doneSendRoutine + + // Send and flush all pending msgs. + // By now, IsRunning == false, + // so any concurrent attempts to send will fail. + // Since sendRoutine has exited, we can call this + // safely + eof := c.sendSomePacketMsgs() + for !eof { + eof = c.sendSomePacketMsgs() + } + c.flush() + + // Now we can close the connection + } + + c.conn.Close() // nolint: errcheck + + // We can't close pong safely here because + // recvRoutine may write to it after we've stopped. + // Though it doesn't need to get closed at all, + // we close it @ recvRoutine. + + // c.Stop() +} + +// OnStop implements BaseService +func (c *MConnection) OnStop() { + if c.stopServices() { + return + } + c.conn.Close() // nolint: errcheck // We can't close pong safely here because @@ -269,7 +339,7 @@ func (c *MConnection) Send(chID byte, msgBytes []byte) bool { default: } } else { - c.Logger.Error("Send failed", "channel", chID, "conn", c, "msgBytes", fmt.Sprintf("%X", msgBytes)) + c.Logger.Debug("Send failed", "channel", chID, "conn", c, "msgBytes", fmt.Sprintf("%X", msgBytes)) } return success } @@ -337,7 +407,7 @@ FOR_LOOP: } case <-c.pingTimer.Chan(): c.Logger.Debug("Send Ping") - _n, err = cdc.MarshalBinaryWriter(c.bufConnWriter, PacketPing{}) + _n, err = cdc.MarshalBinaryLengthPrefixedWriter(c.bufConnWriter, PacketPing{}) if err != nil { break SELECTION } @@ -359,13 +429,13 @@ FOR_LOOP: } case <-c.pong: c.Logger.Debug("Send Pong") - _n, err = cdc.MarshalBinaryWriter(c.bufConnWriter, PacketPong{}) + _n, err = cdc.MarshalBinaryLengthPrefixedWriter(c.bufConnWriter, PacketPong{}) if err != nil { break SELECTION } c.sendMonitor.Update(int(_n)) c.flush() - case <-c.quit: + case <-c.quitSendRoutine: break FOR_LOOP case <-c.send: // Send some PacketMsgs @@ -391,6 +461,7 @@ FOR_LOOP: // Cleanup c.stopPongTimer() + close(c.doneSendRoutine) } // Returns true if messages from channels were exhausted. @@ -477,7 +548,7 @@ FOR_LOOP: var packet Packet var _n int64 var err error - _n, err = cdc.UnmarshalBinaryReader(c.bufConnReader, &packet, int64(c._maxPacketMsgSize)) + _n, err = cdc.UnmarshalBinaryLengthPrefixedReader(c.bufConnReader, &packet, int64(c._maxPacketMsgSize)) c.recvMonitor.Update(int(_n)) if err != nil { if c.IsRunning() { @@ -553,7 +624,7 @@ func (c *MConnection) stopPongTimer() { // maxPacketMsgSize returns a maximum size of PacketMsg, including the overhead // of amino encoding. func (c *MConnection) maxPacketMsgSize() int { - return len(cdc.MustMarshalBinary(PacketMsg{ + return len(cdc.MustMarshalBinaryLengthPrefixed(PacketMsg{ ChannelID: 0x01, EOF: 1, Bytes: make([]byte, c.config.MaxPacketMsgPayloadSize), @@ -723,7 +794,7 @@ func (ch *Channel) nextPacketMsg() PacketMsg { // Not goroutine-safe func (ch *Channel) writePacketMsgTo(w io.Writer) (n int64, err error) { var packet = ch.nextPacketMsg() - n, err = cdc.MarshalBinaryWriter(w, packet) + n, err = cdc.MarshalBinaryLengthPrefixedWriter(w, packet) atomic.AddInt64(&ch.recentlySent, n) return } diff --git a/p2p/conn/connection_test.go b/p2p/conn/connection_test.go index 95b5488a..afad69d1 100644 --- a/p2p/conn/connection_test.go +++ b/p2p/conn/connection_test.go @@ -30,12 +30,49 @@ func createMConnectionWithCallbacks(conn net.Conn, onReceive func(chID byte, msg cfg := DefaultMConnConfig() cfg.PingInterval = 90 * time.Millisecond cfg.PongTimeout = 45 * time.Millisecond - chDescs := []*ChannelDescriptor{&ChannelDescriptor{ID: 0x01, Priority: 1, SendQueueCapacity: 1}} + chDescs := []*ChannelDescriptor{{ID: 0x01, Priority: 1, SendQueueCapacity: 1}} c := NewMConnectionWithConfig(conn, chDescs, onReceive, onError, cfg) c.SetLogger(log.TestingLogger()) return c } +func TestMConnectionSendFlushStop(t *testing.T) { + server, client := NetPipe() + defer server.Close() // nolint: errcheck + defer client.Close() // nolint: errcheck + + clientConn := createTestMConnection(client) + err := clientConn.Start() + require.Nil(t, err) + defer clientConn.Stop() + + msg := []byte("abc") + assert.True(t, clientConn.Send(0x01, msg)) + + aminoMsgLength := 14 + + // start the reader in a new routine, so we can flush + errCh := make(chan error) + go func() { + msgB := make([]byte, aminoMsgLength) + _, err := server.Read(msgB) + if err != nil { + t.Fatal(err) + } + errCh <- err + }() + + // stop the conn - it should flush all conns + clientConn.FlushStop() + + timer := time.NewTimer(3 * time.Second) + select { + case <-errCh: + case <-timer.C: + t.Error("timed out waiting for msgs to be read") + } +} + func TestMConnectionSend(t *testing.T) { server, client := NetPipe() defer server.Close() // nolint: errcheck @@ -140,7 +177,7 @@ func TestMConnectionPongTimeoutResultsInError(t *testing.T) { go func() { // read ping var pkt PacketPing - _, err = cdc.UnmarshalBinaryReader(server, &pkt, maxPingPongPacketSize) + _, err = cdc.UnmarshalBinaryLengthPrefixedReader(server, &pkt, maxPingPongPacketSize) assert.Nil(t, err) serverGotPing <- struct{}{} }() @@ -176,22 +213,22 @@ func TestMConnectionMultiplePongsInTheBeginning(t *testing.T) { defer mconn.Stop() // sending 3 pongs in a row (abuse) - _, err = server.Write(cdc.MustMarshalBinary(PacketPong{})) + _, err = server.Write(cdc.MustMarshalBinaryLengthPrefixed(PacketPong{})) require.Nil(t, err) - _, err = server.Write(cdc.MustMarshalBinary(PacketPong{})) + _, err = server.Write(cdc.MustMarshalBinaryLengthPrefixed(PacketPong{})) require.Nil(t, err) - _, err = server.Write(cdc.MustMarshalBinary(PacketPong{})) + _, err = server.Write(cdc.MustMarshalBinaryLengthPrefixed(PacketPong{})) require.Nil(t, err) serverGotPing := make(chan struct{}) go func() { // read ping (one byte) var packet, err = Packet(nil), error(nil) - _, err = cdc.UnmarshalBinaryReader(server, &packet, maxPingPongPacketSize) + _, err = cdc.UnmarshalBinaryLengthPrefixedReader(server, &packet, maxPingPongPacketSize) require.Nil(t, err) serverGotPing <- struct{}{} // respond with pong - _, err = server.Write(cdc.MustMarshalBinary(PacketPong{})) + _, err = server.Write(cdc.MustMarshalBinaryLengthPrefixed(PacketPong{})) require.Nil(t, err) }() <-serverGotPing @@ -227,18 +264,18 @@ func TestMConnectionMultiplePings(t *testing.T) { // sending 3 pings in a row (abuse) // see https://github.com/tendermint/tendermint/issues/1190 - _, err = server.Write(cdc.MustMarshalBinary(PacketPing{})) + _, err = server.Write(cdc.MustMarshalBinaryLengthPrefixed(PacketPing{})) require.Nil(t, err) var pkt PacketPong - _, err = cdc.UnmarshalBinaryReader(server, &pkt, maxPingPongPacketSize) + _, err = cdc.UnmarshalBinaryLengthPrefixedReader(server, &pkt, maxPingPongPacketSize) require.Nil(t, err) - _, err = server.Write(cdc.MustMarshalBinary(PacketPing{})) + _, err = server.Write(cdc.MustMarshalBinaryLengthPrefixed(PacketPing{})) require.Nil(t, err) - _, err = cdc.UnmarshalBinaryReader(server, &pkt, maxPingPongPacketSize) + _, err = cdc.UnmarshalBinaryLengthPrefixedReader(server, &pkt, maxPingPongPacketSize) require.Nil(t, err) - _, err = server.Write(cdc.MustMarshalBinary(PacketPing{})) + _, err = server.Write(cdc.MustMarshalBinaryLengthPrefixed(PacketPing{})) require.Nil(t, err) - _, err = cdc.UnmarshalBinaryReader(server, &pkt, maxPingPongPacketSize) + _, err = cdc.UnmarshalBinaryLengthPrefixedReader(server, &pkt, maxPingPongPacketSize) require.Nil(t, err) assert.True(t, mconn.IsRunning()) @@ -270,20 +307,20 @@ func TestMConnectionPingPongs(t *testing.T) { go func() { // read ping var pkt PacketPing - _, err = cdc.UnmarshalBinaryReader(server, &pkt, maxPingPongPacketSize) + _, err = cdc.UnmarshalBinaryLengthPrefixedReader(server, &pkt, maxPingPongPacketSize) require.Nil(t, err) serverGotPing <- struct{}{} // respond with pong - _, err = server.Write(cdc.MustMarshalBinary(PacketPong{})) + _, err = server.Write(cdc.MustMarshalBinaryLengthPrefixed(PacketPong{})) require.Nil(t, err) time.Sleep(mconn.config.PingInterval) // read ping - _, err = cdc.UnmarshalBinaryReader(server, &pkt, maxPingPongPacketSize) + _, err = cdc.UnmarshalBinaryLengthPrefixedReader(server, &pkt, maxPingPongPacketSize) require.Nil(t, err) // respond with pong - _, err = server.Write(cdc.MustMarshalBinary(PacketPong{})) + _, err = server.Write(cdc.MustMarshalBinaryLengthPrefixed(PacketPong{})) require.Nil(t, err) }() <-serverGotPing @@ -380,7 +417,7 @@ func TestMConnectionReadErrorBadEncoding(t *testing.T) { client := mconnClient.conn // send badly encoded msgPacket - bz := cdc.MustMarshalBinary(PacketMsg{}) + bz := cdc.MustMarshalBinaryLengthPrefixed(PacketMsg{}) bz[4] += 0x01 // Invalid prefix bytes. // Write it. @@ -428,7 +465,7 @@ func TestMConnectionReadErrorLongMessage(t *testing.T) { EOF: 1, Bytes: make([]byte, mconnClient.config.MaxPacketMsgPayloadSize), } - _, err = cdc.MarshalBinaryWriter(buf, packet) + _, err = cdc.MarshalBinaryLengthPrefixedWriter(buf, packet) assert.Nil(t, err) _, err = client.Write(buf.Bytes()) assert.Nil(t, err) @@ -441,7 +478,7 @@ func TestMConnectionReadErrorLongMessage(t *testing.T) { EOF: 1, Bytes: make([]byte, mconnClient.config.MaxPacketMsgPayloadSize+100), } - _, err = cdc.MarshalBinaryWriter(buf, packet) + _, err = cdc.MarshalBinaryLengthPrefixedWriter(buf, packet) assert.Nil(t, err) _, err = client.Write(buf.Bytes()) assert.NotNil(t, err) diff --git a/p2p/conn/secret_connection.go b/p2p/conn/secret_connection.go index acdd96de..d2ba6fb5 100644 --- a/p2p/conn/secret_connection.go +++ b/p2p/conn/secret_connection.go @@ -4,13 +4,14 @@ import ( "bytes" crand "crypto/rand" "crypto/sha256" + "crypto/subtle" "encoding/binary" "errors" "io" "net" + "sync" "time" - // forked to github.com/tendermint/crypto "golang.org/x/crypto/chacha20poly1305" "golang.org/x/crypto/curve25519" "golang.org/x/crypto/nacl/box" @@ -28,20 +29,38 @@ const aeadSizeOverhead = 16 // overhead of poly 1305 authentication tag const aeadKeySize = chacha20poly1305.KeySize const aeadNonceSize = chacha20poly1305.NonceSize -// SecretConnection implements net.conn. +var ErrSmallOrderRemotePubKey = errors.New("detected low order point from remote peer") + +// SecretConnection implements net.Conn. // It is an implementation of the STS protocol. -// Note we do not (yet) assume that a remote peer's pubkey -// is known ahead of time, and thus we are technically -// still vulnerable to MITM. (TODO!) -// See docs/sts-final.pdf for more info +// See https://github.com/tendermint/tendermint/blob/0.1/docs/sts-final.pdf for +// details on the protocol. +// +// Consumers of the SecretConnection are responsible for authenticating +// the remote peer's pubkey against known information, like a nodeID. +// Otherwise they are vulnerable to MITM. +// (TODO(ismail): see also https://github.com/tendermint/tendermint/issues/3010) type SecretConnection struct { - conn io.ReadWriteCloser - recvBuffer []byte - recvNonce *[aeadNonceSize]byte - sendNonce *[aeadNonceSize]byte + + // immutable recvSecret *[aeadKeySize]byte sendSecret *[aeadKeySize]byte remPubKey crypto.PubKey + conn io.ReadWriteCloser + + // net.Conn must be thread safe: + // https://golang.org/pkg/net/#Conn. + // Since we have internal mutable state, + // we need mtxs. But recv and send states + // are independent, so we can use two mtxs. + // All .Read are covered by recvMtx, + // all .Write are covered by sendMtx. + recvMtx sync.Mutex + recvBuffer []byte + recvNonce *[aeadNonceSize]byte + + sendMtx sync.Mutex + sendNonce *[aeadNonceSize]byte } // MakeSecretConnection performs handshake and returns a new authenticated @@ -110,9 +129,12 @@ func (sc *SecretConnection) RemotePubKey() crypto.PubKey { return sc.remPubKey } -// Writes encrypted frames of `sealedFrameSize` -// CONTRACT: data smaller than dataMaxSize is read atomically. +// Writes encrypted frames of `totalFrameSize + aeadSizeOverhead`. +// CONTRACT: data smaller than dataMaxSize is written atomically. func (sc *SecretConnection) Write(data []byte) (n int, err error) { + sc.sendMtx.Lock() + defer sc.sendMtx.Unlock() + for 0 < len(data) { var frame = make([]byte, totalFrameSize) var chunk []byte @@ -131,6 +153,7 @@ func (sc *SecretConnection) Write(data []byte) (n int, err error) { if err != nil { return n, errors.New("Invalid SecretConnection Key") } + // encrypt the frame var sealedFrame = make([]byte, aeadSizeOverhead+totalFrameSize) aead.Seal(sealedFrame[:0], sc.sendNonce[:], frame, nil) @@ -148,23 +171,30 @@ func (sc *SecretConnection) Write(data []byte) (n int, err error) { // CONTRACT: data smaller than dataMaxSize is read atomically. func (sc *SecretConnection) Read(data []byte) (n int, err error) { + sc.recvMtx.Lock() + defer sc.recvMtx.Unlock() + + // read off and update the recvBuffer, if non-empty if 0 < len(sc.recvBuffer) { n = copy(data, sc.recvBuffer) sc.recvBuffer = sc.recvBuffer[n:] return } + // read off the conn + sealedFrame := make([]byte, totalFrameSize+aeadSizeOverhead) + _, err = io.ReadFull(sc.conn, sealedFrame) + if err != nil { + return + } + aead, err := chacha20poly1305.New(sc.recvSecret[:]) if err != nil { return n, errors.New("Invalid SecretConnection Key") } - sealedFrame := make([]byte, totalFrameSize+aeadSizeOverhead) - _, err = io.ReadFull(sc.conn, sealedFrame) - if err != nil { - return - } - // decrypt the frame + // decrypt the frame. + // reads and updates the sc.recvNonce var frame = make([]byte, totalFrameSize) _, err = aead.Open(frame[:0], sc.recvNonce[:], sealedFrame, nil) if err != nil { @@ -173,12 +203,13 @@ func (sc *SecretConnection) Read(data []byte) (n int, err error) { incrNonce(sc.recvNonce) // end decryption + // copy checkLength worth into data, + // set recvBuffer to the rest. var chunkLength = binary.LittleEndian.Uint32(frame) // read the first four bytes if chunkLength > dataMaxSize { return 0, errors.New("chunkLength is greater than dataMaxSize") } var chunk = frame[dataLenSize : dataLenSize+chunkLength] - n = copy(data, chunk) sc.recvBuffer = chunk[n:] return @@ -211,7 +242,7 @@ func shareEphPubKey(conn io.ReadWriteCloser, locEphPub *[32]byte) (remEphPub *[3 // Send our pubkey and receive theirs in tandem. var trs, _ = cmn.Parallel( func(_ int) (val interface{}, err error, abort bool) { - var _, err1 = cdc.MarshalBinaryWriter(conn, locEphPub) + var _, err1 = cdc.MarshalBinaryLengthPrefixedWriter(conn, locEphPub) if err1 != nil { return nil, err1, true // abort } @@ -219,10 +250,13 @@ func shareEphPubKey(conn io.ReadWriteCloser, locEphPub *[32]byte) (remEphPub *[3 }, func(_ int) (val interface{}, err error, abort bool) { var _remEphPub [32]byte - var _, err2 = cdc.UnmarshalBinaryReader(conn, &_remEphPub, 1024*1024) // TODO + var _, err2 = cdc.UnmarshalBinaryLengthPrefixedReader(conn, &_remEphPub, 1024*1024) // TODO if err2 != nil { return nil, err2, true // abort } + if hasSmallOrder(_remEphPub) { + return nil, ErrSmallOrderRemotePubKey, true + } return _remEphPub, nil, false }, ) @@ -238,6 +272,52 @@ func shareEphPubKey(conn io.ReadWriteCloser, locEphPub *[32]byte) (remEphPub *[3 return &_remEphPub, nil } +// use the samne blacklist as lib sodium (see https://eprint.iacr.org/2017/806.pdf for reference): +// https://github.com/jedisct1/libsodium/blob/536ed00d2c5e0c65ac01e29141d69a30455f2038/src/libsodium/crypto_scalarmult/curve25519/ref10/x25519_ref10.c#L11-L17 +var blacklist = [][32]byte{ + // 0 (order 4) + {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + // 1 (order 1) + {0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + // 325606250916557431795983626356110631294008115727848805560023387167927233504 + // (order 8) + {0xe0, 0xeb, 0x7a, 0x7c, 0x3b, 0x41, 0xb8, 0xae, 0x16, 0x56, 0xe3, + 0xfa, 0xf1, 0x9f, 0xc4, 0x6a, 0xda, 0x09, 0x8d, 0xeb, 0x9c, 0x32, + 0xb1, 0xfd, 0x86, 0x62, 0x05, 0x16, 0x5f, 0x49, 0xb8, 0x00}, + // 39382357235489614581723060781553021112529911719440698176882885853963445705823 + // (order 8) + {0x5f, 0x9c, 0x95, 0xbc, 0xa3, 0x50, 0x8c, 0x24, 0xb1, 0xd0, 0xb1, + 0x55, 0x9c, 0x83, 0xef, 0x5b, 0x04, 0x44, 0x5c, 0xc4, 0x58, 0x1c, + 0x8e, 0x86, 0xd8, 0x22, 0x4e, 0xdd, 0xd0, 0x9f, 0x11, 0x57}, + // p-1 (order 2) + {0xec, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f}, + // p (=0, order 4) + {0xed, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f}, + // p+1 (=1, order 1) + {0xee, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f}, +} + +func hasSmallOrder(pubKey [32]byte) bool { + isSmallOrderPoint := false + for _, bl := range blacklist { + if subtle.ConstantTimeCompare(pubKey[:], bl[:]) == 1 { + isSmallOrderPoint = true + break + } + } + return isSmallOrderPoint +} + func deriveSecretAndChallenge(dhSecret *[32]byte, locIsLeast bool) (recvSecret, sendSecret *[aeadKeySize]byte, challenge *[32]byte) { hash := sha256.New hkdf := hkdf.New(hash, dhSecret[:], nil, []byte("TENDERMINT_SECRET_CONNECTION_KEY_AND_CHALLENGE_GEN")) @@ -305,7 +385,7 @@ func shareAuthSignature(sc *SecretConnection, pubKey crypto.PubKey, signature [] // Send our info and receive theirs in tandem. var trs, _ = cmn.Parallel( func(_ int) (val interface{}, err error, abort bool) { - var _, err1 = cdc.MarshalBinaryWriter(sc, authSigMessage{pubKey, signature}) + var _, err1 = cdc.MarshalBinaryLengthPrefixedWriter(sc, authSigMessage{pubKey, signature}) if err1 != nil { return nil, err1, true // abort } @@ -313,7 +393,7 @@ func shareAuthSignature(sc *SecretConnection, pubKey crypto.PubKey, signature [] }, func(_ int) (val interface{}, err error, abort bool) { var _recvMsg authSigMessage - var _, err2 = cdc.UnmarshalBinaryReader(sc, &_recvMsg, 1024*1024) // TODO + var _, err2 = cdc.UnmarshalBinaryLengthPrefixedReader(sc, &_recvMsg, 1024*1024) // TODO if err2 != nil { return nil, err2, true // abort } diff --git a/p2p/conn/secret_connection_test.go b/p2p/conn/secret_connection_test.go index 75ed8fe0..6b285476 100644 --- a/p2p/conn/secret_connection_test.go +++ b/p2p/conn/secret_connection_test.go @@ -7,10 +7,12 @@ import ( "fmt" "io" "log" + "net" "os" "path/filepath" "strconv" "strings" + "sync" "testing" "github.com/stretchr/testify/assert" @@ -98,6 +100,95 @@ func TestSecretConnectionHandshake(t *testing.T) { } } +func TestShareLowOrderPubkey(t *testing.T) { + var fooConn, barConn = makeKVStoreConnPair() + locEphPub, _ := genEphKeys() + + // all blacklisted low order points: + for _, remLowOrderPubKey := range blacklist { + _, _ = cmn.Parallel( + func(_ int) (val interface{}, err error, abort bool) { + _, err = shareEphPubKey(fooConn, locEphPub) + + require.Error(t, err) + require.Equal(t, err, ErrSmallOrderRemotePubKey) + + return nil, nil, false + }, + func(_ int) (val interface{}, err error, abort bool) { + readRemKey, err := shareEphPubKey(barConn, &remLowOrderPubKey) + + require.NoError(t, err) + require.Equal(t, locEphPub, readRemKey) + + return nil, nil, false + }) + } +} + +func TestConcurrentWrite(t *testing.T) { + fooSecConn, barSecConn := makeSecretConnPair(t) + fooWriteText := cmn.RandStr(dataMaxSize) + + // write from two routines. + // should be safe from race according to net.Conn: + // https://golang.org/pkg/net/#Conn + n := 100 + wg := new(sync.WaitGroup) + wg.Add(3) + go writeLots(t, wg, fooSecConn, fooWriteText, n) + go writeLots(t, wg, fooSecConn, fooWriteText, n) + + // Consume reads from bar's reader + readLots(t, wg, barSecConn, n*2) + wg.Wait() + + if err := fooSecConn.Close(); err != nil { + t.Error(err) + } +} + +func TestConcurrentRead(t *testing.T) { + fooSecConn, barSecConn := makeSecretConnPair(t) + fooWriteText := cmn.RandStr(dataMaxSize) + n := 100 + + // read from two routines. + // should be safe from race according to net.Conn: + // https://golang.org/pkg/net/#Conn + wg := new(sync.WaitGroup) + wg.Add(3) + go readLots(t, wg, fooSecConn, n/2) + go readLots(t, wg, fooSecConn, n/2) + + // write to bar + writeLots(t, wg, barSecConn, fooWriteText, n) + wg.Wait() + + if err := fooSecConn.Close(); err != nil { + t.Error(err) + } +} + +func writeLots(t *testing.T, wg *sync.WaitGroup, conn net.Conn, txt string, n int) { + defer wg.Done() + for i := 0; i < n; i++ { + _, err := conn.Write([]byte(txt)) + if err != nil { + t.Fatalf("Failed to write to fooSecConn: %v", err) + } + } +} + +func readLots(t *testing.T, wg *sync.WaitGroup, conn net.Conn, n int) { + readBuffer := make([]byte, dataMaxSize) + for i := 0; i < n; i++ { + _, err := conn.Read(readBuffer) + assert.NoError(t, err) + } + wg.Done() +} + func TestSecretConnectionReadWrite(t *testing.T) { fooConn, barConn := makeKVStoreConnPair() fooWrites, barWrites := []string{}, []string{} @@ -307,12 +398,3 @@ func BenchmarkSecretConnection(b *testing.B) { } //barSecConn.Close() race condition } - -func fingerprint(bz []byte) []byte { - const fbsize = 40 - if len(bz) < fbsize { - return bz - } else { - return bz[:fbsize] - } -} diff --git a/p2p/conn/wire.go b/p2p/conn/wire.go index 4bd778c7..5231a6ca 100644 --- a/p2p/conn/wire.go +++ b/p2p/conn/wire.go @@ -1,7 +1,7 @@ package conn import ( - "github.com/tendermint/go-amino" + amino "github.com/tendermint/go-amino" cryptoAmino "github.com/tendermint/tendermint/crypto/encoding/amino" ) diff --git a/p2p/conn_set.go b/p2p/conn_set.go index f960c0e8..d6462278 100644 --- a/p2p/conn_set.go +++ b/p2p/conn_set.go @@ -11,6 +11,7 @@ type ConnSet interface { HasIP(net.IP) bool Set(net.Conn, []net.IP) Remove(net.Conn) + RemoveAddr(net.Addr) } type connSetItem struct { @@ -62,6 +63,13 @@ func (cs *connSet) Remove(c net.Conn) { delete(cs.conns, c.RemoteAddr().String()) } +func (cs *connSet) RemoveAddr(addr net.Addr) { + cs.Lock() + defer cs.Unlock() + + delete(cs.conns, addr.String()) +} + func (cs *connSet) Set(c net.Conn, ips []net.IP) { cs.Lock() defer cs.Unlock() diff --git a/p2p/dummy/peer.go b/p2p/dummy/peer.go index bb6e822f..57edafc6 100644 --- a/p2p/dummy/peer.go +++ b/p2p/dummy/peer.go @@ -25,6 +25,11 @@ func NewPeer() *peer { return p } +// FlushStop just calls Stop. +func (p *peer) FlushStop() { + p.Stop() +} + // ID always returns dummy. func (p *peer) ID() p2p.ID { return p2p.ID("dummy") @@ -42,7 +47,7 @@ func (p *peer) IsPersistent() bool { // NodeInfo always returns empty node info. func (p *peer) NodeInfo() p2p.NodeInfo { - return p2p.NodeInfo{} + return p2p.DefaultNodeInfo{} } // RemoteIP always returns localhost. @@ -50,6 +55,16 @@ func (p *peer) RemoteIP() net.IP { return net.ParseIP("127.0.0.1") } +// Addr always returns tcp://localhost:8800. +func (p *peer) RemoteAddr() net.Addr { + return &net.TCPAddr{IP: net.ParseIP("127.0.0.1"), Port: 8800} +} + +// CloseConn always returns nil. +func (p *peer) CloseConn() error { + return nil +} + // Status always returns empry connection status. func (p *peer) Status() tmconn.ConnectionStatus { return tmconn.ConnectionStatus{} diff --git a/p2p/errors.go b/p2p/errors.go index 902d2203..70615094 100644 --- a/p2p/errors.go +++ b/p2p/errors.go @@ -40,13 +40,12 @@ func (e ErrRejected) Error() string { if e.isDuplicate { if e.conn != nil { return fmt.Sprintf( - "duplicate CONN<%s>: %s", + "duplicate CONN<%s>", e.conn.RemoteAddr().String(), - e.err, ) } if e.id != "" { - return fmt.Sprintf("duplicate ID<%v>: %s", e.id, e.err) + return fmt.Sprintf("duplicate ID<%v>", e.id) } } diff --git a/p2p/key.go b/p2p/key.go index 3f38b48a..4e662f9f 100644 --- a/p2p/key.go +++ b/p2p/key.go @@ -6,9 +6,8 @@ import ( "fmt" "io/ioutil" - crypto "github.com/tendermint/tendermint/crypto" + "github.com/tendermint/tendermint/crypto" "github.com/tendermint/tendermint/crypto/ed25519" - "github.com/tendermint/tendermint/crypto/tmhash" cmn "github.com/tendermint/tendermint/libs/common" ) @@ -17,7 +16,7 @@ type ID string // IDByteLength is the length of a crypto.Address. Currently only 20. // TODO: support other length addresses ? -const IDByteLength = tmhash.Size +const IDByteLength = crypto.AddressSize //------------------------------------------------------------------------------ // Persistent peer ID diff --git a/p2p/metrics.go b/p2p/metrics.go index 86a20505..3a6b9568 100644 --- a/p2p/metrics.go +++ b/p2p/metrics.go @@ -7,7 +7,11 @@ import ( stdprometheus "github.com/prometheus/client_golang/prometheus" ) -const MetricsSubsystem = "p2p" +const ( + // MetricsSubsystem is a subsystem shared by all metrics exposed by this + // package. + MetricsSubsystem = "p2p" +) // Metrics contains metrics exposed by this package. type Metrics struct { @@ -24,46 +28,51 @@ type Metrics struct { } // PrometheusMetrics returns Metrics build using Prometheus client library. -func PrometheusMetrics(namespace string) *Metrics { +// Optionally, labels can be provided along with their values ("foo", +// "fooValue"). +func PrometheusMetrics(namespace string, labelsAndValues ...string) *Metrics { + labels := []string{} + for i := 0; i < len(labelsAndValues); i += 2 { + labels = append(labels, labelsAndValues[i]) + } return &Metrics{ Peers: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ Namespace: namespace, Subsystem: MetricsSubsystem, Name: "peers", Help: "Number of peers.", - }, []string{}), + }, labels).With(labelsAndValues...), PeerReceiveBytesTotal: prometheus.NewCounterFrom(stdprometheus.CounterOpts{ Namespace: namespace, Subsystem: MetricsSubsystem, Name: "peer_receive_bytes_total", Help: "Number of bytes received from a given peer.", - }, []string{"peer_id"}), + }, append(labels, "peer_id")).With(labelsAndValues...), PeerSendBytesTotal: prometheus.NewCounterFrom(stdprometheus.CounterOpts{ Namespace: namespace, Subsystem: MetricsSubsystem, Name: "peer_send_bytes_total", Help: "Number of bytes sent to a given peer.", - }, []string{"peer_id"}), + }, append(labels, "peer_id")).With(labelsAndValues...), PeerPendingSendBytes: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ Namespace: namespace, Subsystem: MetricsSubsystem, Name: "peer_pending_send_bytes", Help: "Number of pending bytes to be sent to a given peer.", - }, []string{"peer_id"}), + }, append(labels, "peer_id")).With(labelsAndValues...), NumTxs: prometheus.NewGaugeFrom(stdprometheus.GaugeOpts{ Namespace: namespace, Subsystem: MetricsSubsystem, Name: "num_txs", Help: "Number of transactions submitted by each peer.", - }, []string{"peer_id"}), - + }, append(labels, "peer_id")).With(labelsAndValues...), } } // NopMetrics returns no-op Metrics. func NopMetrics() *Metrics { return &Metrics{ - Peers: discard.NewGauge(), + Peers: discard.NewGauge(), PeerReceiveBytesTotal: discard.NewCounter(), PeerSendBytesTotal: discard.NewCounter(), PeerPendingSendBytes: discard.NewGauge(), diff --git a/p2p/netaddress.go b/p2p/netaddress.go index f848b7a5..5534ded9 100644 --- a/p2p/netaddress.go +++ b/p2p/netaddress.go @@ -14,6 +14,7 @@ import ( "time" "errors" + cmn "github.com/tendermint/tendermint/libs/common" ) @@ -31,8 +32,10 @@ type NetAddress struct { str string } -// IDAddressString returns id@hostPort. -func IDAddressString(id ID, hostPort string) string { +// IDAddressString returns id@hostPort. It strips the leading +// protocol from protocolHostPort if it exists. +func IDAddressString(id ID, protocolHostPort string) string { + hostPort := removeProtocolIfDefined(protocolHostPort) return fmt.Sprintf("%s@%s", id, hostPort) } @@ -172,6 +175,9 @@ func (na *NetAddress) Same(other interface{}) bool { // String representation: @: func (na *NetAddress) String() string { + if na == nil { + return "" + } if na.str == "" { addrStr := na.DialString() if na.ID != "" { @@ -183,6 +189,9 @@ func (na *NetAddress) String() string { } func (na *NetAddress) DialString() string { + if na == nil { + return "" + } return net.JoinHostPort( na.IP.String(), strconv.FormatUint(uint64(na.Port), 10), @@ -217,10 +226,22 @@ func (na *NetAddress) Routable() bool { // For IPv4 these are either a 0 or all bits set address. For IPv6 a zero // address or one that matches the RFC3849 documentation address format. func (na *NetAddress) Valid() bool { + if string(na.ID) != "" { + data, err := hex.DecodeString(string(na.ID)) + if err != nil || len(data) != IDByteLength { + return false + } + } return na.IP != nil && !(na.IP.IsUnspecified() || na.RFC3849() || na.IP.Equal(net.IPv4bcast)) } +// HasID returns true if the address has an ID. +// NOTE: It does not check whether the ID is valid or not. +func (na *NetAddress) HasID() bool { + return string(na.ID) != "" +} + // Local returns true if it is a local address. func (na *NetAddress) Local() bool { return na.IP.IsLoopback() || zero4.Contains(na.IP) diff --git a/p2p/node_info.go b/p2p/node_info.go index a1653594..699fd7f1 100644 --- a/p2p/node_info.go +++ b/p2p/node_info.go @@ -2,13 +2,14 @@ package p2p import ( "fmt" - "strings" + "reflect" cmn "github.com/tendermint/tendermint/libs/common" + "github.com/tendermint/tendermint/version" ) const ( - maxNodeInfoSize = 10240 // 10Kb + maxNodeInfoSize = 10240 // 10KB maxNumChannels = 16 // plenty of room for upgrades, for now ) @@ -17,12 +18,67 @@ func MaxNodeInfoSize() int { return maxNodeInfoSize } -// NodeInfo is the basic node information exchanged +//------------------------------------------------------------- + +// NodeInfo exposes basic info of a node +// and determines if we're compatible. +type NodeInfo interface { + nodeInfoAddress + nodeInfoTransport +} + +// nodeInfoAddress exposes just the core info of a node. +type nodeInfoAddress interface { + ID() ID + NetAddress() *NetAddress +} + +// nodeInfoTransport validates a nodeInfo and checks +// our compatibility with it. It's for use in the handshake. +type nodeInfoTransport interface { + Validate() error + CompatibleWith(other NodeInfo) error +} + +//------------------------------------------------------------- + +// ProtocolVersion contains the protocol versions for the software. +type ProtocolVersion struct { + P2P version.Protocol `json:"p2p"` + Block version.Protocol `json:"block"` + App version.Protocol `json:"app"` +} + +// defaultProtocolVersion populates the Block and P2P versions using +// the global values, but not the App. +var defaultProtocolVersion = NewProtocolVersion( + version.P2PProtocol, + version.BlockProtocol, + 0, +) + +// NewProtocolVersion returns a fully populated ProtocolVersion. +func NewProtocolVersion(p2p, block, app version.Protocol) ProtocolVersion { + return ProtocolVersion{ + P2P: p2p, + Block: block, + App: app, + } +} + +//------------------------------------------------------------- + +// Assert DefaultNodeInfo satisfies NodeInfo +var _ NodeInfo = DefaultNodeInfo{} + +// DefaultNodeInfo is the basic node information exchanged // between two peers during the Tendermint P2P handshake. -type NodeInfo struct { +type DefaultNodeInfo struct { + ProtocolVersion ProtocolVersion `json:"protocol_version"` + // Authenticate // TODO: replace with NetAddress - ID ID `json:"id"` // authenticated identifier + ID_ ID `json:"id"` // authenticated identifier ListenAddr string `json:"listen_addr"` // accepting incoming // Check compatibility. @@ -32,33 +88,22 @@ type NodeInfo struct { Channels cmn.HexBytes `json:"channels"` // channels this node knows about // ASCIIText fields - Moniker string `json:"moniker"` // arbitrary moniker - Other NodeInfoOther `json:"other"` // other application specific data + Moniker string `json:"moniker"` // arbitrary moniker + Other DefaultNodeInfoOther `json:"other"` // other application specific data } -// NodeInfoOther is the misc. applcation specific data -type NodeInfoOther struct { - AminoVersion string `json:"amino_version"` - P2PVersion string `json:"p2p_version"` - ConsensusVersion string `json:"consensus_version"` - RPCVersion string `json:"rpc_version"` - TxIndex string `json:"tx_index"` - RPCAddress string `json:"rpc_address"` +// DefaultNodeInfoOther is the misc. applcation specific data +type DefaultNodeInfoOther struct { + TxIndex string `json:"tx_index"` + RPCAddress string `json:"rpc_address"` } -func (o NodeInfoOther) String() string { - return fmt.Sprintf( - "{amino_version: %v, p2p_version: %v, consensus_version: %v, rpc_version: %v, tx_index: %v, rpc_address: %v}", - o.AminoVersion, - o.P2PVersion, - o.ConsensusVersion, - o.RPCVersion, - o.TxIndex, - o.RPCAddress, - ) +// ID returns the node's peer ID. +func (info DefaultNodeInfo) ID() ID { + return info.ID_ } -// Validate checks the self-reported NodeInfo is safe. +// Validate checks the self-reported DefaultNodeInfo is safe. // It returns an error if there // are too many Channels, if there are any duplicate Channels, // if the ListenAddr is malformed, or if the ListenAddr is a host name @@ -71,36 +116,29 @@ func (o NodeInfoOther) String() string { // International clients could then use punycode (or we could use // url-encoding), and we just need to be careful with how we handle that in our // clients. (e.g. off by default). -func (info NodeInfo) Validate() error { +func (info DefaultNodeInfo) Validate() error { + + // ID is already validated. + + // Validate ListenAddr. + _, err := NewNetAddressString(IDAddressString(info.ID(), info.ListenAddr)) + if err != nil { + return err + } + + // Network is validated in CompatibleWith. + + // Validate Version + if len(info.Version) > 0 && + (!cmn.IsASCIIText(info.Version) || cmn.ASCIITrim(info.Version) == "") { + + return fmt.Errorf("info.Version must be valid ASCII text without tabs, but got %v", info.Version) + } + + // Validate Channels - ensure max and check for duplicates. if len(info.Channels) > maxNumChannels { return fmt.Errorf("info.Channels is too long (%v). Max is %v", len(info.Channels), maxNumChannels) } - - // Sanitize ASCII text fields. - if !cmn.IsASCIIText(info.Moniker) || cmn.ASCIITrim(info.Moniker) == "" { - return fmt.Errorf("info.Moniker must be valid non-empty ASCII text without tabs, but got %v", info.Moniker) - } - - // Sanitize versions - // XXX: Should we be more strict about version and address formats? - other := info.Other - versions := []string{ - other.AminoVersion, - other.P2PVersion, - other.ConsensusVersion, - other.RPCVersion} - for i, v := range versions { - if cmn.ASCIITrim(v) != "" && !cmn.IsASCIIText(v) { - return fmt.Errorf("info.Other[%d]=%v must be valid non-empty ASCII text without tabs", i, v) - } - } - if cmn.ASCIITrim(other.TxIndex) != "" && (other.TxIndex != "on" && other.TxIndex != "off") { - return fmt.Errorf("info.Other.TxIndex should be either 'on' or 'off', got '%v'", other.TxIndex) - } - if cmn.ASCIITrim(other.RPCAddress) != "" && !cmn.IsASCIIText(other.RPCAddress) { - return fmt.Errorf("info.Other.RPCAddress=%v must be valid non-empty ASCII text without tabs", other.RPCAddress) - } - channels := make(map[byte]struct{}) for _, ch := range info.Channels { _, ok := channels[ch] @@ -110,31 +148,40 @@ func (info NodeInfo) Validate() error { channels[ch] = struct{}{} } - // ensure ListenAddr is good - _, err := NewNetAddressString(IDAddressString(info.ID, info.ListenAddr)) - return err + // Validate Moniker. + if !cmn.IsASCIIText(info.Moniker) || cmn.ASCIITrim(info.Moniker) == "" { + return fmt.Errorf("info.Moniker must be valid non-empty ASCII text without tabs, but got %v", info.Moniker) + } + + // Validate Other. + other := info.Other + txIndex := other.TxIndex + switch txIndex { + case "", "on", "off": + default: + return fmt.Errorf("info.Other.TxIndex should be either 'on', 'off', or empty string, got '%v'", txIndex) + } + // XXX: Should we be more strict about address formats? + rpcAddr := other.RPCAddress + if len(rpcAddr) > 0 && (!cmn.IsASCIIText(rpcAddr) || cmn.ASCIITrim(rpcAddr) == "") { + return fmt.Errorf("info.Other.RPCAddress=%v must be valid ASCII text without tabs", rpcAddr) + } + + return nil } -// CompatibleWith checks if two NodeInfo are compatible with eachother. -// CONTRACT: two nodes are compatible if the major version matches and network match +// CompatibleWith checks if two DefaultNodeInfo are compatible with eachother. +// CONTRACT: two nodes are compatible if the Block version and network match // and they have at least one channel in common. -func (info NodeInfo) CompatibleWith(other NodeInfo) error { - iMajor, _, _, iErr := splitVersion(info.Version) - oMajor, _, _, oErr := splitVersion(other.Version) - - // if our own version number is not formatted right, we messed up - if iErr != nil { - return iErr +func (info DefaultNodeInfo) CompatibleWith(other_ NodeInfo) error { + other, ok := other_.(DefaultNodeInfo) + if !ok { + return fmt.Errorf("wrong NodeInfo type. Expected DefaultNodeInfo, got %v", reflect.TypeOf(other_)) } - // version number must be formatted correctly ("x.x.x") - if oErr != nil { - return oErr - } - - // major version must match - if iMajor != oMajor { - return fmt.Errorf("Peer is on a different major version. Got %v, expected %v", oMajor, iMajor) + if info.ProtocolVersion.Block != other.ProtocolVersion.Block { + return fmt.Errorf("Peer is on a different Block version. Got %v, expected %v", + other.ProtocolVersion.Block, info.ProtocolVersion.Block) } // nodes must be on the same network @@ -164,18 +211,19 @@ OUTER_LOOP: return nil } -// NetAddress returns a NetAddress derived from the NodeInfo - +// NetAddress returns a NetAddress derived from the DefaultNodeInfo - // it includes the authenticated peer ID and the self-reported // ListenAddr. Note that the ListenAddr is not authenticated and // may not match that address actually dialed if its an outbound peer. -func (info NodeInfo) NetAddress() *NetAddress { - netAddr, err := NewNetAddressString(IDAddressString(info.ID, info.ListenAddr)) +func (info DefaultNodeInfo) NetAddress() *NetAddress { + idAddr := IDAddressString(info.ID(), info.ListenAddr) + netAddr, err := NewNetAddressString(idAddr) if err != nil { switch err.(type) { case ErrNetAddressLookup: // XXX If the peer provided a host name and the lookup fails here // we're out of luck. - // TODO: use a NetAddress in NodeInfo + // TODO: use a NetAddress in DefaultNodeInfo default: panic(err) // everything should be well formed by now } @@ -183,15 +231,30 @@ func (info NodeInfo) NetAddress() *NetAddress { return netAddr } -func (info NodeInfo) String() string { - return fmt.Sprintf("NodeInfo{id: %v, moniker: %v, network: %v [listen %v], version: %v (%v)}", - info.ID, info.Moniker, info.Network, info.ListenAddr, info.Version, info.Other) +//----------------------------------------------------------- +// These methods are for Protobuf Compatibility + +// Size returns the size of the amino encoding, in bytes. +func (info *DefaultNodeInfo) Size() int { + bs, _ := info.Marshal() + return len(bs) } -func splitVersion(version string) (string, string, string, error) { - spl := strings.Split(version, ".") - if len(spl) != 3 { - return "", "", "", fmt.Errorf("Invalid version format %v", version) - } - return spl[0], spl[1], spl[2], nil +// Marshal returns the amino encoding. +func (info *DefaultNodeInfo) Marshal() ([]byte, error) { + return cdc.MarshalBinaryBare(info) +} + +// MarshalTo calls Marshal and copies to the given buffer. +func (info *DefaultNodeInfo) MarshalTo(data []byte) (int, error) { + bs, err := info.Marshal() + if err != nil { + return -1, err + } + return copy(data, bs), nil +} + +// Unmarshal deserializes from amino encoded form. +func (info *DefaultNodeInfo) Unmarshal(bs []byte) error { + return cdc.UnmarshalBinaryBare(bs, info) } diff --git a/p2p/node_info_test.go b/p2p/node_info_test.go new file mode 100644 index 00000000..19567d2b --- /dev/null +++ b/p2p/node_info_test.go @@ -0,0 +1,123 @@ +package p2p + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/tendermint/tendermint/crypto/ed25519" +) + +func TestNodeInfoValidate(t *testing.T) { + + // empty fails + ni := DefaultNodeInfo{} + assert.Error(t, ni.Validate()) + + channels := make([]byte, maxNumChannels) + for i := 0; i < maxNumChannels; i++ { + channels[i] = byte(i) + } + dupChannels := make([]byte, 5) + copy(dupChannels[:], channels[:5]) + dupChannels = append(dupChannels, testCh) + + nonAscii := "¢§µ" + emptyTab := fmt.Sprintf("\t") + emptySpace := fmt.Sprintf(" ") + + testCases := []struct { + testName string + malleateNodeInfo func(*DefaultNodeInfo) + expectErr bool + }{ + {"Too Many Channels", func(ni *DefaultNodeInfo) { ni.Channels = append(channels, byte(maxNumChannels)) }, true}, + {"Duplicate Channel", func(ni *DefaultNodeInfo) { ni.Channels = dupChannels }, true}, + {"Good Channels", func(ni *DefaultNodeInfo) { ni.Channels = ni.Channels[:5] }, false}, + + {"Invalid NetAddress", func(ni *DefaultNodeInfo) { ni.ListenAddr = "not-an-address" }, true}, + {"Good NetAddress", func(ni *DefaultNodeInfo) { ni.ListenAddr = "0.0.0.0:26656" }, false}, + + {"Non-ASCII Version", func(ni *DefaultNodeInfo) { ni.Version = nonAscii }, true}, + {"Empty tab Version", func(ni *DefaultNodeInfo) { ni.Version = emptyTab }, true}, + {"Empty space Version", func(ni *DefaultNodeInfo) { ni.Version = emptySpace }, true}, + {"Empty Version", func(ni *DefaultNodeInfo) { ni.Version = "" }, false}, + + {"Non-ASCII Moniker", func(ni *DefaultNodeInfo) { ni.Moniker = nonAscii }, true}, + {"Empty tab Moniker", func(ni *DefaultNodeInfo) { ni.Moniker = emptyTab }, true}, + {"Empty space Moniker", func(ni *DefaultNodeInfo) { ni.Moniker = emptySpace }, true}, + {"Empty Moniker", func(ni *DefaultNodeInfo) { ni.Moniker = "" }, true}, + {"Good Moniker", func(ni *DefaultNodeInfo) { ni.Moniker = "hey its me" }, false}, + + {"Non-ASCII TxIndex", func(ni *DefaultNodeInfo) { ni.Other.TxIndex = nonAscii }, true}, + {"Empty tab TxIndex", func(ni *DefaultNodeInfo) { ni.Other.TxIndex = emptyTab }, true}, + {"Empty space TxIndex", func(ni *DefaultNodeInfo) { ni.Other.TxIndex = emptySpace }, true}, + {"Empty TxIndex", func(ni *DefaultNodeInfo) { ni.Other.TxIndex = "" }, false}, + {"Off TxIndex", func(ni *DefaultNodeInfo) { ni.Other.TxIndex = "off" }, false}, + + {"Non-ASCII RPCAddress", func(ni *DefaultNodeInfo) { ni.Other.RPCAddress = nonAscii }, true}, + {"Empty tab RPCAddress", func(ni *DefaultNodeInfo) { ni.Other.RPCAddress = emptyTab }, true}, + {"Empty space RPCAddress", func(ni *DefaultNodeInfo) { ni.Other.RPCAddress = emptySpace }, true}, + {"Empty RPCAddress", func(ni *DefaultNodeInfo) { ni.Other.RPCAddress = "" }, false}, + {"Good RPCAddress", func(ni *DefaultNodeInfo) { ni.Other.RPCAddress = "0.0.0.0:26657" }, false}, + } + + nodeKey := NodeKey{PrivKey: ed25519.GenPrivKey()} + name := "testing" + + // test case passes + ni = testNodeInfo(nodeKey.ID(), name).(DefaultNodeInfo) + ni.Channels = channels + assert.NoError(t, ni.Validate()) + + for _, tc := range testCases { + ni := testNodeInfo(nodeKey.ID(), name).(DefaultNodeInfo) + ni.Channels = channels + tc.malleateNodeInfo(&ni) + err := ni.Validate() + if tc.expectErr { + assert.Error(t, err, tc.testName) + } else { + assert.NoError(t, err, tc.testName) + } + } + +} + +func TestNodeInfoCompatible(t *testing.T) { + + nodeKey1 := NodeKey{PrivKey: ed25519.GenPrivKey()} + nodeKey2 := NodeKey{PrivKey: ed25519.GenPrivKey()} + name := "testing" + + var newTestChannel byte = 0x2 + + // test NodeInfo is compatible + ni1 := testNodeInfo(nodeKey1.ID(), name).(DefaultNodeInfo) + ni2 := testNodeInfo(nodeKey2.ID(), name).(DefaultNodeInfo) + assert.NoError(t, ni1.CompatibleWith(ni2)) + + // add another channel; still compatible + ni2.Channels = []byte{newTestChannel, testCh} + assert.NoError(t, ni1.CompatibleWith(ni2)) + + // wrong NodeInfo type is not compatible + _, netAddr := CreateRoutableAddr() + ni3 := mockNodeInfo{netAddr} + assert.Error(t, ni1.CompatibleWith(ni3)) + + testCases := []struct { + testName string + malleateNodeInfo func(*DefaultNodeInfo) + }{ + {"Wrong block version", func(ni *DefaultNodeInfo) { ni.ProtocolVersion.Block += 1 }}, + {"Wrong network", func(ni *DefaultNodeInfo) { ni.Network += "-wrong" }}, + {"No common channels", func(ni *DefaultNodeInfo) { ni.Channels = []byte{newTestChannel} }}, + } + + for _, tc := range testCases { + ni := testNodeInfo(nodeKey2.ID(), name).(DefaultNodeInfo) + tc.malleateNodeInfo(&ni) + assert.Error(t, ni1.CompatibleWith(ni)) + } +} diff --git a/p2p/peer.go b/p2p/peer.go index ba22695e..73332a2a 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -3,31 +3,33 @@ package p2p import ( "fmt" "net" - "sync/atomic" "time" cmn "github.com/tendermint/tendermint/libs/common" "github.com/tendermint/tendermint/libs/log" - "github.com/tendermint/tendermint/config" tmconn "github.com/tendermint/tendermint/p2p/conn" ) const metricsTickerDuration = 10 * time.Second -var testIPSuffix uint32 - // Peer is an interface representing a peer connected on a reactor. type Peer interface { cmn.Service + FlushStop() + + ID() ID // peer's cryptographic ID + RemoteIP() net.IP // remote IP of the connection + RemoteAddr() net.Addr // remote address of the connection - ID() ID // peer's cryptographic ID - RemoteIP() net.IP // remote IP of the connection IsOutbound() bool // did we dial the peer IsPersistent() bool // do we redial this peer when we disconnect + + CloseConn() error // close original connection + NodeInfo() NodeInfo // peer's info Status() tmconn.ConnectionStatus - OriginalAddr() *NetAddress + OriginalAddr() *NetAddress // original address for outbound peers Send(byte, []byte) bool TrySend(byte, []byte) bool @@ -40,12 +42,28 @@ type Peer interface { // peerConn contains the raw connection and its config. type peerConn struct { - outbound bool - persistent bool - config *config.P2PConfig - conn net.Conn // source connection - ip net.IP + outbound bool + persistent bool + conn net.Conn // source connection + originalAddr *NetAddress // nil for inbound connections + + // cached RemoteIP() + ip net.IP +} + +func newPeerConn( + outbound, persistent bool, + conn net.Conn, + originalAddr *NetAddress, +) peerConn { + + return peerConn{ + outbound: outbound, + persistent: persistent, + conn: conn, + originalAddr: originalAddr, + } } // ID only exists for SecretConnection. @@ -60,14 +78,6 @@ func (pc peerConn) RemoteIP() net.IP { return pc.ip } - // In test cases a conn could not be present at all or be an in-memory - // implementation where we want to return a fake ip. - if pc.conn == nil || pc.conn.RemoteAddr().String() == "pipe" { - pc.ip = net.IP{172, 16, 0, byte(atomic.AddUint32(&testIPSuffix, 1))} - - return pc.ip - } - host, _, err := net.SplitHostPort(pc.conn.RemoteAddr().String()) if err != nil { panic(err) @@ -120,7 +130,7 @@ func newPeer( p := &peer{ peerConn: pc, nodeInfo: nodeInfo, - channels: nodeInfo.Channels, + channels: nodeInfo.(DefaultNodeInfo).Channels, // TODO Data: cmn.NewCMap(), metricsTicker: time.NewTicker(metricsTickerDuration), metrics: NopMetrics(), @@ -142,6 +152,15 @@ func newPeer( return p } +// String representation. +func (p *peer) String() string { + if p.outbound { + return fmt.Sprintf("Peer{%v %v out}", p.mconn, p.ID()) + } + + return fmt.Sprintf("Peer{%v %v in}", p.mconn, p.ID()) +} + //--------------------------------------------------- // Implements cmn.Service @@ -165,6 +184,15 @@ func (p *peer) OnStart() error { return nil } +// FlushStop mimics OnStop but additionally ensures that all successful +// .Send() calls will get flushed before closing the connection. +// NOTE: it is not safe to call this method more than once. +func (p *peer) FlushStop() { + p.metricsTicker.Stop() + p.BaseService.OnStop() + p.mconn.FlushStop() // stop everything and close the conn +} + // OnStop implements BaseService. func (p *peer) OnStop() { p.metricsTicker.Stop() @@ -177,7 +205,7 @@ func (p *peer) OnStop() { // ID returns the peer's ID - the hex encoded hash of its pubkey. func (p *peer) ID() ID { - return p.nodeInfo.ID + return p.nodeInfo.ID() } // IsOutbound returns true if the connection is outbound, false otherwise. @@ -221,7 +249,7 @@ func (p *peer) Send(chID byte, msgBytes []byte) bool { } res := p.mconn.Send(chID, msgBytes) if res { - p.metrics.PeerSendBytesTotal.With("peer-id", string(p.ID())).Add(float64(len(msgBytes))) + p.metrics.PeerSendBytesTotal.With("peer_id", string(p.ID())).Add(float64(len(msgBytes))) } return res } @@ -236,7 +264,7 @@ func (p *peer) TrySend(chID byte, msgBytes []byte) bool { } res := p.mconn.TrySend(chID, msgBytes) if res { - p.metrics.PeerSendBytesTotal.With("peer-id", string(p.ID())).Add(float64(len(msgBytes))) + p.metrics.PeerSendBytesTotal.With("peer_id", string(p.ID())).Add(float64(len(msgBytes))) } return res } @@ -271,56 +299,22 @@ func (p *peer) hasChannel(chID byte) bool { return false } -//--------------------------------------------------- -// methods used by the Switch +// CloseConn closes original connection. Used for cleaning up in cases where the peer had not been started at all. +func (p *peer) CloseConn() error { + return p.peerConn.conn.Close() +} -// CloseConn should be called by the Switch if the peer was created but never -// started. +//--------------------------------------------------- +// methods only used for testing +// TODO: can we remove these? + +// CloseConn closes the underlying connection func (pc *peerConn) CloseConn() { pc.conn.Close() // nolint: errcheck } -// HandshakeTimeout performs the Tendermint P2P handshake between a given node -// and the peer by exchanging their NodeInfo. It sets the received nodeInfo on -// the peer. -// NOTE: blocking -func (pc *peerConn) HandshakeTimeout( - ourNodeInfo NodeInfo, - timeout time.Duration, -) (peerNodeInfo NodeInfo, err error) { - // Set deadline for handshake so we don't block forever on conn.ReadFull - if err := pc.conn.SetDeadline(time.Now().Add(timeout)); err != nil { - return peerNodeInfo, cmn.ErrorWrap(err, "Error setting deadline") - } - - var trs, _ = cmn.Parallel( - func(_ int) (val interface{}, err error, abort bool) { - _, err = cdc.MarshalBinaryWriter(pc.conn, ourNodeInfo) - return - }, - func(_ int) (val interface{}, err error, abort bool) { - _, err = cdc.UnmarshalBinaryReader( - pc.conn, - &peerNodeInfo, - int64(MaxNodeInfoSize()), - ) - return - }, - ) - if err := trs.FirstError(); err != nil { - return peerNodeInfo, cmn.ErrorWrap(err, "Error during handshake") - } - - // Remove deadline - if err := pc.conn.SetDeadline(time.Time{}); err != nil { - return peerNodeInfo, cmn.ErrorWrap(err, "Error removing deadline") - } - - return peerNodeInfo, nil -} - -// Addr returns peer's remote network address. -func (p *peer) Addr() net.Addr { +// RemoteAddr returns peer's remote network address. +func (p *peer) RemoteAddr() net.Addr { return p.peerConn.conn.RemoteAddr() } @@ -332,14 +326,7 @@ func (p *peer) CanSend(chID byte) bool { return p.mconn.CanSend(chID) } -// String representation. -func (p *peer) String() string { - if p.outbound { - return fmt.Sprintf("Peer{%v %v out}", p.mconn, p.ID()) - } - - return fmt.Sprintf("Peer{%v %v in}", p.mconn, p.ID()) -} +//--------------------------------------------------- func PeerMetrics(metrics *Metrics) PeerOption { return func(p *peer) { @@ -357,7 +344,7 @@ func (p *peer) metricsReporter() { sendQueueSize += float64(chStatus.SendQueueSize) } - p.metrics.PeerPendingSendBytes.With("peer-id", string(p.ID())).Set(sendQueueSize) + p.metrics.PeerPendingSendBytes.With("peer_id", string(p.ID())).Set(sendQueueSize) case <-p.Quit(): return } diff --git a/p2p/peer_set.go b/p2p/peer_set.go index 25785615..87cf61da 100644 --- a/p2p/peer_set.go +++ b/p2p/peer_set.go @@ -98,13 +98,15 @@ func (ps *PeerSet) Get(peerKey ID) Peer { } // Remove discards peer by its Key, if the peer was previously memoized. -func (ps *PeerSet) Remove(peer Peer) { +// Returns true if the peer was removed, and false if it was not found. +// in the set. +func (ps *PeerSet) Remove(peer Peer) bool { ps.mtx.Lock() defer ps.mtx.Unlock() item := ps.lookup[peer.ID()] if item == nil { - return + return false } index := item.index @@ -116,7 +118,7 @@ func (ps *PeerSet) Remove(peer Peer) { if index == len(ps.list)-1 { ps.list = newList delete(ps.lookup, peer.ID()) - return + return true } // Replace the popped item with the last item in the old list. @@ -127,6 +129,7 @@ func (ps *PeerSet) Remove(peer Peer) { lastPeerItem.index = index ps.list = newList delete(ps.lookup, peer.ID()) + return true } // Size returns the number of unique items in the peerSet. diff --git a/p2p/peer_set_test.go b/p2p/peer_set_test.go index ee1c52ea..1d2372fb 100644 --- a/p2p/peer_set_test.go +++ b/p2p/peer_set_test.go @@ -1,7 +1,6 @@ package p2p import ( - "fmt" "net" "sync" "testing" @@ -12,24 +11,38 @@ import ( cmn "github.com/tendermint/tendermint/libs/common" ) -// Returns an empty kvstore peer -func randPeer(ip net.IP) *peer { +// mockPeer for testing the PeerSet +type mockPeer struct { + cmn.BaseService + ip net.IP + id ID +} + +func (mp *mockPeer) FlushStop() { mp.Stop() } +func (mp *mockPeer) TrySend(chID byte, msgBytes []byte) bool { return true } +func (mp *mockPeer) Send(chID byte, msgBytes []byte) bool { return true } +func (mp *mockPeer) NodeInfo() NodeInfo { return DefaultNodeInfo{} } +func (mp *mockPeer) Status() ConnectionStatus { return ConnectionStatus{} } +func (mp *mockPeer) ID() ID { return mp.id } +func (mp *mockPeer) IsOutbound() bool { return false } +func (mp *mockPeer) IsPersistent() bool { return true } +func (mp *mockPeer) Get(s string) interface{} { return s } +func (mp *mockPeer) Set(string, interface{}) {} +func (mp *mockPeer) RemoteIP() net.IP { return mp.ip } +func (mp *mockPeer) OriginalAddr() *NetAddress { return nil } +func (mp *mockPeer) RemoteAddr() net.Addr { return &net.TCPAddr{IP: mp.ip, Port: 8800} } +func (mp *mockPeer) CloseConn() error { return nil } + +// Returns a mock peer +func newMockPeer(ip net.IP) *mockPeer { if ip == nil { ip = net.IP{127, 0, 0, 1} } - nodeKey := NodeKey{PrivKey: ed25519.GenPrivKey()} - p := &peer{ - nodeInfo: NodeInfo{ - ID: nodeKey.ID(), - ListenAddr: fmt.Sprintf("%v.%v.%v.%v:26656", cmn.RandInt()%256, cmn.RandInt()%256, cmn.RandInt()%256, cmn.RandInt()%256), - }, - metrics: NopMetrics(), + return &mockPeer{ + ip: ip, + id: nodeKey.ID(), } - - p.ip = ip - - return p } func TestPeerSetAddRemoveOne(t *testing.T) { @@ -39,7 +52,7 @@ func TestPeerSetAddRemoveOne(t *testing.T) { var peerList []Peer for i := 0; i < 5; i++ { - p := randPeer(net.IP{127, 0, 0, byte(i)}) + p := newMockPeer(net.IP{127, 0, 0, byte(i)}) if err := peerSet.Add(p); err != nil { t.Error(err) } @@ -49,13 +62,15 @@ func TestPeerSetAddRemoveOne(t *testing.T) { n := len(peerList) // 1. Test removing from the front for i, peerAtFront := range peerList { - peerSet.Remove(peerAtFront) + removed := peerSet.Remove(peerAtFront) + assert.True(t, removed) wantSize := n - i - 1 for j := 0; j < 2; j++ { assert.Equal(t, false, peerSet.Has(peerAtFront.ID()), "#%d Run #%d: failed to remove peer", i, j) assert.Equal(t, wantSize, peerSet.Size(), "#%d Run #%d: failed to remove peer and decrement size", i, j) // Test the route of removing the now non-existent element - peerSet.Remove(peerAtFront) + removed := peerSet.Remove(peerAtFront) + assert.False(t, removed) } } @@ -70,7 +85,8 @@ func TestPeerSetAddRemoveOne(t *testing.T) { // b) In reverse, remove each element for i := n - 1; i >= 0; i-- { peerAtEnd := peerList[i] - peerSet.Remove(peerAtEnd) + removed := peerSet.Remove(peerAtEnd) + assert.True(t, removed) assert.Equal(t, false, peerSet.Has(peerAtEnd.ID()), "#%d: failed to remove item at end", i) assert.Equal(t, i, peerSet.Size(), "#%d: differing sizes after peerSet.Remove(atEndPeer)", i) } @@ -83,7 +99,7 @@ func TestPeerSetAddRemoveMany(t *testing.T) { peers := []Peer{} N := 100 for i := 0; i < N; i++ { - peer := randPeer(net.IP{127, 0, 0, byte(i)}) + peer := newMockPeer(net.IP{127, 0, 0, byte(i)}) if err := peerSet.Add(peer); err != nil { t.Errorf("Failed to add new peer") } @@ -94,7 +110,8 @@ func TestPeerSetAddRemoveMany(t *testing.T) { } for i, peer := range peers { - peerSet.Remove(peer) + removed := peerSet.Remove(peer) + assert.True(t, removed) if peerSet.Has(peer.ID()) { t.Errorf("Failed to remove peer") } @@ -107,7 +124,7 @@ func TestPeerSetAddRemoveMany(t *testing.T) { func TestPeerSetAddDuplicate(t *testing.T) { t.Parallel() peerSet := NewPeerSet() - peer := randPeer(nil) + peer := newMockPeer(nil) n := 20 errsChan := make(chan error) @@ -149,7 +166,7 @@ func TestPeerSetGet(t *testing.T) { var ( peerSet = NewPeerSet() - peer = randPeer(nil) + peer = newMockPeer(nil) ) assert.Nil(t, peerSet.Get(peer.ID()), "expecting a nil lookup, before .Add") diff --git a/p2p/peer_test.go b/p2p/peer_test.go index a2a2946a..90be3113 100644 --- a/p2p/peer_test.go +++ b/p2p/peer_test.go @@ -10,7 +10,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - crypto "github.com/tendermint/tendermint/crypto" + "github.com/tendermint/tendermint/crypto" "github.com/tendermint/tendermint/crypto/ed25519" cmn "github.com/tendermint/tendermint/libs/common" "github.com/tendermint/tendermint/libs/log" @@ -19,8 +19,6 @@ import ( tmconn "github.com/tendermint/tendermint/p2p/conn" ) -const testCh = 0x01 - func TestPeerBasic(t *testing.T) { assert, require := assert.New(t), require.New(t) @@ -41,7 +39,7 @@ func TestPeerBasic(t *testing.T) { assert.False(p.IsPersistent()) p.persistent = true assert.True(p.IsPersistent()) - assert.Equal(rp.Addr().DialString(), p.Addr().String()) + assert.Equal(rp.Addr().DialString(), p.RemoteAddr().String()) assert.Equal(rp.ID(), p.ID()) } @@ -81,18 +79,14 @@ func createOutboundPeerAndPerformHandshake( if err != nil { return nil, err } - nodeInfo, err := pc.HandshakeTimeout(NodeInfo{ - ID: addr.ID, - Moniker: "host_peer", - Network: "testing", - Version: "123.123.123", - Channels: []byte{testCh}, - }, 1*time.Second) + timeout := 1 * time.Second + ourNodeInfo := testNodeInfo(addr.ID, "host_peer") + peerNodeInfo, err := handshake(pc.conn, timeout, ourNodeInfo) if err != nil { return nil, err } - p := newPeer(pc, mConfig, nodeInfo, reactorsByCh, chDescs, func(p Peer, r interface{}) {}) + p := newPeer(pc, mConfig, peerNodeInfo, reactorsByCh, chDescs, func(p Peer, r interface{}) {}) p.SetLogger(log.TestingLogger().With("peer", addr)) return p, nil } @@ -143,9 +137,9 @@ type remotePeer struct { PrivKey crypto.PrivKey Config *config.P2PConfig addr *NetAddress - quit chan struct{} channels cmn.HexBytes listenAddr string + listener net.Listener } func (rp *remotePeer) Addr() *NetAddress { @@ -165,25 +159,45 @@ func (rp *remotePeer) Start() { if e != nil { golog.Fatalf("net.Listen tcp :0: %+v", e) } + rp.listener = l rp.addr = NewNetAddress(PubKeyToID(rp.PrivKey.PubKey()), l.Addr()) - rp.quit = make(chan struct{}) if rp.channels == nil { rp.channels = []byte{testCh} } - go rp.accept(l) + go rp.accept() } func (rp *remotePeer) Stop() { - close(rp.quit) + rp.listener.Close() } -func (rp *remotePeer) accept(l net.Listener) { +func (rp *remotePeer) Dial(addr *NetAddress) (net.Conn, error) { + conn, err := addr.DialTimeout(1 * time.Second) + if err != nil { + return nil, err + } + pc, err := testInboundPeerConn(conn, rp.Config, rp.PrivKey) + if err != nil { + return nil, err + } + _, err = handshake(pc.conn, time.Second, rp.nodeInfo()) + if err != nil { + return nil, err + } + return conn, err +} + +func (rp *remotePeer) accept() { conns := []net.Conn{} for { - conn, err := l.Accept() + conn, err := rp.listener.Accept() if err != nil { - golog.Fatalf("Failed to accept conn: %+v", err) + golog.Printf("Failed to accept conn: %+v", err) + for _, conn := range conns { + _ = conn.Close() + } + return } pc, err := testInboundPeerConn(conn, rp.Config, rp.PrivKey) @@ -191,29 +205,23 @@ func (rp *remotePeer) accept(l net.Listener) { golog.Fatalf("Failed to create a peer: %+v", err) } - _, err = handshake(pc.conn, time.Second, NodeInfo{ - ID: rp.Addr().ID, - Moniker: "remote_peer", - Network: "testing", - Version: "123.123.123", - ListenAddr: l.Addr().String(), - Channels: rp.channels, - }) + _, err = handshake(pc.conn, time.Second, rp.nodeInfo()) if err != nil { golog.Fatalf("Failed to perform handshake: %+v", err) } conns = append(conns, conn) - - select { - case <-rp.quit: - for _, conn := range conns { - if err := conn.Close(); err != nil { - golog.Fatal(err) - } - } - return - default: - } + } +} + +func (rp *remotePeer) nodeInfo() NodeInfo { + return DefaultNodeInfo{ + ProtocolVersion: defaultProtocolVersion, + ID_: rp.Addr().ID, + ListenAddr: rp.listener.Addr().String(), + Network: "testing", + Version: "1.2.3-rc0-deadbeef", + Channels: rp.channels, + Moniker: "remote_peer", } } diff --git a/p2p/pex/addrbook.go b/p2p/pex/addrbook.go index e0c0e0b9..3cda9ac7 100644 --- a/p2p/pex/addrbook.go +++ b/p2p/pex/addrbook.go @@ -13,7 +13,7 @@ import ( "sync" "time" - crypto "github.com/tendermint/tendermint/crypto" + "github.com/tendermint/tendermint/crypto" cmn "github.com/tendermint/tendermint/libs/common" "github.com/tendermint/tendermint/p2p" ) @@ -164,6 +164,7 @@ func (a *addrBook) FilePath() string { func (a *addrBook) AddOurAddress(addr *p2p.NetAddress) { a.mtx.Lock() defer a.mtx.Unlock() + a.Logger.Info("Add our address to book", "addr", addr) a.ourAddrs[addr.String()] = struct{}{} } @@ -171,14 +172,16 @@ func (a *addrBook) AddOurAddress(addr *p2p.NetAddress) { // OurAddress returns true if it is our address. func (a *addrBook) OurAddress(addr *p2p.NetAddress) bool { a.mtx.Lock() + defer a.mtx.Unlock() + _, ok := a.ourAddrs[addr.String()] - a.mtx.Unlock() return ok } func (a *addrBook) AddPrivateIDs(IDs []string) { a.mtx.Lock() defer a.mtx.Unlock() + for _, id := range IDs { a.privateIDs[p2p.ID(id)] = struct{}{} } @@ -191,6 +194,7 @@ func (a *addrBook) AddPrivateIDs(IDs []string) { func (a *addrBook) AddAddress(addr *p2p.NetAddress, src *p2p.NetAddress) error { a.mtx.Lock() defer a.mtx.Unlock() + return a.addAddress(addr, src) } @@ -198,11 +202,12 @@ func (a *addrBook) AddAddress(addr *p2p.NetAddress, src *p2p.NetAddress) error { func (a *addrBook) RemoveAddress(addr *p2p.NetAddress) { a.mtx.Lock() defer a.mtx.Unlock() + ka := a.addrLookup[addr.ID] if ka == nil { return } - a.Logger.Info("Remove address from book", "addr", ka.Addr, "ID", ka.ID()) + a.Logger.Info("Remove address from book", "addr", addr) a.removeFromAllBuckets(ka) } @@ -211,6 +216,7 @@ func (a *addrBook) RemoveAddress(addr *p2p.NetAddress) { func (a *addrBook) IsGood(addr *p2p.NetAddress) bool { a.mtx.Lock() defer a.mtx.Unlock() + return a.addrLookup[addr.ID].isOld() } @@ -218,6 +224,7 @@ func (a *addrBook) IsGood(addr *p2p.NetAddress) bool { func (a *addrBook) HasAddress(addr *p2p.NetAddress) bool { a.mtx.Lock() defer a.mtx.Unlock() + ka := a.addrLookup[addr.ID] return ka != nil } @@ -292,6 +299,7 @@ func (a *addrBook) PickAddress(biasTowardsNewAddrs int) *p2p.NetAddress { func (a *addrBook) MarkGood(addr *p2p.NetAddress) { a.mtx.Lock() defer a.mtx.Unlock() + ka := a.addrLookup[addr.ID] if ka == nil { return @@ -306,6 +314,7 @@ func (a *addrBook) MarkGood(addr *p2p.NetAddress) { func (a *addrBook) MarkAttempt(addr *p2p.NetAddress) { a.mtx.Lock() defer a.mtx.Unlock() + ka := a.addrLookup[addr.ID] if ka == nil { return @@ -360,6 +369,10 @@ func (a *addrBook) GetSelection() []*p2p.NetAddress { return allAddr[:numAddresses] } +func percentageOfNum(p, n int) int { + return int(math.Round((float64(p) / float64(100)) * float64(n))) +} + // GetSelectionWithBias implements AddrBook. // It randomly selects some addresses (old & new). Suitable for peer-exchange protocols. // Must never return a nil address. @@ -399,11 +412,28 @@ func (a *addrBook) GetSelectionWithBias(biasTowardsNewAddrs int) []*p2p.NetAddre newBucketToAddrsMap := make(map[int]map[string]struct{}) var newIndex int + // initialize counters used to count old and new added addresses. + // len(oldBucketToAddrsMap) cannot be used as multiple addresses can endup in the same bucket. + var oldAddressesAdded int + var newAddressesAdded int + + // number of new addresses that, if possible, should be in the beginning of the selection + numRequiredNewAdd := percentageOfNum(biasTowardsNewAddrs, numAddresses) + selectionIndex := 0 ADDRS_LOOP: for selectionIndex < numAddresses { - pickFromOldBucket := int((float64(selectionIndex)/float64(numAddresses))*100) >= biasTowardsNewAddrs - pickFromOldBucket = (pickFromOldBucket && a.nOld > 0) || a.nNew == 0 + // biasedTowardsOldAddrs indicates if the selection can switch to old addresses + biasedTowardsOldAddrs := selectionIndex >= numRequiredNewAdd + // An old addresses is selected if: + // - the bias is for old and old addressees are still available or, + // - there are no new addresses or all new addresses have been selected. + // numAddresses <= a.nOld + a.nNew therefore it is guaranteed that there are enough + // addresses to fill the selection + pickFromOldBucket := + (biasedTowardsOldAddrs && oldAddressesAdded < a.nOld) || + a.nNew == 0 || newAddressesAdded >= a.nNew + bucket := make(map[string]*knownAddress) // loop until we pick a random non-empty bucket @@ -441,6 +471,7 @@ ADDRS_LOOP: oldBucketToAddrsMap[oldIndex] = make(map[string]struct{}) } oldBucketToAddrsMap[oldIndex][selectedAddr.String()] = struct{}{} + oldAddressesAdded++ } else { if addrsMap, ok := newBucketToAddrsMap[newIndex]; ok { if _, ok = addrsMap[selectedAddr.String()]; ok { @@ -450,6 +481,7 @@ ADDRS_LOOP: newBucketToAddrsMap[newIndex] = make(map[string]struct{}) } newBucketToAddrsMap[newIndex][selectedAddr.String()] = struct{}{} + newAddressesAdded++ } selection[selectionIndex] = selectedAddr @@ -477,6 +509,7 @@ func (a *addrBook) ListOfKnownAddresses() []*knownAddress { func (a *addrBook) Size() int { a.mtx.Lock() defer a.mtx.Unlock() + return a.size() } @@ -648,6 +681,14 @@ func (a *addrBook) addAddress(addr, src *p2p.NetAddress) error { return ErrAddrBookNonRoutable{addr} } + if !addr.Valid() { + return ErrAddrBookInvalidAddr{addr} + } + + if !addr.HasID() { + return ErrAddrBookInvalidAddrNoID{addr} + } + // TODO: we should track ourAddrs by ID and by IP:PORT and refuse both. if _, ok := a.ourAddrs[addr.String()]; ok { return ErrAddrBookSelf{addr} diff --git a/p2p/pex/addrbook_test.go b/p2p/pex/addrbook_test.go index ade02d49..9effa5d0 100644 --- a/p2p/pex/addrbook_test.go +++ b/p2p/pex/addrbook_test.go @@ -4,38 +4,18 @@ import ( "encoding/hex" "fmt" "io/ioutil" + "math" "os" "testing" - "github.com/stretchr/testify/require" - "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" cmn "github.com/tendermint/tendermint/libs/common" "github.com/tendermint/tendermint/libs/log" "github.com/tendermint/tendermint/p2p" ) -func createTempFileName(prefix string) string { - f, err := ioutil.TempFile("", prefix) - if err != nil { - panic(err) - } - fname := f.Name() - err = f.Close() - if err != nil { - panic(err) - } - return fname -} - -func deleteTempFile(fname string) { - err := os.Remove(fname) - if err != nil { - panic(err) - } -} - func TestAddrBookPickAddress(t *testing.T) { fname := createTempFileName("addrbook_test") defer deleteTempFile(fname) @@ -239,6 +219,34 @@ func TestAddrBookRemoveAddress(t *testing.T) { assert.Equal(t, 0, book.Size()) } +func TestAddrBookGetSelectionWithOneMarkedGood(t *testing.T) { + // create a book with 10 addresses, 1 good/old and 9 new + book, fname := createAddrBookWithMOldAndNNewAddrs(t, 1, 9) + defer deleteTempFile(fname) + + addrs := book.GetSelectionWithBias(biasToSelectNewPeers) + assert.NotNil(t, addrs) + assertMOldAndNNewAddrsInSelection(t, 1, 9, addrs, book) +} + +func TestAddrBookGetSelectionWithOneNotMarkedGood(t *testing.T) { + // create a book with 10 addresses, 9 good/old and 1 new + book, fname := createAddrBookWithMOldAndNNewAddrs(t, 9, 1) + defer deleteTempFile(fname) + + addrs := book.GetSelectionWithBias(biasToSelectNewPeers) + assert.NotNil(t, addrs) + assertMOldAndNNewAddrsInSelection(t, 9, 1, addrs, book) +} + +func TestAddrBookGetSelectionReturnsNilWhenAddrBookIsEmpty(t *testing.T) { + book, fname := createAddrBookWithMOldAndNNewAddrs(t, 0, 0) + defer deleteTempFile(fname) + + addrs := book.GetSelectionWithBias(biasToSelectNewPeers) + assert.Nil(t, addrs) +} + func TestAddrBookGetSelection(t *testing.T) { fname := createTempFileName("addrbook_test") defer deleteTempFile(fname) @@ -335,9 +343,16 @@ func TestAddrBookGetSelectionWithBias(t *testing.T) { good++ } } + got, expected := int((float64(good)/float64(len(selection)))*100), (100 - biasTowardsNewAddrs) - if got >= expected { - t.Fatalf("expected more good peers (%% got: %d, %% expected: %d, number of good addrs: %d, total: %d)", got, expected, good, len(selection)) + + // compute some slack to protect against small differences due to rounding: + slack := int(math.Round(float64(100) / float64(len(selection)))) + if got > expected+slack { + t.Fatalf("got more good peers (%% got: %d, %% expected: %d, number of good addrs: %d, total: %d)", got, expected, good, len(selection)) + } + if got < expected-slack { + t.Fatalf("got fewer good peers (%% got: %d, %% expected: %d, number of good addrs: %d, total: %d)", got, expected, good, len(selection)) } } @@ -417,3 +432,199 @@ func TestPrivatePeers(t *testing.T) { assert.True(t, ok) } } + +func testAddrBookAddressSelection(t *testing.T, bookSize int) { + // generate all combinations of old (m) and new addresses + for nOld := 0; nOld <= bookSize; nOld++ { + nNew := bookSize - nOld + dbgStr := fmt.Sprintf("book of size %d (new %d, old %d)", bookSize, nNew, nOld) + + // create book and get selection + book, fname := createAddrBookWithMOldAndNNewAddrs(t, nOld, nNew) + defer deleteTempFile(fname) + addrs := book.GetSelectionWithBias(biasToSelectNewPeers) + assert.NotNil(t, addrs, "%s - expected a non-nil selection", dbgStr) + nAddrs := len(addrs) + assert.NotZero(t, nAddrs, "%s - expected at least one address in selection", dbgStr) + + // check there's no nil addresses + for _, addr := range addrs { + if addr == nil { + t.Fatalf("%s - got nil address in selection %v", dbgStr, addrs) + } + } + + // XXX: shadowing + nOld, nNew := countOldAndNewAddrsInSelection(addrs, book) + + // Given: + // n - num new addrs, m - num old addrs + // k - num new addrs expected in the beginning (based on bias %) + // i=min(n, k), aka expFirstNew + // j=min(m, r-i), aka expOld + // + // We expect this layout: + // indices: 0...i-1 i...i+j-1 i+j...r + // addresses: N0..Ni-1 O0..Oj-1 Ni... + // + // There is at least one partition and at most three. + var ( + k = percentageOfNum(biasToSelectNewPeers, nAddrs) + expFirstNew = cmn.MinInt(nNew, k) + expOld = cmn.MinInt(nOld, nAddrs-expFirstNew) + expNew = nAddrs - expOld + expLastNew = expNew - expFirstNew + ) + + // Verify that the number of old and new addresses are as expected + if nNew < expNew || nNew > expNew { + t.Fatalf("%s - expected new addrs %d, got %d", dbgStr, expNew, nNew) + } + if nOld < expOld || nOld > expOld { + t.Fatalf("%s - expected old addrs %d, got %d", dbgStr, expOld, nOld) + } + + // Verify that the order of addresses is as expected + // Get the sequence types and lengths of the selection + seqLens, seqTypes, err := analyseSelectionLayout(book, addrs) + assert.NoError(t, err, "%s", dbgStr) + + // Build a list with the expected lengths of partitions and another with the expected types, e.g.: + // expSeqLens = [10, 22], expSeqTypes = [1, 2] + // means we expect 10 new (type 1) addresses followed by 22 old (type 2) addresses. + var expSeqLens []int + var expSeqTypes []int + + switch { + case expOld == 0: // all new addresses + expSeqLens = []int{nAddrs} + expSeqTypes = []int{1} + case expFirstNew == 0: // all old addresses + expSeqLens = []int{nAddrs} + expSeqTypes = []int{2} + case nAddrs-expFirstNew-expOld == 0: // new addresses, old addresses + expSeqLens = []int{expFirstNew, expOld} + expSeqTypes = []int{1, 2} + default: // new addresses, old addresses, new addresses + expSeqLens = []int{expFirstNew, expOld, expLastNew} + expSeqTypes = []int{1, 2, 1} + } + + assert.Equal(t, expSeqLens, seqLens, + "%s - expected sequence lengths of old/new %v, got %v", + dbgStr, expSeqLens, seqLens) + assert.Equal(t, expSeqTypes, seqTypes, + "%s - expected sequence types (1-new, 2-old) was %v, got %v", + dbgStr, expSeqTypes, seqTypes) + } +} + +func TestMultipleAddrBookAddressSelection(t *testing.T) { + // test books with smaller size, < N + const N = 32 + for bookSize := 1; bookSize < N; bookSize++ { + testAddrBookAddressSelection(t, bookSize) + } + + // Test for two books with sizes from following ranges + ranges := [...][]int{{33, 100}, {100, 175}} + bookSizes := make([]int, 0, len(ranges)) + for _, r := range ranges { + bookSizes = append(bookSizes, cmn.RandIntn(r[1]-r[0])+r[0]) + } + t.Logf("Testing address selection for the following book sizes %v\n", bookSizes) + for _, bookSize := range bookSizes { + testAddrBookAddressSelection(t, bookSize) + } +} + +func assertMOldAndNNewAddrsInSelection(t *testing.T, m, n int, addrs []*p2p.NetAddress, book *addrBook) { + nOld, nNew := countOldAndNewAddrsInSelection(addrs, book) + assert.Equal(t, m, nOld, "old addresses") + assert.Equal(t, n, nNew, "new addresses") +} + +func createTempFileName(prefix string) string { + f, err := ioutil.TempFile("", prefix) + if err != nil { + panic(err) + } + fname := f.Name() + err = f.Close() + if err != nil { + panic(err) + } + return fname +} + +func deleteTempFile(fname string) { + err := os.Remove(fname) + if err != nil { + panic(err) + } +} + +func createAddrBookWithMOldAndNNewAddrs(t *testing.T, nOld, nNew int) (book *addrBook, fname string) { + fname = createTempFileName("addrbook_test") + + book = NewAddrBook(fname, true) + book.SetLogger(log.TestingLogger()) + assert.Zero(t, book.Size()) + + randAddrs := randNetAddressPairs(t, nOld) + for _, addr := range randAddrs { + book.AddAddress(addr.addr, addr.src) + book.MarkGood(addr.addr) + } + + randAddrs = randNetAddressPairs(t, nNew) + for _, addr := range randAddrs { + book.AddAddress(addr.addr, addr.src) + } + + return +} + +func countOldAndNewAddrsInSelection(addrs []*p2p.NetAddress, book *addrBook) (nOld, nNew int) { + for _, addr := range addrs { + if book.IsGood(addr) { + nOld++ + } else { + nNew++ + } + } + return +} + +// Analyse the layout of the selection specified by 'addrs' +// Returns: +// - seqLens - the lengths of the sequences of addresses of same type +// - seqTypes - the types of sequences in selection +func analyseSelectionLayout(book *addrBook, addrs []*p2p.NetAddress) (seqLens, seqTypes []int, err error) { + // address types are: 0 - nil, 1 - new, 2 - old + var ( + prevType = 0 + currentSeqLen = 0 + ) + + for _, addr := range addrs { + addrType := 0 + if book.IsGood(addr) { + addrType = 2 + } else { + addrType = 1 + } + if addrType != prevType && prevType != 0 { + seqLens = append(seqLens, currentSeqLen) + seqTypes = append(seqTypes, prevType) + currentSeqLen = 0 + } + currentSeqLen++ + prevType = addrType + } + + seqLens = append(seqLens, currentSeqLen) + seqTypes = append(seqTypes, prevType) + + return +} diff --git a/p2p/pex/errors.go b/p2p/pex/errors.go index 7f660bdc..1f44ceee 100644 --- a/p2p/pex/errors.go +++ b/p2p/pex/errors.go @@ -46,3 +46,19 @@ type ErrAddrBookNilAddr struct { func (err ErrAddrBookNilAddr) Error() string { return fmt.Sprintf("Cannot add a nil address. Got (addr, src) = (%v, %v)", err.Addr, err.Src) } + +type ErrAddrBookInvalidAddr struct { + Addr *p2p.NetAddress +} + +func (err ErrAddrBookInvalidAddr) Error() string { + return fmt.Sprintf("Cannot add invalid address %v", err.Addr) +} + +type ErrAddrBookInvalidAddrNoID struct { + Addr *p2p.NetAddress +} + +func (err ErrAddrBookInvalidAddrNoID) Error() string { + return fmt.Sprintf("Cannot add address with no ID %v", err.Addr) +} diff --git a/p2p/pex/pex_reactor.go b/p2p/pex/pex_reactor.go index c919794a..01d1d8db 100644 --- a/p2p/pex/pex_reactor.go +++ b/p2p/pex/pex_reactor.go @@ -208,21 +208,38 @@ func (r *PEXReactor) Receive(chID byte, src Peer, msgBytes []byte) { switch msg := msg.(type) { case *pexRequestMessage: - // Check we're not receiving too many requests - if err := r.receiveRequest(src); err != nil { - r.Switch.StopPeerForError(src, err) - return - } - // Seeds disconnect after sending a batch of addrs - // NOTE: this is a prime candidate for amplification attacks + // NOTE: this is a prime candidate for amplification attacks, // so it's important we // 1) restrict how frequently peers can request // 2) limit the output size - if r.config.SeedMode { + + // If we're a seed and this is an inbound peer, + // respond once and disconnect. + if r.config.SeedMode && !src.IsOutbound() { + id := string(src.ID()) + v := r.lastReceivedRequests.Get(id) + if v != nil { + // FlushStop/StopPeer are already + // running in a go-routine. + return + } + r.lastReceivedRequests.Set(id, time.Now()) + + // Send addrs and disconnect r.SendAddrs(src, r.book.GetSelectionWithBias(biasToSelectNewPeers)) - r.Switch.StopPeerGracefully(src) + go func() { + // In a go-routine so it doesn't block .Receive. + src.FlushStop() + r.Switch.StopPeerGracefully(src) + }() + } else { + // Check we're not receiving requests too frequently. + if err := r.receiveRequest(src); err != nil { + r.Switch.StopPeerForError(src, err) + return + } r.SendAddrs(src, r.book.GetSelection()) } @@ -288,21 +305,37 @@ func (r *PEXReactor) RequestAddrs(p Peer) { func (r *PEXReactor) ReceiveAddrs(addrs []*p2p.NetAddress, src Peer) error { id := string(src.ID()) if !r.requestsSent.Has(id) { - return cmn.NewError("Received unsolicited pexAddrsMessage") + return errors.New("Unsolicited pexAddrsMessage") } r.requestsSent.Delete(id) srcAddr := src.NodeInfo().NetAddress() for _, netAddr := range addrs { - // NOTE: GetSelection methods should never return nil addrs + // Validate netAddr. Disconnect from a peer if it sends us invalid data. if netAddr == nil { - return cmn.NewError("received nil addr") + return errors.New("nil address in pexAddrsMessage") + } + // TODO: extract validating logic from NewNetAddressStringWithOptionalID + // and put it in netAddr#Valid (#2722) + na, err := p2p.NewNetAddressString(netAddr.String()) + if err != nil { + return fmt.Errorf("%s address in pexAddrsMessage is invalid: %v", + netAddr.String(), + err, + ) } - err := r.book.AddAddress(netAddr, srcAddr) - r.logErrAddrBook(err) + // NOTE: we check netAddr validity and routability in book#AddAddress. + err = r.book.AddAddress(na, srcAddr) + if err != nil { + r.logErrAddrBook(err) + // XXX: should we be strict about incoming data and disconnect from a + // peer here too? + continue + } - // If this address came from a seed node, try to connect to it without waiting. + // If this address came from a seed node, try to connect to it without + // waiting. for _, seedAddr := range r.seedAddrs { if seedAddr.Equals(srcAddr) { r.ensurePeers() @@ -438,7 +471,11 @@ func (r *PEXReactor) dialPeer(addr *p2p.NetAddress) { attempts, lastDialed := r.dialAttemptsInfo(addr) if attempts > maxAttemptsToDial { - r.Logger.Error("Reached max attempts to dial", "addr", addr, "attempts", attempts) + // Do not log the message if the addr gets readded. + if attempts+1 == maxAttemptsToDial { + r.Logger.Info("Reached max attempts to dial", "addr", addr, "attempts", attempts) + r.attemptsToDial.Store(addr.DialString(), _attemptsToDial{attempts + 1, time.Now()}) + } r.book.MarkBad(addr) return } @@ -579,10 +616,9 @@ func (of oldestFirst) Less(i, j int) bool { return of[i].LastAttempt.Before(of[j // getPeersToCrawl returns addresses of potential peers that we wish to validate. // NOTE: The status information is ordered as described above. func (r *PEXReactor) getPeersToCrawl() []crawlPeerInfo { - var of oldestFirst - // TODO: be more selective addrs := r.book.ListOfKnownAddresses() + of := make(oldestFirst, 0, len(addrs)) for _, addr := range addrs { if len(addr.ID()) == 0 { continue // dont use peers without id diff --git a/p2p/pex/pex_reactor_test.go b/p2p/pex/pex_reactor_test.go index c22eabdc..4f4ccb03 100644 --- a/p2p/pex/pex_reactor_test.go +++ b/p2p/pex/pex_reactor_test.go @@ -12,7 +12,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - crypto "github.com/tendermint/tendermint/crypto" + "github.com/tendermint/tendermint/crypto" "github.com/tendermint/tendermint/crypto/ed25519" cmn "github.com/tendermint/tendermint/libs/common" "github.com/tendermint/tendermint/libs/log" @@ -316,11 +316,86 @@ func TestPEXReactorCrawlStatus(t *testing.T) { // TODO: test } +// connect a peer to a seed, wait a bit, then stop it. +// this should give it time to request addrs and for the seed +// to call FlushStop, and allows us to test calling Stop concurrently +// with FlushStop. Before a fix, this non-deterministically reproduced +// https://github.com/tendermint/tendermint/issues/3231. +func TestPEXReactorSeedModeFlushStop(t *testing.T) { + N := 2 + switches := make([]*p2p.Switch, N) + + // directory to store address books + dir, err := ioutil.TempDir("", "pex_reactor") + require.Nil(t, err) + defer os.RemoveAll(dir) // nolint: errcheck + + books := make([]*addrBook, N) + logger := log.TestingLogger() + + // create switches + for i := 0; i < N; i++ { + switches[i] = p2p.MakeSwitch(cfg, i, "testing", "123.123.123", func(i int, sw *p2p.Switch) *p2p.Switch { + books[i] = NewAddrBook(filepath.Join(dir, fmt.Sprintf("addrbook%d.json", i)), false) + books[i].SetLogger(logger.With("pex", i)) + sw.SetAddrBook(books[i]) + + sw.SetLogger(logger.With("pex", i)) + + config := &PEXReactorConfig{} + if i == 0 { + // first one is a seed node + config = &PEXReactorConfig{SeedMode: true} + } + r := NewPEXReactor(books[i], config) + r.SetLogger(logger.With("pex", i)) + r.SetEnsurePeersPeriod(250 * time.Millisecond) + sw.AddReactor("pex", r) + + return sw + }) + } + + for _, sw := range switches { + err := sw.Start() // start switch and reactors + require.Nil(t, err) + } + + reactor := switches[0].Reactors()["pex"].(*PEXReactor) + peerID := switches[1].NodeInfo().ID() + + err = switches[1].DialPeerWithAddress(switches[0].NodeInfo().NetAddress(), false) + assert.NoError(t, err) + + // sleep up to a second while waiting for the peer to send us a message. + // this isn't perfect since it's possible the peer sends us a msg and we FlushStop + // before this loop catches it. but non-deterministically it works pretty well. + for i := 0; i < 1000; i++ { + v := reactor.lastReceivedRequests.Get(string(peerID)) + if v != nil { + break + } + time.Sleep(time.Millisecond) + } + + // by now the FlushStop should have happened. Try stopping the peer. + // it should be safe to do this. + peers := switches[0].Peers().List() + for _, peer := range peers { + peer.Stop() + } + + // stop the switches + for _, s := range switches { + s.Stop() + } +} + func TestPEXReactorDoesNotAddPrivatePeersToAddrBook(t *testing.T) { peer := p2p.CreateRandomPeer(false) pexR, book := createReactor(&PEXReactorConfig{}) - book.AddPrivateIDs([]string{string(peer.NodeInfo().ID)}) + book.AddPrivateIDs([]string{string(peer.NodeInfo().ID())}) defer teardownReactor(book) // we have to send a request to receive responses @@ -387,12 +462,13 @@ func newMockPeer() mockPeer { return mp } +func (mp mockPeer) FlushStop() { mp.Stop() } func (mp mockPeer) ID() p2p.ID { return mp.addr.ID } func (mp mockPeer) IsOutbound() bool { return mp.outbound } func (mp mockPeer) IsPersistent() bool { return mp.persistent } func (mp mockPeer) NodeInfo() p2p.NodeInfo { - return p2p.NodeInfo{ - ID: mp.addr.ID, + return p2p.DefaultNodeInfo{ + ID_: mp.addr.ID, ListenAddr: mp.addr.DialString(), } } @@ -403,6 +479,8 @@ func (mockPeer) TrySend(byte, []byte) bool { return false } func (mockPeer) Set(string, interface{}) {} func (mockPeer) Get(string) interface{} { return nil } func (mockPeer) OriginalAddr() *p2p.NetAddress { return nil } +func (mockPeer) RemoteAddr() net.Addr { return &net.TCPAddr{IP: net.ParseIP("127.0.0.1"), Port: 8800} } +func (mockPeer) CloseConn() error { return nil } func assertPeersWithTimeout( t *testing.T, diff --git a/p2p/pex/wire.go b/p2p/pex/wire.go index 57fc9385..c88b1941 100644 --- a/p2p/pex/wire.go +++ b/p2p/pex/wire.go @@ -1,7 +1,7 @@ package pex import ( - "github.com/tendermint/go-amino" + amino "github.com/tendermint/go-amino" ) var cdc *amino.Codec = amino.NewCodec() diff --git a/p2p/switch.go b/p2p/switch.go index 8325d7e8..7d2e6c3f 100644 --- a/p2p/switch.go +++ b/p2p/switch.go @@ -27,6 +27,17 @@ const ( reconnectBackOffBaseSeconds = 3 ) +// MConnConfig returns an MConnConfig with fields updated +// from the P2PConfig. +func MConnConfig(cfg *config.P2PConfig) conn.MConnConfig { + mConfig := conn.DefaultMConnConfig() + mConfig.FlushThrottle = cfg.FlushThrottleTimeout + mConfig.SendRate = cfg.SendRate + mConfig.RecvRate = cfg.RecvRate + mConfig.MaxPacketMsgPayloadSize = cfg.MaxPacketMsgPayloadSize + return mConfig +} + //----------------------------------------------------------------------------- // An AddrBook represents an address book from the pex package, which is used @@ -70,8 +81,6 @@ type Switch struct { filterTimeout time.Duration peerFilters []PeerFilterFunc - mConfig conn.MConnConfig - rng *cmn.Rand // seed for randomizing dial times and orders metrics *Metrics @@ -102,14 +111,6 @@ func NewSwitch( // Ensure we have a completely undeterministic PRNG. sw.rng = cmn.NewRand() - mConfig := conn.DefaultMConnConfig() - mConfig.FlushThrottle = cfg.FlushThrottleTimeout - mConfig.SendRate = cfg.SendRate - mConfig.RecvRate = cfg.RecvRate - mConfig.MaxPacketMsgPayloadSize = cfg.MaxPacketMsgPayloadSize - - sw.mConfig = mConfig - sw.BaseService = *cmn.NewBaseService(nil, "P2P Switch", sw) for _, option := range options { @@ -209,8 +210,11 @@ func (sw *Switch) OnStart() error { func (sw *Switch) OnStop() { // Stop peers for _, p := range sw.peers.List() { + sw.transport.Cleanup(p) p.Stop() - sw.peers.Remove(p) + if sw.peers.Remove(p) { + sw.metrics.Peers.Add(float64(-1)) + } } // Stop reactors @@ -298,8 +302,10 @@ func (sw *Switch) StopPeerGracefully(peer Peer) { } func (sw *Switch) stopAndRemovePeer(peer Peer, reason interface{}) { - sw.peers.Remove(peer) - sw.metrics.Peers.Add(float64(-1)) + if sw.peers.Remove(peer) { + sw.metrics.Peers.Add(float64(-1)) + } + sw.transport.Cleanup(peer) peer.Stop() for _, reactor := range sw.reactors { reactor.RemovePeer(peer, reason) @@ -328,6 +334,11 @@ func (sw *Switch) reconnectToPeer(addr *NetAddress) { return } + if sw.IsDialingOrExistingAddress(addr) { + sw.Logger.Debug("Peer connection has been established or dialed while we waiting next try", "addr", addr) + return + } + err := sw.DialPeerWithAddress(addr, true) if err == nil { return // success @@ -415,12 +426,15 @@ func (sw *Switch) DialPeersAsync(addrBook AddrBook, peers []string, persistent b if addr.Same(ourAddr) { sw.Logger.Debug("Ignore attempt to connect to ourselves", "addr", addr, "ourAddr", ourAddr) return - } else if sw.IsDialingOrExistingAddress(addr) { + } + + sw.randomSleep(0) + + if sw.IsDialingOrExistingAddress(addr) { sw.Logger.Debug("Ignore attempt to connect to an existing peer", "addr", addr) return } - sw.randomSleep(0) err := sw.DialPeerWithAddress(addr, persistent) if err != nil { switch err.(type) { @@ -466,14 +480,12 @@ func (sw *Switch) acceptRoutine() { metrics: sw.metrics, }) if err != nil { - switch err.(type) { + switch err := err.(type) { case ErrRejected: - rErr := err.(ErrRejected) - - if rErr.IsSelf() { + if err.IsSelf() { // Remove the given address from the address book and add to our addresses // to avoid dialing in the future. - addr := rErr.Addr() + addr := err.Addr() sw.addrBook.RemoveAddress(&addr) sw.addrBook.AddOurAddress(&addr) } @@ -496,6 +508,12 @@ func (sw *Switch) acceptRoutine() { "err", err, "numPeers", sw.peers.Size(), ) + // We could instead have a retry loop around the acceptRoutine, + // but that would need to stop and let the node shutdown eventually. + // So might as well panic and let process managers restart the node. + // There's no point in letting the node run without the acceptRoutine, + // since it won't be able to accept new connections. + panic(fmt.Errorf("accept routine exited: %v", err)) } break @@ -511,13 +529,16 @@ func (sw *Switch) acceptRoutine() { "max", sw.config.MaxNumInboundPeers, ) - _ = p.Stop() + sw.transport.Cleanup(p) continue } if err := sw.addPeer(p); err != nil { - _ = p.Stop() + sw.transport.Cleanup(p) + if p.IsRunning() { + _ = p.Stop() + } sw.Logger.Info( "Ignoring inbound connection: error while adding peer", "err", err, @@ -560,9 +581,13 @@ func (sw *Switch) addOutboundPeerWithConfig( // to avoid dialing in the future. sw.addrBook.RemoveAddress(addr) sw.addrBook.AddOurAddress(addr) + + return err } } + // retry persistent peers after + // any dial error besides IsSelf() if persistent { go sw.reconnectToPeer(addr) } @@ -571,7 +596,10 @@ func (sw *Switch) addOutboundPeerWithConfig( } if err := sw.addPeer(p); err != nil { - _ = p.Stop() + sw.transport.Cleanup(p) + if p.IsRunning() { + _ = p.Stop() + } return err } @@ -606,19 +634,24 @@ func (sw *Switch) filterPeer(p Peer) error { return nil } -// addPeer starts up the Peer and adds it to the Switch. +// addPeer starts up the Peer and adds it to the Switch. Error is returned if +// the peer is filtered out or failed to start or can't be added. func (sw *Switch) addPeer(p Peer) error { if err := sw.filterPeer(p); err != nil { return err } - p.SetLogger(sw.Logger.With("peer", p.NodeInfo().NetAddress().String)) + p.SetLogger(sw.Logger.With("peer", p.NodeInfo().NetAddress())) - // All good. Start peer + // Handle the shut down case where the switch has stopped but we're + // concurrently trying to add a peer. if sw.IsRunning() { + // All good. Start peer if err := sw.startInitPeer(p); err != nil { return err } + } else { + sw.Logger.Error("Won't start a peer - switch is not running", "peer", p) } // Add the peer to .peers. diff --git a/p2p/switch_test.go b/p2p/switch_test.go index 4fea3cfe..35866161 100644 --- a/p2p/switch_test.go +++ b/p2p/switch_test.go @@ -3,10 +3,18 @@ package p2p import ( "bytes" "fmt" + "io" + "io/ioutil" + "net" + "net/http" + "net/http/httptest" + "regexp" + "strconv" "sync" "testing" "time" + stdprometheus "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -143,6 +151,7 @@ func assertMsgReceivedWithTimeout(t *testing.T, msgBytes []byte, channel byte, r } return } + case <-time.After(timeout): t.Fatalf("Expected to have received 1 message in channel #%v, got zero", channel) } @@ -334,6 +343,54 @@ func TestSwitchStopsNonPersistentPeerOnError(t *testing.T) { assert.False(p.IsRunning()) } +func TestSwitchStopPeerForError(t *testing.T) { + s := httptest.NewServer(stdprometheus.UninstrumentedHandler()) + defer s.Close() + + scrapeMetrics := func() string { + resp, _ := http.Get(s.URL) + buf, _ := ioutil.ReadAll(resp.Body) + return string(buf) + } + + namespace, subsystem, name := config.TestInstrumentationConfig().Namespace, MetricsSubsystem, "peers" + re := regexp.MustCompile(namespace + `_` + subsystem + `_` + name + ` ([0-9\.]+)`) + peersMetricValue := func() float64 { + matches := re.FindStringSubmatch(scrapeMetrics()) + f, _ := strconv.ParseFloat(matches[1], 64) + return f + } + + p2pMetrics := PrometheusMetrics(namespace) + + // make two connected switches + sw1, sw2 := MakeSwitchPair(t, func(i int, sw *Switch) *Switch { + // set metrics on sw1 + if i == 0 { + opt := WithMetrics(p2pMetrics) + opt(sw) + } + return initSwitchFunc(i, sw) + }) + + assert.Equal(t, len(sw1.Peers().List()), 1) + assert.EqualValues(t, 1, peersMetricValue()) + + // send messages to the peer from sw1 + p := sw1.Peers().List()[0] + p.Send(0x1, []byte("here's a message to send")) + + // stop sw2. this should cause the p to fail, + // which results in calling StopPeerForError internally + sw2.Stop() + + // now call StopPeerForError explicitly, eg. from a reactor + sw1.StopPeerForError(p, fmt.Errorf("some err")) + + assert.Equal(t, len(sw1.Peers().List()), 0) + assert.EqualValues(t, 0, peersMetricValue()) +} + func TestSwitchReconnectsToPersistentPeer(t *testing.T) { assert, require := assert.New(t), require.New(t) @@ -421,6 +478,58 @@ func TestSwitchFullConnectivity(t *testing.T) { } } +func TestSwitchAcceptRoutine(t *testing.T) { + cfg.MaxNumInboundPeers = 5 + + // make switch + sw := MakeSwitch(cfg, 1, "testing", "123.123.123", initSwitchFunc) + err := sw.Start() + require.NoError(t, err) + defer sw.Stop() + + remotePeers := make([]*remotePeer, 0) + assert.Equal(t, 0, sw.Peers().Size()) + + // 1. check we connect up to MaxNumInboundPeers + for i := 0; i < cfg.MaxNumInboundPeers; i++ { + rp := &remotePeer{PrivKey: ed25519.GenPrivKey(), Config: cfg} + remotePeers = append(remotePeers, rp) + rp.Start() + c, err := rp.Dial(sw.NodeInfo().NetAddress()) + require.NoError(t, err) + // spawn a reading routine to prevent connection from closing + go func(c net.Conn) { + for { + one := make([]byte, 1) + _, err := c.Read(one) + if err != nil { + return + } + } + }(c) + } + time.Sleep(10 * time.Millisecond) + assert.Equal(t, cfg.MaxNumInboundPeers, sw.Peers().Size()) + + // 2. check we close new connections if we already have MaxNumInboundPeers peers + rp := &remotePeer{PrivKey: ed25519.GenPrivKey(), Config: cfg} + rp.Start() + conn, err := rp.Dial(sw.NodeInfo().NetAddress()) + require.NoError(t, err) + // check conn is closed + one := make([]byte, 1) + conn.SetReadDeadline(time.Now().Add(10 * time.Millisecond)) + _, err = conn.Read(one) + assert.Equal(t, io.EOF, err) + assert.Equal(t, cfg.MaxNumInboundPeers, sw.Peers().Size()) + rp.Stop() + + // stop remote peers + for _, rp := range remotePeers { + rp.Stop() + } +} + func BenchmarkSwitchBroadcast(b *testing.B) { s1, s2 := MakeSwitchPair(b, func(i int, sw *Switch) *Switch { // Make bar reactors of bar channels each diff --git a/p2p/test_util.go b/p2p/test_util.go index e35e0989..2d320df8 100644 --- a/p2p/test_util.go +++ b/p2p/test_util.go @@ -5,7 +5,7 @@ import ( "net" "time" - crypto "github.com/tendermint/tendermint/crypto" + "github.com/tendermint/tendermint/crypto" "github.com/tendermint/tendermint/crypto/ed25519" cmn "github.com/tendermint/tendermint/libs/common" "github.com/tendermint/tendermint/libs/log" @@ -14,6 +14,19 @@ import ( "github.com/tendermint/tendermint/p2p/conn" ) +const testCh = 0x01 + +//------------------------------------------------ + +type mockNodeInfo struct { + addr *NetAddress +} + +func (ni mockNodeInfo) ID() ID { return ni.addr.ID } +func (ni mockNodeInfo) NetAddress() *NetAddress { return ni.addr } +func (ni mockNodeInfo) Validate() error { return nil } +func (ni mockNodeInfo) CompatibleWith(other NodeInfo) error { return nil } + func AddPeerToSwitch(sw *Switch, peer Peer) { sw.peers.Add(peer) } @@ -24,12 +37,9 @@ func CreateRandomPeer(outbound bool) *peer { peerConn: peerConn{ outbound: outbound, }, - nodeInfo: NodeInfo{ - ID: netAddr.ID, - ListenAddr: netAddr.DialString(), - }, - mconn: &conn.MConnection{}, - metrics: NopMetrics(), + nodeInfo: mockNodeInfo{netAddr}, + mconn: &conn.MConnection{}, + metrics: NopMetrics(), } p.SetLogger(log.TestingLogger().With("peer", addr)) return p @@ -115,7 +125,7 @@ func (sw *Switch) addPeerWithConnection(conn net.Conn) error { return err } - ni, err := handshake(conn, 50*time.Millisecond, sw.nodeInfo) + ni, err := handshake(conn, time.Second, sw.nodeInfo) if err != nil { if err := conn.Close(); err != nil { sw.Logger.Error("Error closing connection", "err", err) @@ -125,7 +135,7 @@ func (sw *Switch) addPeerWithConnection(conn net.Conn) error { p := newPeer( pc, - sw.mConfig, + MConnConfig(sw.config), ni, sw.reactorsByCh, sw.chDescs, @@ -159,53 +169,34 @@ func MakeSwitch( initSwitch func(int, *Switch) *Switch, opts ...SwitchOption, ) *Switch { - var ( - nodeKey = NodeKey{ - PrivKey: ed25519.GenPrivKey(), - } - ni = NodeInfo{ - ID: nodeKey.ID(), - Moniker: fmt.Sprintf("switch%d", i), - Network: network, - Version: version, - ListenAddr: fmt.Sprintf("127.0.0.1:%d", cmn.RandIntn(64512)+1023), - Other: NodeInfoOther{ - AminoVersion: "1.0", - P2PVersion: "1.0", - ConsensusVersion: "1.0", - RPCVersion: "1.0", - TxIndex: "off", - RPCAddress: fmt.Sprintf("127.0.0.1:%d", cmn.RandIntn(64512)+1023), - }, - } - ) - addr, err := NewNetAddressStringWithOptionalID( - IDAddressString(nodeKey.ID(), ni.ListenAddr), - ) - if err != nil { - panic(err) + nodeKey := NodeKey{ + PrivKey: ed25519.GenPrivKey(), } + nodeInfo := testNodeInfo(nodeKey.ID(), fmt.Sprintf("node%d", i)) - t := NewMultiplexTransport(ni, nodeKey) + t := NewMultiplexTransport(nodeInfo, nodeKey, MConnConfig(cfg)) + addr := nodeInfo.NetAddress() if err := t.Listen(*addr); err != nil { panic(err) } // TODO: let the config be passed in? sw := initSwitch(i, NewSwitch(cfg, t, opts...)) - sw.SetLogger(log.TestingLogger()) + sw.SetLogger(log.TestingLogger().With("switch", i)) sw.SetNodeKey(&nodeKey) + ni := nodeInfo.(DefaultNodeInfo) for ch := range sw.reactorsByCh { ni.Channels = append(ni.Channels, ch) } + nodeInfo = ni // TODO: We need to setup reactors ahead of time so the NodeInfo is properly // populated and we don't have to do those awkward overrides and setters. - t.nodeInfo = ni - sw.SetNodeInfo(ni) + t.nodeInfo = nodeInfo + sw.SetNodeInfo(nodeInfo) return sw } @@ -241,10 +232,40 @@ func testPeerConn( // Only the information we already have return peerConn{ - config: cfg, outbound: outbound, persistent: persistent, conn: conn, originalAddr: originalAddr, }, nil } + +//---------------------------------------------------------------- +// rand node info + +func testNodeInfo(id ID, name string) NodeInfo { + return testNodeInfoWithNetwork(id, name, "testing") +} + +func testNodeInfoWithNetwork(id ID, name, network string) NodeInfo { + return DefaultNodeInfo{ + ProtocolVersion: defaultProtocolVersion, + ID_: id, + ListenAddr: fmt.Sprintf("127.0.0.1:%d", getFreePort()), + Network: network, + Version: "1.2.3-rc0-deadbeef", + Channels: []byte{testCh}, + Moniker: name, + Other: DefaultNodeInfoOther{ + TxIndex: "on", + RPCAddress: fmt.Sprintf("127.0.0.1:%d", getFreePort()), + }, + } +} + +func getFreePort() int { + port, err := cmn.GetFreePort() + if err != nil { + panic(err) + } + return port +} diff --git a/p2p/transport.go b/p2p/transport.go index 6f097b4f..2d4420a1 100644 --- a/p2p/transport.go +++ b/p2p/transport.go @@ -6,7 +6,6 @@ import ( "net" "time" - "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/crypto" "github.com/tendermint/tendermint/p2p/conn" ) @@ -53,6 +52,9 @@ type Transport interface { // Dial connects to the Peer for the address. Dial(NetAddress, peerConfig) (Peer, error) + + // Cleanup any resources associated with Peer. + Cleanup(Peer) } // transportLifecycle bundles the methods for callers to control start and stop @@ -129,11 +131,10 @@ type MultiplexTransport struct { nodeKey NodeKey resolver IPResolver - // TODO(xla): Those configs are still needed as we parameterise peerConn and + // TODO(xla): This config is still needed as we parameterise peerConn and // peer currently. All relevant configuration should be refactored into options // with sane defaults. - mConfig conn.MConnConfig - p2pConfig config.P2PConfig + mConfig conn.MConnConfig } // Test multiplexTransport for interface completeness. @@ -144,6 +145,7 @@ var _ transportLifecycle = (*MultiplexTransport)(nil) func NewMultiplexTransport( nodeInfo NodeInfo, nodeKey NodeKey, + mConfig conn.MConnConfig, ) *MultiplexTransport { return &MultiplexTransport{ acceptc: make(chan accept), @@ -151,7 +153,7 @@ func NewMultiplexTransport( dialTimeout: defaultDialTimeout, filterTimeout: defaultFilterTimeout, handshakeTimeout: defaultHandshakeTimeout, - mConfig: conn.DefaultMConnConfig(), + mConfig: mConfig, nodeInfo: nodeInfo, nodeKey: nodeKey, conns: NewConnSet(), @@ -171,7 +173,7 @@ func (mt *MultiplexTransport) Accept(cfg peerConfig) (Peer, error) { cfg.outbound = false - return mt.wrapPeer(a.conn, a.nodeInfo, cfg), nil + return mt.wrapPeer(a.conn, a.nodeInfo, cfg, nil), nil case <-mt.closec: return nil, &ErrTransportClosed{} } @@ -199,7 +201,7 @@ func (mt *MultiplexTransport) Dial( cfg.outbound = true - p := mt.wrapPeer(secretConn, nodeInfo, cfg) + p := mt.wrapPeer(secretConn, nodeInfo, cfg, &addr) return p, nil } @@ -275,6 +277,13 @@ func (mt *MultiplexTransport) acceptPeers() { } } +// Cleanup removes the given address from the connections set and +// closes the connection. +func (mt *MultiplexTransport) Cleanup(peer Peer) { + mt.conns.RemoveAddr(peer.RemoteAddr()) + _ = peer.CloseConn() +} + func (mt *MultiplexTransport) cleanup(c net.Conn) error { mt.conns.Remove(c) @@ -335,7 +344,7 @@ func (mt *MultiplexTransport) upgrade( secretConn, err = upgradeSecretConn(c, mt.handshakeTimeout, mt.nodeKey.PrivKey) if err != nil { - return nil, NodeInfo{}, ErrRejected{ + return nil, nil, ErrRejected{ conn: c, err: fmt.Errorf("secrect conn failed: %v", err), isAuthFailure: true, @@ -344,7 +353,7 @@ func (mt *MultiplexTransport) upgrade( nodeInfo, err = handshake(secretConn, mt.handshakeTimeout, mt.nodeInfo) if err != nil { - return nil, NodeInfo{}, ErrRejected{ + return nil, nil, ErrRejected{ conn: c, err: fmt.Errorf("handshake failed: %v", err), isAuthFailure: true, @@ -352,7 +361,7 @@ func (mt *MultiplexTransport) upgrade( } if err := nodeInfo.Validate(); err != nil { - return nil, NodeInfo{}, ErrRejected{ + return nil, nil, ErrRejected{ conn: c, err: err, isNodeInfoInvalid: true, @@ -360,34 +369,34 @@ func (mt *MultiplexTransport) upgrade( } // Ensure connection key matches self reported key. - if connID := PubKeyToID(secretConn.RemotePubKey()); connID != nodeInfo.ID { - return nil, NodeInfo{}, ErrRejected{ + if connID := PubKeyToID(secretConn.RemotePubKey()); connID != nodeInfo.ID() { + return nil, nil, ErrRejected{ conn: c, id: connID, err: fmt.Errorf( "conn.ID (%v) NodeInfo.ID (%v) missmatch", connID, - nodeInfo.ID, + nodeInfo.ID(), ), isAuthFailure: true, } } // Reject self. - if mt.nodeInfo.ID == nodeInfo.ID { - return nil, NodeInfo{}, ErrRejected{ - addr: *NewNetAddress(nodeInfo.ID, c.RemoteAddr()), + if mt.nodeInfo.ID() == nodeInfo.ID() { + return nil, nil, ErrRejected{ + addr: *NewNetAddress(nodeInfo.ID(), c.RemoteAddr()), conn: c, - id: nodeInfo.ID, + id: nodeInfo.ID(), isSelf: true, } } if err := mt.nodeInfo.CompatibleWith(nodeInfo); err != nil { - return nil, NodeInfo{}, ErrRejected{ + return nil, nil, ErrRejected{ conn: c, err: err, - id: nodeInfo.ID, + id: nodeInfo.ID(), isIncompatible: true, } } @@ -399,14 +408,18 @@ func (mt *MultiplexTransport) wrapPeer( c net.Conn, ni NodeInfo, cfg peerConfig, + dialedAddr *NetAddress, ) Peer { + + peerConn := newPeerConn( + cfg.outbound, + cfg.persistent, + c, + dialedAddr, + ) + p := newPeer( - peerConn{ - conn: c, - config: &mt.p2pConfig, - outbound: cfg.outbound, - persistent: cfg.persistent, - }, + peerConn, mt.mConfig, ni, cfg.reactorsByCh, @@ -415,12 +428,6 @@ func (mt *MultiplexTransport) wrapPeer( PeerMetrics(cfg.metrics), ) - // Wait for Peer to Stop so we can cleanup. - go func(c net.Conn) { - <-p.Quit() - _ = mt.cleanup(c) - }(c) - return p } @@ -430,21 +437,22 @@ func handshake( nodeInfo NodeInfo, ) (NodeInfo, error) { if err := c.SetDeadline(time.Now().Add(timeout)); err != nil { - return NodeInfo{}, err + return nil, err } var ( errc = make(chan error, 2) - peerNodeInfo NodeInfo + peerNodeInfo DefaultNodeInfo + ourNodeInfo = nodeInfo.(DefaultNodeInfo) ) go func(errc chan<- error, c net.Conn) { - _, err := cdc.MarshalBinaryWriter(c, nodeInfo) + _, err := cdc.MarshalBinaryLengthPrefixedWriter(c, ourNodeInfo) errc <- err }(errc, c) go func(errc chan<- error, c net.Conn) { - _, err := cdc.UnmarshalBinaryReader( + _, err := cdc.UnmarshalBinaryLengthPrefixedReader( c, &peerNodeInfo, int64(MaxNodeInfoSize()), @@ -455,7 +463,7 @@ func handshake( for i := 0; i < cap(errc); i++ { err := <-errc if err != nil { - return NodeInfo{}, err + return nil, err } } diff --git a/p2p/transport_test.go b/p2p/transport_test.go index 9e3cc467..7d9c17fb 100644 --- a/p2p/transport_test.go +++ b/p2p/transport_test.go @@ -9,11 +9,30 @@ import ( "time" "github.com/tendermint/tendermint/crypto/ed25519" + "github.com/tendermint/tendermint/p2p/conn" ) +var defaultNodeName = "host_peer" + +func emptyNodeInfo() NodeInfo { + return DefaultNodeInfo{} +} + +// newMultiplexTransport returns a tcp connected multiplexed peer +// using the default MConnConfig. It's a convenience function used +// for testing. +func newMultiplexTransport( + nodeInfo NodeInfo, + nodeKey NodeKey, +) *MultiplexTransport { + return NewMultiplexTransport( + nodeInfo, nodeKey, conn.DefaultMConnConfig(), + ) +} + func TestTransportMultiplexConnFilter(t *testing.T) { - mt := NewMultiplexTransport( - NodeInfo{}, + mt := newMultiplexTransport( + emptyNodeInfo(), NodeKey{ PrivKey: ed25519.GenPrivKey(), }, @@ -69,8 +88,8 @@ func TestTransportMultiplexConnFilter(t *testing.T) { } func TestTransportMultiplexConnFilterTimeout(t *testing.T) { - mt := NewMultiplexTransport( - NodeInfo{}, + mt := newMultiplexTransport( + emptyNodeInfo(), NodeKey{ PrivKey: ed25519.GenPrivKey(), }, @@ -120,6 +139,7 @@ func TestTransportMultiplexConnFilterTimeout(t *testing.T) { t.Errorf("expected ErrFilterTimeout") } } + func TestTransportMultiplexAcceptMultiple(t *testing.T) { mt := testSetupMultiplexTransport(t) @@ -133,13 +153,8 @@ func TestTransportMultiplexAcceptMultiple(t *testing.T) { go func() { var ( pv = ed25519.GenPrivKey() - dialer = NewMultiplexTransport( - NodeInfo{ - ID: PubKeyToID(pv.PubKey()), - ListenAddr: "127.0.0.1:0", - Moniker: "dialer", - Version: "1.0.0", - }, + dialer = newMultiplexTransport( + testNodeInfo(PubKeyToID(pv.PubKey()), defaultNodeName), NodeKey{ PrivKey: pv, }, @@ -207,15 +222,10 @@ func TestTransportMultiplexAcceptNonBlocking(t *testing.T) { var ( fastNodePV = ed25519.GenPrivKey() - fastNodeInfo = NodeInfo{ - ID: PubKeyToID(fastNodePV.PubKey()), - ListenAddr: "127.0.0.1:0", - Moniker: "fastNode", - Version: "1.0.0", - } - errc = make(chan error) - fastc = make(chan struct{}) - slowc = make(chan struct{}) + fastNodeInfo = testNodeInfo(PubKeyToID(fastNodePV.PubKey()), "fastnode") + errc = make(chan error) + fastc = make(chan struct{}) + slowc = make(chan struct{}) ) // Simulate slow Peer. @@ -248,11 +258,11 @@ func TestTransportMultiplexAcceptNonBlocking(t *testing.T) { return } - _, err = handshake(sc, 20*time.Millisecond, NodeInfo{ - ID: PubKeyToID(ed25519.GenPrivKey().PubKey()), - ListenAddr: "127.0.0.1:0", - Moniker: "slow_peer", - }) + _, err = handshake(sc, 20*time.Millisecond, + testNodeInfo( + PubKeyToID(ed25519.GenPrivKey().PubKey()), + "slow_peer", + )) if err != nil { errc <- err return @@ -264,7 +274,7 @@ func TestTransportMultiplexAcceptNonBlocking(t *testing.T) { <-slowc var ( - dialer = NewMultiplexTransport( + dialer = newMultiplexTransport( fastNodeInfo, NodeKey{ PrivKey: fastNodePV, @@ -310,13 +320,8 @@ func TestTransportMultiplexValidateNodeInfo(t *testing.T) { go func() { var ( pv = ed25519.GenPrivKey() - dialer = NewMultiplexTransport( - NodeInfo{ - ID: PubKeyToID(pv.PubKey()), - ListenAddr: "127.0.0.1:0", - Moniker: "", // Should not be empty. - Version: "1.0.0", - }, + dialer = newMultiplexTransport( + testNodeInfo(PubKeyToID(pv.PubKey()), ""), // Should not be empty NodeKey{ PrivKey: pv, }, @@ -358,13 +363,10 @@ func TestTransportMultiplexRejectMissmatchID(t *testing.T) { errc := make(chan error) go func() { - dialer := NewMultiplexTransport( - NodeInfo{ - ID: PubKeyToID(ed25519.GenPrivKey().PubKey()), - ListenAddr: "127.0.0.1:0", - Moniker: "dialer", - Version: "1.0.0", - }, + dialer := newMultiplexTransport( + testNodeInfo( + PubKeyToID(ed25519.GenPrivKey().PubKey()), "dialer", + ), NodeKey{ PrivKey: ed25519.GenPrivKey(), }, @@ -407,13 +409,8 @@ func TestTransportMultiplexRejectIncompatible(t *testing.T) { go func() { var ( pv = ed25519.GenPrivKey() - dialer = NewMultiplexTransport( - NodeInfo{ - ID: PubKeyToID(pv.PubKey()), - ListenAddr: "127.0.0.1:0", - Moniker: "dialer", - Version: "2.0.0", - }, + dialer = newMultiplexTransport( + testNodeInfoWithNetwork(PubKeyToID(pv.PubKey()), "dialer", "incompatible-network"), NodeKey{ PrivKey: pv, }, @@ -501,13 +498,13 @@ func TestTransportConnDuplicateIPFilter(t *testing.T) { ) cs.Set(c, []net.IP{ - net.IP{10, 0, 10, 1}, - net.IP{10, 0, 10, 2}, - net.IP{10, 0, 10, 3}, + {10, 0, 10, 1}, + {10, 0, 10, 2}, + {10, 0, 10, 3}, }) if err := filter(cs, c, []net.IP{ - net.IP{10, 0, 10, 2}, + {10, 0, 10, 2}, }); err == nil { t.Errorf("expected Peer to be rejected as duplicate") } @@ -521,9 +518,7 @@ func TestTransportHandshake(t *testing.T) { var ( peerPV = ed25519.GenPrivKey() - peerNodeInfo = NodeInfo{ - ID: PubKeyToID(peerPV.PubKey()), - } + peerNodeInfo = testNodeInfo(PubKeyToID(peerPV.PubKey()), defaultNodeName) ) go func() { @@ -534,15 +529,15 @@ func TestTransportHandshake(t *testing.T) { } go func(c net.Conn) { - _, err := cdc.MarshalBinaryWriter(c, peerNodeInfo) + _, err := cdc.MarshalBinaryLengthPrefixedWriter(c, peerNodeInfo.(DefaultNodeInfo)) if err != nil { t.Error(err) } }(c) go func(c net.Conn) { - ni := NodeInfo{} + var ni DefaultNodeInfo - _, err := cdc.UnmarshalBinaryReader( + _, err := cdc.UnmarshalBinaryLengthPrefixedReader( c, &ni, int64(MaxNodeInfoSize()), @@ -558,7 +553,7 @@ func TestTransportHandshake(t *testing.T) { t.Fatal(err) } - ni, err := handshake(c, 20*time.Millisecond, NodeInfo{}) + ni, err := handshake(c, 20*time.Millisecond, emptyNodeInfo()) if err != nil { t.Fatal(err) } @@ -571,13 +566,10 @@ func TestTransportHandshake(t *testing.T) { func testSetupMultiplexTransport(t *testing.T) *MultiplexTransport { var ( pv = ed25519.GenPrivKey() - mt = NewMultiplexTransport( - NodeInfo{ - ID: PubKeyToID(pv.PubKey()), - ListenAddr: "127.0.0.1:0", - Moniker: "transport", - Version: "1.0.0", - }, + mt = newMultiplexTransport( + testNodeInfo( + PubKeyToID(pv.PubKey()), "transport", + ), NodeKey{ PrivKey: pv, }, diff --git a/p2p/trust/metric_test.go b/p2p/trust/metric_test.go index f690ce55..0273dad6 100644 --- a/p2p/trust/metric_test.go +++ b/p2p/trust/metric_test.go @@ -65,6 +65,7 @@ func TestTrustMetricCopyNilPointer(t *testing.T) { } // XXX: This test fails non-deterministically +//nolint:unused,deadcode func _TestTrustMetricStopPause(t *testing.T) { // The TestTicker will provide manual control over // the passing of time within the metric diff --git a/p2p/version.go b/p2p/version.go deleted file mode 100644 index 9a4c7bba..00000000 --- a/p2p/version.go +++ /dev/null @@ -1,3 +0,0 @@ -package p2p - -const Version = "0.5.0" diff --git a/p2p/wire.go b/p2p/wire.go index 40176e3a..191e3c52 100644 --- a/p2p/wire.go +++ b/p2p/wire.go @@ -1,7 +1,7 @@ package p2p import ( - "github.com/tendermint/go-amino" + amino "github.com/tendermint/go-amino" cryptoAmino "github.com/tendermint/tendermint/crypto/encoding/amino" ) diff --git a/privval/client.go b/privval/client.go new file mode 100644 index 00000000..11151fee --- /dev/null +++ b/privval/client.go @@ -0,0 +1,240 @@ +package privval + +import ( + "errors" + "fmt" + "net" + "sync" + "time" + + "github.com/tendermint/tendermint/crypto" + cmn "github.com/tendermint/tendermint/libs/common" + "github.com/tendermint/tendermint/libs/log" + "github.com/tendermint/tendermint/types" +) + +const ( + defaultConnHeartBeatSeconds = 2 + defaultDialRetries = 10 +) + +// Socket errors. +var ( + ErrUnexpectedResponse = errors.New("received unexpected response") +) + +var ( + connHeartbeat = time.Second * defaultConnHeartBeatSeconds +) + +// SocketValOption sets an optional parameter on the SocketVal. +type SocketValOption func(*SocketVal) + +// SocketValHeartbeat sets the period on which to check the liveness of the +// connected Signer connections. +func SocketValHeartbeat(period time.Duration) SocketValOption { + return func(sc *SocketVal) { sc.connHeartbeat = period } +} + +// SocketVal implements PrivValidator. +// It listens for an external process to dial in and uses +// the socket to request signatures. +type SocketVal struct { + cmn.BaseService + + listener net.Listener + + // ping + cancelPing chan struct{} + pingTicker *time.Ticker + connHeartbeat time.Duration + + // signer is mutable since it can be + // reset if the connection fails. + // failures are detected by a background + // ping routine. + // All messages are request/response, so we hold the mutex + // so only one request/response pair can happen at a time. + // Methods on the underlying net.Conn itself + // are already gorountine safe. + mtx sync.Mutex + signer *RemoteSignerClient +} + +// Check that SocketVal implements PrivValidator. +var _ types.PrivValidator = (*SocketVal)(nil) + +// NewSocketVal returns an instance of SocketVal. +func NewSocketVal( + logger log.Logger, + listener net.Listener, +) *SocketVal { + sc := &SocketVal{ + listener: listener, + connHeartbeat: connHeartbeat, + } + + sc.BaseService = *cmn.NewBaseService(logger, "SocketVal", sc) + + return sc +} + +//-------------------------------------------------------- +// Implement PrivValidator + +// GetPubKey implements PrivValidator. +func (sc *SocketVal) GetPubKey() crypto.PubKey { + sc.mtx.Lock() + defer sc.mtx.Unlock() + return sc.signer.GetPubKey() +} + +// SignVote implements PrivValidator. +func (sc *SocketVal) SignVote(chainID string, vote *types.Vote) error { + sc.mtx.Lock() + defer sc.mtx.Unlock() + return sc.signer.SignVote(chainID, vote) +} + +// SignProposal implements PrivValidator. +func (sc *SocketVal) SignProposal(chainID string, proposal *types.Proposal) error { + sc.mtx.Lock() + defer sc.mtx.Unlock() + return sc.signer.SignProposal(chainID, proposal) +} + +//-------------------------------------------------------- +// More thread safe methods proxied to the signer + +// Ping is used to check connection health. +func (sc *SocketVal) Ping() error { + sc.mtx.Lock() + defer sc.mtx.Unlock() + return sc.signer.Ping() +} + +// Close closes the underlying net.Conn. +func (sc *SocketVal) Close() { + sc.mtx.Lock() + defer sc.mtx.Unlock() + if sc.signer != nil { + if err := sc.signer.Close(); err != nil { + sc.Logger.Error("OnStop", "err", err) + } + } + + if sc.listener != nil { + if err := sc.listener.Close(); err != nil { + sc.Logger.Error("OnStop", "err", err) + } + } +} + +//-------------------------------------------------------- +// Service start and stop + +// OnStart implements cmn.Service. +func (sc *SocketVal) OnStart() error { + if closed, err := sc.reset(); err != nil { + sc.Logger.Error("OnStart", "err", err) + return err + } else if closed { + return fmt.Errorf("listener is closed") + } + + // Start a routine to keep the connection alive + sc.cancelPing = make(chan struct{}, 1) + sc.pingTicker = time.NewTicker(sc.connHeartbeat) + go func() { + for { + select { + case <-sc.pingTicker.C: + err := sc.Ping() + if err != nil { + sc.Logger.Error("Ping", "err", err) + if err == ErrUnexpectedResponse { + return + } + + closed, err := sc.reset() + if err != nil { + sc.Logger.Error("Reconnecting to remote signer failed", "err", err) + continue + } + if closed { + sc.Logger.Info("listener is closing") + return + } + + sc.Logger.Info("Re-created connection to remote signer", "impl", sc) + } + case <-sc.cancelPing: + sc.pingTicker.Stop() + return + } + } + }() + + return nil +} + +// OnStop implements cmn.Service. +func (sc *SocketVal) OnStop() { + if sc.cancelPing != nil { + close(sc.cancelPing) + } + sc.Close() +} + +//-------------------------------------------------------- +// Connection and signer management + +// waits to accept and sets a new connection. +// connection is closed in OnStop. +// returns true if the listener is closed +// (ie. it returns a nil conn). +func (sc *SocketVal) reset() (closed bool, err error) { + sc.mtx.Lock() + defer sc.mtx.Unlock() + + // first check if the conn already exists and close it. + if sc.signer != nil { + if err := sc.signer.Close(); err != nil { + sc.Logger.Error("error closing socket val connection during reset", "err", err) + } + } + + // wait for a new conn + conn, err := sc.acceptConnection() + if err != nil { + return false, err + } + + // listener is closed + if conn == nil { + return true, nil + } + + sc.signer, err = NewRemoteSignerClient(conn) + if err != nil { + // failed to fetch the pubkey. close out the connection. + if err := conn.Close(); err != nil { + sc.Logger.Error("error closing connection", "err", err) + } + return false, err + } + return false, nil +} + +// Attempt to accept a connection. +// Times out after the listener's acceptDeadline +func (sc *SocketVal) acceptConnection() (net.Conn, error) { + conn, err := sc.listener.Accept() + if err != nil { + if !sc.IsRunning() { + return nil, nil // Ignore error from listener closing. + } + return nil, err + } + return conn, nil +} diff --git a/privval/client_test.go b/privval/client_test.go new file mode 100644 index 00000000..1aea58cf --- /dev/null +++ b/privval/client_test.go @@ -0,0 +1,461 @@ +package privval + +import ( + "fmt" + "net" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/tendermint/tendermint/crypto/ed25519" + cmn "github.com/tendermint/tendermint/libs/common" + "github.com/tendermint/tendermint/libs/log" + + "github.com/tendermint/tendermint/types" +) + +var ( + testAcceptDeadline = defaultAcceptDeadlineSeconds * time.Second + + testConnDeadline = 100 * time.Millisecond + testConnDeadline2o3 = 66 * time.Millisecond // 2/3 of the other one + + testHeartbeatTimeout = 10 * time.Millisecond + testHeartbeatTimeout3o2 = 6 * time.Millisecond // 3/2 of the other one +) + +type socketTestCase struct { + addr string + dialer Dialer +} + +func socketTestCases(t *testing.T) []socketTestCase { + tcpAddr := fmt.Sprintf("tcp://%s", testFreeTCPAddr(t)) + unixFilePath, err := testUnixAddr() + require.NoError(t, err) + unixAddr := fmt.Sprintf("unix://%s", unixFilePath) + return []socketTestCase{ + { + addr: tcpAddr, + dialer: DialTCPFn(tcpAddr, testConnDeadline, ed25519.GenPrivKey()), + }, + { + addr: unixAddr, + dialer: DialUnixFn(unixFilePath), + }, + } +} + +func TestSocketPVAddress(t *testing.T) { + for _, tc := range socketTestCases(t) { + // Execute the test within a closure to ensure the deferred statements + // are called between each for loop iteration, for isolated test cases. + func() { + var ( + chainID = cmn.RandStr(12) + sc, rs = testSetupSocketPair(t, chainID, types.NewMockPV(), tc.addr, tc.dialer) + ) + defer sc.Stop() + defer rs.Stop() + + serverAddr := rs.privVal.GetPubKey().Address() + clientAddr := sc.GetPubKey().Address() + + assert.Equal(t, serverAddr, clientAddr) + }() + } +} + +func TestSocketPVPubKey(t *testing.T) { + for _, tc := range socketTestCases(t) { + func() { + var ( + chainID = cmn.RandStr(12) + sc, rs = testSetupSocketPair(t, chainID, types.NewMockPV(), tc.addr, tc.dialer) + ) + defer sc.Stop() + defer rs.Stop() + + clientKey := sc.GetPubKey() + + privvalPubKey := rs.privVal.GetPubKey() + + assert.Equal(t, privvalPubKey, clientKey) + }() + } +} + +func TestSocketPVProposal(t *testing.T) { + for _, tc := range socketTestCases(t) { + func() { + var ( + chainID = cmn.RandStr(12) + sc, rs = testSetupSocketPair(t, chainID, types.NewMockPV(), tc.addr, tc.dialer) + + ts = time.Now() + privProposal = &types.Proposal{Timestamp: ts} + clientProposal = &types.Proposal{Timestamp: ts} + ) + defer sc.Stop() + defer rs.Stop() + + require.NoError(t, rs.privVal.SignProposal(chainID, privProposal)) + require.NoError(t, sc.SignProposal(chainID, clientProposal)) + assert.Equal(t, privProposal.Signature, clientProposal.Signature) + }() + } +} + +func TestSocketPVVote(t *testing.T) { + for _, tc := range socketTestCases(t) { + func() { + var ( + chainID = cmn.RandStr(12) + sc, rs = testSetupSocketPair(t, chainID, types.NewMockPV(), tc.addr, tc.dialer) + + ts = time.Now() + vType = types.PrecommitType + want = &types.Vote{Timestamp: ts, Type: vType} + have = &types.Vote{Timestamp: ts, Type: vType} + ) + defer sc.Stop() + defer rs.Stop() + + require.NoError(t, rs.privVal.SignVote(chainID, want)) + require.NoError(t, sc.SignVote(chainID, have)) + assert.Equal(t, want.Signature, have.Signature) + }() + } +} + +func TestSocketPVVoteResetDeadline(t *testing.T) { + for _, tc := range socketTestCases(t) { + func() { + var ( + chainID = cmn.RandStr(12) + sc, rs = testSetupSocketPair(t, chainID, types.NewMockPV(), tc.addr, tc.dialer) + + ts = time.Now() + vType = types.PrecommitType + want = &types.Vote{Timestamp: ts, Type: vType} + have = &types.Vote{Timestamp: ts, Type: vType} + ) + defer sc.Stop() + defer rs.Stop() + + time.Sleep(testConnDeadline2o3) + + require.NoError(t, rs.privVal.SignVote(chainID, want)) + require.NoError(t, sc.SignVote(chainID, have)) + assert.Equal(t, want.Signature, have.Signature) + + // This would exceed the deadline if it was not extended by the previous message + time.Sleep(testConnDeadline2o3) + + require.NoError(t, rs.privVal.SignVote(chainID, want)) + require.NoError(t, sc.SignVote(chainID, have)) + assert.Equal(t, want.Signature, have.Signature) + }() + } +} + +func TestSocketPVVoteKeepalive(t *testing.T) { + for _, tc := range socketTestCases(t) { + func() { + var ( + chainID = cmn.RandStr(12) + sc, rs = testSetupSocketPair(t, chainID, types.NewMockPV(), tc.addr, tc.dialer) + + ts = time.Now() + vType = types.PrecommitType + want = &types.Vote{Timestamp: ts, Type: vType} + have = &types.Vote{Timestamp: ts, Type: vType} + ) + defer sc.Stop() + defer rs.Stop() + + time.Sleep(testConnDeadline * 2) + + require.NoError(t, rs.privVal.SignVote(chainID, want)) + require.NoError(t, sc.SignVote(chainID, have)) + assert.Equal(t, want.Signature, have.Signature) + }() + } +} + +func TestSocketPVDeadline(t *testing.T) { + for _, tc := range socketTestCases(t) { + func() { + var ( + listenc = make(chan struct{}) + thisConnTimeout = 100 * time.Millisecond + sc = newSocketVal(log.TestingLogger(), tc.addr, thisConnTimeout) + ) + + go func(sc *SocketVal) { + defer close(listenc) + + // Note: the TCP connection times out at the accept() phase, + // whereas the Unix domain sockets connection times out while + // attempting to fetch the remote signer's public key. + assert.True(t, IsConnTimeout(sc.Start())) + + assert.False(t, sc.IsRunning()) + }(sc) + + for { + _, err := cmn.Connect(tc.addr) + if err == nil { + break + } + } + + <-listenc + }() + } +} + +func TestRemoteSignVoteErrors(t *testing.T) { + for _, tc := range socketTestCases(t) { + func() { + var ( + chainID = cmn.RandStr(12) + sc, rs = testSetupSocketPair(t, chainID, types.NewErroringMockPV(), tc.addr, tc.dialer) + + ts = time.Now() + vType = types.PrecommitType + vote = &types.Vote{Timestamp: ts, Type: vType} + ) + defer sc.Stop() + defer rs.Stop() + + err := sc.SignVote("", vote) + require.Equal(t, err.(*RemoteSignerError).Description, types.ErroringMockPVErr.Error()) + + err = rs.privVal.SignVote(chainID, vote) + require.Error(t, err) + err = sc.SignVote(chainID, vote) + require.Error(t, err) + }() + } +} + +func TestRemoteSignProposalErrors(t *testing.T) { + for _, tc := range socketTestCases(t) { + func() { + var ( + chainID = cmn.RandStr(12) + sc, rs = testSetupSocketPair(t, chainID, types.NewErroringMockPV(), tc.addr, tc.dialer) + + ts = time.Now() + proposal = &types.Proposal{Timestamp: ts} + ) + defer sc.Stop() + defer rs.Stop() + + err := sc.SignProposal("", proposal) + require.Equal(t, err.(*RemoteSignerError).Description, types.ErroringMockPVErr.Error()) + + err = rs.privVal.SignProposal(chainID, proposal) + require.Error(t, err) + + err = sc.SignProposal(chainID, proposal) + require.Error(t, err) + }() + } +} + +func TestErrUnexpectedResponse(t *testing.T) { + for _, tc := range socketTestCases(t) { + func() { + var ( + logger = log.TestingLogger() + chainID = cmn.RandStr(12) + readyc = make(chan struct{}) + errc = make(chan error, 1) + + rs = NewRemoteSigner( + logger, + chainID, + types.NewMockPV(), + tc.dialer, + ) + sc = newSocketVal(logger, tc.addr, testConnDeadline) + ) + + testStartSocketPV(t, readyc, sc) + defer sc.Stop() + RemoteSignerConnDeadline(time.Millisecond)(rs) + RemoteSignerConnRetries(100)(rs) + // we do not want to Start() the remote signer here and instead use the connection to + // reply with intentionally wrong replies below: + rsConn, err := rs.connect() + defer rsConn.Close() + require.NoError(t, err) + require.NotNil(t, rsConn) + // send over public key to get the remote signer running: + go testReadWriteResponse(t, &PubKeyResponse{}, rsConn) + <-readyc + + // Proposal: + go func(errc chan error) { + errc <- sc.SignProposal(chainID, &types.Proposal{}) + }(errc) + // read request and write wrong response: + go testReadWriteResponse(t, &SignedVoteResponse{}, rsConn) + err = <-errc + require.Error(t, err) + require.Equal(t, err, ErrUnexpectedResponse) + + // Vote: + go func(errc chan error) { + errc <- sc.SignVote(chainID, &types.Vote{}) + }(errc) + // read request and write wrong response: + go testReadWriteResponse(t, &SignedProposalResponse{}, rsConn) + err = <-errc + require.Error(t, err) + require.Equal(t, err, ErrUnexpectedResponse) + }() + } +} + +func TestRetryConnToRemoteSigner(t *testing.T) { + for _, tc := range socketTestCases(t) { + func() { + var ( + logger = log.TestingLogger() + chainID = cmn.RandStr(12) + readyc = make(chan struct{}) + + rs = NewRemoteSigner( + logger, + chainID, + types.NewMockPV(), + tc.dialer, + ) + thisConnTimeout = testConnDeadline + sc = newSocketVal(logger, tc.addr, thisConnTimeout) + ) + // Ping every: + SocketValHeartbeat(testHeartbeatTimeout)(sc) + + RemoteSignerConnDeadline(testConnDeadline)(rs) + RemoteSignerConnRetries(10)(rs) + + testStartSocketPV(t, readyc, sc) + defer sc.Stop() + require.NoError(t, rs.Start()) + assert.True(t, rs.IsRunning()) + + <-readyc + time.Sleep(testHeartbeatTimeout * 2) + + rs.Stop() + rs2 := NewRemoteSigner( + logger, + chainID, + types.NewMockPV(), + tc.dialer, + ) + // let some pings pass + time.Sleep(testHeartbeatTimeout3o2) + require.NoError(t, rs2.Start()) + assert.True(t, rs2.IsRunning()) + defer rs2.Stop() + + // give the client some time to re-establish the conn to the remote signer + // should see sth like this in the logs: + // + // E[10016-01-10|17:12:46.128] Ping err="remote signer timed out" + // I[10016-01-10|17:16:42.447] Re-created connection to remote signer impl=SocketVal + time.Sleep(testConnDeadline * 2) + }() + } +} + +func newSocketVal(logger log.Logger, addr string, connDeadline time.Duration) *SocketVal { + proto, address := cmn.ProtocolAndAddress(addr) + ln, err := net.Listen(proto, address) + logger.Info("Listening at", "proto", proto, "address", address) + if err != nil { + panic(err) + } + var svln net.Listener + if proto == "unix" { + unixLn := NewUnixListener(ln) + UnixListenerAcceptDeadline(testAcceptDeadline)(unixLn) + UnixListenerConnDeadline(connDeadline)(unixLn) + svln = unixLn + } else { + tcpLn := NewTCPListener(ln, ed25519.GenPrivKey()) + TCPListenerAcceptDeadline(testAcceptDeadline)(tcpLn) + TCPListenerConnDeadline(connDeadline)(tcpLn) + svln = tcpLn + } + return NewSocketVal(logger, svln) +} + +func testSetupSocketPair( + t *testing.T, + chainID string, + privValidator types.PrivValidator, + addr string, + dialer Dialer, +) (*SocketVal, *RemoteSigner) { + var ( + logger = log.TestingLogger() + privVal = privValidator + readyc = make(chan struct{}) + rs = NewRemoteSigner( + logger, + chainID, + privVal, + dialer, + ) + + thisConnTimeout = testConnDeadline + sc = newSocketVal(logger, addr, thisConnTimeout) + ) + + SocketValHeartbeat(testHeartbeatTimeout)(sc) + RemoteSignerConnDeadline(testConnDeadline)(rs) + RemoteSignerConnRetries(1e6)(rs) + + testStartSocketPV(t, readyc, sc) + + require.NoError(t, rs.Start()) + assert.True(t, rs.IsRunning()) + + <-readyc + + return sc, rs +} + +func testReadWriteResponse(t *testing.T, resp RemoteSignerMsg, rsConn net.Conn) { + _, err := readMsg(rsConn) + require.NoError(t, err) + + err = writeMsg(rsConn, resp) + require.NoError(t, err) +} + +func testStartSocketPV(t *testing.T, readyc chan struct{}, sc *SocketVal) { + go func(sc *SocketVal) { + require.NoError(t, sc.Start()) + assert.True(t, sc.IsRunning()) + + readyc <- struct{}{} + }(sc) +} + +// testFreeTCPAddr claims a free port so we don't block on listener being ready. +func testFreeTCPAddr(t *testing.T) string { + ln, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + defer ln.Close() + + return fmt.Sprintf("127.0.0.1:%d", ln.Addr().(*net.TCPAddr).Port) +} diff --git a/privval/doc.go b/privval/doc.go new file mode 100644 index 00000000..ed378c19 --- /dev/null +++ b/privval/doc.go @@ -0,0 +1,21 @@ +/* + +Package privval provides different implementations of the types.PrivValidator. + +FilePV + +FilePV is the simplest implementation and developer default. It uses one file for the private key and another to store state. + +SocketVal + +SocketVal establishes a connection to an external process, like a Key Management Server (KMS), using a socket. +SocketVal listens for the external KMS process to dial in. +SocketVal takes a listener, which determines the type of connection +(ie. encrypted over tcp, or unencrypted over unix). + +RemoteSigner + +RemoteSigner is a simple wrapper around a net.Conn. It's used by both IPCVal and TCPVal. + +*/ +package privval diff --git a/privval/file.go b/privval/file.go new file mode 100644 index 00000000..d27d7a78 --- /dev/null +++ b/privval/file.go @@ -0,0 +1,421 @@ +package privval + +import ( + "bytes" + "errors" + "fmt" + "io/ioutil" + "time" + + "github.com/tendermint/tendermint/crypto" + "github.com/tendermint/tendermint/crypto/ed25519" + cmn "github.com/tendermint/tendermint/libs/common" + "github.com/tendermint/tendermint/types" + tmtime "github.com/tendermint/tendermint/types/time" +) + +// TODO: type ? +const ( + stepNone int8 = 0 // Used to distinguish the initial state + stepPropose int8 = 1 + stepPrevote int8 = 2 + stepPrecommit int8 = 3 +) + +// A vote is either stepPrevote or stepPrecommit. +func voteToStep(vote *types.Vote) int8 { + switch vote.Type { + case types.PrevoteType: + return stepPrevote + case types.PrecommitType: + return stepPrecommit + default: + panic("Unknown vote type") + return 0 + } +} + +//------------------------------------------------------------------------------- + +// FilePVKey stores the immutable part of PrivValidator. +type FilePVKey struct { + Address types.Address `json:"address"` + PubKey crypto.PubKey `json:"pub_key"` + PrivKey crypto.PrivKey `json:"priv_key"` + + filePath string +} + +// Save persists the FilePVKey to its filePath. +func (pvKey FilePVKey) Save() { + outFile := pvKey.filePath + if outFile == "" { + panic("Cannot save PrivValidator key: filePath not set") + } + + jsonBytes, err := cdc.MarshalJSONIndent(pvKey, "", " ") + if err != nil { + panic(err) + } + err = cmn.WriteFileAtomic(outFile, jsonBytes, 0600) + if err != nil { + panic(err) + } + +} + +//------------------------------------------------------------------------------- + +// FilePVLastSignState stores the mutable part of PrivValidator. +type FilePVLastSignState struct { + Height int64 `json:"height"` + Round int `json:"round"` + Step int8 `json:"step"` + Signature []byte `json:"signature,omitempty"` + SignBytes cmn.HexBytes `json:"signbytes,omitempty"` + + filePath string +} + +// CheckHRS checks the given height, round, step (HRS) against that of the +// FilePVLastSignState. It returns an error if the arguments constitute a regression, +// or if they match but the SignBytes are empty. +// The returned boolean indicates whether the last Signature should be reused - +// it returns true if the HRS matches the arguments and the SignBytes are not empty (indicating +// we have already signed for this HRS, and can reuse the existing signature). +// It panics if the HRS matches the arguments, there's a SignBytes, but no Signature. +func (lss *FilePVLastSignState) CheckHRS(height int64, round int, step int8) (bool, error) { + + if lss.Height > height { + return false, fmt.Errorf("Height regression. Got %v, last height %v", height, lss.Height) + } + + if lss.Height == height { + if lss.Round > round { + return false, fmt.Errorf("Round regression at height %v. Got %v, last round %v", height, round, lss.Round) + } + + if lss.Round == round { + if lss.Step > step { + return false, fmt.Errorf("Step regression at height %v round %v. Got %v, last step %v", height, round, step, lss.Step) + } else if lss.Step == step { + if lss.SignBytes != nil { + if lss.Signature == nil { + panic("pv: Signature is nil but SignBytes is not!") + } + return true, nil + } + return false, errors.New("No SignBytes found") + } + } + } + return false, nil +} + +// Save persists the FilePvLastSignState to its filePath. +func (lss *FilePVLastSignState) Save() { + outFile := lss.filePath + if outFile == "" { + panic("Cannot save FilePVLastSignState: filePath not set") + } + jsonBytes, err := cdc.MarshalJSONIndent(lss, "", " ") + if err != nil { + panic(err) + } + err = cmn.WriteFileAtomic(outFile, jsonBytes, 0600) + if err != nil { + panic(err) + } +} + +//------------------------------------------------------------------------------- + +// FilePV implements PrivValidator using data persisted to disk +// to prevent double signing. +// NOTE: the directories containing pv.Key.filePath and pv.LastSignState.filePath must already exist. +// It includes the LastSignature and LastSignBytes so we don't lose the signature +// if the process crashes after signing but before the resulting consensus message is processed. +type FilePV struct { + Key FilePVKey + LastSignState FilePVLastSignState +} + +// GenFilePV generates a new validator with randomly generated private key +// and sets the filePaths, but does not call Save(). +func GenFilePV(keyFilePath, stateFilePath string) *FilePV { + privKey := ed25519.GenPrivKey() + + return &FilePV{ + Key: FilePVKey{ + Address: privKey.PubKey().Address(), + PubKey: privKey.PubKey(), + PrivKey: privKey, + filePath: keyFilePath, + }, + LastSignState: FilePVLastSignState{ + Step: stepNone, + filePath: stateFilePath, + }, + } +} + +// LoadFilePV loads a FilePV from the filePaths. The FilePV handles double +// signing prevention by persisting data to the stateFilePath. If either file path +// does not exist, the program will exit. +func LoadFilePV(keyFilePath, stateFilePath string) *FilePV { + return loadFilePV(keyFilePath, stateFilePath, true) +} + +// LoadFilePVEmptyState loads a FilePV from the given keyFilePath, with an empty LastSignState. +// If the keyFilePath does not exist, the program will exit. +func LoadFilePVEmptyState(keyFilePath, stateFilePath string) *FilePV { + return loadFilePV(keyFilePath, stateFilePath, false) +} + +// If loadState is true, we load from the stateFilePath. Otherwise, we use an empty LastSignState. +func loadFilePV(keyFilePath, stateFilePath string, loadState bool) *FilePV { + keyJSONBytes, err := ioutil.ReadFile(keyFilePath) + if err != nil { + cmn.Exit(err.Error()) + } + pvKey := FilePVKey{} + err = cdc.UnmarshalJSON(keyJSONBytes, &pvKey) + if err != nil { + cmn.Exit(fmt.Sprintf("Error reading PrivValidator key from %v: %v\n", keyFilePath, err)) + } + + // overwrite pubkey and address for convenience + pvKey.PubKey = pvKey.PrivKey.PubKey() + pvKey.Address = pvKey.PubKey.Address() + pvKey.filePath = keyFilePath + + pvState := FilePVLastSignState{} + if loadState { + stateJSONBytes, err := ioutil.ReadFile(stateFilePath) + if err != nil { + cmn.Exit(err.Error()) + } + err = cdc.UnmarshalJSON(stateJSONBytes, &pvState) + if err != nil { + cmn.Exit(fmt.Sprintf("Error reading PrivValidator state from %v: %v\n", stateFilePath, err)) + } + } + + pvState.filePath = stateFilePath + + return &FilePV{ + Key: pvKey, + LastSignState: pvState, + } +} + +// LoadOrGenFilePV loads a FilePV from the given filePaths +// or else generates a new one and saves it to the filePaths. +func LoadOrGenFilePV(keyFilePath, stateFilePath string) *FilePV { + var pv *FilePV + if cmn.FileExists(keyFilePath) { + pv = LoadFilePV(keyFilePath, stateFilePath) + } else { + pv = GenFilePV(keyFilePath, stateFilePath) + pv.Save() + } + return pv +} + +// GetAddress returns the address of the validator. +// Implements PrivValidator. +func (pv *FilePV) GetAddress() types.Address { + return pv.Key.Address +} + +// GetPubKey returns the public key of the validator. +// Implements PrivValidator. +func (pv *FilePV) GetPubKey() crypto.PubKey { + return pv.Key.PubKey +} + +// SignVote signs a canonical representation of the vote, along with the +// chainID. Implements PrivValidator. +func (pv *FilePV) SignVote(chainID string, vote *types.Vote) error { + if err := pv.signVote(chainID, vote); err != nil { + return fmt.Errorf("Error signing vote: %v", err) + } + return nil +} + +// SignProposal signs a canonical representation of the proposal, along with +// the chainID. Implements PrivValidator. +func (pv *FilePV) SignProposal(chainID string, proposal *types.Proposal) error { + if err := pv.signProposal(chainID, proposal); err != nil { + return fmt.Errorf("Error signing proposal: %v", err) + } + return nil +} + +// Save persists the FilePV to disk. +func (pv *FilePV) Save() { + pv.Key.Save() + pv.LastSignState.Save() +} + +// Reset resets all fields in the FilePV. +// NOTE: Unsafe! +func (pv *FilePV) Reset() { + var sig []byte + pv.LastSignState.Height = 0 + pv.LastSignState.Round = 0 + pv.LastSignState.Step = 0 + pv.LastSignState.Signature = sig + pv.LastSignState.SignBytes = nil + pv.Save() +} + +// String returns a string representation of the FilePV. +func (pv *FilePV) String() string { + return fmt.Sprintf("PrivValidator{%v LH:%v, LR:%v, LS:%v}", pv.GetAddress(), pv.LastSignState.Height, pv.LastSignState.Round, pv.LastSignState.Step) +} + +//------------------------------------------------------------------------------------ + +// signVote checks if the vote is good to sign and sets the vote signature. +// It may need to set the timestamp as well if the vote is otherwise the same as +// a previously signed vote (ie. we crashed after signing but before the vote hit the WAL). +func (pv *FilePV) signVote(chainID string, vote *types.Vote) error { + height, round, step := vote.Height, vote.Round, voteToStep(vote) + + lss := pv.LastSignState + + sameHRS, err := lss.CheckHRS(height, round, step) + if err != nil { + return err + } + + signBytes := vote.SignBytes(chainID) + + // We might crash before writing to the wal, + // causing us to try to re-sign for the same HRS. + // If signbytes are the same, use the last signature. + // If they only differ by timestamp, use last timestamp and signature + // Otherwise, return error + if sameHRS { + if bytes.Equal(signBytes, lss.SignBytes) { + vote.Signature = lss.Signature + } else if timestamp, ok := checkVotesOnlyDifferByTimestamp(lss.SignBytes, signBytes); ok { + vote.Timestamp = timestamp + vote.Signature = lss.Signature + } else { + err = fmt.Errorf("Conflicting data") + } + return err + } + + // It passed the checks. Sign the vote + sig, err := pv.Key.PrivKey.Sign(signBytes) + if err != nil { + return err + } + pv.saveSigned(height, round, step, signBytes, sig) + vote.Signature = sig + return nil +} + +// signProposal checks if the proposal is good to sign and sets the proposal signature. +// It may need to set the timestamp as well if the proposal is otherwise the same as +// a previously signed proposal ie. we crashed after signing but before the proposal hit the WAL). +func (pv *FilePV) signProposal(chainID string, proposal *types.Proposal) error { + height, round, step := proposal.Height, proposal.Round, stepPropose + + lss := pv.LastSignState + + sameHRS, err := lss.CheckHRS(height, round, step) + if err != nil { + return err + } + + signBytes := proposal.SignBytes(chainID) + + // We might crash before writing to the wal, + // causing us to try to re-sign for the same HRS. + // If signbytes are the same, use the last signature. + // If they only differ by timestamp, use last timestamp and signature + // Otherwise, return error + if sameHRS { + if bytes.Equal(signBytes, lss.SignBytes) { + proposal.Signature = lss.Signature + } else if timestamp, ok := checkProposalsOnlyDifferByTimestamp(lss.SignBytes, signBytes); ok { + proposal.Timestamp = timestamp + proposal.Signature = lss.Signature + } else { + err = fmt.Errorf("Conflicting data") + } + return err + } + + // It passed the checks. Sign the proposal + sig, err := pv.Key.PrivKey.Sign(signBytes) + if err != nil { + return err + } + pv.saveSigned(height, round, step, signBytes, sig) + proposal.Signature = sig + return nil +} + +// Persist height/round/step and signature +func (pv *FilePV) saveSigned(height int64, round int, step int8, + signBytes []byte, sig []byte) { + + pv.LastSignState.Height = height + pv.LastSignState.Round = round + pv.LastSignState.Step = step + pv.LastSignState.Signature = sig + pv.LastSignState.SignBytes = signBytes + pv.LastSignState.Save() +} + +//----------------------------------------------------------------------------------------- + +// returns the timestamp from the lastSignBytes. +// returns true if the only difference in the votes is their timestamp. +func checkVotesOnlyDifferByTimestamp(lastSignBytes, newSignBytes []byte) (time.Time, bool) { + var lastVote, newVote types.CanonicalVote + if err := cdc.UnmarshalBinaryLengthPrefixed(lastSignBytes, &lastVote); err != nil { + panic(fmt.Sprintf("LastSignBytes cannot be unmarshalled into vote: %v", err)) + } + if err := cdc.UnmarshalBinaryLengthPrefixed(newSignBytes, &newVote); err != nil { + panic(fmt.Sprintf("signBytes cannot be unmarshalled into vote: %v", err)) + } + + lastTime := lastVote.Timestamp + + // set the times to the same value and check equality + now := tmtime.Now() + lastVote.Timestamp = now + newVote.Timestamp = now + lastVoteBytes, _ := cdc.MarshalJSON(lastVote) + newVoteBytes, _ := cdc.MarshalJSON(newVote) + + return lastTime, bytes.Equal(newVoteBytes, lastVoteBytes) +} + +// returns the timestamp from the lastSignBytes. +// returns true if the only difference in the proposals is their timestamp +func checkProposalsOnlyDifferByTimestamp(lastSignBytes, newSignBytes []byte) (time.Time, bool) { + var lastProposal, newProposal types.CanonicalProposal + if err := cdc.UnmarshalBinaryLengthPrefixed(lastSignBytes, &lastProposal); err != nil { + panic(fmt.Sprintf("LastSignBytes cannot be unmarshalled into proposal: %v", err)) + } + if err := cdc.UnmarshalBinaryLengthPrefixed(newSignBytes, &newProposal); err != nil { + panic(fmt.Sprintf("signBytes cannot be unmarshalled into proposal: %v", err)) + } + + lastTime := lastProposal.Timestamp + // set the times to the same value and check equality + now := tmtime.Now() + lastProposal.Timestamp = now + newProposal.Timestamp = now + lastProposalBytes, _ := cdc.MarshalBinaryLengthPrefixed(lastProposal) + newProposalBytes, _ := cdc.MarshalBinaryLengthPrefixed(newProposal) + + return lastTime, bytes.Equal(newProposalBytes, lastProposalBytes) +} diff --git a/privval/priv_validator_test.go b/privval/file_test.go similarity index 57% rename from privval/priv_validator_test.go rename to privval/file_test.go index 404ff770..06d75a80 100644 --- a/privval/priv_validator_test.go +++ b/privval/file_test.go @@ -18,36 +18,100 @@ import ( func TestGenLoadValidator(t *testing.T) { assert := assert.New(t) - tempFile, err := ioutil.TempFile("", "priv_validator_") + tempKeyFile, err := ioutil.TempFile("", "priv_validator_key_") require.Nil(t, err) - privVal := GenFilePV(tempFile.Name()) + tempStateFile, err := ioutil.TempFile("", "priv_validator_state_") + require.Nil(t, err) + + privVal := GenFilePV(tempKeyFile.Name(), tempStateFile.Name()) height := int64(100) - privVal.LastHeight = height + privVal.LastSignState.Height = height privVal.Save() addr := privVal.GetAddress() - privVal = LoadFilePV(tempFile.Name()) + privVal = LoadFilePV(tempKeyFile.Name(), tempStateFile.Name()) assert.Equal(addr, privVal.GetAddress(), "expected privval addr to be the same") - assert.Equal(height, privVal.LastHeight, "expected privval.LastHeight to have been saved") + assert.Equal(height, privVal.LastSignState.Height, "expected privval.LastHeight to have been saved") +} + +func TestResetValidator(t *testing.T) { + tempKeyFile, err := ioutil.TempFile("", "priv_validator_key_") + require.Nil(t, err) + tempStateFile, err := ioutil.TempFile("", "priv_validator_state_") + require.Nil(t, err) + + privVal := GenFilePV(tempKeyFile.Name(), tempStateFile.Name()) + emptyState := FilePVLastSignState{filePath: tempStateFile.Name()} + + // new priv val has empty state + assert.Equal(t, privVal.LastSignState, emptyState) + + // test vote + height, round := int64(10), 1 + voteType := byte(types.PrevoteType) + blockID := types.BlockID{[]byte{1, 2, 3}, types.PartSetHeader{}} + vote := newVote(privVal.Key.Address, 0, height, round, voteType, blockID) + err = privVal.SignVote("mychainid", vote) + assert.NoError(t, err, "expected no error signing vote") + + // priv val after signing is not same as empty + assert.NotEqual(t, privVal.LastSignState, emptyState) + + // priv val after reset is same as empty + privVal.Reset() + assert.Equal(t, privVal.LastSignState, emptyState) } func TestLoadOrGenValidator(t *testing.T) { assert := assert.New(t) - tempFile, err := ioutil.TempFile("", "priv_validator_") + tempKeyFile, err := ioutil.TempFile("", "priv_validator_key_") require.Nil(t, err) - tempFilePath := tempFile.Name() - if err := os.Remove(tempFilePath); err != nil { + tempStateFile, err := ioutil.TempFile("", "priv_validator_state_") + require.Nil(t, err) + + tempKeyFilePath := tempKeyFile.Name() + if err := os.Remove(tempKeyFilePath); err != nil { t.Error(err) } - privVal := LoadOrGenFilePV(tempFilePath) + tempStateFilePath := tempStateFile.Name() + if err := os.Remove(tempStateFilePath); err != nil { + t.Error(err) + } + + privVal := LoadOrGenFilePV(tempKeyFilePath, tempStateFilePath) addr := privVal.GetAddress() - privVal = LoadOrGenFilePV(tempFilePath) + privVal = LoadOrGenFilePV(tempKeyFilePath, tempStateFilePath) assert.Equal(addr, privVal.GetAddress(), "expected privval addr to be the same") } -func TestUnmarshalValidator(t *testing.T) { +func TestUnmarshalValidatorState(t *testing.T) { + assert, require := assert.New(t), require.New(t) + + // create some fixed values + serialized := `{ + "height": "1", + "round": "1", + "step": 1 + }` + + val := FilePVLastSignState{} + err := cdc.UnmarshalJSON([]byte(serialized), &val) + require.Nil(err, "%+v", err) + + // make sure the values match + assert.EqualValues(val.Height, 1) + assert.EqualValues(val.Round, 1) + assert.EqualValues(val.Step, 1) + + // export it and make sure it is the same + out, err := cdc.MarshalJSON(val) + require.Nil(err, "%+v", err) + assert.JSONEq(serialized, string(out)) +} + +func TestUnmarshalValidatorKey(t *testing.T) { assert, require := assert.New(t), require.New(t) // create some fixed values @@ -67,22 +131,19 @@ func TestUnmarshalValidator(t *testing.T) { "type": "tendermint/PubKeyEd25519", "value": "%s" }, - "last_height": "0", - "last_round": "0", - "last_step": 0, "priv_key": { "type": "tendermint/PrivKeyEd25519", "value": "%s" } }`, addr, pubB64, privB64) - val := FilePV{} + val := FilePVKey{} err := cdc.UnmarshalJSON([]byte(serialized), &val) require.Nil(err, "%+v", err) // make sure the values match - assert.EqualValues(addr, val.GetAddress()) - assert.EqualValues(pubKey, val.GetPubKey()) + assert.EqualValues(addr, val.Address) + assert.EqualValues(pubKey, val.PubKey) assert.EqualValues(privKey, val.PrivKey) // export it and make sure it is the same @@ -94,17 +155,20 @@ func TestUnmarshalValidator(t *testing.T) { func TestSignVote(t *testing.T) { assert := assert.New(t) - tempFile, err := ioutil.TempFile("", "priv_validator_") + tempKeyFile, err := ioutil.TempFile("", "priv_validator_key_") require.Nil(t, err) - privVal := GenFilePV(tempFile.Name()) + tempStateFile, err := ioutil.TempFile("", "priv_validator_state_") + require.Nil(t, err) + + privVal := GenFilePV(tempKeyFile.Name(), tempStateFile.Name()) block1 := types.BlockID{[]byte{1, 2, 3}, types.PartSetHeader{}} block2 := types.BlockID{[]byte{3, 2, 1}, types.PartSetHeader{}} height, round := int64(10), 1 - voteType := types.VoteTypePrevote + voteType := byte(types.PrevoteType) // sign a vote for first time - vote := newVote(privVal.Address, 0, height, round, voteType, block1) + vote := newVote(privVal.Key.Address, 0, height, round, voteType, block1) err = privVal.SignVote("mychainid", vote) assert.NoError(err, "expected no error signing vote") @@ -114,10 +178,10 @@ func TestSignVote(t *testing.T) { // now try some bad votes cases := []*types.Vote{ - newVote(privVal.Address, 0, height, round-1, voteType, block1), // round regression - newVote(privVal.Address, 0, height-1, round, voteType, block1), // height regression - newVote(privVal.Address, 0, height-2, round+4, voteType, block1), // height regression and different round - newVote(privVal.Address, 0, height, round, voteType, block2), // different block + newVote(privVal.Key.Address, 0, height, round-1, voteType, block1), // round regression + newVote(privVal.Key.Address, 0, height-1, round, voteType, block1), // height regression + newVote(privVal.Key.Address, 0, height-2, round+4, voteType, block1), // height regression and different round + newVote(privVal.Key.Address, 0, height, round, voteType, block2), // different block } for _, c := range cases { @@ -136,12 +200,15 @@ func TestSignVote(t *testing.T) { func TestSignProposal(t *testing.T) { assert := assert.New(t) - tempFile, err := ioutil.TempFile("", "priv_validator_") + tempKeyFile, err := ioutil.TempFile("", "priv_validator_key_") + require.Nil(t, err) + tempStateFile, err := ioutil.TempFile("", "priv_validator_state_") require.Nil(t, err) - privVal := GenFilePV(tempFile.Name()) - block1 := types.PartSetHeader{5, []byte{1, 2, 3}} - block2 := types.PartSetHeader{10, []byte{3, 2, 1}} + privVal := GenFilePV(tempKeyFile.Name(), tempStateFile.Name()) + + block1 := types.BlockID{[]byte{1, 2, 3}, types.PartSetHeader{5, []byte{1, 2, 3}}} + block2 := types.BlockID{[]byte{3, 2, 1}, types.PartSetHeader{10, []byte{3, 2, 1}}} height, round := int64(10), 1 // sign a proposal for first time @@ -175,11 +242,14 @@ func TestSignProposal(t *testing.T) { } func TestDifferByTimestamp(t *testing.T) { - tempFile, err := ioutil.TempFile("", "priv_validator_") + tempKeyFile, err := ioutil.TempFile("", "priv_validator_key_") + require.Nil(t, err) + tempStateFile, err := ioutil.TempFile("", "priv_validator_state_") require.Nil(t, err) - privVal := GenFilePV(tempFile.Name()) - block1 := types.PartSetHeader{5, []byte{1, 2, 3}} + privVal := GenFilePV(tempKeyFile.Name(), tempStateFile.Name()) + + block1 := types.BlockID{[]byte{1, 2, 3}, types.PartSetHeader{5, []byte{1, 2, 3}}} height, round := int64(10), 1 chainID := "mychainid" @@ -206,9 +276,9 @@ func TestDifferByTimestamp(t *testing.T) { // test vote { - voteType := types.VoteTypePrevote + voteType := byte(types.PrevoteType) blockID := types.BlockID{[]byte{1, 2, 3}, types.PartSetHeader{}} - vote := newVote(privVal.Address, 0, height, round, voteType, blockID) + vote := newVote(privVal.Key.Address, 0, height, round, voteType, blockID) err := privVal.SignVote("mychainid", vote) assert.NoError(t, err, "expected no error signing vote") @@ -235,17 +305,17 @@ func newVote(addr types.Address, idx int, height int64, round int, typ byte, blo ValidatorIndex: idx, Height: height, Round: round, - Type: typ, + Type: types.SignedMsgType(typ), Timestamp: tmtime.Now(), BlockID: blockID, } } -func newProposal(height int64, round int, partsHeader types.PartSetHeader) *types.Proposal { +func newProposal(height int64, round int, blockID types.BlockID) *types.Proposal { return &types.Proposal{ - Height: height, - Round: round, - BlockPartsHeader: partsHeader, - Timestamp: tmtime.Now(), + Height: height, + Round: round, + BlockID: blockID, + Timestamp: tmtime.Now(), } } diff --git a/privval/old_file.go b/privval/old_file.go new file mode 100644 index 00000000..ec72c183 --- /dev/null +++ b/privval/old_file.go @@ -0,0 +1,80 @@ +package privval + +import ( + "io/ioutil" + "os" + + "github.com/tendermint/tendermint/crypto" + cmn "github.com/tendermint/tendermint/libs/common" + "github.com/tendermint/tendermint/types" +) + +// OldFilePV is the old version of the FilePV, pre v0.28.0. +type OldFilePV struct { + Address types.Address `json:"address"` + PubKey crypto.PubKey `json:"pub_key"` + LastHeight int64 `json:"last_height"` + LastRound int `json:"last_round"` + LastStep int8 `json:"last_step"` + LastSignature []byte `json:"last_signature,omitempty"` + LastSignBytes cmn.HexBytes `json:"last_signbytes,omitempty"` + PrivKey crypto.PrivKey `json:"priv_key"` + + filePath string +} + +// LoadOldFilePV loads an OldFilePV from the filePath. +func LoadOldFilePV(filePath string) (*OldFilePV, error) { + pvJSONBytes, err := ioutil.ReadFile(filePath) + if err != nil { + return nil, err + } + pv := &OldFilePV{} + err = cdc.UnmarshalJSON(pvJSONBytes, &pv) + if err != nil { + return nil, err + } + + // overwrite pubkey and address for convenience + pv.PubKey = pv.PrivKey.PubKey() + pv.Address = pv.PubKey.Address() + + pv.filePath = filePath + return pv, nil +} + +// Upgrade convets the OldFilePV to the new FilePV, separating the immutable and mutable components, +// and persisting them to the keyFilePath and stateFilePath, respectively. +// It renames the original file by adding ".bak". +func (oldFilePV *OldFilePV) Upgrade(keyFilePath, stateFilePath string) *FilePV { + privKey := oldFilePV.PrivKey + pvKey := FilePVKey{ + PrivKey: privKey, + PubKey: privKey.PubKey(), + Address: privKey.PubKey().Address(), + filePath: keyFilePath, + } + + pvState := FilePVLastSignState{ + Height: oldFilePV.LastHeight, + Round: oldFilePV.LastRound, + Step: oldFilePV.LastStep, + Signature: oldFilePV.LastSignature, + SignBytes: oldFilePV.LastSignBytes, + filePath: stateFilePath, + } + + // Save the new PV files + pv := &FilePV{ + Key: pvKey, + LastSignState: pvState, + } + pv.Save() + + // Rename the old PV file + err := os.Rename(oldFilePV.filePath, oldFilePV.filePath+".bak") + if err != nil { + panic(err) + } + return pv +} diff --git a/privval/old_file_test.go b/privval/old_file_test.go new file mode 100644 index 00000000..46391a3f --- /dev/null +++ b/privval/old_file_test.go @@ -0,0 +1,77 @@ +package privval_test + +import ( + "io/ioutil" + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/tendermint/tendermint/privval" +) + +const oldPrivvalContent = `{ + "address": "1D8089FAFDFAE4A637F3D616E17B92905FA2D91D", + "pub_key": { + "type": "tendermint/PubKeyEd25519", + "value": "r3Yg2AhDZ745CNTpavsGU+mRZ8WpRXqoJuyqjN8mJq0=" + }, + "last_height": "5", + "last_round": "0", + "last_step": 3, + "last_signature": "CTr7b9ZQlrJJf+12rPl5t/YSCUc/KqV7jQogCfFJA24e7hof69X6OMT7eFLVQHyodPjD/QTA298XHV5ejxInDQ==", + "last_signbytes": "750802110500000000000000220B08B398F3E00510F48DA6402A480A20FC258973076512999C3E6839A22E9FBDB1B77CF993E8A9955412A41A59D4CAD312240A20C971B286ACB8AAA6FCA0365EB0A660B189EDC08B46B5AF2995DEFA51A28D215B10013211746573742D636861696E2D533245415533", + "priv_key": { + "type": "tendermint/PrivKeyEd25519", + "value": "7MwvTGEWWjsYwjn2IpRb+GYsWi9nnFsw8jPLLY1UtP6vdiDYCENnvjkI1Olq+wZT6ZFnxalFeqgm7KqM3yYmrQ==" + } +}` + +func TestLoadAndUpgrade(t *testing.T) { + + oldFilePath := initTmpOldFile(t) + defer os.Remove(oldFilePath) + newStateFile, err := ioutil.TempFile("", "priv_validator_state*.json") + defer os.Remove(newStateFile.Name()) + require.NoError(t, err) + newKeyFile, err := ioutil.TempFile("", "priv_validator_key*.json") + defer os.Remove(newKeyFile.Name()) + require.NoError(t, err) + + oldPV, err := privval.LoadOldFilePV(oldFilePath) + assert.NoError(t, err) + newPV := oldPV.Upgrade(newKeyFile.Name(), newStateFile.Name()) + + assertEqualPV(t, oldPV, newPV) + assert.NoError(t, err) + upgradedPV := privval.LoadFilePV(newKeyFile.Name(), newStateFile.Name()) + assertEqualPV(t, oldPV, upgradedPV) + oldPV, err = privval.LoadOldFilePV(oldFilePath + ".bak") + require.NoError(t, err) + assertEqualPV(t, oldPV, upgradedPV) +} + +func assertEqualPV(t *testing.T, oldPV *privval.OldFilePV, newPV *privval.FilePV) { + assert.Equal(t, oldPV.Address, newPV.Key.Address) + assert.Equal(t, oldPV.Address, newPV.GetAddress()) + assert.Equal(t, oldPV.PubKey, newPV.Key.PubKey) + assert.Equal(t, oldPV.PubKey, newPV.GetPubKey()) + assert.Equal(t, oldPV.PrivKey, newPV.Key.PrivKey) + + assert.Equal(t, oldPV.LastHeight, newPV.LastSignState.Height) + assert.Equal(t, oldPV.LastRound, newPV.LastSignState.Round) + assert.Equal(t, oldPV.LastSignature, newPV.LastSignState.Signature) + assert.Equal(t, oldPV.LastSignBytes, newPV.LastSignState.SignBytes) + assert.Equal(t, oldPV.LastStep, newPV.LastSignState.Step) +} + +func initTmpOldFile(t *testing.T) string { + tmpfile, err := ioutil.TempFile("", "priv_validator_*.json") + require.NoError(t, err) + t.Logf("created test file %s", tmpfile.Name()) + _, err = tmpfile.WriteString(oldPrivvalContent) + require.NoError(t, err) + + return tmpfile.Name() +} diff --git a/privval/priv_validator.go b/privval/priv_validator.go deleted file mode 100644 index e606b826..00000000 --- a/privval/priv_validator.go +++ /dev/null @@ -1,356 +0,0 @@ -package privval - -import ( - "bytes" - "errors" - "fmt" - "io/ioutil" - "sync" - "time" - - "github.com/tendermint/tendermint/crypto" - "github.com/tendermint/tendermint/crypto/ed25519" - cmn "github.com/tendermint/tendermint/libs/common" - "github.com/tendermint/tendermint/types" - tmtime "github.com/tendermint/tendermint/types/time" -) - -// TODO: type ? -const ( - stepNone int8 = 0 // Used to distinguish the initial state - stepPropose int8 = 1 - stepPrevote int8 = 2 - stepPrecommit int8 = 3 -) - -func voteToStep(vote *types.Vote) int8 { - switch vote.Type { - case types.VoteTypePrevote: - return stepPrevote - case types.VoteTypePrecommit: - return stepPrecommit - default: - cmn.PanicSanity("Unknown vote type") - return 0 - } -} - -// FilePV implements PrivValidator using data persisted to disk -// to prevent double signing. -// NOTE: the directory containing the pv.filePath must already exist. -// It includes the LastSignature and LastSignBytes so we don't lose the signature -// if the process crashes after signing but before the resulting consensus message is processed. -type FilePV struct { - Address types.Address `json:"address"` - PubKey crypto.PubKey `json:"pub_key"` - LastHeight int64 `json:"last_height"` - LastRound int `json:"last_round"` - LastStep int8 `json:"last_step"` - LastSignature []byte `json:"last_signature,omitempty"` - LastSignBytes cmn.HexBytes `json:"last_signbytes,omitempty"` - PrivKey crypto.PrivKey `json:"priv_key"` - - // For persistence. - // Overloaded for testing. - filePath string - mtx sync.Mutex -} - -// GetAddress returns the address of the validator. -// Implements PrivValidator. -func (pv *FilePV) GetAddress() types.Address { - return pv.Address -} - -// GetPubKey returns the public key of the validator. -// Implements PrivValidator. -func (pv *FilePV) GetPubKey() crypto.PubKey { - return pv.PubKey -} - -// GenFilePV generates a new validator with randomly generated private key -// and sets the filePath, but does not call Save(). -func GenFilePV(filePath string) *FilePV { - privKey := ed25519.GenPrivKey() - return &FilePV{ - Address: privKey.PubKey().Address(), - PubKey: privKey.PubKey(), - PrivKey: privKey, - LastStep: stepNone, - filePath: filePath, - } -} - -// LoadFilePV loads a FilePV from the filePath. The FilePV handles double -// signing prevention by persisting data to the filePath. If the filePath does -// not exist, the FilePV must be created manually and saved. -func LoadFilePV(filePath string) *FilePV { - pvJSONBytes, err := ioutil.ReadFile(filePath) - if err != nil { - cmn.Exit(err.Error()) - } - pv := &FilePV{} - err = cdc.UnmarshalJSON(pvJSONBytes, &pv) - if err != nil { - cmn.Exit(fmt.Sprintf("Error reading PrivValidator from %v: %v\n", filePath, err)) - } - - // overwrite pubkey and address for convenience - pv.PubKey = pv.PrivKey.PubKey() - pv.Address = pv.PubKey.Address() - - pv.filePath = filePath - return pv -} - -// LoadOrGenFilePV loads a FilePV from the given filePath -// or else generates a new one and saves it to the filePath. -func LoadOrGenFilePV(filePath string) *FilePV { - var pv *FilePV - if cmn.FileExists(filePath) { - pv = LoadFilePV(filePath) - } else { - pv = GenFilePV(filePath) - pv.Save() - } - return pv -} - -// Save persists the FilePV to disk. -func (pv *FilePV) Save() { - pv.mtx.Lock() - defer pv.mtx.Unlock() - pv.save() -} - -func (pv *FilePV) save() { - outFile := pv.filePath - if outFile == "" { - panic("Cannot save PrivValidator: filePath not set") - } - jsonBytes, err := cdc.MarshalJSONIndent(pv, "", " ") - if err != nil { - panic(err) - } - err = cmn.WriteFileAtomic(outFile, jsonBytes, 0600) - if err != nil { - panic(err) - } -} - -// Reset resets all fields in the FilePV. -// NOTE: Unsafe! -func (pv *FilePV) Reset() { - var sig []byte - pv.LastHeight = 0 - pv.LastRound = 0 - pv.LastStep = 0 - pv.LastSignature = sig - pv.LastSignBytes = nil - pv.Save() -} - -// SignVote signs a canonical representation of the vote, along with the -// chainID. Implements PrivValidator. -func (pv *FilePV) SignVote(chainID string, vote *types.Vote) error { - pv.mtx.Lock() - defer pv.mtx.Unlock() - if err := pv.signVote(chainID, vote); err != nil { - return fmt.Errorf("Error signing vote: %v", err) - } - return nil -} - -// SignProposal signs a canonical representation of the proposal, along with -// the chainID. Implements PrivValidator. -func (pv *FilePV) SignProposal(chainID string, proposal *types.Proposal) error { - pv.mtx.Lock() - defer pv.mtx.Unlock() - if err := pv.signProposal(chainID, proposal); err != nil { - return fmt.Errorf("Error signing proposal: %v", err) - } - return nil -} - -// returns error if HRS regression or no LastSignBytes. returns true if HRS is unchanged -func (pv *FilePV) checkHRS(height int64, round int, step int8) (bool, error) { - if pv.LastHeight > height { - return false, errors.New("Height regression") - } - - if pv.LastHeight == height { - if pv.LastRound > round { - return false, errors.New("Round regression") - } - - if pv.LastRound == round { - if pv.LastStep > step { - return false, errors.New("Step regression") - } else if pv.LastStep == step { - if pv.LastSignBytes != nil { - if pv.LastSignature == nil { - panic("pv: LastSignature is nil but LastSignBytes is not!") - } - return true, nil - } - return false, errors.New("No LastSignature found") - } - } - } - return false, nil -} - -// signVote checks if the vote is good to sign and sets the vote signature. -// It may need to set the timestamp as well if the vote is otherwise the same as -// a previously signed vote (ie. we crashed after signing but before the vote hit the WAL). -func (pv *FilePV) signVote(chainID string, vote *types.Vote) error { - height, round, step := vote.Height, vote.Round, voteToStep(vote) - signBytes := vote.SignBytes(chainID) - - sameHRS, err := pv.checkHRS(height, round, step) - if err != nil { - return err - } - - // We might crash before writing to the wal, - // causing us to try to re-sign for the same HRS. - // If signbytes are the same, use the last signature. - // If they only differ by timestamp, use last timestamp and signature - // Otherwise, return error - if sameHRS { - if bytes.Equal(signBytes, pv.LastSignBytes) { - vote.Signature = pv.LastSignature - } else if timestamp, ok := checkVotesOnlyDifferByTimestamp(pv.LastSignBytes, signBytes); ok { - vote.Timestamp = timestamp - vote.Signature = pv.LastSignature - } else { - err = fmt.Errorf("Conflicting data") - } - return err - } - - // It passed the checks. Sign the vote - sig, err := pv.PrivKey.Sign(signBytes) - if err != nil { - return err - } - pv.saveSigned(height, round, step, signBytes, sig) - vote.Signature = sig - return nil -} - -// signProposal checks if the proposal is good to sign and sets the proposal signature. -// It may need to set the timestamp as well if the proposal is otherwise the same as -// a previously signed proposal ie. we crashed after signing but before the proposal hit the WAL). -func (pv *FilePV) signProposal(chainID string, proposal *types.Proposal) error { - height, round, step := proposal.Height, proposal.Round, stepPropose - signBytes := proposal.SignBytes(chainID) - - sameHRS, err := pv.checkHRS(height, round, step) - if err != nil { - return err - } - - // We might crash before writing to the wal, - // causing us to try to re-sign for the same HRS. - // If signbytes are the same, use the last signature. - // If they only differ by timestamp, use last timestamp and signature - // Otherwise, return error - if sameHRS { - if bytes.Equal(signBytes, pv.LastSignBytes) { - proposal.Signature = pv.LastSignature - } else if timestamp, ok := checkProposalsOnlyDifferByTimestamp(pv.LastSignBytes, signBytes); ok { - proposal.Timestamp = timestamp - proposal.Signature = pv.LastSignature - } else { - err = fmt.Errorf("Conflicting data") - } - return err - } - - // It passed the checks. Sign the proposal - sig, err := pv.PrivKey.Sign(signBytes) - if err != nil { - return err - } - pv.saveSigned(height, round, step, signBytes, sig) - proposal.Signature = sig - return nil -} - -// Persist height/round/step and signature -func (pv *FilePV) saveSigned(height int64, round int, step int8, - signBytes []byte, sig []byte) { - - pv.LastHeight = height - pv.LastRound = round - pv.LastStep = step - pv.LastSignature = sig - pv.LastSignBytes = signBytes - pv.save() -} - -// SignHeartbeat signs a canonical representation of the heartbeat, along with the chainID. -// Implements PrivValidator. -func (pv *FilePV) SignHeartbeat(chainID string, heartbeat *types.Heartbeat) error { - pv.mtx.Lock() - defer pv.mtx.Unlock() - sig, err := pv.PrivKey.Sign(heartbeat.SignBytes(chainID)) - if err != nil { - return err - } - heartbeat.Signature = sig - return nil -} - -// String returns a string representation of the FilePV. -func (pv *FilePV) String() string { - return fmt.Sprintf("PrivValidator{%v LH:%v, LR:%v, LS:%v}", pv.GetAddress(), pv.LastHeight, pv.LastRound, pv.LastStep) -} - -//------------------------------------- - -// returns the timestamp from the lastSignBytes. -// returns true if the only difference in the votes is their timestamp. -func checkVotesOnlyDifferByTimestamp(lastSignBytes, newSignBytes []byte) (time.Time, bool) { - var lastVote, newVote types.CanonicalVote - if err := cdc.UnmarshalBinary(lastSignBytes, &lastVote); err != nil { - panic(fmt.Sprintf("LastSignBytes cannot be unmarshalled into vote: %v", err)) - } - if err := cdc.UnmarshalBinary(newSignBytes, &newVote); err != nil { - panic(fmt.Sprintf("signBytes cannot be unmarshalled into vote: %v", err)) - } - - lastTime := lastVote.Timestamp - - // set the times to the same value and check equality - now := tmtime.Now() - lastVote.Timestamp = now - newVote.Timestamp = now - lastVoteBytes, _ := cdc.MarshalJSON(lastVote) - newVoteBytes, _ := cdc.MarshalJSON(newVote) - - return lastTime, bytes.Equal(newVoteBytes, lastVoteBytes) -} - -// returns the timestamp from the lastSignBytes. -// returns true if the only difference in the proposals is their timestamp -func checkProposalsOnlyDifferByTimestamp(lastSignBytes, newSignBytes []byte) (time.Time, bool) { - var lastProposal, newProposal types.CanonicalProposal - if err := cdc.UnmarshalBinary(lastSignBytes, &lastProposal); err != nil { - panic(fmt.Sprintf("LastSignBytes cannot be unmarshalled into proposal: %v", err)) - } - if err := cdc.UnmarshalBinary(newSignBytes, &newProposal); err != nil { - panic(fmt.Sprintf("signBytes cannot be unmarshalled into proposal: %v", err)) - } - - lastTime := lastProposal.Timestamp - // set the times to the same value and check equality - now := tmtime.Now() - lastProposal.Timestamp = now - newProposal.Timestamp = now - lastProposalBytes, _ := cdc.MarshalBinary(lastProposal) - newProposalBytes, _ := cdc.MarshalBinary(newProposal) - - return lastTime, bytes.Equal(newProposalBytes, lastProposalBytes) -} diff --git a/privval/remote_signer.go b/privval/remote_signer.go new file mode 100644 index 00000000..a5b8cac6 --- /dev/null +++ b/privval/remote_signer.go @@ -0,0 +1,275 @@ +package privval + +import ( + "fmt" + "io" + "net" + + "github.com/pkg/errors" + + amino "github.com/tendermint/go-amino" + "github.com/tendermint/tendermint/crypto" + cmn "github.com/tendermint/tendermint/libs/common" + "github.com/tendermint/tendermint/types" +) + +// Socket errors. +var ( + ErrConnTimeout = errors.New("remote signer timed out") +) + +// RemoteSignerClient implements PrivValidator. +// It uses a net.Conn to request signatures +// from an external process. +type RemoteSignerClient struct { + conn net.Conn + + // memoized + consensusPubKey crypto.PubKey +} + +// Check that RemoteSignerClient implements PrivValidator. +var _ types.PrivValidator = (*RemoteSignerClient)(nil) + +// NewRemoteSignerClient returns an instance of RemoteSignerClient. +func NewRemoteSignerClient(conn net.Conn) (*RemoteSignerClient, error) { + + // retrieve and memoize the consensus public key once. + pubKey, err := getPubKey(conn) + if err != nil { + return nil, cmn.ErrorWrap(err, "error while retrieving public key for remote signer") + } + return &RemoteSignerClient{ + conn: conn, + consensusPubKey: pubKey, + }, nil +} + +// Close calls Close on the underlying net.Conn. +func (sc *RemoteSignerClient) Close() error { + return sc.conn.Close() +} + +// GetPubKey implements PrivValidator. +func (sc *RemoteSignerClient) GetPubKey() crypto.PubKey { + return sc.consensusPubKey +} + +// not thread-safe (only called on startup). +func getPubKey(conn net.Conn) (crypto.PubKey, error) { + err := writeMsg(conn, &PubKeyRequest{}) + if err != nil { + return nil, err + } + + res, err := readMsg(conn) + if err != nil { + return nil, err + } + pubKeyResp, ok := res.(*PubKeyResponse) + if !ok { + return nil, errors.Wrap(ErrUnexpectedResponse, "response is not PubKeyResponse") + } + + if pubKeyResp.Error != nil { + return nil, errors.Wrap(pubKeyResp.Error, "failed to get private validator's public key") + } + + return pubKeyResp.PubKey, nil +} + +// SignVote implements PrivValidator. +func (sc *RemoteSignerClient) SignVote(chainID string, vote *types.Vote) error { + err := writeMsg(sc.conn, &SignVoteRequest{Vote: vote}) + if err != nil { + return err + } + + res, err := readMsg(sc.conn) + if err != nil { + return err + } + + resp, ok := res.(*SignedVoteResponse) + if !ok { + return ErrUnexpectedResponse + } + if resp.Error != nil { + return resp.Error + } + *vote = *resp.Vote + + return nil +} + +// SignProposal implements PrivValidator. +func (sc *RemoteSignerClient) SignProposal( + chainID string, + proposal *types.Proposal, +) error { + err := writeMsg(sc.conn, &SignProposalRequest{Proposal: proposal}) + if err != nil { + return err + } + + res, err := readMsg(sc.conn) + if err != nil { + return err + } + resp, ok := res.(*SignedProposalResponse) + if !ok { + return ErrUnexpectedResponse + } + if resp.Error != nil { + return resp.Error + } + *proposal = *resp.Proposal + + return nil +} + +// Ping is used to check connection health. +func (sc *RemoteSignerClient) Ping() error { + err := writeMsg(sc.conn, &PingRequest{}) + if err != nil { + return err + } + + res, err := readMsg(sc.conn) + if err != nil { + return err + } + _, ok := res.(*PingResponse) + if !ok { + return ErrUnexpectedResponse + } + + return nil +} + +// RemoteSignerMsg is sent between RemoteSigner and the RemoteSigner client. +type RemoteSignerMsg interface{} + +func RegisterRemoteSignerMsg(cdc *amino.Codec) { + cdc.RegisterInterface((*RemoteSignerMsg)(nil), nil) + cdc.RegisterConcrete(&PubKeyRequest{}, "tendermint/remotesigner/PubKeyRequest", nil) + cdc.RegisterConcrete(&PubKeyResponse{}, "tendermint/remotesigner/PubKeyResponse", nil) + cdc.RegisterConcrete(&SignVoteRequest{}, "tendermint/remotesigner/SignVoteRequest", nil) + cdc.RegisterConcrete(&SignedVoteResponse{}, "tendermint/remotesigner/SignedVoteResponse", nil) + cdc.RegisterConcrete(&SignProposalRequest{}, "tendermint/remotesigner/SignProposalRequest", nil) + cdc.RegisterConcrete(&SignedProposalResponse{}, "tendermint/remotesigner/SignedProposalResponse", nil) + cdc.RegisterConcrete(&PingRequest{}, "tendermint/remotesigner/PingRequest", nil) + cdc.RegisterConcrete(&PingResponse{}, "tendermint/remotesigner/PingResponse", nil) +} + +// PubKeyRequest requests the consensus public key from the remote signer. +type PubKeyRequest struct{} + +// PubKeyResponse is a PrivValidatorSocket message containing the public key. +type PubKeyResponse struct { + PubKey crypto.PubKey + Error *RemoteSignerError +} + +// SignVoteRequest is a PrivValidatorSocket message containing a vote. +type SignVoteRequest struct { + Vote *types.Vote +} + +// SignedVoteResponse is a PrivValidatorSocket message containing a signed vote along with a potenial error message. +type SignedVoteResponse struct { + Vote *types.Vote + Error *RemoteSignerError +} + +// SignProposalRequest is a PrivValidatorSocket message containing a Proposal. +type SignProposalRequest struct { + Proposal *types.Proposal +} + +type SignedProposalResponse struct { + Proposal *types.Proposal + Error *RemoteSignerError +} + +// PingRequest is a PrivValidatorSocket message to keep the connection alive. +type PingRequest struct { +} + +type PingResponse struct { +} + +// RemoteSignerError allows (remote) validators to include meaningful error descriptions in their reply. +type RemoteSignerError struct { + // TODO(ismail): create an enum of known errors + Code int + Description string +} + +func (e *RemoteSignerError) Error() string { + return fmt.Sprintf("RemoteSigner returned error #%d: %s", e.Code, e.Description) +} + +func readMsg(r io.Reader) (msg RemoteSignerMsg, err error) { + const maxRemoteSignerMsgSize = 1024 * 10 + _, err = cdc.UnmarshalBinaryLengthPrefixedReader(r, &msg, maxRemoteSignerMsgSize) + if _, ok := err.(timeoutError); ok { + err = cmn.ErrorWrap(ErrConnTimeout, err.Error()) + } + return +} + +func writeMsg(w io.Writer, msg interface{}) (err error) { + _, err = cdc.MarshalBinaryLengthPrefixedWriter(w, msg) + if _, ok := err.(timeoutError); ok { + err = cmn.ErrorWrap(ErrConnTimeout, err.Error()) + } + return +} + +func handleRequest(req RemoteSignerMsg, chainID string, privVal types.PrivValidator) (RemoteSignerMsg, error) { + var res RemoteSignerMsg + var err error + + switch r := req.(type) { + case *PubKeyRequest: + var p crypto.PubKey + p = privVal.GetPubKey() + res = &PubKeyResponse{p, nil} + case *SignVoteRequest: + err = privVal.SignVote(chainID, r.Vote) + if err != nil { + res = &SignedVoteResponse{nil, &RemoteSignerError{0, err.Error()}} + } else { + res = &SignedVoteResponse{r.Vote, nil} + } + case *SignProposalRequest: + err = privVal.SignProposal(chainID, r.Proposal) + if err != nil { + res = &SignedProposalResponse{nil, &RemoteSignerError{0, err.Error()}} + } else { + res = &SignedProposalResponse{r.Proposal, nil} + } + case *PingRequest: + res = &PingResponse{} + default: + err = fmt.Errorf("unknown msg: %v", r) + } + + return res, err +} + +// IsConnTimeout returns a boolean indicating whether the error is known to +// report that a connection timeout occurred. This detects both fundamental +// network timeouts, as well as ErrConnTimeout errors. +func IsConnTimeout(err error) bool { + if cmnErr, ok := err.(cmn.Error); ok { + if cmnErr.Data() == ErrConnTimeout { + return true + } + } + if _, ok := err.(timeoutError); ok { + return true + } + return false +} diff --git a/privval/remote_signer_test.go b/privval/remote_signer_test.go new file mode 100644 index 00000000..cb2a600d --- /dev/null +++ b/privval/remote_signer_test.go @@ -0,0 +1,90 @@ +package privval + +import ( + "net" + "testing" + "time" + + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/tendermint/tendermint/crypto/ed25519" + cmn "github.com/tendermint/tendermint/libs/common" + "github.com/tendermint/tendermint/libs/log" + "github.com/tendermint/tendermint/types" +) + +// TestRemoteSignerRetryTCPOnly will test connection retry attempts over TCP. We +// don't need this for Unix sockets because the OS instantly knows the state of +// both ends of the socket connection. This basically causes the +// RemoteSigner.dialer() call inside RemoteSigner.connect() to return +// successfully immediately, putting an instant stop to any retry attempts. +func TestRemoteSignerRetryTCPOnly(t *testing.T) { + var ( + attemptc = make(chan int) + retries = 2 + ) + + ln, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + + go func(ln net.Listener, attemptc chan<- int) { + attempts := 0 + + for { + conn, err := ln.Accept() + require.NoError(t, err) + + err = conn.Close() + require.NoError(t, err) + + attempts++ + + if attempts == retries { + attemptc <- attempts + break + } + } + }(ln, attemptc) + + rs := NewRemoteSigner( + log.TestingLogger(), + cmn.RandStr(12), + types.NewMockPV(), + DialTCPFn(ln.Addr().String(), testConnDeadline, ed25519.GenPrivKey()), + ) + defer rs.Stop() + + RemoteSignerConnDeadline(time.Millisecond)(rs) + RemoteSignerConnRetries(retries)(rs) + + assert.Equal(t, rs.Start(), ErrDialRetryMax) + + select { + case attempts := <-attemptc: + assert.Equal(t, retries, attempts) + case <-time.After(100 * time.Millisecond): + t.Error("expected remote to observe connection attempts") + } +} + +func TestIsConnTimeoutForFundamentalTimeouts(t *testing.T) { + // Generate a networking timeout + dialer := DialTCPFn(testFreeTCPAddr(t), time.Millisecond, ed25519.GenPrivKey()) + _, err := dialer() + assert.Error(t, err) + assert.True(t, IsConnTimeout(err)) +} + +func TestIsConnTimeoutForWrappedConnTimeouts(t *testing.T) { + dialer := DialTCPFn(testFreeTCPAddr(t), time.Millisecond, ed25519.GenPrivKey()) + _, err := dialer() + assert.Error(t, err) + err = cmn.ErrorWrap(ErrConnTimeout, err.Error()) + assert.True(t, IsConnTimeout(err)) +} + +func TestIsConnTimeoutForNonTimeoutErrors(t *testing.T) { + assert.False(t, IsConnTimeout(cmn.ErrorWrap(ErrDialRetryMax, "max retries exceeded"))) + assert.False(t, IsConnTimeout(errors.New("completely irrelevant error"))) +} diff --git a/privval/server.go b/privval/server.go new file mode 100644 index 00000000..8b22c69e --- /dev/null +++ b/privval/server.go @@ -0,0 +1,168 @@ +package privval + +import ( + "io" + "net" + "time" + + "github.com/pkg/errors" + "github.com/tendermint/tendermint/crypto/ed25519" + cmn "github.com/tendermint/tendermint/libs/common" + "github.com/tendermint/tendermint/libs/log" + p2pconn "github.com/tendermint/tendermint/p2p/conn" + "github.com/tendermint/tendermint/types" +) + +// Socket errors. +var ( + ErrDialRetryMax = errors.New("dialed maximum retries") +) + +// RemoteSignerOption sets an optional parameter on the RemoteSigner. +type RemoteSignerOption func(*RemoteSigner) + +// RemoteSignerConnDeadline sets the read and write deadline for connections +// from external signing processes. +func RemoteSignerConnDeadline(deadline time.Duration) RemoteSignerOption { + return func(ss *RemoteSigner) { ss.connDeadline = deadline } +} + +// RemoteSignerConnRetries sets the amount of attempted retries to connect. +func RemoteSignerConnRetries(retries int) RemoteSignerOption { + return func(ss *RemoteSigner) { ss.connRetries = retries } +} + +// RemoteSigner dials using its dialer and responds to any +// signature requests using its privVal. +type RemoteSigner struct { + cmn.BaseService + + chainID string + connDeadline time.Duration + connRetries int + privVal types.PrivValidator + + dialer Dialer + conn net.Conn +} + +// Dialer dials a remote address and returns a net.Conn or an error. +type Dialer func() (net.Conn, error) + +// DialTCPFn dials the given tcp addr, using the given connTimeout and privKey for the +// authenticated encryption handshake. +func DialTCPFn(addr string, connTimeout time.Duration, privKey ed25519.PrivKeyEd25519) Dialer { + return func() (net.Conn, error) { + conn, err := cmn.Connect(addr) + if err == nil { + err = conn.SetDeadline(time.Now().Add(connTimeout)) + } + if err == nil { + conn, err = p2pconn.MakeSecretConnection(conn, privKey) + } + return conn, err + } +} + +// DialUnixFn dials the given unix socket. +func DialUnixFn(addr string) Dialer { + return func() (net.Conn, error) { + unixAddr := &net.UnixAddr{addr, "unix"} + return net.DialUnix("unix", nil, unixAddr) + } +} + +// NewRemoteSigner return a RemoteSigner that will dial using the given +// dialer and respond to any signature requests over the connection +// using the given privVal. +func NewRemoteSigner( + logger log.Logger, + chainID string, + privVal types.PrivValidator, + dialer Dialer, +) *RemoteSigner { + rs := &RemoteSigner{ + chainID: chainID, + connDeadline: time.Second * defaultConnDeadlineSeconds, + connRetries: defaultDialRetries, + privVal: privVal, + dialer: dialer, + } + + rs.BaseService = *cmn.NewBaseService(logger, "RemoteSigner", rs) + return rs +} + +// OnStart implements cmn.Service. +func (rs *RemoteSigner) OnStart() error { + conn, err := rs.connect() + if err != nil { + rs.Logger.Error("OnStart", "err", err) + return err + } + rs.conn = conn + + go rs.handleConnection(conn) + + return nil +} + +// OnStop implements cmn.Service. +func (rs *RemoteSigner) OnStop() { + if rs.conn == nil { + return + } + + if err := rs.conn.Close(); err != nil { + rs.Logger.Error("OnStop", "err", cmn.ErrorWrap(err, "closing listener failed")) + } +} + +func (rs *RemoteSigner) connect() (net.Conn, error) { + for retries := rs.connRetries; retries > 0; retries-- { + // Don't sleep if it is the first retry. + if retries != rs.connRetries { + time.Sleep(rs.connDeadline) + } + conn, err := rs.dialer() + if err != nil { + rs.Logger.Error("dialing", "err", err) + continue + } + return conn, nil + } + + return nil, ErrDialRetryMax +} + +func (rs *RemoteSigner) handleConnection(conn net.Conn) { + for { + if !rs.IsRunning() { + return // Ignore error from listener closing. + } + + // Reset the connection deadline + conn.SetDeadline(time.Now().Add(rs.connDeadline)) + + req, err := readMsg(conn) + if err != nil { + if err != io.EOF { + rs.Logger.Error("handleConnection readMsg", "err", err) + } + return + } + + res, err := handleRequest(req, rs.chainID, rs.privVal) + + if err != nil { + // only log the error; we'll reply with an error in res + rs.Logger.Error("handleConnection handleRequest", "err", err) + } + + err = writeMsg(conn, res) + if err != nil { + rs.Logger.Error("handleConnection writeMsg", "err", err) + return + } + } +} diff --git a/privval/socket.go b/privval/socket.go index da95f8fb..bd9cd920 100644 --- a/privval/socket.go +++ b/privval/socket.go @@ -1,603 +1,184 @@ package privval import ( - "errors" - "fmt" - "io" "net" "time" - "github.com/tendermint/go-amino" - - "github.com/tendermint/tendermint/crypto" "github.com/tendermint/tendermint/crypto/ed25519" - cmn "github.com/tendermint/tendermint/libs/common" - "github.com/tendermint/tendermint/libs/log" p2pconn "github.com/tendermint/tendermint/p2p/conn" - "github.com/tendermint/tendermint/types" ) const ( defaultAcceptDeadlineSeconds = 3 defaultConnDeadlineSeconds = 3 - defaultConnHeartBeatSeconds = 30 - defaultConnWaitSeconds = 60 - defaultDialRetries = 10 ) -// Socket errors. -var ( - ErrDialRetryMax = errors.New("dialed maximum retries") - ErrConnWaitTimeout = errors.New("waited for remote signer for too long") - ErrConnTimeout = errors.New("remote signer timed out") - ErrUnexpectedResponse = errors.New("received unexpected response") -) +// timeoutError can be used to check if an error returned from the netp package +// was due to a timeout. +type timeoutError interface { + Timeout() bool +} -var ( - acceptDeadline = time.Second * defaultAcceptDeadlineSeconds - connDeadline = time.Second * defaultConnDeadlineSeconds - connHeartbeat = time.Second * defaultConnHeartBeatSeconds -) +//------------------------------------------------------------------ +// TCP Listener -// SocketPVOption sets an optional parameter on the SocketPV. -type SocketPVOption func(*SocketPV) +// TCPListenerOption sets an optional parameter on the tcpListener. +type TCPListenerOption func(*tcpListener) -// SocketPVAcceptDeadline sets the deadline for the SocketPV listener. +// TCPListenerAcceptDeadline sets the deadline for the listener. // A zero time value disables the deadline. -func SocketPVAcceptDeadline(deadline time.Duration) SocketPVOption { - return func(sc *SocketPV) { sc.acceptDeadline = deadline } +func TCPListenerAcceptDeadline(deadline time.Duration) TCPListenerOption { + return func(tl *tcpListener) { tl.acceptDeadline = deadline } } -// SocketPVConnDeadline sets the read and write deadline for connections +// TCPListenerConnDeadline sets the read and write deadline for connections // from external signing processes. -func SocketPVConnDeadline(deadline time.Duration) SocketPVOption { - return func(sc *SocketPV) { sc.connDeadline = deadline } +func TCPListenerConnDeadline(deadline time.Duration) TCPListenerOption { + return func(tl *tcpListener) { tl.connDeadline = deadline } } -// SocketPVHeartbeat sets the period on which to check the liveness of the -// connected Signer connections. -func SocketPVHeartbeat(period time.Duration) SocketPVOption { - return func(sc *SocketPV) { sc.connHeartbeat = period } +// tcpListener implements net.Listener. +var _ net.Listener = (*tcpListener)(nil) + +// tcpListener wraps a *net.TCPListener to standardise protocol timeouts +// and potentially other tuning parameters. It also returns encrypted connections. +type tcpListener struct { + *net.TCPListener + + secretConnKey ed25519.PrivKeyEd25519 + + acceptDeadline time.Duration + connDeadline time.Duration } -// SocketPVConnWait sets the timeout duration before connection of external -// signing processes are considered to be unsuccessful. -func SocketPVConnWait(timeout time.Duration) SocketPVOption { - return func(sc *SocketPV) { sc.connWaitTimeout = timeout } -} - -// SocketPV implements PrivValidator, it uses a socket to request signatures -// from an external process. -type SocketPV struct { - cmn.BaseService - - addr string - acceptDeadline time.Duration - connDeadline time.Duration - connHeartbeat time.Duration - connWaitTimeout time.Duration - privKey ed25519.PrivKeyEd25519 - - conn net.Conn - listener net.Listener -} - -// Check that SocketPV implements PrivValidator. -var _ types.PrivValidator = (*SocketPV)(nil) - -// NewSocketPV returns an instance of SocketPV. -func NewSocketPV( - logger log.Logger, - socketAddr string, - privKey ed25519.PrivKeyEd25519, -) *SocketPV { - sc := &SocketPV{ - addr: socketAddr, - acceptDeadline: acceptDeadline, - connDeadline: connDeadline, - connHeartbeat: connHeartbeat, - connWaitTimeout: time.Second * defaultConnWaitSeconds, - privKey: privKey, +// NewTCPListener returns a listener that accepts authenticated encrypted connections +// using the given secretConnKey and the default timeout values. +func NewTCPListener(ln net.Listener, secretConnKey ed25519.PrivKeyEd25519) *tcpListener { + return &tcpListener{ + TCPListener: ln.(*net.TCPListener), + secretConnKey: secretConnKey, + acceptDeadline: time.Second * defaultAcceptDeadlineSeconds, + connDeadline: time.Second * defaultConnDeadlineSeconds, } - - sc.BaseService = *cmn.NewBaseService(logger, "SocketPV", sc) - - return sc } -// GetAddress implements PrivValidator. -func (sc *SocketPV) GetAddress() types.Address { - addr, err := sc.getAddress() - if err != nil { - panic(err) - } - - return addr -} - -// Address is an alias for PubKey().Address(). -func (sc *SocketPV) getAddress() (cmn.HexBytes, error) { - p, err := sc.getPubKey() +// Accept implements net.Listener. +func (ln *tcpListener) Accept() (net.Conn, error) { + err := ln.SetDeadline(time.Now().Add(ln.acceptDeadline)) if err != nil { return nil, err } - return p.Address(), nil -} - -// GetPubKey implements PrivValidator. -func (sc *SocketPV) GetPubKey() crypto.PubKey { - pubKey, err := sc.getPubKey() - if err != nil { - panic(err) - } - - return pubKey -} - -func (sc *SocketPV) getPubKey() (crypto.PubKey, error) { - err := writeMsg(sc.conn, &PubKeyMsg{}) + tc, err := ln.AcceptTCP() if err != nil { return nil, err } - res, err := readMsg(sc.conn) + // Wrap the conn in our timeout and encryption wrappers + timeoutConn := newTimeoutConn(tc, ln.connDeadline) + secretConn, err := p2pconn.MakeSecretConnection(timeoutConn, ln.secretConnKey) if err != nil { return nil, err } - return res.(*PubKeyMsg).PubKey, nil + return secretConn, nil } -// SignVote implements PrivValidator. -func (sc *SocketPV) SignVote(chainID string, vote *types.Vote) error { - err := writeMsg(sc.conn, &SignVoteRequest{Vote: vote}) - if err != nil { - return err - } +//------------------------------------------------------------------ +// Unix Listener - res, err := readMsg(sc.conn) - if err != nil { - return err - } +// unixListener implements net.Listener. +var _ net.Listener = (*unixListener)(nil) - resp, ok := res.(*SignedVoteResponse) - if !ok { - return ErrUnexpectedResponse - } - if resp.Error != nil { - return fmt.Errorf("remote error occurred: code: %v, description: %s", - resp.Error.Code, - resp.Error.Description) - } - *vote = *resp.Vote +type UnixListenerOption func(*unixListener) - return nil +// UnixListenerAcceptDeadline sets the deadline for the listener. +// A zero time value disables the deadline. +func UnixListenerAcceptDeadline(deadline time.Duration) UnixListenerOption { + return func(ul *unixListener) { ul.acceptDeadline = deadline } } -// SignProposal implements PrivValidator. -func (sc *SocketPV) SignProposal( - chainID string, - proposal *types.Proposal, -) error { - err := writeMsg(sc.conn, &SignProposalRequest{Proposal: proposal}) - if err != nil { - return err - } - - res, err := readMsg(sc.conn) - if err != nil { - return err - } - resp, ok := res.(*SignedProposalResponse) - if !ok { - return ErrUnexpectedResponse - } - if resp.Error != nil { - return fmt.Errorf("remote error occurred: code: %v, description: %s", - resp.Error.Code, - resp.Error.Description) - } - *proposal = *resp.Proposal - - return nil +// UnixListenerConnDeadline sets the read and write deadline for connections +// from external signing processes. +func UnixListenerConnDeadline(deadline time.Duration) UnixListenerOption { + return func(ul *unixListener) { ul.connDeadline = deadline } } -// SignHeartbeat implements PrivValidator. -func (sc *SocketPV) SignHeartbeat( - chainID string, - heartbeat *types.Heartbeat, -) error { - err := writeMsg(sc.conn, &SignHeartbeatRequest{Heartbeat: heartbeat}) - if err != nil { - return err - } +// unixListener wraps a *net.UnixListener to standardise protocol timeouts +// and potentially other tuning parameters. It returns unencrypted connections. +type unixListener struct { + *net.UnixListener - res, err := readMsg(sc.conn) - if err != nil { - return err - } - resp, ok := res.(*SignedHeartbeatResponse) - if !ok { - return ErrUnexpectedResponse - } - if resp.Error != nil { - return fmt.Errorf("remote error occurred: code: %v, description: %s", - resp.Error.Code, - resp.Error.Description) - } - *heartbeat = *resp.Heartbeat - - return nil + acceptDeadline time.Duration + connDeadline time.Duration } -// OnStart implements cmn.Service. -func (sc *SocketPV) OnStart() error { - if err := sc.listen(); err != nil { - err = cmn.ErrorWrap(err, "failed to listen") - sc.Logger.Error( - "OnStart", - "err", err, - ) - return err - } - - conn, err := sc.waitConnection() - if err != nil { - err = cmn.ErrorWrap(err, "failed to accept connection") - sc.Logger.Error( - "OnStart", - "err", err, - ) - - return err - } - - sc.conn = conn - - return nil -} - -// OnStop implements cmn.Service. -func (sc *SocketPV) OnStop() { - if sc.conn != nil { - if err := sc.conn.Close(); err != nil { - err = cmn.ErrorWrap(err, "failed to close connection") - sc.Logger.Error( - "OnStop", - "err", err, - ) - } - } - - if sc.listener != nil { - if err := sc.listener.Close(); err != nil { - err = cmn.ErrorWrap(err, "failed to close listener") - sc.Logger.Error( - "OnStop", - "err", err, - ) - } +// NewUnixListener returns a listener that accepts unencrypted connections +// using the default timeout values. +func NewUnixListener(ln net.Listener) *unixListener { + return &unixListener{ + UnixListener: ln.(*net.UnixListener), + acceptDeadline: time.Second * defaultAcceptDeadlineSeconds, + connDeadline: time.Second * defaultConnDeadlineSeconds, } } -func (sc *SocketPV) acceptConnection() (net.Conn, error) { - conn, err := sc.listener.Accept() - if err != nil { - if !sc.IsRunning() { - return nil, nil // Ignore error from listener closing. - } - return nil, err - - } - - conn, err = p2pconn.MakeSecretConnection(conn, sc.privKey) +// Accept implements net.Listener. +func (ln *unixListener) Accept() (net.Conn, error) { + err := ln.SetDeadline(time.Now().Add(ln.acceptDeadline)) if err != nil { return nil, err } + tc, err := ln.AcceptUnix() + if err != nil { + return nil, err + } + + // Wrap the conn in our timeout wrapper + conn := newTimeoutConn(tc, ln.connDeadline) + + // TODO: wrap in something that authenticates + // with a MAC - https://github.com/tendermint/tendermint/issues/3099 + return conn, nil } -func (sc *SocketPV) listen() error { - ln, err := net.Listen(cmn.ProtocolAndAddress(sc.addr)) - if err != nil { - return err - } +//------------------------------------------------------------------ +// Connection - sc.listener = newTCPTimeoutListener( - ln, - sc.acceptDeadline, - sc.connDeadline, - sc.connHeartbeat, - ) +// timeoutConn implements net.Conn. +var _ net.Conn = (*timeoutConn)(nil) - return nil -} +// timeoutConn wraps a net.Conn to standardise protocol timeouts / deadline resets. +type timeoutConn struct { + net.Conn -// waitConnection uses the configured wait timeout to error if no external -// process connects in the time period. -func (sc *SocketPV) waitConnection() (net.Conn, error) { - var ( - connc = make(chan net.Conn, 1) - errc = make(chan error, 1) - ) - - go func(connc chan<- net.Conn, errc chan<- error) { - conn, err := sc.acceptConnection() - if err != nil { - errc <- err - return - } - - connc <- conn - }(connc, errc) - - select { - case conn := <-connc: - return conn, nil - case err := <-errc: - if _, ok := err.(timeoutError); ok { - return nil, cmn.ErrorWrap(ErrConnWaitTimeout, err.Error()) - } - return nil, err - case <-time.After(sc.connWaitTimeout): - return nil, ErrConnWaitTimeout - } -} - -//--------------------------------------------------------- - -// RemoteSignerOption sets an optional parameter on the RemoteSigner. -type RemoteSignerOption func(*RemoteSigner) - -// RemoteSignerConnDeadline sets the read and write deadline for connections -// from external signing processes. -func RemoteSignerConnDeadline(deadline time.Duration) RemoteSignerOption { - return func(ss *RemoteSigner) { ss.connDeadline = deadline } -} - -// RemoteSignerConnRetries sets the amount of attempted retries to connect. -func RemoteSignerConnRetries(retries int) RemoteSignerOption { - return func(ss *RemoteSigner) { ss.connRetries = retries } -} - -// RemoteSigner implements PrivValidator by dialing to a socket. -type RemoteSigner struct { - cmn.BaseService - - addr string - chainID string connDeadline time.Duration - connRetries int - privKey ed25519.PrivKeyEd25519 - privVal types.PrivValidator - - conn net.Conn } -// NewRemoteSigner returns an instance of RemoteSigner. -func NewRemoteSigner( - logger log.Logger, - chainID, socketAddr string, - privVal types.PrivValidator, - privKey ed25519.PrivKeyEd25519, -) *RemoteSigner { - rs := &RemoteSigner{ - addr: socketAddr, - chainID: chainID, - connDeadline: time.Second * defaultConnDeadlineSeconds, - connRetries: defaultDialRetries, - privKey: privKey, - privVal: privVal, - } - - rs.BaseService = *cmn.NewBaseService(logger, "RemoteSigner", rs) - - return rs -} - -// OnStart implements cmn.Service. -func (rs *RemoteSigner) OnStart() error { - conn, err := rs.connect() - if err != nil { - err = cmn.ErrorWrap(err, "connect") - rs.Logger.Error("OnStart", "err", err) - return err - } - - go rs.handleConnection(conn) - - return nil -} - -// OnStop implements cmn.Service. -func (rs *RemoteSigner) OnStop() { - if rs.conn == nil { - return - } - - if err := rs.conn.Close(); err != nil { - rs.Logger.Error("OnStop", "err", cmn.ErrorWrap(err, "closing listener failed")) +// newTimeoutConn returns an instance of timeoutConn. +func newTimeoutConn( + conn net.Conn, + connDeadline time.Duration) *timeoutConn { + return &timeoutConn{ + conn, + connDeadline, } } -func (rs *RemoteSigner) connect() (net.Conn, error) { - for retries := rs.connRetries; retries > 0; retries-- { - // Don't sleep if it is the first retry. - if retries != rs.connRetries { - time.Sleep(rs.connDeadline) - } +// Read implements net.Conn. +func (c timeoutConn) Read(b []byte) (n int, err error) { + // Reset deadline + c.Conn.SetReadDeadline(time.Now().Add(c.connDeadline)) - conn, err := cmn.Connect(rs.addr) - if err != nil { - err = cmn.ErrorWrap(err, "connection failed") - rs.Logger.Error( - "connect", - "addr", rs.addr, - "err", err, - ) - - continue - } - - if err := conn.SetDeadline(time.Now().Add(connDeadline)); err != nil { - err = cmn.ErrorWrap(err, "setting connection timeout failed") - rs.Logger.Error( - "connect", - "err", err, - ) - continue - } - - conn, err = p2pconn.MakeSecretConnection(conn, rs.privKey) - if err != nil { - err = cmn.ErrorWrap(err, "encrypting connection failed") - rs.Logger.Error( - "connect", - "err", err, - ) - - continue - } - - return conn, nil - } - - return nil, ErrDialRetryMax + return c.Conn.Read(b) } -func (rs *RemoteSigner) handleConnection(conn net.Conn) { - for { - if !rs.IsRunning() { - return // Ignore error from listener closing. - } +// Write implements net.Conn. +func (c timeoutConn) Write(b []byte) (n int, err error) { + // Reset deadline + c.Conn.SetWriteDeadline(time.Now().Add(c.connDeadline)) - req, err := readMsg(conn) - if err != nil { - if err != io.EOF { - rs.Logger.Error("handleConnection", "err", err) - } - return - } - - var res SocketPVMsg - - switch r := req.(type) { - case *PubKeyMsg: - var p crypto.PubKey - p = rs.privVal.GetPubKey() - res = &PubKeyMsg{p} - case *SignVoteRequest: - err = rs.privVal.SignVote(rs.chainID, r.Vote) - if err != nil { - res = &SignedVoteResponse{nil, &RemoteSignerError{0, err.Error()}} - } else { - res = &SignedVoteResponse{r.Vote, nil} - } - case *SignProposalRequest: - err = rs.privVal.SignProposal(rs.chainID, r.Proposal) - if err != nil { - res = &SignedProposalResponse{nil, &RemoteSignerError{0, err.Error()}} - } else { - res = &SignedProposalResponse{r.Proposal, nil} - } - case *SignHeartbeatRequest: - err = rs.privVal.SignHeartbeat(rs.chainID, r.Heartbeat) - if err != nil { - res = &SignedHeartbeatResponse{nil, &RemoteSignerError{0, err.Error()}} - } else { - res = &SignedHeartbeatResponse{r.Heartbeat, nil} - } - default: - err = fmt.Errorf("unknown msg: %v", r) - } - - if err != nil { - // only log the error; we'll reply with an error in res - rs.Logger.Error("handleConnection", "err", err) - } - - err = writeMsg(conn, res) - if err != nil { - rs.Logger.Error("handleConnection", "err", err) - return - } - } -} - -//--------------------------------------------------------- - -// SocketPVMsg is sent between RemoteSigner and SocketPV. -type SocketPVMsg interface{} - -func RegisterSocketPVMsg(cdc *amino.Codec) { - cdc.RegisterInterface((*SocketPVMsg)(nil), nil) - cdc.RegisterConcrete(&PubKeyMsg{}, "tendermint/socketpv/PubKeyMsg", nil) - cdc.RegisterConcrete(&SignVoteRequest{}, "tendermint/socketpv/SignVoteRequest", nil) - cdc.RegisterConcrete(&SignedVoteResponse{}, "tendermint/socketpv/SignedVoteResponse", nil) - cdc.RegisterConcrete(&SignProposalRequest{}, "tendermint/socketpv/SignProposalRequest", nil) - cdc.RegisterConcrete(&SignedProposalResponse{}, "tendermint/socketpv/SignedProposalResponse", nil) - cdc.RegisterConcrete(&SignHeartbeatRequest{}, "tendermint/socketpv/SignHeartbeatRequest", nil) - cdc.RegisterConcrete(&SignedHeartbeatResponse{}, "tendermint/socketpv/SignedHeartbeatResponse", nil) -} - -// PubKeyMsg is a PrivValidatorSocket message containing the public key. -type PubKeyMsg struct { - PubKey crypto.PubKey -} - -// SignVoteRequest is a PrivValidatorSocket message containing a vote. -type SignVoteRequest struct { - Vote *types.Vote -} - -// SignedVoteResponse is a PrivValidatorSocket message containing a signed vote along with a potenial error message. -type SignedVoteResponse struct { - Vote *types.Vote - Error *RemoteSignerError -} - -// SignProposalRequest is a PrivValidatorSocket message containing a Proposal. -type SignProposalRequest struct { - Proposal *types.Proposal -} - -type SignedProposalResponse struct { - Proposal *types.Proposal - Error *RemoteSignerError -} - -// SignHeartbeatRequest is a PrivValidatorSocket message containing a Heartbeat. -type SignHeartbeatRequest struct { - Heartbeat *types.Heartbeat -} - -type SignedHeartbeatResponse struct { - Heartbeat *types.Heartbeat - Error *RemoteSignerError -} - -// RemoteSignerError allows (remote) validators to include meaningful error descriptions in their reply. -type RemoteSignerError struct { - // TODO(ismail): create an enum of known errors - Code int - Description string -} - -func readMsg(r io.Reader) (msg SocketPVMsg, err error) { - const maxSocketPVMsgSize = 1024 * 10 - _, err = cdc.UnmarshalBinaryReader(r, &msg, maxSocketPVMsgSize) - if _, ok := err.(timeoutError); ok { - err = cmn.ErrorWrap(ErrConnTimeout, err.Error()) - } - return -} - -func writeMsg(w io.Writer, msg interface{}) (err error) { - _, err = cdc.MarshalBinaryWriter(w, msg) - if _, ok := err.(timeoutError); ok { - err = cmn.ErrorWrap(ErrConnTimeout, err.Error()) - } - return + return c.Conn.Write(b) } diff --git a/privval/socket_tcp.go b/privval/socket_tcp.go deleted file mode 100644 index b26db00c..00000000 --- a/privval/socket_tcp.go +++ /dev/null @@ -1,66 +0,0 @@ -package privval - -import ( - "net" - "time" -) - -// timeoutError can be used to check if an error returned from the netp package -// was due to a timeout. -type timeoutError interface { - Timeout() bool -} - -// tcpTimeoutListener implements net.Listener. -var _ net.Listener = (*tcpTimeoutListener)(nil) - -// tcpTimeoutListener wraps a *net.TCPListener to standardise protocol timeouts -// and potentially other tuning parameters. -type tcpTimeoutListener struct { - *net.TCPListener - - acceptDeadline time.Duration - connDeadline time.Duration - period time.Duration -} - -// newTCPTimeoutListener returns an instance of tcpTimeoutListener. -func newTCPTimeoutListener( - ln net.Listener, - acceptDeadline, connDeadline time.Duration, - period time.Duration, -) tcpTimeoutListener { - return tcpTimeoutListener{ - TCPListener: ln.(*net.TCPListener), - acceptDeadline: acceptDeadline, - connDeadline: connDeadline, - period: period, - } -} - -// Accept implements net.Listener. -func (ln tcpTimeoutListener) Accept() (net.Conn, error) { - err := ln.SetDeadline(time.Now().Add(ln.acceptDeadline)) - if err != nil { - return nil, err - } - - tc, err := ln.AcceptTCP() - if err != nil { - return nil, err - } - - if err := tc.SetDeadline(time.Now().Add(ln.connDeadline)); err != nil { - return nil, err - } - - if err := tc.SetKeepAlive(true); err != nil { - return nil, err - } - - if err := tc.SetKeepAlivePeriod(ln.period); err != nil { - return nil, err - } - - return tc, nil -} diff --git a/privval/socket_tcp_test.go b/privval/socket_tcp_test.go deleted file mode 100644 index 44a673c0..00000000 --- a/privval/socket_tcp_test.go +++ /dev/null @@ -1,64 +0,0 @@ -package privval - -import ( - "net" - "testing" - "time" -) - -func TestTCPTimeoutListenerAcceptDeadline(t *testing.T) { - ln, err := net.Listen("tcp", "127.0.0.1:0") - if err != nil { - t.Fatal(err) - } - - ln = newTCPTimeoutListener(ln, time.Millisecond, time.Second, time.Second) - - _, err = ln.Accept() - opErr, ok := err.(*net.OpError) - if !ok { - t.Fatalf("have %v, want *net.OpError", err) - } - - if have, want := opErr.Op, "accept"; have != want { - t.Errorf("have %v, want %v", have, want) - } -} - -func TestTCPTimeoutListenerConnDeadline(t *testing.T) { - ln, err := net.Listen("tcp", "127.0.0.1:0") - if err != nil { - t.Fatal(err) - } - - ln = newTCPTimeoutListener(ln, time.Second, time.Millisecond, time.Second) - - donec := make(chan struct{}) - go func(ln net.Listener) { - defer close(donec) - - c, err := ln.Accept() - if err != nil { - t.Fatal(err) - } - - time.Sleep(2 * time.Millisecond) - - _, err = c.Write([]byte("foo")) - opErr, ok := err.(*net.OpError) - if !ok { - t.Fatalf("have %v, want *net.OpError", err) - } - - if have, want := opErr.Op, "write"; have != want { - t.Errorf("have %v, want %v", have, want) - } - }(ln) - - _, err = net.Dial("tcp", ln.Addr().String()) - if err != nil { - t.Fatal(err) - } - - <-donec -} diff --git a/privval/socket_test.go b/privval/socket_test.go index 84e721be..7f7bbd89 100644 --- a/privval/socket_test.go +++ b/privval/socket_test.go @@ -1,443 +1,126 @@ package privval import ( - "fmt" + "io/ioutil" "net" + "os" "testing" "time" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/tendermint/tendermint/crypto/ed25519" - cmn "github.com/tendermint/tendermint/libs/common" - "github.com/tendermint/tendermint/libs/log" - - p2pconn "github.com/tendermint/tendermint/p2p/conn" - "github.com/tendermint/tendermint/types" ) -func TestSocketPVAddress(t *testing.T) { - var ( - chainID = cmn.RandStr(12) - sc, rs = testSetupSocketPair(t, chainID, types.NewMockPV()) - ) - defer sc.Stop() - defer rs.Stop() - - serverAddr := rs.privVal.GetAddress() - - clientAddr, err := sc.getAddress() - require.NoError(t, err) - - assert.Equal(t, serverAddr, clientAddr) - - // TODO(xla): Remove when PrivValidator2 replaced PrivValidator. - assert.Equal(t, serverAddr, sc.GetAddress()) +//------------------------------------------- +// helper funcs +func newPrivKey() ed25519.PrivKeyEd25519 { + return ed25519.GenPrivKey() } -func TestSocketPVPubKey(t *testing.T) { - var ( - chainID = cmn.RandStr(12) - sc, rs = testSetupSocketPair(t, chainID, types.NewMockPV()) - ) - defer sc.Stop() - defer rs.Stop() +//------------------------------------------- +// tests - clientKey, err := sc.getPubKey() - require.NoError(t, err) - - privKey := rs.privVal.GetPubKey() - - assert.Equal(t, privKey, clientKey) - - // TODO(xla): Remove when PrivValidator2 replaced PrivValidator. - assert.Equal(t, privKey, sc.GetPubKey()) +type listenerTestCase struct { + description string // For test reporting purposes. + listener net.Listener + dialer Dialer } -func TestSocketPVProposal(t *testing.T) { - var ( - chainID = cmn.RandStr(12) - sc, rs = testSetupSocketPair(t, chainID, types.NewMockPV()) - - ts = time.Now() - privProposal = &types.Proposal{Timestamp: ts} - clientProposal = &types.Proposal{Timestamp: ts} - ) - defer sc.Stop() - defer rs.Stop() - - require.NoError(t, rs.privVal.SignProposal(chainID, privProposal)) - require.NoError(t, sc.SignProposal(chainID, clientProposal)) - assert.Equal(t, privProposal.Signature, clientProposal.Signature) +// testUnixAddr will attempt to obtain a platform-independent temporary file +// name for a Unix socket +func testUnixAddr() (string, error) { + f, err := ioutil.TempFile("", "tendermint-privval-test-*") + if err != nil { + return "", err + } + addr := f.Name() + f.Close() + os.Remove(addr) + return addr, nil } -func TestSocketPVVote(t *testing.T) { - var ( - chainID = cmn.RandStr(12) - sc, rs = testSetupSocketPair(t, chainID, types.NewMockPV()) - - ts = time.Now() - vType = types.VoteTypePrecommit - want = &types.Vote{Timestamp: ts, Type: vType} - have = &types.Vote{Timestamp: ts, Type: vType} - ) - defer sc.Stop() - defer rs.Stop() - - require.NoError(t, rs.privVal.SignVote(chainID, want)) - require.NoError(t, sc.SignVote(chainID, have)) - assert.Equal(t, want.Signature, have.Signature) -} - -func TestSocketPVHeartbeat(t *testing.T) { - var ( - chainID = cmn.RandStr(12) - sc, rs = testSetupSocketPair(t, chainID, types.NewMockPV()) - - want = &types.Heartbeat{} - have = &types.Heartbeat{} - ) - defer sc.Stop() - defer rs.Stop() - - require.NoError(t, rs.privVal.SignHeartbeat(chainID, want)) - require.NoError(t, sc.SignHeartbeat(chainID, have)) - assert.Equal(t, want.Signature, have.Signature) -} - -func TestSocketPVAcceptDeadline(t *testing.T) { - var ( - sc = NewSocketPV( - log.TestingLogger(), - "127.0.0.1:0", - ed25519.GenPrivKey(), - ) - ) - defer sc.Stop() - - SocketPVAcceptDeadline(time.Millisecond)(sc) - - assert.Equal(t, sc.Start().(cmn.Error).Data(), ErrConnWaitTimeout) -} - -func TestSocketPVDeadline(t *testing.T) { - var ( - addr = testFreeAddr(t) - listenc = make(chan struct{}) - sc = NewSocketPV( - log.TestingLogger(), - addr, - ed25519.GenPrivKey(), - ) - ) - - SocketPVConnDeadline(100 * time.Millisecond)(sc) - SocketPVConnWait(500 * time.Millisecond)(sc) - - go func(sc *SocketPV) { - defer close(listenc) - - require.NoError(t, sc.Start()) - - assert.True(t, sc.IsRunning()) - }(sc) - - for { - conn, err := cmn.Connect(addr) - if err != nil { - continue - } - - _, err = p2pconn.MakeSecretConnection( - conn, - ed25519.GenPrivKey(), - ) - if err == nil { - break - } +func tcpListenerTestCase(t *testing.T, acceptDeadline, connectDeadline time.Duration) listenerTestCase { + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) } - <-listenc - - // Sleep to guarantee deadline has been hit. - time.Sleep(20 * time.Microsecond) - - _, err := sc.getPubKey() - assert.Equal(t, err.(cmn.Error).Data(), ErrConnTimeout) + tcpLn := NewTCPListener(ln, newPrivKey()) + TCPListenerAcceptDeadline(acceptDeadline)(tcpLn) + TCPListenerConnDeadline(connectDeadline)(tcpLn) + return listenerTestCase{ + description: "TCP", + listener: tcpLn, + dialer: DialTCPFn(ln.Addr().String(), testConnDeadline, newPrivKey()), + } } -func TestSocketPVWait(t *testing.T) { - sc := NewSocketPV( - log.TestingLogger(), - "127.0.0.1:0", - ed25519.GenPrivKey(), - ) - defer sc.Stop() +func unixListenerTestCase(t *testing.T, acceptDeadline, connectDeadline time.Duration) listenerTestCase { + addr, err := testUnixAddr() + if err != nil { + t.Fatal(err) + } + ln, err := net.Listen("unix", addr) + if err != nil { + t.Fatal(err) + } - SocketPVConnWait(time.Millisecond)(sc) - - assert.Equal(t, sc.Start().(cmn.Error).Data(), ErrConnWaitTimeout) + unixLn := NewUnixListener(ln) + UnixListenerAcceptDeadline(acceptDeadline)(unixLn) + UnixListenerConnDeadline(connectDeadline)(unixLn) + return listenerTestCase{ + description: "Unix", + listener: unixLn, + dialer: DialUnixFn(addr), + } } -func TestRemoteSignerRetry(t *testing.T) { - var ( - attemptc = make(chan int) - retries = 2 - ) +func listenerTestCases(t *testing.T, acceptDeadline, connectDeadline time.Duration) []listenerTestCase { + return []listenerTestCase{ + tcpListenerTestCase(t, acceptDeadline, connectDeadline), + unixListenerTestCase(t, acceptDeadline, connectDeadline), + } +} - ln, err := net.Listen("tcp", "127.0.0.1:0") - require.NoError(t, err) +func TestListenerAcceptDeadlines(t *testing.T) { + for _, tc := range listenerTestCases(t, time.Millisecond, time.Second) { + _, err := tc.listener.Accept() + opErr, ok := err.(*net.OpError) + if !ok { + t.Fatalf("for %s listener, have %v, want *net.OpError", tc.description, err) + } - go func(ln net.Listener, attemptc chan<- int) { - attempts := 0 + if have, want := opErr.Op, "accept"; have != want { + t.Errorf("for %s listener, have %v, want %v", tc.description, have, want) + } + } +} - for { - conn, err := ln.Accept() - require.NoError(t, err) - - err = conn.Close() - require.NoError(t, err) - - attempts++ - - if attempts == retries { - attemptc <- attempts - break +func TestListenerConnectDeadlines(t *testing.T) { + for _, tc := range listenerTestCases(t, time.Second, time.Millisecond) { + go func(dialer Dialer) { + _, err := dialer() + if err != nil { + panic(err) } + }(tc.dialer) + + c, err := tc.listener.Accept() + if err != nil { + t.Fatal(err) } - }(ln, attemptc) - rs := NewRemoteSigner( - log.TestingLogger(), - cmn.RandStr(12), - ln.Addr().String(), - types.NewMockPV(), - ed25519.GenPrivKey(), - ) - defer rs.Stop() + time.Sleep(2 * time.Millisecond) - RemoteSignerConnDeadline(time.Millisecond)(rs) - RemoteSignerConnRetries(retries)(rs) + msg := make([]byte, 200) + _, err = c.Read(msg) + opErr, ok := err.(*net.OpError) + if !ok { + t.Fatalf("for %s listener, have %v, want *net.OpError", tc.description, err) + } - assert.Equal(t, rs.Start().(cmn.Error).Data(), ErrDialRetryMax) - - select { - case attempts := <-attemptc: - assert.Equal(t, retries, attempts) - case <-time.After(100 * time.Millisecond): - t.Error("expected remote to observe connection attempts") + if have, want := opErr.Op, "read"; have != want { + t.Errorf("for %s listener, have %v, want %v", tc.description, have, want) + } } } - -func TestRemoteSignVoteErrors(t *testing.T) { - var ( - chainID = cmn.RandStr(12) - sc, rs = testSetupSocketPair(t, chainID, types.NewErroringMockPV()) - - ts = time.Now() - vType = types.VoteTypePrecommit - vote = &types.Vote{Timestamp: ts, Type: vType} - ) - defer sc.Stop() - defer rs.Stop() - - err := writeMsg(sc.conn, &SignVoteRequest{Vote: vote}) - require.NoError(t, err) - - res, err := readMsg(sc.conn) - require.NoError(t, err) - - resp := *res.(*SignedVoteResponse) - require.NotNil(t, resp.Error) - require.Equal(t, resp.Error.Description, types.ErroringMockPVErr.Error()) - - err = rs.privVal.SignVote(chainID, vote) - require.Error(t, err) - err = sc.SignVote(chainID, vote) - require.Error(t, err) -} - -func TestRemoteSignProposalErrors(t *testing.T) { - var ( - chainID = cmn.RandStr(12) - sc, rs = testSetupSocketPair(t, chainID, types.NewErroringMockPV()) - - ts = time.Now() - proposal = &types.Proposal{Timestamp: ts} - ) - defer sc.Stop() - defer rs.Stop() - - err := writeMsg(sc.conn, &SignProposalRequest{Proposal: proposal}) - require.NoError(t, err) - - res, err := readMsg(sc.conn) - require.NoError(t, err) - - resp := *res.(*SignedProposalResponse) - require.NotNil(t, resp.Error) - require.Equal(t, resp.Error.Description, types.ErroringMockPVErr.Error()) - - err = rs.privVal.SignProposal(chainID, proposal) - require.Error(t, err) - - err = sc.SignProposal(chainID, proposal) - require.Error(t, err) -} - -func TestRemoteSignHeartbeatErrors(t *testing.T) { - var ( - chainID = cmn.RandStr(12) - sc, rs = testSetupSocketPair(t, chainID, types.NewErroringMockPV()) - hb = &types.Heartbeat{} - ) - defer sc.Stop() - defer rs.Stop() - - err := writeMsg(sc.conn, &SignHeartbeatRequest{Heartbeat: hb}) - require.NoError(t, err) - - res, err := readMsg(sc.conn) - require.NoError(t, err) - - resp := *res.(*SignedHeartbeatResponse) - require.NotNil(t, resp.Error) - require.Equal(t, resp.Error.Description, types.ErroringMockPVErr.Error()) - - err = rs.privVal.SignHeartbeat(chainID, hb) - require.Error(t, err) - - err = sc.SignHeartbeat(chainID, hb) - require.Error(t, err) -} - -func TestErrUnexpectedResponse(t *testing.T) { - var ( - addr = testFreeAddr(t) - logger = log.TestingLogger() - chainID = cmn.RandStr(12) - readyc = make(chan struct{}) - errc = make(chan error, 1) - - rs = NewRemoteSigner( - logger, - chainID, - addr, - types.NewMockPV(), - ed25519.GenPrivKey(), - ) - sc = NewSocketPV( - logger, - addr, - ed25519.GenPrivKey(), - ) - ) - - testStartSocketPV(t, readyc, sc) - defer sc.Stop() - RemoteSignerConnDeadline(time.Millisecond)(rs) - RemoteSignerConnRetries(1e6)(rs) - - // we do not want to Start() the remote signer here and instead use the connection to - // reply with intentionally wrong replies below: - rsConn, err := rs.connect() - defer rsConn.Close() - require.NoError(t, err) - require.NotNil(t, rsConn) - <-readyc - - // Heartbeat: - go func(errc chan error) { - errc <- sc.SignHeartbeat(chainID, &types.Heartbeat{}) - }(errc) - // read request and write wrong response: - go testReadWriteResponse(t, &SignedVoteResponse{}, rsConn) - err = <-errc - require.Error(t, err) - require.Equal(t, err, ErrUnexpectedResponse) - - // Proposal: - go func(errc chan error) { - errc <- sc.SignProposal(chainID, &types.Proposal{}) - }(errc) - // read request and write wrong response: - go testReadWriteResponse(t, &SignedHeartbeatResponse{}, rsConn) - err = <-errc - require.Error(t, err) - require.Equal(t, err, ErrUnexpectedResponse) - - // Vote: - go func(errc chan error) { - errc <- sc.SignVote(chainID, &types.Vote{}) - }(errc) - // read request and write wrong response: - go testReadWriteResponse(t, &SignedHeartbeatResponse{}, rsConn) - err = <-errc - require.Error(t, err) - require.Equal(t, err, ErrUnexpectedResponse) -} - -func testSetupSocketPair( - t *testing.T, - chainID string, - privValidator types.PrivValidator, -) (*SocketPV, *RemoteSigner) { - var ( - addr = testFreeAddr(t) - logger = log.TestingLogger() - privVal = privValidator - readyc = make(chan struct{}) - rs = NewRemoteSigner( - logger, - chainID, - addr, - privVal, - ed25519.GenPrivKey(), - ) - sc = NewSocketPV( - logger, - addr, - ed25519.GenPrivKey(), - ) - ) - - testStartSocketPV(t, readyc, sc) - - RemoteSignerConnDeadline(time.Millisecond)(rs) - RemoteSignerConnRetries(1e6)(rs) - - require.NoError(t, rs.Start()) - assert.True(t, rs.IsRunning()) - - <-readyc - - return sc, rs -} - -func testReadWriteResponse(t *testing.T, resp SocketPVMsg, rsConn net.Conn) { - _, err := readMsg(rsConn) - require.NoError(t, err) - - err = writeMsg(rsConn, resp) - require.NoError(t, err) -} - -func testStartSocketPV(t *testing.T, readyc chan struct{}, sc *SocketPV) { - go func(sc *SocketPV) { - require.NoError(t, sc.Start()) - assert.True(t, sc.IsRunning()) - - readyc <- struct{}{} - }(sc) -} - -// testFreeAddr claims a free port so we don't block on listener being ready. -func testFreeAddr(t *testing.T) string { - ln, err := net.Listen("tcp", "127.0.0.1:0") - require.NoError(t, err) - defer ln.Close() - - return fmt.Sprintf("127.0.0.1:%d", ln.Addr().(*net.TCPAddr).Port) -} diff --git a/privval/wire.go b/privval/wire.go index 50660ff3..637039b7 100644 --- a/privval/wire.go +++ b/privval/wire.go @@ -1,7 +1,7 @@ package privval import ( - "github.com/tendermint/go-amino" + amino "github.com/tendermint/go-amino" cryptoAmino "github.com/tendermint/tendermint/crypto/encoding/amino" ) @@ -9,5 +9,5 @@ var cdc = amino.NewCodec() func init() { cryptoAmino.RegisterAmino(cdc) - RegisterSocketPVMsg(cdc) + RegisterRemoteSignerMsg(cdc) } diff --git a/proxy/app_conn_test.go b/proxy/app_conn_test.go index 5eadb032..ca98f1be 100644 --- a/proxy/app_conn_test.go +++ b/proxy/app_conn_test.go @@ -143,7 +143,7 @@ func TestInfo(t *testing.T) { proxy := NewAppConnTest(cli) t.Log("Connected") - resInfo, err := proxy.InfoSync(types.RequestInfo{Version: ""}) + resInfo, err := proxy.InfoSync(RequestInfo) if err != nil { t.Errorf("Unexpected error: %v", err) } diff --git a/proxy/client.go b/proxy/client.go index 87f4e716..c5ee5fe1 100644 --- a/proxy/client.go +++ b/proxy/client.go @@ -6,6 +6,7 @@ import ( "github.com/pkg/errors" abcicli "github.com/tendermint/tendermint/abci/client" + "github.com/tendermint/tendermint/abci/example/counter" "github.com/tendermint/tendermint/abci/example/kvstore" "github.com/tendermint/tendermint/abci/types" ) @@ -64,15 +65,15 @@ func (r *remoteClientCreator) NewABCIClient() (abcicli.Client, error) { func DefaultClientCreator(addr, transport, dbDir string) ClientCreator { switch addr { + case "counter": + return NewLocalClientCreator(counter.NewCounterApplication(false)) + case "counter_serial": + return NewLocalClientCreator(counter.NewCounterApplication(true)) case "kvstore": - fallthrough - case "dummy": return NewLocalClientCreator(kvstore.NewKVStoreApplication()) case "persistent_kvstore": - fallthrough - case "persistent_dummy": return NewLocalClientCreator(kvstore.NewPersistentKVStoreApplication(dbDir)) - case "nilapp": + case "noop": return NewLocalClientCreator(types.NewBaseApplication()) default: mustConnect := false // loop retrying diff --git a/proxy/version.go b/proxy/version.go new file mode 100644 index 00000000..fb506e65 --- /dev/null +++ b/proxy/version.go @@ -0,0 +1,15 @@ +package proxy + +import ( + abci "github.com/tendermint/tendermint/abci/types" + "github.com/tendermint/tendermint/version" +) + +// RequestInfo contains all the information for sending +// the abci.RequestInfo message during handshake with the app. +// It contains only compile-time version information. +var RequestInfo = abci.RequestInfo{ + Version: version.Version, + BlockVersion: version.BlockProtocol.Uint64(), + P2PVersion: version.P2PProtocol.Uint64(), +} diff --git a/rpc/client/helpers.go b/rpc/client/helpers.go index 7e64d116..2e80a306 100644 --- a/rpc/client/helpers.go +++ b/rpc/client/helpers.go @@ -69,7 +69,18 @@ func WaitForOneEvent(c EventsClient, evtTyp string, timeout time.Duration) (type } // make sure to unregister after the test is over - defer c.UnsubscribeAll(ctx, subscriber) + defer func() { + // drain evts to make sure we don't block + LOOP: + for { + select { + case <-evts: + default: + break LOOP + } + } + c.UnsubscribeAll(ctx, subscriber) + }() select { case evt := <-evts: diff --git a/rpc/client/httpclient.go b/rpc/client/httpclient.go index 322569df..715444dd 100644 --- a/rpc/client/httpclient.go +++ b/rpc/client/httpclient.go @@ -110,6 +110,24 @@ func (c *HTTP) broadcastTX(route string, tx types.Tx) (*ctypes.ResultBroadcastTx return result, nil } +func (c *HTTP) UnconfirmedTxs(limit int) (*ctypes.ResultUnconfirmedTxs, error) { + result := new(ctypes.ResultUnconfirmedTxs) + _, err := c.rpc.Call("unconfirmed_txs", map[string]interface{}{"limit": limit}, result) + if err != nil { + return nil, errors.Wrap(err, "unconfirmed_txs") + } + return result, nil +} + +func (c *HTTP) NumUnconfirmedTxs() (*ctypes.ResultUnconfirmedTxs, error) { + result := new(ctypes.ResultUnconfirmedTxs) + _, err := c.rpc.Call("num_unconfirmed_txs", map[string]interface{}{}, result) + if err != nil { + return nil, errors.Wrap(err, "num_unconfirmed_txs") + } + return result, nil +} + func (c *HTTP) NetInfo() (*ctypes.ResultNetInfo, error) { result := new(ctypes.ResultNetInfo) _, err := c.rpc.Call("net_info", map[string]interface{}{}, result) diff --git a/rpc/client/interface.go b/rpc/client/interface.go index a92b762d..df9064a7 100644 --- a/rpc/client/interface.go +++ b/rpc/client/interface.go @@ -95,3 +95,9 @@ type NetworkClient interface { type EventsClient interface { types.EventBusSubscriber } + +// MempoolClient shows us data about current mempool state. +type MempoolClient interface { + UnconfirmedTxs(limit int) (*ctypes.ResultUnconfirmedTxs, error) + NumUnconfirmedTxs() (*ctypes.ResultUnconfirmedTxs, error) +} diff --git a/rpc/client/localclient.go b/rpc/client/localclient.go index 28f69204..e02ef487 100644 --- a/rpc/client/localclient.go +++ b/rpc/client/localclient.go @@ -77,6 +77,14 @@ func (Local) BroadcastTxSync(tx types.Tx) (*ctypes.ResultBroadcastTx, error) { return core.BroadcastTxSync(tx) } +func (Local) UnconfirmedTxs(limit int) (*ctypes.ResultUnconfirmedTxs, error) { + return core.UnconfirmedTxs(limit) +} + +func (Local) NumUnconfirmedTxs() (*ctypes.ResultUnconfirmedTxs, error) { + return core.NumUnconfirmedTxs() +} + func (Local) NetInfo() (*ctypes.ResultNetInfo, error) { return core.NetInfo() } diff --git a/rpc/client/mock/abci.go b/rpc/client/mock/abci.go index 3a0ed79c..e63d22e0 100644 --- a/rpc/client/mock/abci.go +++ b/rpc/client/mock/abci.go @@ -3,10 +3,10 @@ package mock import ( abci "github.com/tendermint/tendermint/abci/types" cmn "github.com/tendermint/tendermint/libs/common" + "github.com/tendermint/tendermint/proxy" "github.com/tendermint/tendermint/rpc/client" ctypes "github.com/tendermint/tendermint/rpc/core/types" "github.com/tendermint/tendermint/types" - "github.com/tendermint/tendermint/version" ) // ABCIApp will send all abci related request to the named app, @@ -23,7 +23,7 @@ var ( ) func (a ABCIApp) ABCIInfo() (*ctypes.ResultABCIInfo, error) { - return &ctypes.ResultABCIInfo{a.App.Info(abci.RequestInfo{Version: version.Version})}, nil + return &ctypes.ResultABCIInfo{a.App.Info(proxy.RequestInfo)}, nil } func (a ABCIApp) ABCIQuery(path string, data cmn.HexBytes) (*ctypes.ResultABCIQuery, error) { diff --git a/rpc/client/rpc_test.go b/rpc/client/rpc_test.go index dc0c3948..222a9a9b 100644 --- a/rpc/client/rpc_test.go +++ b/rpc/client/rpc_test.go @@ -4,6 +4,7 @@ import ( "bytes" "fmt" "math/rand" + "net/http" "strings" "testing" @@ -36,6 +37,21 @@ func GetClients() []client.Client { } } +func TestCorsEnabled(t *testing.T) { + origin := rpctest.GetConfig().RPC.CORSAllowedOrigins[0] + remote := strings.Replace(rpctest.GetConfig().RPC.ListenAddress, "tcp", "http", -1) + + req, err := http.NewRequest("GET", remote, nil) + require.Nil(t, err, "%+v", err) + req.Header.Set("Origin", origin) + c := &http.Client{} + resp, err := c.Do(req) + defer resp.Body.Close() + + require.Nil(t, err, "%+v", err) + assert.Equal(t, resp.Header.Get("Access-Control-Allow-Origin"), origin) +} + // Make sure status is correct (we connect properly) func TestStatus(t *testing.T) { for i, c := range GetClients() { @@ -269,6 +285,42 @@ func TestBroadcastTxCommit(t *testing.T) { } } +func TestUnconfirmedTxs(t *testing.T) { + _, _, tx := MakeTxKV() + + mempool := node.MempoolReactor().Mempool + _ = mempool.CheckTx(tx, nil) + + for i, c := range GetClients() { + mc, ok := c.(client.MempoolClient) + require.True(t, ok, "%d", i) + txs, err := mc.UnconfirmedTxs(1) + require.Nil(t, err, "%d: %+v", i, err) + assert.Exactly(t, types.Txs{tx}, types.Txs(txs.Txs)) + } + + mempool.Flush() +} + +func TestNumUnconfirmedTxs(t *testing.T) { + _, _, tx := MakeTxKV() + + mempool := node.MempoolReactor().Mempool + _ = mempool.CheckTx(tx, nil) + mempoolSize := mempool.Size() + + for i, c := range GetClients() { + mc, ok := c.(client.MempoolClient) + require.True(t, ok, "%d", i) + res, err := mc.NumUnconfirmedTxs() + require.Nil(t, err, "%d: %+v", i, err) + + assert.Equal(t, mempoolSize, res.N) + } + + mempool.Flush() +} + func TestTx(t *testing.T) { // first we broadcast a tx c := getHTTPClient() @@ -358,21 +410,33 @@ func TestTxSearch(t *testing.T) { } // query by height - result, err = c.TxSearch(fmt.Sprintf("tx.height >= %d", txHeight), true, 1, 30) + result, err = c.TxSearch(fmt.Sprintf("tx.height=%d", txHeight), true, 1, 30) require.Nil(t, err, "%+v", err) require.Len(t, result.Txs, 1) - // we query for non existing tx + // query for non existing tx result, err = c.TxSearch(fmt.Sprintf("tx.hash='%X'", anotherTxHash), false, 1, 30) require.Nil(t, err, "%+v", err) require.Len(t, result.Txs, 0) - // we query using a tag (see kvstore application) + // query using a tag (see kvstore application) result, err = c.TxSearch("app.creator='Cosmoshi Netowoko'", false, 1, 30) require.Nil(t, err, "%+v", err) if len(result.Txs) == 0 { t.Fatal("expected a lot of transactions") } + + // query using a tag (see kvstore application) and height + result, err = c.TxSearch("app.creator='Cosmoshi Netowoko' AND tx.height<10000", true, 1, 30) + require.Nil(t, err, "%+v", err) + if len(result.Txs) == 0 { + t.Fatal("expected a lot of transactions") + } + + // query a non existing tx with page 1 and txsPerPage 1 + result, err = c.TxSearch("app.creator='Cosmoshi Neetowoko'", true, 1, 1) + require.Nil(t, err, "%+v", err) + require.Len(t, result.Txs, 0) } } diff --git a/rpc/core/abci.go b/rpc/core/abci.go index 47219563..c9d516f9 100644 --- a/rpc/core/abci.go +++ b/rpc/core/abci.go @@ -3,8 +3,8 @@ package core import ( abci "github.com/tendermint/tendermint/abci/types" cmn "github.com/tendermint/tendermint/libs/common" + "github.com/tendermint/tendermint/proxy" ctypes "github.com/tendermint/tendermint/rpc/core/types" - "github.com/tendermint/tendermint/version" ) // Query the application for some information. @@ -15,6 +15,11 @@ import ( // // ```go // client := client.NewHTTP("tcp://0.0.0.0:26657", "/websocket") +// err := client.Start() +// if err != nil { +// // handle error +// } +// defer client.Stop() // result, err := client.ABCIQuery("", "abcd", true) // ``` // @@ -69,6 +74,11 @@ func ABCIQuery(path string, data cmn.HexBytes, height int64, prove bool) (*ctype // // ```go // client := client.NewHTTP("tcp://0.0.0.0:26657", "/websocket") +// err := client.Start() +// if err != nil { +// // handle error +// } +// defer client.Stop() // info, err := client.ABCIInfo() // ``` // @@ -87,7 +97,7 @@ func ABCIQuery(path string, data cmn.HexBytes, height int64, prove bool) (*ctype // } // ``` func ABCIInfo() (*ctypes.ResultABCIInfo, error) { - resInfo, err := proxyAppQuery.InfoSync(abci.RequestInfo{Version: version.Version}) + resInfo, err := proxyAppQuery.InfoSync(proxy.RequestInfo) if err != nil { return nil, err } diff --git a/rpc/core/blocks.go b/rpc/core/blocks.go index a9252f55..ee4009e5 100644 --- a/rpc/core/blocks.go +++ b/rpc/core/blocks.go @@ -18,6 +18,11 @@ import ( // // ```go // client := client.NewHTTP("tcp://0.0.0.0:26657", "/websocket") +// err := client.Start() +// if err != nil { +// // handle error +// } +// defer client.Stop() // info, err := client.BlockchainInfo(10, 10) // ``` // @@ -123,6 +128,11 @@ func filterMinMax(height, min, max, limit int64) (int64, int64, error) { // // ```go // client := client.NewHTTP("tcp://0.0.0.0:26657", "/websocket") +// err := client.Start() +// if err != nil { +// // handle error +// } +// defer client.Stop() // info, err := client.Block(10) // ``` // @@ -235,6 +245,11 @@ func Block(heightPtr *int64) (*ctypes.ResultBlock, error) { // // ```go // client := client.NewHTTP("tcp://0.0.0.0:26657", "/websocket") +// err := client.Start() +// if err != nil { +// // handle error +// } +// defer client.Stop() // info, err := client.Commit(11) // ``` // @@ -329,6 +344,11 @@ func Commit(heightPtr *int64) (*ctypes.ResultCommit, error) { // // ```go // client := client.NewHTTP("tcp://0.0.0.0:26657", "/websocket") +// err := client.Start() +// if err != nil { +// // handle error +// } +// defer client.Stop() // info, err := client.BlockResults(10) // ``` // diff --git a/rpc/core/consensus.go b/rpc/core/consensus.go index 1d5f9275..9968a1b2 100644 --- a/rpc/core/consensus.go +++ b/rpc/core/consensus.go @@ -2,7 +2,6 @@ package core import ( cm "github.com/tendermint/tendermint/consensus" - "github.com/tendermint/tendermint/p2p" ctypes "github.com/tendermint/tendermint/rpc/core/types" sm "github.com/tendermint/tendermint/state" "github.com/tendermint/tendermint/types" @@ -17,6 +16,11 @@ import ( // // ```go // client := client.NewHTTP("tcp://0.0.0.0:26657", "/websocket") +// err := client.Start() +// if err != nil { +// // handle error +// } +// defer client.Stop() // state, err := client.Validators() // ``` // @@ -28,7 +32,7 @@ import ( // "result": { // "validators": [ // { -// "accum": "0", +// "proposer_priority": "0", // "voting_power": "10", // "pub_key": { // "data": "68DFDA7E50F82946E7E8546BED37944A422CD1B831E70DF66BA3B8430593944D", @@ -68,6 +72,11 @@ func Validators(heightPtr *int64) (*ctypes.ResultValidators, error) { // // ```go // client := client.NewHTTP("tcp://0.0.0.0:26657", "/websocket") +// err := client.Start() +// if err != nil { +// // handle error +// } +// defer client.Stop() // state, err := client.DumpConsensusState() // ``` // @@ -93,7 +102,7 @@ func Validators(heightPtr *int64) (*ctypes.ResultValidators, error) { // "value": "SBctdhRBcXtBgdI/8a/alTsUhGXqGs9k5ylV1u5iKHg=" // }, // "voting_power": "10", -// "accum": "0" +// "proposer_priority": "0" // } // ], // "proposer": { @@ -103,7 +112,7 @@ func Validators(heightPtr *int64) (*ctypes.ResultValidators, error) { // "value": "SBctdhRBcXtBgdI/8a/alTsUhGXqGs9k5ylV1u5iKHg=" // }, // "voting_power": "10", -// "accum": "0" +// "proposer_priority": "0" // } // }, // "proposal": null, @@ -139,7 +148,7 @@ func Validators(heightPtr *int64) (*ctypes.ResultValidators, error) { // "value": "SBctdhRBcXtBgdI/8a/alTsUhGXqGs9k5ylV1u5iKHg=" // }, // "voting_power": "10", -// "accum": "0" +// "proposer_priority": "0" // } // ], // "proposer": { @@ -149,7 +158,7 @@ func Validators(heightPtr *int64) (*ctypes.ResultValidators, error) { // "value": "SBctdhRBcXtBgdI/8a/alTsUhGXqGs9k5ylV1u5iKHg=" // }, // "voting_power": "10", -// "accum": "0" +// "proposer_priority": "0" // } // } // }, @@ -194,14 +203,17 @@ func DumpConsensusState() (*ctypes.ResultDumpConsensusState, error) { peers := p2pPeers.Peers().List() peerStates := make([]ctypes.PeerStateInfo, len(peers)) for i, peer := range peers { - peerState := peer.Get(types.PeerStateKey).(*cm.PeerState) + peerState, ok := peer.Get(types.PeerStateKey).(*cm.PeerState) + if !ok { // peer does not have a state yet + continue + } peerStateJSON, err := peerState.ToJSON() if err != nil { return nil, err } peerStates[i] = ctypes.PeerStateInfo{ // Peer basic info. - NodeAddress: p2p.IDAddressString(peer.ID(), peer.NodeInfo().ListenAddr), + NodeAddress: peer.NodeInfo().NetAddress().String(), // Peer consensus state. PeerState: peerStateJSON, } @@ -223,6 +235,11 @@ func DumpConsensusState() (*ctypes.ResultDumpConsensusState, error) { // // ```go // client := client.NewHTTP("tcp://0.0.0.0:26657", "/websocket") +// err := client.Start() +// if err != nil { +// // handle error +// } +// defer client.Stop() // state, err := client.ConsensusState() // ``` // @@ -271,6 +288,11 @@ func ConsensusState() (*ctypes.ResultConsensusState, error) { // // ```go // client := client.NewHTTP("tcp://0.0.0.0:26657", "/websocket") +// err := client.Start() +// if err != nil { +// // handle error +// } +// defer client.Stop() // state, err := client.ConsensusParams() // ``` // diff --git a/rpc/core/doc.go b/rpc/core/doc.go index 5378dde2..ec79c8e1 100644 --- a/rpc/core/doc.go +++ b/rpc/core/doc.go @@ -12,7 +12,10 @@ See it here: https://github.com/tendermint/tendermint/tree/master/rpc/lib ## Configuration -Set the `laddr` config parameter under `[rpc]` table in the `$TMHOME/config/config.toml` file or the `--rpc.laddr` command-line flag to the desired protocol://host:port setting. Default: `tcp://0.0.0.0:26657`. +RPC can be configured by tuning parameters under `[rpc]` table in the `$TMHOME/config/config.toml` file or by using the `--rpc.X` command-line flags. + +Default rpc listen address is `tcp://0.0.0.0:26657`. To set another address, set the `laddr` config parameter to desired value. +CORS (Cross-Origin Resource Sharing) can be enabled by setting `cors_allowed_origins`, `cors_allowed_methods`, `cors_allowed_headers` config parameters. ## Arguments diff --git a/rpc/core/events.go b/rpc/core/events.go index 6f679e33..e4fd2041 100644 --- a/rpc/core/events.go +++ b/rpc/core/events.go @@ -2,6 +2,7 @@ package core import ( "context" + "fmt" "github.com/pkg/errors" @@ -53,11 +54,16 @@ import ( // import "github.com/tendermint/tendermint/types" // // client := client.NewHTTP("tcp://0.0.0.0:26657", "/websocket") +// err := client.Start() +// if err != nil { +// // handle error +// } +// defer client.Stop() // ctx, cancel := context.WithTimeout(context.Background(), timeout) // defer cancel() // query := query.MustParse("tm.event = 'Tx' AND tx.height = 3") // txs := make(chan interface{}) -// err := client.Subscribe(ctx, "test-client", query, txs) +// err = client.Subscribe(ctx, "test-client", query, txs) // // go func() { // for e := range txs { @@ -104,7 +110,7 @@ func Subscribe(wsCtx rpctypes.WSRPCContext, query string) (*ctypes.ResultSubscri go func() { for event := range ch { tmResult := &ctypes.ResultEvent{query, event.(tmtypes.TMEventData)} - wsCtx.TryWriteRPCResponse(rpctypes.NewRPCSuccessResponse(wsCtx.Codec(), wsCtx.Request.ID+"#event", tmResult)) + wsCtx.TryWriteRPCResponse(rpctypes.NewRPCSuccessResponse(wsCtx.Codec(), rpctypes.JSONRPCStringID(fmt.Sprintf("%v#event", wsCtx.Request.ID)), tmResult)) } }() @@ -115,7 +121,12 @@ func Subscribe(wsCtx rpctypes.WSRPCContext, query string) (*ctypes.ResultSubscri // // ```go // client := client.NewHTTP("tcp://0.0.0.0:26657", "/websocket") -// err := client.Unsubscribe("test-client", query) +// err := client.Start() +// if err != nil { +// // handle error +// } +// defer client.Stop() +// err = client.Unsubscribe("test-client", query) // ``` // // > The above command returns JSON structured like this: @@ -154,7 +165,12 @@ func Unsubscribe(wsCtx rpctypes.WSRPCContext, query string) (*ctypes.ResultUnsub // // ```go // client := client.NewHTTP("tcp://0.0.0.0:26657", "/websocket") -// err := client.UnsubscribeAll("test-client") +// err := client.Start() +// if err != nil { +// // handle error +// } +// defer client.Stop() +// err = client.UnsubscribeAll("test-client") // ``` // // > The above command returns JSON structured like this: diff --git a/rpc/core/health.go b/rpc/core/health.go index 0ec4b5b4..eeb8686b 100644 --- a/rpc/core/health.go +++ b/rpc/core/health.go @@ -13,6 +13,11 @@ import ( // // ```go // client := client.NewHTTP("tcp://0.0.0.0:26657", "/websocket") +// err := client.Start() +// if err != nil { +// // handle error +// } +// defer client.Stop() // result, err := client.Health() // ``` // diff --git a/rpc/core/mempool.go b/rpc/core/mempool.go index c015363a..ff6b029c 100644 --- a/rpc/core/mempool.go +++ b/rpc/core/mempool.go @@ -8,8 +8,8 @@ import ( "github.com/pkg/errors" abci "github.com/tendermint/tendermint/abci/types" - cmn "github.com/tendermint/tendermint/libs/common" ctypes "github.com/tendermint/tendermint/rpc/core/types" + rpcserver "github.com/tendermint/tendermint/rpc/lib/server" "github.com/tendermint/tendermint/types" ) @@ -24,6 +24,11 @@ import ( // // ```go // client := client.NewHTTP("tcp://0.0.0.0:26657", "/websocket") +// err := client.Start() +// if err != nil { +// // handle error +// } +// defer client.Stop() // result, err := client.BroadcastTxAsync("123") // ``` // @@ -51,7 +56,7 @@ import ( func BroadcastTxAsync(tx types.Tx) (*ctypes.ResultBroadcastTx, error) { err := mempool.CheckTx(tx, nil) if err != nil { - return nil, fmt.Errorf("Error broadcasting transaction: %v", err) + return nil, err } return &ctypes.ResultBroadcastTx{Hash: tx.Hash()}, nil } @@ -64,6 +69,11 @@ func BroadcastTxAsync(tx types.Tx) (*ctypes.ResultBroadcastTx, error) { // // ```go // client := client.NewHTTP("tcp://0.0.0.0:26657", "/websocket") +// err := client.Start() +// if err != nil { +// // handle error +// } +// defer client.Stop() // result, err := client.BroadcastTxSync("456") // ``` // @@ -94,7 +104,7 @@ func BroadcastTxSync(tx types.Tx) (*ctypes.ResultBroadcastTx, error) { resCh <- res }) if err != nil { - return nil, fmt.Errorf("Error broadcasting transaction: %v", err) + return nil, err } res := <-resCh r := res.GetCheckTx() @@ -106,8 +116,9 @@ func BroadcastTxSync(tx types.Tx) (*ctypes.ResultBroadcastTx, error) { }, nil } -// CONTRACT: only returns error if mempool.BroadcastTx errs (ie. problem with the app) -// or if we timeout waiting for tx to commit. +// CONTRACT: only returns error if mempool.CheckTx() errs or if we timeout +// waiting for tx to commit. +// // If CheckTx or DeliverTx fail, no error will be returned, but the returned result // will contain a non-OK ABCI code. // @@ -117,6 +128,11 @@ func BroadcastTxSync(tx types.Tx) (*ctypes.ResultBroadcastTx, error) { // // ```go // client := client.NewHTTP("tcp://0.0.0.0:26657", "/websocket") +// err := client.Start() +// if err != nil { +// // handle error +// } +// defer client.Stop() // result, err := client.BroadcastTxCommit("789") // ``` // @@ -150,20 +166,31 @@ func BroadcastTxSync(tx types.Tx) (*ctypes.ResultBroadcastTx, error) { // |-----------+------+---------+----------+-----------------| // | tx | Tx | nil | true | The transaction | func BroadcastTxCommit(tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) { - // subscribe to tx being committed in block + // Subscribe to tx being committed in block. ctx, cancel := context.WithTimeout(context.Background(), subscribeTimeout) defer cancel() - deliverTxResCh := make(chan interface{}) + deliverTxResCh := make(chan interface{}, 1) q := types.EventQueryTxFor(tx) err := eventBus.Subscribe(ctx, "mempool", q, deliverTxResCh) if err != nil { err = errors.Wrap(err, "failed to subscribe to tx") - logger.Error("Error on broadcastTxCommit", "err", err) - return nil, fmt.Errorf("Error on broadcastTxCommit: %v", err) + logger.Error("Error on broadcast_tx_commit", "err", err) + return nil, err } - defer eventBus.Unsubscribe(context.Background(), "mempool", q) + defer func() { + // drain deliverTxResCh to make sure we don't block + LOOP: + for { + select { + case <-deliverTxResCh: + default: + break LOOP + } + } + eventBus.Unsubscribe(context.Background(), "mempool", q) + }() - // broadcast the tx and register checktx callback + // Broadcast tx and wait for CheckTx result checkTxResCh := make(chan *abci.Response, 1) err = mempool.CheckTx(tx, func(res *abci.Response) { checkTxResCh <- res @@ -172,40 +199,39 @@ func BroadcastTxCommit(tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) { logger.Error("Error on broadcastTxCommit", "err", err) return nil, fmt.Errorf("Error on broadcastTxCommit: %v", err) } - checkTxRes := <-checkTxResCh - checkTxR := checkTxRes.GetCheckTx() - if checkTxR.Code != abci.CodeTypeOK { - // CheckTx failed! + checkTxResMsg := <-checkTxResCh + checkTxRes := checkTxResMsg.GetCheckTx() + if checkTxRes.Code != abci.CodeTypeOK { return &ctypes.ResultBroadcastTxCommit{ - CheckTx: *checkTxR, + CheckTx: *checkTxRes, DeliverTx: abci.ResponseDeliverTx{}, Hash: tx.Hash(), }, nil } - // Wait for the tx to be included in a block, - // timeout after something reasonable. + // Wait for the tx to be included in a block or timeout. // TODO: configurable? - timer := time.NewTimer(60 * 2 * time.Second) + var deliverTxTimeout = rpcserver.WriteTimeout / 2 select { - case deliverTxResMsg := <-deliverTxResCh: + case deliverTxResMsg, ok := <-deliverTxResCh: // The tx was included in a block. + if !ok { + return nil, errors.New("Error on broadcastTxCommit: expected DeliverTxResult, got nil. Did the Tendermint stop?") + } deliverTxRes := deliverTxResMsg.(types.EventDataTx) - // The tx was included in a block. - deliverTxR := deliverTxRes.Result - logger.Info("DeliverTx passed ", "tx", cmn.HexBytes(tx), "response", deliverTxR) return &ctypes.ResultBroadcastTxCommit{ - CheckTx: *checkTxR, - DeliverTx: deliverTxR, + CheckTx: *checkTxRes, + DeliverTx: deliverTxRes.Result, Hash: tx.Hash(), Height: deliverTxRes.Height, }, nil - case <-timer.C: - logger.Error("failed to include tx") + case <-time.After(deliverTxTimeout): + err = errors.New("Timed out waiting for tx to be included in a block") + logger.Error("Error on broadcastTxCommit", "err", err) return &ctypes.ResultBroadcastTxCommit{ - CheckTx: *checkTxR, + CheckTx: *checkTxRes, DeliverTx: abci.ResponseDeliverTx{}, Hash: tx.Hash(), - }, fmt.Errorf("Timed out waiting for transaction to be included in a block") + }, err } } @@ -217,6 +243,11 @@ func BroadcastTxCommit(tx types.Tx) (*ctypes.ResultBroadcastTxCommit, error) { // // ```go // client := client.NewHTTP("tcp://0.0.0.0:26657", "/websocket") +// err := client.Start() +// if err != nil { +// // handle error +// } +// defer client.Stop() // result, err := client.UnconfirmedTxs() // ``` // @@ -255,6 +286,11 @@ func UnconfirmedTxs(limit int) (*ctypes.ResultUnconfirmedTxs, error) { // // ```go // client := client.NewHTTP("tcp://0.0.0.0:26657", "/websocket") +// err := client.Start() +// if err != nil { +// // handle error +// } +// defer client.Stop() // result, err := client.UnconfirmedTxs() // ``` // diff --git a/rpc/core/net.go b/rpc/core/net.go index 9816d2f6..4d95c2ef 100644 --- a/rpc/core/net.go +++ b/rpc/core/net.go @@ -1,8 +1,11 @@ package core import ( + "fmt" + "github.com/pkg/errors" + "github.com/tendermint/tendermint/p2p" ctypes "github.com/tendermint/tendermint/rpc/core/types" ) @@ -14,6 +17,11 @@ import ( // // ```go // client := client.NewHTTP("tcp://0.0.0.0:26657", "/websocket") +// err := client.Start() +// if err != nil { +// // handle error +// } +// defer client.Stop() // info, err := client.NetInfo() // ``` // @@ -37,10 +45,15 @@ import ( func NetInfo() (*ctypes.ResultNetInfo, error) { peers := []ctypes.Peer{} for _, peer := range p2pPeers.Peers().List() { + nodeInfo, ok := peer.NodeInfo().(p2p.DefaultNodeInfo) + if !ok { + return nil, fmt.Errorf("peer.NodeInfo() is not DefaultNodeInfo") + } peers = append(peers, ctypes.Peer{ - NodeInfo: peer.NodeInfo(), + NodeInfo: nodeInfo, IsOutbound: peer.IsOutbound(), ConnectionStatus: peer.Status(), + RemoteIP: peer.RemoteIP(), }) } // TODO: Should we include PersistentPeers and Seeds in here? @@ -88,6 +101,11 @@ func UnsafeDialPeers(peers []string, persistent bool) (*ctypes.ResultDialPeers, // // ```go // client := client.NewHTTP("tcp://0.0.0.0:26657", "/websocket") +// err := client.Start() +// if err != nil { +// // handle error +// } +// defer client.Stop() // genesis, err := client.Genesis() // ``` // diff --git a/rpc/core/pipe.go b/rpc/core/pipe.go index 188ea1c3..23649544 100644 --- a/rpc/core/pipe.go +++ b/rpc/core/pipe.go @@ -1,15 +1,14 @@ package core import ( - "time" - "github.com/tendermint/tendermint/consensus" - crypto "github.com/tendermint/tendermint/crypto" + "github.com/tendermint/tendermint/crypto" dbm "github.com/tendermint/tendermint/libs/db" "github.com/tendermint/tendermint/libs/log" mempl "github.com/tendermint/tendermint/mempool" "github.com/tendermint/tendermint/p2p" "github.com/tendermint/tendermint/proxy" + rpcserver "github.com/tendermint/tendermint/rpc/lib/server" sm "github.com/tendermint/tendermint/state" "github.com/tendermint/tendermint/state/txindex" "github.com/tendermint/tendermint/types" @@ -21,7 +20,7 @@ const ( maxPerPage = 100 ) -var subscribeTimeout = 5 * time.Second +var subscribeTimeout = rpcserver.WriteTimeout / 2 //---------------------------------------------- // These interfaces are used by RPC and must be thread safe @@ -150,8 +149,19 @@ func validatePage(page, perPage, totalCount int) int { } func validatePerPage(perPage int) int { - if perPage < 1 || perPage > maxPerPage { + if perPage < 1 { return defaultPerPage + } else if perPage > maxPerPage { + return maxPerPage } return perPage } + +func validateSkipCount(page, perPage int) int { + skipCount := (page - 1) * perPage + if skipCount < 0 { + return 0 + } + + return skipCount +} diff --git a/rpc/core/pipe_test.go b/rpc/core/pipe_test.go index 225e3649..19ed11fc 100644 --- a/rpc/core/pipe_test.go +++ b/rpc/core/pipe_test.go @@ -47,7 +47,6 @@ func TestPaginationPage(t *testing.T) { } func TestPaginationPerPage(t *testing.T) { - cases := []struct { totalCount int perPage int @@ -59,7 +58,7 @@ func TestPaginationPerPage(t *testing.T) { {5, defaultPerPage, defaultPerPage}, {5, maxPerPage - 1, maxPerPage - 1}, {5, maxPerPage, maxPerPage}, - {5, maxPerPage + 1, defaultPerPage}, + {5, maxPerPage + 1, maxPerPage}, } for _, c := range cases { diff --git a/rpc/core/status.go b/rpc/core/status.go index 17fb2f34..224857d0 100644 --- a/rpc/core/status.go +++ b/rpc/core/status.go @@ -5,6 +5,7 @@ import ( "time" cmn "github.com/tendermint/tendermint/libs/common" + "github.com/tendermint/tendermint/p2p" ctypes "github.com/tendermint/tendermint/rpc/core/types" sm "github.com/tendermint/tendermint/state" "github.com/tendermint/tendermint/types" @@ -19,6 +20,11 @@ import ( // // ```go // client := client.NewHTTP("tcp://0.0.0.0:26657", "/websocket") +// err := client.Start() +// if err != nil { +// // handle error +// } +// defer client.Stop() // result, err := client.Status() // ``` // @@ -30,6 +36,11 @@ import ( // "id": "", // "result": { // "node_info": { +// "protocol_version": { +// "p2p": "4", +// "block": "7", +// "app": "0" +// }, // "id": "53729852020041b956e86685e24394e0bee4373f", // "listen_addr": "10.0.2.15:26656", // "network": "test-chain-Y1OHx6", @@ -37,10 +48,6 @@ import ( // "channels": "4020212223303800", // "moniker": "ubuntu-xenial", // "other": { -// "amino_version": "0.12.0", -// "p2p_version": "0.5.0", -// "consensus_version": "v1/0.2.2", -// "rpc_version": "0.7.0/3", // "tx_index": "on", // "rpc_addr": "tcp://0.0.0.0:26657" // } @@ -91,7 +98,7 @@ func Status() (*ctypes.ResultStatus, error) { } result := &ctypes.ResultStatus{ - NodeInfo: p2pTransport.NodeInfo(), + NodeInfo: p2pTransport.NodeInfo().(p2p.DefaultNodeInfo), SyncInfo: ctypes.SyncInfo{ LatestBlockHash: latestBlockHash, LatestAppHash: latestAppHash, diff --git a/rpc/core/tx.go b/rpc/core/tx.go index ba632001..f1bfd56a 100644 --- a/rpc/core/tx.go +++ b/rpc/core/tx.go @@ -21,6 +21,11 @@ import ( // // ```go // client := client.NewHTTP("tcp://0.0.0.0:26657", "/websocket") +// err := client.Start() +// if err != nil { +// // handle error +// } +// defer client.Stop() // tx, err := client.Tx([]byte("2B8EC32BA2579B3B8606E42C06DE2F7AFA2556EF"), true) // ``` // @@ -115,6 +120,11 @@ func Tx(hash []byte, prove bool) (*ctypes.ResultTx, error) { // // ```go // client := client.NewHTTP("tcp://0.0.0.0:26657", "/websocket") +// err := client.Start() +// if err != nil { +// // handle error +// } +// defer client.Stop() // q, err := tmquery.New("account.owner='Ivan'") // tx, err := client.TxSearch(q, true) // ``` @@ -191,10 +201,11 @@ func TxSearch(query string, prove bool, page, perPage int) (*ctypes.ResultTxSear totalCount := len(results) perPage = validatePerPage(perPage) page = validatePage(page, perPage, totalCount) - skipCount := (page - 1) * perPage + skipCount := validateSkipCount(page, perPage) apiResults := make([]*ctypes.ResultTx, cmn.MinInt(perPage, totalCount-skipCount)) var proof types.TxProof + // if there's no tx in the results array, we don't need to loop through the apiResults array for i := 0; i < len(apiResults); i++ { r := results[skipCount+i] height := r.Height diff --git a/rpc/core/types/responses.go b/rpc/core/types/responses.go index f9a5a5d1..18142c32 100644 --- a/rpc/core/types/responses.go +++ b/rpc/core/types/responses.go @@ -2,10 +2,11 @@ package core_types import ( "encoding/json" + "net" "time" abci "github.com/tendermint/tendermint/abci/types" - crypto "github.com/tendermint/tendermint/crypto" + "github.com/tendermint/tendermint/crypto" cmn "github.com/tendermint/tendermint/libs/common" "github.com/tendermint/tendermint/p2p" @@ -74,9 +75,9 @@ type ValidatorInfo struct { // Node Status type ResultStatus struct { - NodeInfo p2p.NodeInfo `json:"node_info"` - SyncInfo SyncInfo `json:"sync_info"` - ValidatorInfo ValidatorInfo `json:"validator_info"` + NodeInfo p2p.DefaultNodeInfo `json:"node_info"` + SyncInfo SyncInfo `json:"sync_info"` + ValidatorInfo ValidatorInfo `json:"validator_info"` } // Is TxIndexing enabled @@ -107,9 +108,10 @@ type ResultDialPeers struct { // A peer type Peer struct { - p2p.NodeInfo `json:"node_info"` + NodeInfo p2p.DefaultNodeInfo `json:"node_info"` IsOutbound bool `json:"is_outbound"` ConnectionStatus p2p.ConnectionStatus `json:"connection_status"` + RemoteIP net.IP `json:"remote_ip"` } // Validators for a height diff --git a/rpc/core/types/responses_test.go b/rpc/core/types/responses_test.go index c6c86e1f..796299d3 100644 --- a/rpc/core/types/responses_test.go +++ b/rpc/core/types/responses_test.go @@ -15,17 +15,17 @@ func TestStatusIndexer(t *testing.T) { status = &ResultStatus{} assert.False(t, status.TxIndexEnabled()) - status.NodeInfo = p2p.NodeInfo{} + status.NodeInfo = p2p.DefaultNodeInfo{} assert.False(t, status.TxIndexEnabled()) cases := []struct { expected bool - other p2p.NodeInfoOther + other p2p.DefaultNodeInfoOther }{ - {false, p2p.NodeInfoOther{}}, - {false, p2p.NodeInfoOther{TxIndex: "aa"}}, - {false, p2p.NodeInfoOther{TxIndex: "off"}}, - {true, p2p.NodeInfoOther{TxIndex: "on"}}, + {false, p2p.DefaultNodeInfoOther{}}, + {false, p2p.DefaultNodeInfoOther{TxIndex: "aa"}}, + {false, p2p.DefaultNodeInfoOther{TxIndex: "off"}}, + {true, p2p.DefaultNodeInfoOther{TxIndex: "on"}}, } for _, tc := range cases { diff --git a/rpc/core/types/wire.go b/rpc/core/types/wire.go index ef1fa800..2180b5a5 100644 --- a/rpc/core/types/wire.go +++ b/rpc/core/types/wire.go @@ -1,7 +1,7 @@ package core_types import ( - "github.com/tendermint/go-amino" + amino "github.com/tendermint/go-amino" "github.com/tendermint/tendermint/types" ) diff --git a/rpc/core/version.go b/rpc/core/version.go deleted file mode 100644 index e283de47..00000000 --- a/rpc/core/version.go +++ /dev/null @@ -1,5 +0,0 @@ -package core - -// a single integer is sufficient here - -const Version = "3" // rpc routes for profiling, setting config diff --git a/rpc/grpc/client_server.go b/rpc/grpc/client_server.go index c8898968..2bc89864 100644 --- a/rpc/grpc/client_server.go +++ b/rpc/grpc/client_server.go @@ -1,12 +1,9 @@ package core_grpc import ( - "fmt" "net" - "strings" "time" - "golang.org/x/net/netutil" "google.golang.org/grpc" cmn "github.com/tendermint/tendermint/libs/common" @@ -17,28 +14,12 @@ type Config struct { MaxOpenConnections int } -// StartGRPCServer starts a new gRPC BroadcastAPIServer, listening on -// protoAddr, in a goroutine. Returns a listener and an error, if it fails to -// parse an address. -func StartGRPCServer(protoAddr string, config Config) (net.Listener, error) { - parts := strings.SplitN(protoAddr, "://", 2) - if len(parts) != 2 { - return nil, fmt.Errorf("Invalid listen address for grpc server (did you forget a tcp:// prefix?) : %s", protoAddr) - } - proto, addr := parts[0], parts[1] - ln, err := net.Listen(proto, addr) - if err != nil { - return nil, err - } - if config.MaxOpenConnections > 0 { - ln = netutil.LimitListener(ln, config.MaxOpenConnections) - } - +// StartGRPCServer starts a new gRPC BroadcastAPIServer using the given net.Listener. +// NOTE: This function blocks - you may want to call it in a go-routine. +func StartGRPCServer(ln net.Listener) error { grpcServer := grpc.NewServer() RegisterBroadcastAPIServer(grpcServer, &broadcastAPI{}) - go grpcServer.Serve(ln) // nolint: errcheck - - return ln, nil + return grpcServer.Serve(ln) } // StartGRPCClient dials the gRPC server using protoAddr and returns a new diff --git a/rpc/grpc/grpc_test.go b/rpc/grpc/grpc_test.go index eda3896f..ff05c835 100644 --- a/rpc/grpc/grpc_test.go +++ b/rpc/grpc/grpc_test.go @@ -8,8 +8,8 @@ import ( "github.com/stretchr/testify/require" "github.com/tendermint/tendermint/abci/example/kvstore" - "github.com/tendermint/tendermint/rpc/grpc" - "github.com/tendermint/tendermint/rpc/test" + core_grpc "github.com/tendermint/tendermint/rpc/grpc" + rpctest "github.com/tendermint/tendermint/rpc/test" ) func TestMain(m *testing.M) { diff --git a/rpc/lib/client/args_test.go b/rpc/lib/client/args_test.go index cb7a56bd..e3dd09e8 100644 --- a/rpc/lib/client/args_test.go +++ b/rpc/lib/client/args_test.go @@ -5,8 +5,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - - "github.com/tendermint/go-amino" + amino "github.com/tendermint/go-amino" ) type Tx []byte diff --git a/rpc/lib/client/http_client.go b/rpc/lib/client/http_client.go index bd440289..97b8dfe7 100644 --- a/rpc/lib/client/http_client.go +++ b/rpc/lib/client/http_client.go @@ -12,7 +12,7 @@ import ( "strings" "github.com/pkg/errors" - "github.com/tendermint/go-amino" + amino "github.com/tendermint/go-amino" types "github.com/tendermint/tendermint/rpc/lib/types" ) @@ -99,7 +99,7 @@ func NewJSONRPCClient(remote string) *JSONRPCClient { } func (c *JSONRPCClient) Call(method string, params map[string]interface{}, result interface{}) (interface{}, error) { - request, err := types.MapToRequest(c.cdc, "jsonrpc-client", method, params) + request, err := types.MapToRequest(c.cdc, types.JSONRPCStringID("jsonrpc-client"), method, params) if err != nil { return nil, err } diff --git a/rpc/lib/client/ws_client.go b/rpc/lib/client/ws_client.go index 6da996e2..e3b55956 100644 --- a/rpc/lib/client/ws_client.go +++ b/rpc/lib/client/ws_client.go @@ -13,7 +13,7 @@ import ( "github.com/pkg/errors" metrics "github.com/rcrowley/go-metrics" - "github.com/tendermint/go-amino" + amino "github.com/tendermint/go-amino" cmn "github.com/tendermint/tendermint/libs/common" types "github.com/tendermint/tendermint/rpc/lib/types" ) @@ -214,7 +214,7 @@ func (c *WSClient) Send(ctx context.Context, request types.RPCRequest) error { // Call the given method. See Send description. func (c *WSClient) Call(ctx context.Context, method string, params map[string]interface{}) error { - request, err := types.MapToRequest(c.cdc, "ws-client", method, params) + request, err := types.MapToRequest(c.cdc, types.JSONRPCStringID("ws-client"), method, params) if err != nil { return err } @@ -224,7 +224,7 @@ func (c *WSClient) Call(ctx context.Context, method string, params map[string]in // CallWithArrayParams the given method with params in a form of array. See // Send description. func (c *WSClient) CallWithArrayParams(ctx context.Context, method string, params []interface{}) error { - request, err := types.ArrayToRequest(c.cdc, "ws-client", method, params) + request, err := types.ArrayToRequest(c.cdc, types.JSONRPCStringID("ws-client"), method, params) if err != nil { return err } diff --git a/rpc/lib/doc.go b/rpc/lib/doc.go index dbdb362d..aa9638bf 100644 --- a/rpc/lib/doc.go +++ b/rpc/lib/doc.go @@ -70,12 +70,9 @@ // wm := rpcserver.NewWebsocketManager(Routes) // mux.HandleFunc("/websocket", wm.WebsocketHandler) // logger := log.NewTMLogger(log.NewSyncWriter(os.Stdout)) -// go func() { -// _, err := rpcserver.StartHTTPServer("0.0.0.0:8008", mux, logger) -// if err != nil { -// panic(err) -// } -// }() +// listener, err := rpc.Listen("0.0.0.0:8080", rpcserver.Config{}) +// if err != nil { panic(err) } +// go rpcserver.StartHTTPServer(listener, mux, logger) // // Note that unix sockets are supported as well (eg. `/path/to/socket` instead of `0.0.0.0:8008`) // Now see all available endpoints by sending a GET request to `0.0.0.0:8008`. diff --git a/rpc/lib/rpc_test.go b/rpc/lib/rpc_test.go index 3d76db32..794ab462 100644 --- a/rpc/lib/rpc_test.go +++ b/rpc/lib/rpc_test.go @@ -121,12 +121,11 @@ func setup() { wm := server.NewWebsocketManager(Routes, RoutesCdc, server.ReadWait(5*time.Second), server.PingPeriod(1*time.Second)) wm.SetLogger(tcpLogger) mux.HandleFunc(websocketEndpoint, wm.WebsocketHandler) - go func() { - _, err := server.StartHTTPServer(tcpAddr, mux, tcpLogger, server.Config{}) - if err != nil { - panic(err) - } - }() + listener1, err := server.Listen(tcpAddr, server.Config{}) + if err != nil { + panic(err) + } + go server.StartHTTPServer(listener1, mux, tcpLogger) unixLogger := logger.With("socket", "unix") mux2 := http.NewServeMux() @@ -134,12 +133,11 @@ func setup() { wm = server.NewWebsocketManager(Routes, RoutesCdc) wm.SetLogger(unixLogger) mux2.HandleFunc(websocketEndpoint, wm.WebsocketHandler) - go func() { - _, err := server.StartHTTPServer(unixAddr, mux2, unixLogger, server.Config{}) - if err != nil { - panic(err) - } - }() + listener2, err := server.Listen(unixAddr, server.Config{}) + if err != nil { + panic(err) + } + go server.StartHTTPServer(listener2, mux2, unixLogger) // wait for servers to start time.Sleep(time.Second * 2) diff --git a/rpc/lib/server/handlers.go b/rpc/lib/server/handlers.go index 3ec5f81e..edab88fe 100644 --- a/rpc/lib/server/handlers.go +++ b/rpc/lib/server/handlers.go @@ -103,7 +103,7 @@ func makeJSONRPCHandler(funcMap map[string]*RPCFunc, cdc *amino.Codec, logger lo return func(w http.ResponseWriter, r *http.Request) { b, err := ioutil.ReadAll(r.Body) if err != nil { - WriteRPCResponseHTTP(w, types.RPCInvalidRequestError("", errors.Wrap(err, "Error reading request body"))) + WriteRPCResponseHTTP(w, types.RPCInvalidRequestError(types.JSONRPCStringID(""), errors.Wrap(err, "Error reading request body"))) return } // if its an empty request (like from a browser), @@ -116,12 +116,12 @@ func makeJSONRPCHandler(funcMap map[string]*RPCFunc, cdc *amino.Codec, logger lo var request types.RPCRequest err = json.Unmarshal(b, &request) if err != nil { - WriteRPCResponseHTTP(w, types.RPCParseError("", errors.Wrap(err, "Error unmarshalling request"))) + WriteRPCResponseHTTP(w, types.RPCParseError(types.JSONRPCStringID(""), errors.Wrap(err, "Error unmarshalling request"))) return } // A Notification is a Request object without an "id" member. // The Server MUST NOT reply to a Notification, including those that are within a batch request. - if request.ID == "" { + if request.ID == types.JSONRPCStringID("") { logger.Debug("HTTPJSONRPC received a notification, skipping... (please send a non-empty ID if you want to call a method)") return } @@ -255,7 +255,7 @@ func makeHTTPHandler(rpcFunc *RPCFunc, cdc *amino.Codec, logger log.Logger) func // Exception for websocket endpoints if rpcFunc.ws { return func(w http.ResponseWriter, r *http.Request) { - WriteRPCResponseHTTP(w, types.RPCMethodNotFoundError("")) + WriteRPCResponseHTTP(w, types.RPCMethodNotFoundError(types.JSONRPCStringID(""))) } } // All other endpoints @@ -263,17 +263,17 @@ func makeHTTPHandler(rpcFunc *RPCFunc, cdc *amino.Codec, logger log.Logger) func logger.Debug("HTTP HANDLER", "req", r) args, err := httpParamsToArgs(rpcFunc, cdc, r) if err != nil { - WriteRPCResponseHTTP(w, types.RPCInvalidParamsError("", errors.Wrap(err, "Error converting http params to arguments"))) + WriteRPCResponseHTTP(w, types.RPCInvalidParamsError(types.JSONRPCStringID(""), errors.Wrap(err, "Error converting http params to arguments"))) return } returns := rpcFunc.f.Call(args) logger.Info("HTTPRestRPC", "method", r.URL.Path, "args", args, "returns", returns) result, err := unreflectResult(returns) if err != nil { - WriteRPCResponseHTTP(w, types.RPCInternalError("", err)) + WriteRPCResponseHTTP(w, types.RPCInternalError(types.JSONRPCStringID(""), err)) return } - WriteRPCResponseHTTP(w, types.NewRPCSuccessResponse(cdc, "", result)) + WriteRPCResponseHTTP(w, types.NewRPCSuccessResponse(cdc, types.JSONRPCStringID(""), result)) } } @@ -580,7 +580,7 @@ func (wsc *wsConnection) readRoutine() { err = fmt.Errorf("WSJSONRPC: %v", r) } wsc.Logger.Error("Panic in WSJSONRPC handler", "err", err, "stack", string(debug.Stack())) - wsc.WriteRPCResponse(types.RPCInternalError("unknown", err)) + wsc.WriteRPCResponse(types.RPCInternalError(types.JSONRPCStringID("unknown"), err)) go wsc.readRoutine() } else { wsc.baseConn.Close() // nolint: errcheck @@ -615,13 +615,13 @@ func (wsc *wsConnection) readRoutine() { var request types.RPCRequest err = json.Unmarshal(in, &request) if err != nil { - wsc.WriteRPCResponse(types.RPCParseError("", errors.Wrap(err, "Error unmarshaling request"))) + wsc.WriteRPCResponse(types.RPCParseError(types.JSONRPCStringID(""), errors.Wrap(err, "Error unmarshaling request"))) continue } // A Notification is a Request object without an "id" member. // The Server MUST NOT reply to a Notification, including those that are within a batch request. - if request.ID == "" { + if request.ID == types.JSONRPCStringID("") { wsc.Logger.Debug("WSJSONRPC received a notification, skipping... (please send a non-empty ID if you want to call a method)") continue } diff --git a/rpc/lib/server/handlers_test.go b/rpc/lib/server/handlers_test.go index 6004959a..b1d3c788 100644 --- a/rpc/lib/server/handlers_test.go +++ b/rpc/lib/server/handlers_test.go @@ -47,21 +47,22 @@ func statusOK(code int) bool { return code >= 200 && code <= 299 } func TestRPCParams(t *testing.T) { mux := testMux() tests := []struct { - payload string - wantErr string + payload string + wantErr string + expectedId interface{} }{ // bad - {`{"jsonrpc": "2.0", "id": "0"}`, "Method not found"}, - {`{"jsonrpc": "2.0", "method": "y", "id": "0"}`, "Method not found"}, - {`{"method": "c", "id": "0", "params": a}`, "invalid character"}, - {`{"method": "c", "id": "0", "params": ["a"]}`, "got 1"}, - {`{"method": "c", "id": "0", "params": ["a", "b"]}`, "invalid character"}, - {`{"method": "c", "id": "0", "params": [1, 1]}`, "of type string"}, + {`{"jsonrpc": "2.0", "id": "0"}`, "Method not found", types.JSONRPCStringID("0")}, + {`{"jsonrpc": "2.0", "method": "y", "id": "0"}`, "Method not found", types.JSONRPCStringID("0")}, + {`{"method": "c", "id": "0", "params": a}`, "invalid character", types.JSONRPCStringID("")}, // id not captured in JSON parsing failures + {`{"method": "c", "id": "0", "params": ["a"]}`, "got 1", types.JSONRPCStringID("0")}, + {`{"method": "c", "id": "0", "params": ["a", "b"]}`, "invalid character", types.JSONRPCStringID("0")}, + {`{"method": "c", "id": "0", "params": [1, 1]}`, "of type string", types.JSONRPCStringID("0")}, // good - {`{"jsonrpc": "2.0", "method": "c", "id": "0", "params": null}`, ""}, - {`{"method": "c", "id": "0", "params": {}}`, ""}, - {`{"method": "c", "id": "0", "params": ["a", "10"]}`, ""}, + {`{"jsonrpc": "2.0", "method": "c", "id": "0", "params": null}`, "", types.JSONRPCStringID("0")}, + {`{"method": "c", "id": "0", "params": {}}`, "", types.JSONRPCStringID("0")}, + {`{"method": "c", "id": "0", "params": ["a", "10"]}`, "", types.JSONRPCStringID("0")}, } for i, tt := range tests { @@ -80,7 +81,7 @@ func TestRPCParams(t *testing.T) { recv := new(types.RPCResponse) assert.Nil(t, json.Unmarshal(blob, recv), "#%d: expecting successful parsing of an RPCResponse:\nblob: %s", i, blob) assert.NotEqual(t, recv, new(types.RPCResponse), "#%d: not expecting a blank RPCResponse", i) - + assert.Equal(t, tt.expectedId, recv.ID, "#%d: expected ID not matched in RPCResponse", i) if tt.wantErr == "" { assert.Nil(t, recv.Error, "#%d: not expecting an error", i) } else { @@ -91,9 +92,56 @@ func TestRPCParams(t *testing.T) { } } +func TestJSONRPCID(t *testing.T) { + mux := testMux() + tests := []struct { + payload string + wantErr bool + expectedId interface{} + }{ + // good id + {`{"jsonrpc": "2.0", "method": "c", "id": "0", "params": ["a", "10"]}`, false, types.JSONRPCStringID("0")}, + {`{"jsonrpc": "2.0", "method": "c", "id": "abc", "params": ["a", "10"]}`, false, types.JSONRPCStringID("abc")}, + {`{"jsonrpc": "2.0", "method": "c", "id": 0, "params": ["a", "10"]}`, false, types.JSONRPCIntID(0)}, + {`{"jsonrpc": "2.0", "method": "c", "id": 1, "params": ["a", "10"]}`, false, types.JSONRPCIntID(1)}, + {`{"jsonrpc": "2.0", "method": "c", "id": 1.3, "params": ["a", "10"]}`, false, types.JSONRPCIntID(1)}, + {`{"jsonrpc": "2.0", "method": "c", "id": -1, "params": ["a", "10"]}`, false, types.JSONRPCIntID(-1)}, + {`{"jsonrpc": "2.0", "method": "c", "id": null, "params": ["a", "10"]}`, false, nil}, + + // bad id + {`{"jsonrpc": "2.0", "method": "c", "id": {}, "params": ["a", "10"]}`, true, nil}, + {`{"jsonrpc": "2.0", "method": "c", "id": [], "params": ["a", "10"]}`, true, nil}, + } + + for i, tt := range tests { + req, _ := http.NewRequest("POST", "http://localhost/", strings.NewReader(tt.payload)) + rec := httptest.NewRecorder() + mux.ServeHTTP(rec, req) + res := rec.Result() + // Always expecting back a JSONRPCResponse + assert.True(t, statusOK(res.StatusCode), "#%d: should always return 2XX", i) + blob, err := ioutil.ReadAll(res.Body) + if err != nil { + t.Errorf("#%d: err reading body: %v", i, err) + continue + } + + recv := new(types.RPCResponse) + err = json.Unmarshal(blob, recv) + assert.Nil(t, err, "#%d: expecting successful parsing of an RPCResponse:\nblob: %s", i, blob) + if !tt.wantErr { + assert.NotEqual(t, recv, new(types.RPCResponse), "#%d: not expecting a blank RPCResponse", i) + assert.Equal(t, tt.expectedId, recv.ID, "#%d: expected ID not matched in RPCResponse", i) + assert.Nil(t, recv.Error, "#%d: not expecting an error", i) + } else { + assert.True(t, recv.Error.Code < 0, "#%d: not expecting a positive JSONRPC code", i) + } + } +} + func TestRPCNotification(t *testing.T) { mux := testMux() - body := strings.NewReader(`{"jsonrpc": "2.0"}`) + body := strings.NewReader(`{"jsonrpc": "2.0", "id": ""}`) req, _ := http.NewRequest("POST", "http://localhost/", body) rec := httptest.NewRecorder() mux.ServeHTTP(rec, req) @@ -134,7 +182,7 @@ func TestWebsocketManagerHandler(t *testing.T) { } // check basic functionality works - req, err := types.MapToRequest(amino.NewCodec(), "TestWebsocketManager", "c", map[string]interface{}{"s": "a", "i": 10}) + req, err := types.MapToRequest(amino.NewCodec(), types.JSONRPCStringID("TestWebsocketManager"), "c", map[string]interface{}{"s": "a", "i": 10}) require.NoError(t, err) err = c.WriteJSON(req) require.NoError(t, err) diff --git a/rpc/lib/server/http_server.go b/rpc/lib/server/http_server.go index 8069a81d..9db69b6f 100644 --- a/rpc/lib/server/http_server.go +++ b/rpc/lib/server/http_server.go @@ -27,92 +27,56 @@ const ( // maxBodyBytes controls the maximum number of bytes the // server will read parsing the request body. maxBodyBytes = int64(1000000) // 1MB + + // same as the net/http default + maxHeaderBytes = 1 << 20 + + // Timeouts for reading/writing to the http connection. + // Public so handlers can read them - + // /broadcast_tx_commit has it's own timeout, which should + // be less than the WriteTimeout here. + // TODO: use a config instead. + ReadTimeout = 3 * time.Second + WriteTimeout = 20 * time.Second ) -// StartHTTPServer starts an HTTP server on listenAddr with the given handler. +// StartHTTPServer takes a listener and starts an HTTP server with the given handler. // It wraps handler with RecoverAndLogHandler. -func StartHTTPServer( - listenAddr string, - handler http.Handler, - logger log.Logger, - config Config, -) (listener net.Listener, err error) { - var proto, addr string - parts := strings.SplitN(listenAddr, "://", 2) - if len(parts) != 2 { - return nil, errors.Errorf( - "Invalid listening address %s (use fully formed addresses, including the tcp:// or unix:// prefix)", - listenAddr, - ) +// NOTE: This function blocks - you may want to call it in a go-routine. +func StartHTTPServer(listener net.Listener, handler http.Handler, logger log.Logger) error { + logger.Info(fmt.Sprintf("Starting RPC HTTP server on %s", listener.Addr())) + s := &http.Server{ + Handler: RecoverAndLogHandler(maxBytesHandler{h: handler, n: maxBodyBytes}, logger), + ReadTimeout: ReadTimeout, + WriteTimeout: WriteTimeout, + MaxHeaderBytes: maxHeaderBytes, } - proto, addr = parts[0], parts[1] - - logger.Info(fmt.Sprintf("Starting RPC HTTP server on %s", listenAddr)) - listener, err = net.Listen(proto, addr) - if err != nil { - return nil, errors.Errorf("Failed to listen on %v: %v", listenAddr, err) - } - if config.MaxOpenConnections > 0 { - listener = netutil.LimitListener(listener, config.MaxOpenConnections) - } - - go func() { - err := http.Serve( - listener, - RecoverAndLogHandler(maxBytesHandler{h: handler, n: maxBodyBytes}, logger), - ) - logger.Info("RPC HTTP server stopped", "err", err) - }() - return listener, nil + err := s.Serve(listener) + logger.Info("RPC HTTP server stopped", "err", err) + return err } -// StartHTTPAndTLSServer starts an HTTPS server on listenAddr with the given -// handler. +// StartHTTPAndTLSServer takes a listener and starts an HTTPS server with the given handler. // It wraps handler with RecoverAndLogHandler. +// NOTE: This function blocks - you may want to call it in a go-routine. func StartHTTPAndTLSServer( - listenAddr string, + listener net.Listener, handler http.Handler, certFile, keyFile string, logger log.Logger, - config Config, -) (listener net.Listener, err error) { - var proto, addr string - parts := strings.SplitN(listenAddr, "://", 2) - if len(parts) != 2 { - return nil, errors.Errorf( - "Invalid listening address %s (use fully formed addresses, including the tcp:// or unix:// prefix)", - listenAddr, - ) +) error { + logger.Info(fmt.Sprintf("Starting RPC HTTPS server on %s (cert: %q, key: %q)", + listener.Addr(), certFile, keyFile)) + s := &http.Server{ + Handler: RecoverAndLogHandler(maxBytesHandler{h: handler, n: maxBodyBytes}, logger), + ReadTimeout: ReadTimeout, + WriteTimeout: WriteTimeout, + MaxHeaderBytes: maxHeaderBytes, } - proto, addr = parts[0], parts[1] + err := s.ServeTLS(listener, certFile, keyFile) - logger.Info( - fmt.Sprintf( - "Starting RPC HTTPS server on %s (cert: %q, key: %q)", - listenAddr, - certFile, - keyFile, - ), - ) - listener, err = net.Listen(proto, addr) - if err != nil { - return nil, errors.Errorf("Failed to listen on %v: %v", listenAddr, err) - } - if config.MaxOpenConnections > 0 { - listener = netutil.LimitListener(listener, config.MaxOpenConnections) - } - - err = http.ServeTLS( - listener, - RecoverAndLogHandler(maxBytesHandler{h: handler, n: maxBodyBytes}, logger), - certFile, - keyFile, - ) - if err != nil { - logger.Error("RPC HTTPS server stopped", "err", err) - return nil, err - } - return listener, nil + logger.Error("RPC HTTPS server stopped", "err", err) + return err } func WriteRPCResponseHTTPError( @@ -151,11 +115,6 @@ func RecoverAndLogHandler(handler http.Handler, logger log.Logger) http.Handler rww := &ResponseWriterWrapper{-1, w} begin := time.Now() - // Common headers - origin := r.Header.Get("Origin") - rww.Header().Set("Access-Control-Allow-Origin", origin) - rww.Header().Set("Access-Control-Allow-Credentials", "true") - rww.Header().Set("Access-Control-Expose-Headers", "X-Server-Time") rww.Header().Set("X-Server-Time", fmt.Sprintf("%v", begin.Unix())) defer func() { @@ -173,8 +132,7 @@ func RecoverAndLogHandler(handler http.Handler, logger log.Logger) http.Handler "Panic in RPC HTTP handler", "err", e, "stack", string(debug.Stack()), ) - rww.WriteHeader(http.StatusInternalServerError) - WriteRPCResponseHTTP(rww, types.RPCInternalError("", e.(error))) + WriteRPCResponseHTTPError(rww, http.StatusInternalServerError, types.RPCInternalError(types.JSONRPCStringID(""), e.(error))) } } @@ -219,3 +177,25 @@ func (h maxBytesHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { r.Body = http.MaxBytesReader(w, r.Body, h.n) h.h.ServeHTTP(w, r) } + +// Listen starts a new net.Listener on the given address. +// It returns an error if the address is invalid or the call to Listen() fails. +func Listen(addr string, config Config) (listener net.Listener, err error) { + parts := strings.SplitN(addr, "://", 2) + if len(parts) != 2 { + return nil, errors.Errorf( + "Invalid listening address %s (use fully formed addresses, including the tcp:// or unix:// prefix)", + addr, + ) + } + proto, addr := parts[0], parts[1] + listener, err = net.Listen(proto, addr) + if err != nil { + return nil, errors.Errorf("Failed to listen on %v: %v", addr, err) + } + if config.MaxOpenConnections > 0 { + listener = netutil.LimitListener(listener, config.MaxOpenConnections) + } + + return listener, nil +} diff --git a/rpc/lib/server/http_server_test.go b/rpc/lib/server/http_server_test.go index 73ebc2e7..6b852afa 100644 --- a/rpc/lib/server/http_server_test.go +++ b/rpc/lib/server/http_server_test.go @@ -30,11 +30,10 @@ func TestMaxOpenConnections(t *testing.T) { time.Sleep(10 * time.Millisecond) fmt.Fprint(w, "some body") }) - l, err := StartHTTPServer("tcp://127.0.0.1:0", mux, log.TestingLogger(), Config{MaxOpenConnections: max}) - if err != nil { - t.Fatal(err) - } + l, err := Listen("tcp://127.0.0.1:0", Config{MaxOpenConnections: max}) + require.NoError(t, err) defer l.Close() + go StartHTTPServer(l, mux, log.TestingLogger()) // Make N GET calls to the server. attempts := max * 2 @@ -67,11 +66,14 @@ func TestMaxOpenConnections(t *testing.T) { func TestStartHTTPAndTLSServer(t *testing.T) { // set up fixtures listenerAddr := "tcp://0.0.0.0:0" + listener, err := Listen(listenerAddr, Config{MaxOpenConnections: 1}) + require.NoError(t, err) mux := http.NewServeMux() mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {}) // test failure - gotListener, err := StartHTTPAndTLSServer(listenerAddr, mux, "", "", log.TestingLogger(), Config{MaxOpenConnections: 1}) - require.Nil(t, gotListener) + err = StartHTTPAndTLSServer(listener, mux, "", "", log.TestingLogger()) require.IsType(t, (*os.PathError)(nil), err) + + // TODO: test that starting the server can actually work } diff --git a/rpc/lib/test/main.go b/rpc/lib/test/main.go index 544284b9..0a9684d7 100644 --- a/rpc/lib/test/main.go +++ b/rpc/lib/test/main.go @@ -28,11 +28,11 @@ func main() { cdc := amino.NewCodec() logger := log.NewTMLogger(log.NewSyncWriter(os.Stdout)) rpcserver.RegisterRPCFuncs(mux, routes, cdc, logger) - _, err := rpcserver.StartHTTPServer("0.0.0.0:8008", mux, logger, rpcserver.Config{}) + listener, err := rpcserver.Listen("0.0.0.0:8008", rpcserver.Config{}) if err != nil { cmn.Exit(err.Error()) } - + go rpcserver.StartHTTPServer(listener, mux, logger) // Wait forever cmn.TrapSignal(func() { }) diff --git a/rpc/lib/types/types.go b/rpc/lib/types/types.go index fe9a9253..e0753a03 100644 --- a/rpc/lib/types/types.go +++ b/rpc/lib/types/types.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "reflect" "strings" "github.com/pkg/errors" @@ -13,17 +14,75 @@ import ( tmpubsub "github.com/tendermint/tendermint/libs/pubsub" ) +// a wrapper to emulate a sum type: jsonrpcid = string | int +// TODO: refactor when Go 2.0 arrives https://github.com/golang/go/issues/19412 +type jsonrpcid interface { + isJSONRPCID() +} + +// JSONRPCStringID a wrapper for JSON-RPC string IDs +type JSONRPCStringID string + +func (JSONRPCStringID) isJSONRPCID() {} + +// JSONRPCIntID a wrapper for JSON-RPC integer IDs +type JSONRPCIntID int + +func (JSONRPCIntID) isJSONRPCID() {} + +func idFromInterface(idInterface interface{}) (jsonrpcid, error) { + switch id := idInterface.(type) { + case string: + return JSONRPCStringID(id), nil + case float64: + // json.Unmarshal uses float64 for all numbers + // (https://golang.org/pkg/encoding/json/#Unmarshal), + // but the JSONRPC2.0 spec says the id SHOULD NOT contain + // decimals - so we truncate the decimals here. + return JSONRPCIntID(int(id)), nil + default: + typ := reflect.TypeOf(id) + return nil, fmt.Errorf("JSON-RPC ID (%v) is of unknown type (%v)", id, typ) + } +} + //---------------------------------------- // REQUEST type RPCRequest struct { JSONRPC string `json:"jsonrpc"` - ID string `json:"id"` + ID jsonrpcid `json:"id"` Method string `json:"method"` Params json.RawMessage `json:"params"` // must be map[string]interface{} or []interface{} } -func NewRPCRequest(id string, method string, params json.RawMessage) RPCRequest { +// UnmarshalJSON custom JSON unmarshalling due to jsonrpcid being string or int +func (request *RPCRequest) UnmarshalJSON(data []byte) error { + unsafeReq := &struct { + JSONRPC string `json:"jsonrpc"` + ID interface{} `json:"id"` + Method string `json:"method"` + Params json.RawMessage `json:"params"` // must be map[string]interface{} or []interface{} + }{} + err := json.Unmarshal(data, &unsafeReq) + if err != nil { + return err + } + request.JSONRPC = unsafeReq.JSONRPC + request.Method = unsafeReq.Method + request.Params = unsafeReq.Params + if unsafeReq.ID == nil { + return nil + } + id, err := idFromInterface(unsafeReq.ID) + if err != nil { + return err + } + request.ID = id + return nil +} + +func NewRPCRequest(id jsonrpcid, method string, params json.RawMessage) RPCRequest { return RPCRequest{ JSONRPC: "2.0", ID: id, @@ -36,7 +95,7 @@ func (req RPCRequest) String() string { return fmt.Sprintf("[%s %s]", req.ID, req.Method) } -func MapToRequest(cdc *amino.Codec, id string, method string, params map[string]interface{}) (RPCRequest, error) { +func MapToRequest(cdc *amino.Codec, id jsonrpcid, method string, params map[string]interface{}) (RPCRequest, error) { var params_ = make(map[string]json.RawMessage, len(params)) for name, value := range params { valueJSON, err := cdc.MarshalJSON(value) @@ -53,7 +112,7 @@ func MapToRequest(cdc *amino.Codec, id string, method string, params map[string] return request, nil } -func ArrayToRequest(cdc *amino.Codec, id string, method string, params []interface{}) (RPCRequest, error) { +func ArrayToRequest(cdc *amino.Codec, id jsonrpcid, method string, params []interface{}) (RPCRequest, error) { var params_ = make([]json.RawMessage, len(params)) for i, value := range params { valueJSON, err := cdc.MarshalJSON(value) @@ -89,12 +148,38 @@ func (err RPCError) Error() string { type RPCResponse struct { JSONRPC string `json:"jsonrpc"` - ID string `json:"id"` + ID jsonrpcid `json:"id"` Result json.RawMessage `json:"result,omitempty"` Error *RPCError `json:"error,omitempty"` } -func NewRPCSuccessResponse(cdc *amino.Codec, id string, res interface{}) RPCResponse { +// UnmarshalJSON custom JSON unmarshalling due to jsonrpcid being string or int +func (response *RPCResponse) UnmarshalJSON(data []byte) error { + unsafeResp := &struct { + JSONRPC string `json:"jsonrpc"` + ID interface{} `json:"id"` + Result json.RawMessage `json:"result,omitempty"` + Error *RPCError `json:"error,omitempty"` + }{} + err := json.Unmarshal(data, &unsafeResp) + if err != nil { + return err + } + response.JSONRPC = unsafeResp.JSONRPC + response.Error = unsafeResp.Error + response.Result = unsafeResp.Result + if unsafeResp.ID == nil { + return nil + } + id, err := idFromInterface(unsafeResp.ID) + if err != nil { + return err + } + response.ID = id + return nil +} + +func NewRPCSuccessResponse(cdc *amino.Codec, id jsonrpcid, res interface{}) RPCResponse { var rawMsg json.RawMessage if res != nil { @@ -109,7 +194,7 @@ func NewRPCSuccessResponse(cdc *amino.Codec, id string, res interface{}) RPCResp return RPCResponse{JSONRPC: "2.0", ID: id, Result: rawMsg} } -func NewRPCErrorResponse(id string, code int, msg string, data string) RPCResponse { +func NewRPCErrorResponse(id jsonrpcid, code int, msg string, data string) RPCResponse { return RPCResponse{ JSONRPC: "2.0", ID: id, @@ -124,27 +209,27 @@ func (resp RPCResponse) String() string { return fmt.Sprintf("[%s %s]", resp.ID, resp.Error) } -func RPCParseError(id string, err error) RPCResponse { +func RPCParseError(id jsonrpcid, err error) RPCResponse { return NewRPCErrorResponse(id, -32700, "Parse error. Invalid JSON", err.Error()) } -func RPCInvalidRequestError(id string, err error) RPCResponse { +func RPCInvalidRequestError(id jsonrpcid, err error) RPCResponse { return NewRPCErrorResponse(id, -32600, "Invalid Request", err.Error()) } -func RPCMethodNotFoundError(id string) RPCResponse { +func RPCMethodNotFoundError(id jsonrpcid) RPCResponse { return NewRPCErrorResponse(id, -32601, "Method not found", "") } -func RPCInvalidParamsError(id string, err error) RPCResponse { +func RPCInvalidParamsError(id jsonrpcid, err error) RPCResponse { return NewRPCErrorResponse(id, -32602, "Invalid params", err.Error()) } -func RPCInternalError(id string, err error) RPCResponse { +func RPCInternalError(id jsonrpcid, err error) RPCResponse { return NewRPCErrorResponse(id, -32603, "Internal error", err.Error()) } -func RPCServerError(id string, err error) RPCResponse { +func RPCServerError(id jsonrpcid, err error) RPCResponse { return NewRPCErrorResponse(id, -32000, "Server error", err.Error()) } diff --git a/rpc/lib/types/types_test.go b/rpc/lib/types/types_test.go index 9dd1b7a1..a5b2da9c 100644 --- a/rpc/lib/types/types_test.go +++ b/rpc/lib/types/types_test.go @@ -8,31 +8,64 @@ import ( "github.com/pkg/errors" "github.com/stretchr/testify/assert" - "github.com/tendermint/go-amino" + amino "github.com/tendermint/go-amino" ) type SampleResult struct { Value string } +type responseTest struct { + id jsonrpcid + expected string +} + +var responseTests = []responseTest{ + {JSONRPCStringID("1"), `"1"`}, + {JSONRPCStringID("alphabet"), `"alphabet"`}, + {JSONRPCStringID(""), `""`}, + {JSONRPCStringID("àáâ"), `"àáâ"`}, + {JSONRPCIntID(-1), "-1"}, + {JSONRPCIntID(0), "0"}, + {JSONRPCIntID(1), "1"}, + {JSONRPCIntID(100), "100"}, +} + func TestResponses(t *testing.T) { assert := assert.New(t) cdc := amino.NewCodec() + for _, tt := range responseTests { + jsonid := tt.id + a := NewRPCSuccessResponse(cdc, jsonid, &SampleResult{"hello"}) + b, _ := json.Marshal(a) + s := fmt.Sprintf(`{"jsonrpc":"2.0","id":%v,"result":{"Value":"hello"}}`, tt.expected) + assert.Equal(string(s), string(b)) - a := NewRPCSuccessResponse(cdc, "1", &SampleResult{"hello"}) - b, _ := json.Marshal(a) - s := `{"jsonrpc":"2.0","id":"1","result":{"Value":"hello"}}` - assert.Equal(string(s), string(b)) + d := RPCParseError(jsonid, errors.New("Hello world")) + e, _ := json.Marshal(d) + f := fmt.Sprintf(`{"jsonrpc":"2.0","id":%v,"error":{"code":-32700,"message":"Parse error. Invalid JSON","data":"Hello world"}}`, tt.expected) + assert.Equal(string(f), string(e)) - d := RPCParseError("1", errors.New("Hello world")) - e, _ := json.Marshal(d) - f := `{"jsonrpc":"2.0","id":"1","error":{"code":-32700,"message":"Parse error. Invalid JSON","data":"Hello world"}}` - assert.Equal(string(f), string(e)) + g := RPCMethodNotFoundError(jsonid) + h, _ := json.Marshal(g) + i := fmt.Sprintf(`{"jsonrpc":"2.0","id":%v,"error":{"code":-32601,"message":"Method not found"}}`, tt.expected) + assert.Equal(string(h), string(i)) + } +} - g := RPCMethodNotFoundError("2") - h, _ := json.Marshal(g) - i := `{"jsonrpc":"2.0","id":"2","error":{"code":-32601,"message":"Method not found"}}` - assert.Equal(string(h), string(i)) +func TestUnmarshallResponses(t *testing.T) { + assert := assert.New(t) + cdc := amino.NewCodec() + for _, tt := range responseTests { + response := &RPCResponse{} + err := json.Unmarshal([]byte(fmt.Sprintf(`{"jsonrpc":"2.0","id":%v,"result":{"Value":"hello"}}`, tt.expected)), response) + assert.Nil(err) + a := NewRPCSuccessResponse(cdc, tt.id, &SampleResult{"hello"}) + assert.Equal(*response, a) + } + response := &RPCResponse{} + err := json.Unmarshal([]byte(`{"jsonrpc":"2.0","id":true,"result":{"Value":"hello"}}`), response) + assert.NotNil(err) } func TestRPCError(t *testing.T) { diff --git a/rpc/lib/version.go b/rpc/lib/version.go deleted file mode 100644 index 8828f260..00000000 --- a/rpc/lib/version.go +++ /dev/null @@ -1,7 +0,0 @@ -package rpc - -const Maj = "0" -const Min = "7" -const Fix = "0" - -const Version = Maj + "." + Min + "." + Fix diff --git a/rpc/test/helpers.go b/rpc/test/helpers.go index 0a9cd984..67439b1d 100644 --- a/rpc/test/helpers.go +++ b/rpc/test/helpers.go @@ -11,9 +11,9 @@ import ( "github.com/tendermint/tendermint/libs/log" abci "github.com/tendermint/tendermint/abci/types" - cmn "github.com/tendermint/tendermint/libs/common" cfg "github.com/tendermint/tendermint/config" + cmn "github.com/tendermint/tendermint/libs/common" nm "github.com/tendermint/tendermint/node" "github.com/tendermint/tendermint/p2p" "github.com/tendermint/tendermint/privval" @@ -64,14 +64,17 @@ func makePathname() string { } func randPort() int { - return int(cmn.RandUint16()/2 + 10000) + port, err := cmn.GetFreePort() + if err != nil { + panic(err) + } + return port } func makeAddrs() (string, string, string) { - start := randPort() - return fmt.Sprintf("tcp://0.0.0.0:%d", start), - fmt.Sprintf("tcp://0.0.0.0:%d", start+1), - fmt.Sprintf("tcp://0.0.0.0:%d", start+2) + return fmt.Sprintf("tcp://0.0.0.0:%d", randPort()), + fmt.Sprintf("tcp://0.0.0.0:%d", randPort()), + fmt.Sprintf("tcp://0.0.0.0:%d", randPort()) } // GetConfig returns a config for the test cases as a singleton @@ -84,6 +87,7 @@ func GetConfig() *cfg.Config { tm, rpc, grpc := makeAddrs() globalConfig.P2P.ListenAddress = tm globalConfig.RPC.ListenAddress = rpc + globalConfig.RPC.CORSAllowedOrigins = []string{"https://tendermint.com/"} globalConfig.RPC.GRPCListenAddress = grpc globalConfig.TxIndex.IndexTags = "app.creator,tx.height" // see kvstore application } @@ -118,8 +122,9 @@ func NewTendermint(app abci.Application) *nm.Node { config := GetConfig() logger := log.NewTMLogger(log.NewSyncWriter(os.Stdout)) logger = log.NewFilter(logger, log.AllowError()) - pvFile := config.PrivValidatorFile() - pv := privval.LoadOrGenFilePV(pvFile) + pvKeyFile := config.PrivValidatorKeyFile() + pvKeyStateFile := config.PrivValidatorStateFile() + pv := privval.LoadOrGenFilePV(pvKeyFile, pvKeyStateFile) papp := proxy.NewLocalClientCreator(app) nodeKey, err := p2p.LoadOrGenNodeKey(config.NodeKeyFile()) if err != nil { diff --git a/scripts/authors.sh b/scripts/authors.sh new file mode 100755 index 00000000..7aafb012 --- /dev/null +++ b/scripts/authors.sh @@ -0,0 +1,16 @@ +#! /bin/bash + +# Usage: +# `./authors.sh` +# Print a list of all authors who have committed to develop since master. +# +# `./authors.sh ` +# Lookup the email address on Github and print the associated username + +author=$1 + +if [[ "$author" == "" ]]; then + git log master..develop | grep Author | sort | uniq +else + curl -s "https://api.github.com/search/users?q=$author+in%3Aemail&type=Users&utf8=%E2%9C%93" | jq .items[0].login +fi diff --git a/scripts/dist.sh b/scripts/dist.sh index 40aa71e9..f999c537 100755 --- a/scripts/dist.sh +++ b/scripts/dist.sh @@ -6,7 +6,7 @@ set -e # Get the version from the environment, or try to figure it out. if [ -z $VERSION ]; then - VERSION=$(awk -F\" '/Version =/ { print $2; exit }' < version/version.go) + VERSION=$(awk -F\" 'TMCoreSemVer =/ { print $2; exit }' < version/version.go) fi if [ -z "$VERSION" ]; then echo "Please specify a version." diff --git a/scripts/get_tools.sh b/scripts/get_tools.sh index 955ec943..dd956691 100755 --- a/scripts/get_tools.sh +++ b/scripts/get_tools.sh @@ -5,11 +5,14 @@ set -e # specific git hash. # # repos it installs: -# github.com/mitchellh/gox # github.com/golang/dep/cmd/dep -# gopkg.in/alecthomas/gometalinter.v2 # github.com/gogo/protobuf/protoc-gen-gogo # github.com/square/certstrap +# github.com/mitchellh/gox +# github.com/golangci/golangci-lint +# github.com/petermattis/goid +# github.com/sasha-s/go-deadlock +# goimports ## check if GOPATH is set if [ -z ${GOPATH+x} ]; then @@ -45,9 +48,22 @@ installFromGithub() { echo "" } -installFromGithub mitchellh/gox 51ed453898ca5579fea9ad1f08dff6b121d9f2e8 +######################## COMMON TOOLS ######################################## installFromGithub golang/dep 22125cfaa6ddc71e145b1535d4b7ee9744fefff2 cmd/dep -## gometalinter v2.0.11 -installFromGithub alecthomas/gometalinter 17a7ffa42374937bfecabfb8d2efbd4db0c26741 + +######################## DEVELOPER TOOLS ##################################### installFromGithub gogo/protobuf 61dbc136cf5d2f08d68a011382652244990a53a9 protoc-gen-gogo + installFromGithub square/certstrap e27060a3643e814151e65b9807b6b06d169580a7 + +# used to build tm-monitor & tm-bench binaries +installFromGithub mitchellh/gox 51ed453898ca5579fea9ad1f08dff6b121d9f2e8 + +## golangci-lint v1.13.2 +installFromGithub golangci/golangci-lint 7b2421d55194c9dc385eff7720a037aa9244ca3c cmd/golangci-lint + +## make test_with_deadlock +## XXX: https://github.com/tendermint/tendermint/issues/3242 +installFromGithub petermattis/goid b0b1615b78e5ee59739545bb38426383b2cda4c9 +installFromGithub sasha-s/go-deadlock d68e2bc52ae3291765881b9056f2c1527f245f1e +go get golang.org/x/tools/cmd/goimports diff --git a/scripts/install/install_tendermint_arm.sh b/scripts/install/install_tendermint_arm.sh new file mode 100644 index 00000000..06a20f14 --- /dev/null +++ b/scripts/install/install_tendermint_arm.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +REPO=github.com/tendermint/tendermint + +# change this to a specific release or branch +BRANCH=master + +GO_VERSION=1.11.4 + +sudo apt-get update -y + +# get and unpack golang +curl -O https://storage.googleapis.com/golang/go$GO_VERSION.linux-armv6l.tar.gz +tar -xvf go$GO_VERSION.linux-armv6l.tar.gz + +# move go folder and add go binary to path +sudo mv go /usr/local +echo "export PATH=\$PATH:/usr/local/go/bin" >> ~/.profile + +# create the go directory, set GOPATH, and put it on PATH +mkdir go +echo "export GOPATH=$HOME/go" >> ~/.profile +echo "export PATH=\$PATH:\$GOPATH/bin" >> ~/.profile +source ~/.profile + +# get the code and move into repo +go get $REPO +cd "$GOPATH/src/$REPO" + +# build & install +git checkout $BRANCH +# XXX: uncomment if branch isn't master +# git fetch origin $BRANCH +make get_tools +make get_vendor_deps +make install + +# the binary is located in $GOPATH/bin +# run `source ~/.profile` or reset your terminal +# to persist the changes diff --git a/scripts/install/install_tendermint_bsd.sh b/scripts/install/install_tendermint_bsd.sh index aba584f2..c3834058 100644 --- a/scripts/install/install_tendermint_bsd.sh +++ b/scripts/install/install_tendermint_bsd.sh @@ -14,19 +14,21 @@ # change this to a specific release or branch set BRANCH=master +set REPO=github.com/tendermint/tendermint + +set GO_VERSION=1.11.4 sudo pkg update -sudo pkg upgrade -y sudo pkg install -y gmake sudo pkg install -y git # get and unpack golang -curl -O https://storage.googleapis.com/golang/go1.10.freebsd-amd64.tar.gz -tar -xvf go1.10.freebsd-amd64.tar.gz +curl -O https://storage.googleapis.com/golang/go$GO_VERSION.freebsd-amd64.tar.gz +tar -xvf go$GO_VERSION.freebsd-amd64.tar.gz -# move go binary and add to path -mv go /usr/local +# move go folder and add go binary to path +sudo mv go /usr/local set path=($path /usr/local/go/bin) @@ -39,9 +41,8 @@ echo "set path=($path $GOPATH/bin)" >> ~/.tcshrc source ~/.tcshrc # get the code and move into repo -set REPO=github.com/tendermint/tendermint go get $REPO -cd $GOPATH/src/$REPO +cd "$GOPATH/src/$REPO" # build & install master git checkout $BRANCH diff --git a/scripts/install/install_tendermint_ubuntu.sh b/scripts/install/install_tendermint_ubuntu.sh index 0e1de117..59ab7a0a 100644 --- a/scripts/install/install_tendermint_ubuntu.sh +++ b/scripts/install/install_tendermint_ubuntu.sh @@ -13,28 +13,28 @@ REPO=github.com/tendermint/tendermint # change this to a specific release or branch BRANCH=master +GO_VERSION=1.11.4 + sudo apt-get update -y -sudo apt-get upgrade -y sudo apt-get install -y make # get and unpack golang -curl -O https://storage.googleapis.com/golang/go1.10.linux-amd64.tar.gz -tar -xvf go1.10.linux-amd64.tar.gz +curl -O https://storage.googleapis.com/golang/go$GO_VERSION.linux-amd64.tar.gz +tar -xvf go$GO_VERSION.linux-amd64.tar.gz -# move go binary and add to path -mv go /usr/local +# move go folder and add go binary to path +sudo mv go /usr/local echo "export PATH=\$PATH:/usr/local/go/bin" >> ~/.profile -# create the goApps directory, set GOPATH, and put it on PATH -mkdir goApps -echo "export GOPATH=/root/goApps" >> ~/.profile +# create the go directory, set GOPATH, and put it on PATH +mkdir go +echo "export GOPATH=$HOME/go" >> ~/.profile echo "export PATH=\$PATH:\$GOPATH/bin" >> ~/.profile - source ~/.profile # get the code and move into repo go get $REPO -cd $GOPATH/src/$REPO +cd "$GOPATH/src/$REPO" # build & install git checkout $BRANCH diff --git a/scripts/json2wal/main.go b/scripts/json2wal/main.go index be3487e5..dd3e29d0 100644 --- a/scripts/json2wal/main.go +++ b/scripts/json2wal/main.go @@ -14,7 +14,7 @@ import ( "os" "strings" - "github.com/tendermint/go-amino" + amino "github.com/tendermint/go-amino" cs "github.com/tendermint/tendermint/consensus" "github.com/tendermint/tendermint/types" ) @@ -45,7 +45,10 @@ func main() { } defer walFile.Close() - br := bufio.NewReader(f) + // the length of tendermint/wal/MsgInfo in the wal.json may exceed the defaultBufSize(4096) of bufio + // because of the byte array in BlockPart + // leading to unmarshal error: unexpected end of JSON input + br := bufio.NewReaderSize(f, 2*types.BlockPartSizeBytes) dec := cs.NewWALEncoder(walFile) for { diff --git a/scripts/privValUpgrade.go b/scripts/privValUpgrade.go new file mode 100644 index 00000000..72ce505e --- /dev/null +++ b/scripts/privValUpgrade.go @@ -0,0 +1,41 @@ +package main + +import ( + "fmt" + "os" + + "github.com/tendermint/tendermint/libs/log" + "github.com/tendermint/tendermint/privval" +) + +var ( + logger = log.NewTMLogger(log.NewSyncWriter(os.Stdout)) +) + +func main() { + args := os.Args[1:] + if len(args) != 3 { + fmt.Println("Expected three args: ") + fmt.Println("Eg. ~/.tendermint/config/priv_validator.json ~/.tendermint/config/priv_validator_key.json ~/.tendermint/data/priv_validator_state.json") + os.Exit(1) + } + err := loadAndUpgrade(args[0], args[1], args[2]) + if err != nil { + fmt.Println(err) + os.Exit(1) + } +} + +func loadAndUpgrade(oldPVPath, newPVKeyPath, newPVStatePath string) error { + oldPV, err := privval.LoadOldFilePV(oldPVPath) + if err != nil { + return fmt.Errorf("Error reading OldPrivValidator from %v: %v\n", oldPVPath, err) + } + logger.Info("Upgrading PrivValidator file", + "old", oldPVPath, + "newKey", newPVKeyPath, + "newState", newPVStatePath, + ) + oldPV.Upgrade(newPVKeyPath, newPVStatePath) + return nil +} diff --git a/scripts/privValUpgrade_test.go b/scripts/privValUpgrade_test.go new file mode 100644 index 00000000..bac4d315 --- /dev/null +++ b/scripts/privValUpgrade_test.go @@ -0,0 +1,121 @@ +package main + +import ( + "fmt" + "io/ioutil" + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/tendermint/tendermint/privval" +) + +const oldPrivvalContent = `{ + "address": "1D8089FAFDFAE4A637F3D616E17B92905FA2D91D", + "pub_key": { + "type": "tendermint/PubKeyEd25519", + "value": "r3Yg2AhDZ745CNTpavsGU+mRZ8WpRXqoJuyqjN8mJq0=" + }, + "last_height": "5", + "last_round": "0", + "last_step": 3, + "last_signature": "CTr7b9ZQlrJJf+12rPl5t/YSCUc/KqV7jQogCfFJA24e7hof69X6OMT7eFLVQHyodPjD/QTA298XHV5ejxInDQ==", + "last_signbytes": "750802110500000000000000220B08B398F3E00510F48DA6402A480A20FC258973076512999C3E6839A22E9FBDB1B77CF993E8A9955412A41A59D4CAD312240A20C971B286ACB8AAA6FCA0365EB0A660B189EDC08B46B5AF2995DEFA51A28D215B10013211746573742D636861696E2D533245415533", + "priv_key": { + "type": "tendermint/PrivKeyEd25519", + "value": "7MwvTGEWWjsYwjn2IpRb+GYsWi9nnFsw8jPLLY1UtP6vdiDYCENnvjkI1Olq+wZT6ZFnxalFeqgm7KqM3yYmrQ==" + } +}` + +func TestLoadAndUpgrade(t *testing.T) { + + oldFilePath := initTmpOldFile(t) + defer os.Remove(oldFilePath) + newStateFile, err := ioutil.TempFile("", "priv_validator_state*.json") + defer os.Remove(newStateFile.Name()) + require.NoError(t, err) + newKeyFile, err := ioutil.TempFile("", "priv_validator_key*.json") + defer os.Remove(newKeyFile.Name()) + require.NoError(t, err) + emptyOldFile, err := ioutil.TempFile("", "priv_validator_empty*.json") + require.NoError(t, err) + defer os.Remove(emptyOldFile.Name()) + + type args struct { + oldPVPath string + newPVKeyPath string + newPVStatePath string + } + tests := []struct { + name string + args args + wantErr bool + wantPanic bool + }{ + {"successful upgrade", + args{oldPVPath: oldFilePath, newPVKeyPath: newKeyFile.Name(), newPVStatePath: newStateFile.Name()}, + false, false, + }, + {"unsuccessful upgrade: empty old privval file", + args{oldPVPath: emptyOldFile.Name(), newPVKeyPath: newKeyFile.Name(), newPVStatePath: newStateFile.Name()}, + true, false, + }, + {"unsuccessful upgrade: invalid new paths (1/3)", + args{oldPVPath: oldFilePath, newPVKeyPath: "", newPVStatePath: newStateFile.Name()}, + false, true, + }, + {"unsuccessful upgrade: invalid new paths (2/3)", + args{oldPVPath: oldFilePath, newPVKeyPath: newKeyFile.Name(), newPVStatePath: ""}, + false, true, + }, + {"unsuccessful upgrade: invalid new paths (3/3)", + args{oldPVPath: oldFilePath, newPVKeyPath: "", newPVStatePath: ""}, + false, true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // need to re-write the file everytime because upgrading renames it + err := ioutil.WriteFile(oldFilePath, []byte(oldPrivvalContent), 0600) + require.NoError(t, err) + if tt.wantPanic { + require.Panics(t, func() { loadAndUpgrade(tt.args.oldPVPath, tt.args.newPVKeyPath, tt.args.newPVStatePath) }) + } else { + err = loadAndUpgrade(tt.args.oldPVPath, tt.args.newPVKeyPath, tt.args.newPVStatePath) + if tt.wantErr { + assert.Error(t, err) + fmt.Println("ERR", err) + } else { + assert.NoError(t, err) + upgradedPV := privval.LoadFilePV(tt.args.newPVKeyPath, tt.args.newPVStatePath) + oldPV, err := privval.LoadOldFilePV(tt.args.oldPVPath + ".bak") + require.NoError(t, err) + + assert.Equal(t, oldPV.Address, upgradedPV.Key.Address) + assert.Equal(t, oldPV.Address, upgradedPV.GetAddress()) + assert.Equal(t, oldPV.PubKey, upgradedPV.Key.PubKey) + assert.Equal(t, oldPV.PubKey, upgradedPV.GetPubKey()) + assert.Equal(t, oldPV.PrivKey, upgradedPV.Key.PrivKey) + + assert.Equal(t, oldPV.LastHeight, upgradedPV.LastSignState.Height) + assert.Equal(t, oldPV.LastRound, upgradedPV.LastSignState.Round) + assert.Equal(t, oldPV.LastSignature, upgradedPV.LastSignState.Signature) + assert.Equal(t, oldPV.LastSignBytes, upgradedPV.LastSignState.SignBytes) + assert.Equal(t, oldPV.LastStep, upgradedPV.LastSignState.Step) + + } + } + }) + } +} + +func initTmpOldFile(t *testing.T) string { + tmpfile, err := ioutil.TempFile("", "priv_validator_*.json") + require.NoError(t, err) + t.Logf("created test file %s", tmpfile.Name()) + _, err = tmpfile.WriteString(oldPrivvalContent) + require.NoError(t, err) + + return tmpfile.Name() +} diff --git a/scripts/publish.sh b/scripts/publish.sh index ba944087..7da299aa 100755 --- a/scripts/publish.sh +++ b/scripts/publish.sh @@ -6,7 +6,7 @@ DIST_DIR=./build/dist # Get the version from the environment, or try to figure it out. if [ -z $VERSION ]; then - VERSION=$(awk -F\" '/Version =/ { print $2; exit }' < version/version.go) + VERSION=$(awk -F\" 'TMCoreSemVer =/ { print $2; exit }' < version/version.go) fi if [ -z "$VERSION" ]; then echo "Please specify a version." diff --git a/scripts/release.sh b/scripts/release.sh index 9a4e508e..8c40d36b 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -3,7 +3,7 @@ set -e # Get the version from the environment, or try to figure it out. if [ -z $VERSION ]; then - VERSION=$(awk -F\" '/Version =/ { print $2; exit }' < version/version.go) + VERSION=$(awk -F\" 'TMCoreSemVer =/ { print $2; exit }' < version/version.go) fi if [ -z "$VERSION" ]; then echo "Please specify a version." diff --git a/scripts/wal2json/main.go b/scripts/wal2json/main.go index cf8ae86c..ee90ecaa 100644 --- a/scripts/wal2json/main.go +++ b/scripts/wal2json/main.go @@ -12,7 +12,7 @@ import ( "io" "os" - "github.com/tendermint/go-amino" + amino "github.com/tendermint/go-amino" cs "github.com/tendermint/tendermint/consensus" "github.com/tendermint/tendermint/types" ) diff --git a/scripts/wire2amino.go b/scripts/wire2amino.go deleted file mode 100644 index 26069b50..00000000 --- a/scripts/wire2amino.go +++ /dev/null @@ -1,182 +0,0 @@ -package main - -import ( - "encoding/json" - "fmt" - "io/ioutil" - "os" - "path/filepath" - "time" - - "github.com/tendermint/go-amino" - "github.com/tendermint/tendermint/crypto/ed25519" - cryptoAmino "github.com/tendermint/tendermint/crypto/encoding/amino" - - cmn "github.com/tendermint/tendermint/libs/common" - - "github.com/tendermint/tendermint/p2p" - "github.com/tendermint/tendermint/privval" - "github.com/tendermint/tendermint/types" -) - -type GenesisValidator struct { - PubKey Data `json:"pub_key"` - Power int64 `json:"power"` - Name string `json:"name"` -} - -type Genesis struct { - GenesisTime time.Time `json:"genesis_time"` - ChainID string `json:"chain_id"` - ConsensusParams *types.ConsensusParams `json:"consensus_params,omitempty"` - Validators []GenesisValidator `json:"validators"` - AppHash cmn.HexBytes `json:"app_hash"` - AppState json.RawMessage `json:"app_state,omitempty"` - AppOptions json.RawMessage `json:"app_options,omitempty"` // DEPRECATED -} - -type NodeKey struct { - PrivKey Data `json:"priv_key"` -} - -type PrivVal struct { - Address cmn.HexBytes `json:"address"` - LastHeight int64 `json:"last_height"` - LastRound int `json:"last_round"` - LastStep int8 `json:"last_step"` - PubKey Data `json:"pub_key"` - PrivKey Data `json:"priv_key"` -} - -type Data struct { - Type string `json:"type"` - Data cmn.HexBytes `json:"data"` -} - -func convertNodeKey(cdc *amino.Codec, jsonBytes []byte) ([]byte, error) { - var nodeKey NodeKey - err := json.Unmarshal(jsonBytes, &nodeKey) - if err != nil { - return nil, err - } - - var privKey ed25519.PrivKeyEd25519 - copy(privKey[:], nodeKey.PrivKey.Data) - - nodeKeyNew := p2p.NodeKey{privKey} - - bz, err := cdc.MarshalJSON(nodeKeyNew) - if err != nil { - return nil, err - } - return bz, nil -} - -func convertPrivVal(cdc *amino.Codec, jsonBytes []byte) ([]byte, error) { - var privVal PrivVal - err := json.Unmarshal(jsonBytes, &privVal) - if err != nil { - return nil, err - } - - var privKey ed25519.PrivKeyEd25519 - copy(privKey[:], privVal.PrivKey.Data) - - var pubKey ed25519.PubKeyEd25519 - copy(pubKey[:], privVal.PubKey.Data) - - privValNew := privval.FilePV{ - Address: pubKey.Address(), - PubKey: pubKey, - LastHeight: privVal.LastHeight, - LastRound: privVal.LastRound, - LastStep: privVal.LastStep, - PrivKey: privKey, - } - - bz, err := cdc.MarshalJSON(privValNew) - if err != nil { - return nil, err - } - return bz, nil -} - -func convertGenesis(cdc *amino.Codec, jsonBytes []byte) ([]byte, error) { - var genesis Genesis - err := json.Unmarshal(jsonBytes, &genesis) - if err != nil { - return nil, err - } - - genesisNew := types.GenesisDoc{ - GenesisTime: genesis.GenesisTime, - ChainID: genesis.ChainID, - ConsensusParams: genesis.ConsensusParams, - // Validators - AppHash: genesis.AppHash, - AppState: genesis.AppState, - } - - if genesis.AppOptions != nil { - genesisNew.AppState = genesis.AppOptions - } - - for _, v := range genesis.Validators { - var pubKey ed25519.PubKeyEd25519 - copy(pubKey[:], v.PubKey.Data) - genesisNew.Validators = append( - genesisNew.Validators, - types.GenesisValidator{ - PubKey: pubKey, - Power: v.Power, - Name: v.Name, - }, - ) - - } - - bz, err := cdc.MarshalJSON(genesisNew) - if err != nil { - return nil, err - } - return bz, nil -} - -func main() { - cdc := amino.NewCodec() - cryptoAmino.RegisterAmino(cdc) - - args := os.Args[1:] - if len(args) != 1 { - fmt.Println("Please specify a file to convert") - os.Exit(1) - } - - filePath := args[0] - fileName := filepath.Base(filePath) - - fileBytes, err := ioutil.ReadFile(filePath) - if err != nil { - panic(err) - } - - var bz []byte - - switch fileName { - case "node_key.json": - bz, err = convertNodeKey(cdc, fileBytes) - case "priv_validator.json": - bz, err = convertPrivVal(cdc, fileBytes) - case "genesis.json": - bz, err = convertGenesis(cdc, fileBytes) - default: - fmt.Println("Expected file name to be in (node_key.json, priv_validator.json, genesis.json)") - os.Exit(1) - } - - if err != nil { - panic(err) - } - fmt.Println(string(bz)) - -} diff --git a/state/execution.go b/state/execution.go index d5a1a161..470e22bc 100644 --- a/state/execution.go +++ b/state/execution.go @@ -4,11 +4,10 @@ import ( "fmt" "time" - "github.com/ebuchman/fail-test" abci "github.com/tendermint/tendermint/abci/types" dbm "github.com/tendermint/tendermint/libs/db" + "github.com/tendermint/tendermint/libs/fail" "github.com/tendermint/tendermint/libs/log" - "github.com/tendermint/tendermint/mempool" "github.com/tendermint/tendermint/proxy" "github.com/tendermint/tendermint/types" ) @@ -29,7 +28,8 @@ type BlockExecutor struct { // events eventBus types.BlockEventPublisher - // update these with block results after commit + // manage the mempool lock during commit + // and update both with block results after commit. mempool Mempool evpool EvidencePool @@ -48,8 +48,7 @@ func BlockExecutorWithMetrics(metrics *Metrics) BlockExecutorOption { // NewBlockExecutor returns a new BlockExecutor with a NopEventBus. // Call SetEventBus to provide one. -func NewBlockExecutor(db dbm.DB, logger log.Logger, proxyApp proxy.AppConnConsensus, - mempool Mempool, evpool EvidencePool, options ...BlockExecutorOption) *BlockExecutor { +func NewBlockExecutor(db dbm.DB, logger log.Logger, proxyApp proxy.AppConnConsensus, mempool Mempool, evpool EvidencePool, options ...BlockExecutorOption) *BlockExecutor { res := &BlockExecutor{ db: db, proxyApp: proxyApp, @@ -73,6 +72,31 @@ func (blockExec *BlockExecutor) SetEventBus(eventBus types.BlockEventPublisher) blockExec.eventBus = eventBus } +// CreateProposalBlock calls state.MakeBlock with evidence from the evpool +// and txs from the mempool. The max bytes must be big enough to fit the commit. +// Up to 1/10th of the block space is allcoated for maximum sized evidence. +// The rest is given to txs, up to the max gas. +func (blockExec *BlockExecutor) CreateProposalBlock( + height int64, + state State, commit *types.Commit, + proposerAddr []byte, +) (*types.Block, *types.PartSet) { + + maxBytes := state.ConsensusParams.BlockSize.MaxBytes + maxGas := state.ConsensusParams.BlockSize.MaxGas + + // Fetch a limited amount of valid evidence + maxNumEvidence, _ := types.MaxEvidencePerBlock(maxBytes) + evidence := blockExec.evpool.PendingEvidence(maxNumEvidence) + + // Fetch a limited amount of valid txs + maxDataBytes := types.MaxDataBytes(maxBytes, state.Validators.Size(), len(evidence)) + txs := blockExec.mempool.ReapMaxBytesMaxGas(maxDataBytes, maxGas) + + return state.MakeBlock(height, txs, commit, evidence, proposerAddr) + +} + // ValidateBlock validates the given block against the given state. // If the block is invalid, it returns an error. // Validation does not mutate state, but does require historical information from the stateDB, @@ -95,7 +119,7 @@ func (blockExec *BlockExecutor) ApplyBlock(state State, blockID types.BlockID, b startTime := time.Now().UnixNano() abciResponses, err := execBlockOnProxyApp(blockExec.logger, blockExec.proxyApp, block, state.LastValidators, blockExec.db) endTime := time.Now().UnixNano() - blockExec.metrics.BlockProcessingTime.Observe(float64(endTime - startTime) / 1000000) + blockExec.metrics.BlockProcessingTime.Observe(float64(endTime-startTime) / 1000000) if err != nil { return state, ErrProxyAppConn(err) } @@ -107,8 +131,22 @@ func (blockExec *BlockExecutor) ApplyBlock(state State, blockID types.BlockID, b fail.Fail() // XXX + // validate the validator updates and convert to tendermint types + abciValUpdates := abciResponses.EndBlock.ValidatorUpdates + err = validateValidatorUpdates(abciValUpdates, state.ConsensusParams.Validator) + if err != nil { + return state, fmt.Errorf("Error in validator updates: %v", err) + } + validatorUpdates, err := types.PB2TM.ValidatorUpdates(abciValUpdates) + if err != nil { + return state, err + } + if len(validatorUpdates) > 0 { + blockExec.logger.Info("Updates to validators", "updates", types.ValidatorListString(validatorUpdates)) + } + // Update the state with the block and responses. - state, err = updateState(state, blockID, &block.Header, abciResponses) + state, err = updateState(state, blockID, &block.Header, abciResponses, validatorUpdates) if err != nil { return state, fmt.Errorf("Commit failed for application: %v", err) } @@ -132,7 +170,7 @@ func (blockExec *BlockExecutor) ApplyBlock(state State, blockID types.BlockID, b // Events are fired after everything else. // NOTE: if we crash between Commit and Save, events wont be fired during replay - fireEvents(blockExec.logger, blockExec.eventBus, block, abciResponses) + fireEvents(blockExec.logger, blockExec.eventBus, block, abciResponses, validatorUpdates) return state, nil } @@ -180,13 +218,8 @@ func (blockExec *BlockExecutor) Commit( err = blockExec.mempool.Update( block.Height, block.Txs, - mempool.PreCheckAminoMaxBytes( - types.MaxDataBytesUnknownEvidence( - state.ConsensusParams.BlockSize.MaxBytes, - state.Validators.Size(), - ), - ), - mempool.PostCheckMaxGas(state.ConsensusParams.MaxGas), + TxPreCheck(state), + TxPostCheck(state), ) return res.Data, err @@ -198,11 +231,11 @@ func (blockExec *BlockExecutor) Commit( // Executes block's transactions on proxyAppConn. // Returns a list of transaction results and updates to the validator set func execBlockOnProxyApp( - logger log.Logger, - proxyAppConn proxy.AppConnConsensus, - block *types.Block, - lastValSet *types.ValidatorSet, - stateDB dbm.DB, + logger log.Logger, + proxyAppConn proxy.AppConnConsensus, + block *types.Block, + lastValSet *types.ValidatorSet, + stateDB dbm.DB, ) (*ABCIResponses, error) { var validTxs, invalidTxs = 0, 0 @@ -231,8 +264,9 @@ func execBlockOnProxyApp( commitInfo, byzVals := getBeginBlockValidatorInfo(block, lastValSet, stateDB) - // Begin block. - _, err := proxyAppConn.BeginBlockSync(abci.RequestBeginBlock{ + // Begin block + var err error + abciResponses.BeginBlock, err = proxyAppConn.BeginBlockSync(abci.RequestBeginBlock{ Hash: block.Hash(), Header: types.TM2PB.Header(&block.Header), LastCommitInfo: commitInfo, @@ -260,12 +294,6 @@ func execBlockOnProxyApp( logger.Info("Executed block", "height", block.Height, "validTxs", validTxs, "invalidTxs", invalidTxs) - valUpdates := abciResponses.EndBlock.ValidatorUpdates - if len(valUpdates) > 0 { - // TODO: cleanup the formatting - logger.Info("Updates to validators", "updates", valUpdates) - } - return abciResponses, nil } @@ -286,7 +314,7 @@ func getBeginBlockValidatorInfo(block *types.Block, lastValSet *types.ValidatorS // Collect the vote info (list of validators and whether or not they signed). voteInfos := make([]abci.VoteInfo, len(lastValSet.Validators)) for i, val := range lastValSet.Validators { - var vote *types.Vote + var vote *types.CommitSig if i < len(block.LastCommit.Precommits) { vote = block.LastCommit.Precommits[i] } @@ -318,41 +346,22 @@ func getBeginBlockValidatorInfo(block *types.Block, lastValSet *types.ValidatorS } -// If more or equal than 1/3 of total voting power changed in one block, then -// a light client could never prove the transition externally. See -// ./lite/doc.go for details on how a light client tracks validators. -func updateValidators(currentSet *types.ValidatorSet, abciUpdates []abci.ValidatorUpdate) error { - updates, err := types.PB2TM.ValidatorUpdates(abciUpdates) - if err != nil { - return err - } - - // these are tendermint types now - for _, valUpdate := range updates { - if valUpdate.VotingPower < 0 { +func validateValidatorUpdates(abciUpdates []abci.ValidatorUpdate, + params types.ValidatorParams) error { + for _, valUpdate := range abciUpdates { + if valUpdate.GetPower() < 0 { return fmt.Errorf("Voting power can't be negative %v", valUpdate) + } else if valUpdate.GetPower() == 0 { + // continue, since this is deleting the validator, and thus there is no + // pubkey to check + continue } - address := valUpdate.Address - _, val := currentSet.GetByAddress(address) - if valUpdate.VotingPower == 0 { - // remove val - _, removed := currentSet.Remove(address) - if !removed { - return fmt.Errorf("Failed to remove validator %X", address) - } - } else if val == nil { - // add val - added := currentSet.Add(valUpdate) - if !added { - return fmt.Errorf("Failed to add new validator %v", valUpdate) - } - } else { - // update val - updated := currentSet.Update(valUpdate) - if !updated { - return fmt.Errorf("Failed to update validator %X to %v", address, valUpdate) - } + // Check if validator's pubkey matches an ABCI type in the consensus params + thisKeyType := valUpdate.PubKey.Type + if !params.IsValidPubkeyType(thisKeyType) { + return fmt.Errorf("Validator %v is using pubkey %s, which is unsupported for consensus", + valUpdate, thisKeyType) } } return nil @@ -360,10 +369,11 @@ func updateValidators(currentSet *types.ValidatorSet, abciUpdates []abci.Validat // updateState returns a new State updated according to the header and responses. func updateState( - state State, - blockID types.BlockID, - header *types.Header, - abciResponses *ABCIResponses, + state State, + blockID types.BlockID, + header *types.Header, + abciResponses *ABCIResponses, + validatorUpdates []*types.Validator, ) (State, error) { // Copy the valset so we can apply changes from EndBlock @@ -372,8 +382,8 @@ func updateState( // Update the validator set with the latest abciResponses. lastHeightValsChanged := state.LastHeightValidatorsChanged - if len(abciResponses.EndBlock.ValidatorUpdates) > 0 { - err := updateValidators(nValSet, abciResponses.EndBlock.ValidatorUpdates) + if len(validatorUpdates) > 0 { + err := nValSet.UpdateWithChangeSet(validatorUpdates) if err != nil { return state, fmt.Errorf("Error changing validator set: %v", err) } @@ -381,8 +391,8 @@ func updateState( lastHeightValsChanged = header.Height + 1 + 1 } - // Update validator accums and set state variables. - nValSet.IncrementAccum(1) + // Update validator proposer priority and set state variables. + nValSet.IncrementProposerPriority(1) // Update the params with the latest abciResponses. nextParams := state.ConsensusParams @@ -398,9 +408,13 @@ func updateState( lastHeightParamsChanged = header.Height + 1 } + // TODO: allow app to upgrade version + nextVersion := state.Version + // NOTE: the AppHash has not been populated. // It will be filled on state.Save. return State{ + Version: nextVersion, ChainID: state.ChainID, LastBlockHeight: header.Height, LastBlockTotalTx: state.LastBlockTotalTx + header.NumTxs, @@ -420,9 +434,17 @@ func updateState( // Fire NewBlock, NewBlockHeader. // Fire TxEvent for every tx. // NOTE: if Tendermint crashes before commit, some or all of these events may be published again. -func fireEvents(logger log.Logger, eventBus types.BlockEventPublisher, block *types.Block, abciResponses *ABCIResponses) { - eventBus.PublishEventNewBlock(types.EventDataNewBlock{block}) - eventBus.PublishEventNewBlockHeader(types.EventDataNewBlockHeader{block.Header}) +func fireEvents(logger log.Logger, eventBus types.BlockEventPublisher, block *types.Block, abciResponses *ABCIResponses, validatorUpdates []*types.Validator) { + eventBus.PublishEventNewBlock(types.EventDataNewBlock{ + Block: block, + ResultBeginBlock: *abciResponses.BeginBlock, + ResultEndBlock: *abciResponses.EndBlock, + }) + eventBus.PublishEventNewBlockHeader(types.EventDataNewBlockHeader{ + Header: block.Header, + ResultBeginBlock: *abciResponses.BeginBlock, + ResultEndBlock: *abciResponses.EndBlock, + }) for i, tx := range block.Data.Txs { eventBus.PublishEventTx(types.EventDataTx{types.TxResult{ @@ -433,12 +455,9 @@ func fireEvents(logger log.Logger, eventBus types.BlockEventPublisher, block *ty }}) } - abciValUpdates := abciResponses.EndBlock.ValidatorUpdates - if len(abciValUpdates) > 0 { - // if there were an error, we would've stopped in updateValidators - updates, _ := types.PB2TM.ValidatorUpdates(abciValUpdates) + if len(validatorUpdates) > 0 { eventBus.PublishEventValidatorSetUpdates( - types.EventDataValidatorSetUpdates{ValidatorUpdates: updates}) + types.EventDataValidatorSetUpdates{ValidatorUpdates: validatorUpdates}) } } @@ -448,11 +467,11 @@ func fireEvents(logger log.Logger, eventBus types.BlockEventPublisher, block *ty // ExecCommitBlock executes and commits a block on the proxyApp without validating or mutating the state. // It returns the application root hash (result of abci.Commit). func ExecCommitBlock( - appConnConsensus proxy.AppConnConsensus, - block *types.Block, - logger log.Logger, - lastValSet *types.ValidatorSet, - stateDB dbm.DB, + appConnConsensus proxy.AppConnConsensus, + block *types.Block, + logger log.Logger, + lastValSet *types.ValidatorSet, + stateDB dbm.DB, ) ([]byte, error) { _, err := execBlockOnProxyApp(logger, appConnConsensus, block, lastValSet, stateDB) if err != nil { diff --git a/state/execution_test.go b/state/execution_test.go index e93c9bfd..e0c9b4b9 100644 --- a/state/execution_test.go +++ b/state/execution_test.go @@ -12,6 +12,7 @@ import ( "github.com/tendermint/tendermint/abci/example/kvstore" abci "github.com/tendermint/tendermint/abci/types" "github.com/tendermint/tendermint/crypto/ed25519" + "github.com/tendermint/tendermint/crypto/secp256k1" cmn "github.com/tendermint/tendermint/libs/common" dbm "github.com/tendermint/tendermint/libs/db" "github.com/tendermint/tendermint/libs/log" @@ -64,17 +65,17 @@ func TestBeginBlockValidators(t *testing.T) { prevBlockID := types.BlockID{prevHash, prevParts} now := tmtime.Now() - vote0 := &types.Vote{ValidatorIndex: 0, Timestamp: now, Type: types.VoteTypePrecommit} - vote1 := &types.Vote{ValidatorIndex: 1, Timestamp: now} + commitSig0 := (&types.Vote{ValidatorIndex: 0, Timestamp: now, Type: types.PrecommitType}).CommitSig() + commitSig1 := (&types.Vote{ValidatorIndex: 1, Timestamp: now}).CommitSig() testCases := []struct { desc string - lastCommitPrecommits []*types.Vote + lastCommitPrecommits []*types.CommitSig expectedAbsentValidators []int }{ - {"none absent", []*types.Vote{vote0, vote1}, []int{}}, - {"one absent", []*types.Vote{vote0, nil}, []int{1}}, - {"multiple absent", []*types.Vote{nil, nil}, []int{0, 1}}, + {"none absent", []*types.CommitSig{commitSig0, commitSig1}, []int{}}, + {"one absent", []*types.CommitSig{commitSig0, nil}, []int{1}}, + {"multiple absent", []*types.CommitSig{nil, nil}, []int{0, 1}}, } for _, tc := range testCases { @@ -135,10 +136,10 @@ func TestBeginBlockByzantineValidators(t *testing.T) { types.TM2PB.Evidence(ev2, valSet, now)}}, } - vote0 := &types.Vote{ValidatorIndex: 0, Timestamp: now, Type: types.VoteTypePrecommit} - vote1 := &types.Vote{ValidatorIndex: 1, Timestamp: now} - votes := []*types.Vote{vote0, vote1} - lastCommit := &types.Commit{BlockID: prevBlockID, Precommits: votes} + commitSig0 := (&types.Vote{ValidatorIndex: 0, Timestamp: now, Type: types.PrecommitType}).CommitSig() + commitSig1 := (&types.Vote{ValidatorIndex: 1, Timestamp: now}).CommitSig() + commitSigs := []*types.CommitSig{commitSig0, commitSig1} + lastCommit := &types.Commit{BlockID: prevBlockID, Precommits: commitSigs} for _, tc := range testCases { block, _ := state.MakeBlock(10, makeTxs(2), lastCommit, nil, state.Validators.GetProposer().Address) @@ -152,6 +153,76 @@ func TestBeginBlockByzantineValidators(t *testing.T) { } } +func TestValidateValidatorUpdates(t *testing.T) { + pubkey1 := ed25519.GenPrivKey().PubKey() + pubkey2 := ed25519.GenPrivKey().PubKey() + + secpKey := secp256k1.GenPrivKey().PubKey() + + defaultValidatorParams := types.ValidatorParams{[]string{types.ABCIPubKeyTypeEd25519}} + + testCases := []struct { + name string + + abciUpdates []abci.ValidatorUpdate + validatorParams types.ValidatorParams + + shouldErr bool + }{ + { + "adding a validator is OK", + + []abci.ValidatorUpdate{{PubKey: types.TM2PB.PubKey(pubkey2), Power: 20}}, + defaultValidatorParams, + + false, + }, + { + "updating a validator is OK", + + []abci.ValidatorUpdate{{PubKey: types.TM2PB.PubKey(pubkey1), Power: 20}}, + defaultValidatorParams, + + false, + }, + { + "removing a validator is OK", + + []abci.ValidatorUpdate{{PubKey: types.TM2PB.PubKey(pubkey2), Power: 0}}, + defaultValidatorParams, + + false, + }, + { + "adding a validator with negative power results in error", + + []abci.ValidatorUpdate{{PubKey: types.TM2PB.PubKey(pubkey2), Power: -100}}, + defaultValidatorParams, + + true, + }, + { + "adding a validator with pubkey thats not in validator params results in error", + + []abci.ValidatorUpdate{{PubKey: types.TM2PB.PubKey(secpKey), Power: -100}}, + defaultValidatorParams, + + true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + err := validateValidatorUpdates(tc.abciUpdates, tc.validatorParams) + if tc.shouldErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + }) + } +} + func TestUpdateValidators(t *testing.T) { pubkey1 := ed25519.GenPrivKey().PubKey() val1 := types.NewValidator(pubkey1, 10) @@ -194,7 +265,6 @@ func TestUpdateValidators(t *testing.T) { types.NewValidatorSet([]*types.Validator{val1}), false, }, - { "removing a non-existing validator results in error", @@ -204,24 +274,17 @@ func TestUpdateValidators(t *testing.T) { types.NewValidatorSet([]*types.Validator{val1}), true, }, - - { - "adding a validator with negative power results in error", - - types.NewValidatorSet([]*types.Validator{val1}), - []abci.ValidatorUpdate{{PubKey: types.TM2PB.PubKey(pubkey2), Power: -100}}, - - types.NewValidatorSet([]*types.Validator{val1}), - true, - }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - err := updateValidators(tc.currentSet, tc.abciUpdates) + updates, err := types.PB2TM.ValidatorUpdates(tc.abciUpdates) + assert.NoError(t, err) + err = tc.currentSet.UpdateWithChangeSet(updates) if tc.shouldErr { assert.Error(t, err) } else { + assert.NoError(t, err) require.Equal(t, tc.resultingSet.Size(), tc.currentSet.Size()) assert.Equal(t, tc.resultingSet.TotalVotingPower(), tc.currentSet.TotalVotingPower()) @@ -246,8 +309,8 @@ func TestEndBlockValidatorUpdates(t *testing.T) { state, stateDB := state(1, 1) - blockExec := NewBlockExecutor(stateDB, log.TestingLogger(), proxyApp.Consensus(), - MockMempool{}, MockEvidencePool{}) + blockExec := NewBlockExecutor(stateDB, log.TestingLogger(), proxyApp.Consensus(), MockMempool{}, MockEvidencePool{}) + eventBus := types.NewEventBus() err = eventBus.Start() require.NoError(t, err) diff --git a/state/metrics.go b/state/metrics.go index 7acbafa3..bcd713f5 100644 --- a/state/metrics.go +++ b/state/metrics.go @@ -2,19 +2,31 @@ package state import ( "github.com/go-kit/kit/metrics" + "github.com/go-kit/kit/metrics/discard" "github.com/go-kit/kit/metrics/prometheus" stdprometheus "github.com/prometheus/client_golang/prometheus" - "github.com/go-kit/kit/metrics/discard" ) -const MetricsSubsystem = "state" +const ( + // MetricsSubsystem is a subsystem shared by all metrics exposed by this + // package. + MetricsSubsystem = "state" +) +// Metrics contains metrics exposed by this package. type Metrics struct { // Time between BeginBlock and EndBlock. BlockProcessingTime metrics.Histogram } -func PrometheusMetrics(namespace string) *Metrics { +// PrometheusMetrics returns Metrics build using Prometheus client library. +// Optionally, labels can be provided along with their values ("foo", +// "fooValue"). +func PrometheusMetrics(namespace string, labelsAndValues ...string) *Metrics { + labels := []string{} + for i := 0; i < len(labelsAndValues); i += 2 { + labels = append(labels, labelsAndValues[i]) + } return &Metrics{ BlockProcessingTime: prometheus.NewHistogramFrom(stdprometheus.HistogramOpts{ Namespace: namespace, @@ -22,10 +34,11 @@ func PrometheusMetrics(namespace string) *Metrics { Name: "block_processing_time", Help: "Time between BeginBlock and EndBlock in ms.", Buckets: stdprometheus.LinearBuckets(1, 10, 10), - }, []string{}), + }, labels).With(labelsAndValues...), } } +// NopMetrics returns no-op Metrics. func NopMetrics() *Metrics { return &Metrics{ BlockProcessingTime: discard.NewHistogram(), diff --git a/state/state.go b/state/state.go index 26510816..b6253b64 100644 --- a/state/state.go +++ b/state/state.go @@ -8,6 +8,7 @@ import ( "github.com/tendermint/tendermint/types" tmtime "github.com/tendermint/tendermint/types/time" + "github.com/tendermint/tendermint/version" ) // database keys @@ -17,6 +18,29 @@ var ( //----------------------------------------------------------------------------- +// Version is for versioning the State. +// It holds the Block and App version needed for making blocks, +// and the software version to support upgrades to the format of +// the State as stored on disk. +type Version struct { + Consensus version.Consensus + Software string +} + +// initStateVersion sets the Consensus.Block and Software versions, +// but leaves the Consensus.App version blank. +// The Consensus.App version will be set during the Handshake, once +// we hear from the app what protocol version it is running. +var initStateVersion = Version{ + Consensus: version.Consensus{ + Block: version.BlockProtocol, + App: 0, + }, + Software: version.TMCoreSemVer, +} + +//----------------------------------------------------------------------------- + // State is a short description of the latest committed block of the Tendermint consensus. // It keeps all information necessary to validate new blocks, // including the last validator set and the consensus params. @@ -25,6 +49,8 @@ var ( // Instead, use state.Copy() or state.NextState(...). // NOTE: not goroutine-safe. type State struct { + Version Version + // immutable ChainID string @@ -38,7 +64,8 @@ type State struct { // Validators are persisted to the database separately every time they change, // so we can query for historical validator sets. // Note that if s.LastBlockHeight causes a valset change, - // we set s.LastHeightValidatorsChanged = s.LastBlockHeight + 1 + // we set s.LastHeightValidatorsChanged = s.LastBlockHeight + 1 + 1 + // Extra +1 due to nextValSet delay. NextValidators *types.ValidatorSet Validators *types.ValidatorSet LastValidators *types.ValidatorSet @@ -59,6 +86,7 @@ type State struct { // Copy makes a copy of the State for mutating. func (state State) Copy() State { return State{ + Version: state.Version, ChainID: state.ChainID, LastBlockHeight: state.LastBlockHeight, @@ -101,7 +129,7 @@ func (state State) IsEmpty() bool { // MakeBlock builds a block from the current state with the given txs, commit, // and evidence. Note it also takes a proposerAddress because the state does not -// track rounds, and hence doesn't know the correct proposer. TODO: alleviate this! +// track rounds, and hence does not know the correct proposer. TODO: fix this! func (state State) MakeBlock( height int64, txs []types.Tx, @@ -113,31 +141,22 @@ func (state State) MakeBlock( // Build base block with block data. block := types.MakeBlock(height, txs, commit, evidence) - // Fill rest of header with state data. - block.ChainID = state.ChainID - - // Set time + // Set time. + var timestamp time.Time if height == 1 { - block.Time = tmtime.Now() - if block.Time.Before(state.LastBlockTime) { - block.Time = state.LastBlockTime // state.LastBlockTime for height == 1 is genesis time - } + timestamp = state.LastBlockTime // genesis time } else { - block.Time = MedianTime(commit, state.LastValidators) + timestamp = MedianTime(commit, state.LastValidators) } - block.LastBlockID = state.LastBlockID - block.TotalTxs = state.LastBlockTotalTx + block.NumTxs - - block.ValidatorsHash = state.Validators.Hash() - block.NextValidatorsHash = state.NextValidators.Hash() - block.ConsensusHash = state.ConsensusParams.Hash() - block.AppHash = state.AppHash - block.LastResultsHash = state.LastResultsHash - - // NOTE: we can't use the state.Validators because we don't - // IncrementAccum for rounds there. - block.ProposerAddress = proposerAddress + // Fill rest of header with state data. + block.Header.Populate( + state.Version.Consensus, state.ChainID, + timestamp, state.LastBlockID, state.LastBlockTotalTx+block.NumTxs, + state.Validators.Hash(), state.NextValidators.Hash(), + state.ConsensusParams.Hash(), state.AppHash, state.LastResultsHash, + proposerAddress, + ) return block, block.MakePartSet(types.BlockPartSizeBytes) } @@ -197,30 +216,29 @@ func MakeGenesisState(genDoc *types.GenesisDoc) (State, error) { return State{}, fmt.Errorf("Error in genesis file: %v", err) } - // Make validators slice - validators := make([]*types.Validator, len(genDoc.Validators)) - for i, val := range genDoc.Validators { - pubKey := val.PubKey - address := pubKey.Address() - - // Make validator - validators[i] = &types.Validator{ - Address: address, - PubKey: pubKey, - VotingPower: val.Power, + var validatorSet, nextValidatorSet *types.ValidatorSet + if genDoc.Validators == nil { + validatorSet = types.NewValidatorSet(nil) + nextValidatorSet = types.NewValidatorSet(nil) + } else { + validators := make([]*types.Validator, len(genDoc.Validators)) + for i, val := range genDoc.Validators { + validators[i] = types.NewValidator(val.PubKey, val.Power) } + validatorSet = types.NewValidatorSet(validators) + nextValidatorSet = types.NewValidatorSet(validators).CopyIncrementProposerPriority(1) } return State{ - + Version: initStateVersion, ChainID: genDoc.ChainID, LastBlockHeight: 0, LastBlockID: types.BlockID{}, LastBlockTime: genDoc.GenesisTime, - NextValidators: types.NewValidatorSet(validators).CopyIncrementAccum(1), - Validators: types.NewValidatorSet(validators), + NextValidators: nextValidatorSet, + Validators: validatorSet, LastValidators: types.NewValidatorSet(nil), LastHeightValidatorsChanged: 1, diff --git a/state/state_test.go b/state/state_test.go index 1ab470b0..9cbe8342 100644 --- a/state/state_test.go +++ b/state/state_test.go @@ -3,12 +3,15 @@ package state import ( "bytes" "fmt" + "math" + "math/big" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + abci "github.com/tendermint/tendermint/abci/types" - crypto "github.com/tendermint/tendermint/crypto" + "github.com/tendermint/tendermint/crypto" "github.com/tendermint/tendermint/crypto/ed25519" cmn "github.com/tendermint/tendermint/libs/common" dbm "github.com/tendermint/tendermint/libs/db" @@ -48,6 +51,19 @@ func TestStateCopy(t *testing.T) { %v`, state)) } +//TestMakeGenesisStateNilValidators tests state's consistency when genesis file's validators field is nil. +func TestMakeGenesisStateNilValidators(t *testing.T) { + doc := types.GenesisDoc{ + ChainID: "dummy", + Validators: nil, + } + require.Nil(t, doc.ValidateAndComplete()) + state, err := MakeGenesisState(&doc) + require.Nil(t, err) + require.Equal(t, 0, len(state.Validators.Validators)) + require.Equal(t, 0, len(state.NextValidators.Validators)) +} + // TestStateSaveLoad tests saving and loading State from a db. func TestStateSaveLoad(t *testing.T) { tearDown, stateDB, state := setupTestCase(t) @@ -119,8 +135,8 @@ func TestABCIResponsesSaveLoad2(t *testing.T) { {Code: 383}, {Data: []byte("Gotcha!"), Tags: []cmn.KVPair{ - cmn.KVPair{Key: []byte("a"), Value: []byte("1")}, - cmn.KVPair{Key: []byte("build"), Value: []byte("stuff")}, + {Key: []byte("a"), Value: []byte("1")}, + {Key: []byte("build"), Value: []byte("stuff")}, }}, }, types.ABCIResults{ @@ -208,6 +224,7 @@ func TestOneValidatorChangesSaveLoad(t *testing.T) { _, val := state.Validators.GetByIndex(0) power := val.VotingPower var err error + var validatorUpdates []*types.Validator for i := int64(1); i < highestHeight; i++ { // When we get to a change height, use the next pubkey. if changeIndex < len(changeHeights) && i == changeHeights[changeIndex] { @@ -215,8 +232,10 @@ func TestOneValidatorChangesSaveLoad(t *testing.T) { power++ } header, blockID, responses := makeHeaderPartsResponsesValPowerChange(state, i, power) - state, err = updateState(state, blockID, &header, responses) - assert.Nil(t, err) + validatorUpdates, err = types.PB2TM.ValidatorUpdates(responses.EndBlock.ValidatorUpdates) + require.NoError(t, err) + state, err = updateState(state, blockID, &header, responses, validatorUpdates) + require.NoError(t, err) nextHeight := state.LastBlockHeight + 1 saveValidatorsInfo(stateDB, nextHeight+1, state.LastHeightValidatorsChanged, state.NextValidators) } @@ -246,6 +265,561 @@ func TestOneValidatorChangesSaveLoad(t *testing.T) { } } +func TestProposerFrequency(t *testing.T) { + + // some explicit test cases + testCases := []struct { + powers []int64 + }{ + // 2 vals + {[]int64{1, 1}}, + {[]int64{1, 2}}, + {[]int64{1, 100}}, + {[]int64{5, 5}}, + {[]int64{5, 100}}, + {[]int64{50, 50}}, + {[]int64{50, 100}}, + {[]int64{1, 1000}}, + + // 3 vals + {[]int64{1, 1, 1}}, + {[]int64{1, 2, 3}}, + {[]int64{1, 2, 3}}, + {[]int64{1, 1, 10}}, + {[]int64{1, 1, 100}}, + {[]int64{1, 10, 100}}, + {[]int64{1, 1, 1000}}, + {[]int64{1, 10, 1000}}, + {[]int64{1, 100, 1000}}, + + // 4 vals + {[]int64{1, 1, 1, 1}}, + {[]int64{1, 2, 3, 4}}, + {[]int64{1, 1, 1, 10}}, + {[]int64{1, 1, 1, 100}}, + {[]int64{1, 1, 1, 1000}}, + {[]int64{1, 1, 10, 100}}, + {[]int64{1, 1, 10, 1000}}, + {[]int64{1, 1, 100, 1000}}, + {[]int64{1, 10, 100, 1000}}, + } + + for caseNum, testCase := range testCases { + // run each case 5 times to sample different + // initial priorities + for i := 0; i < 5; i++ { + valSet := genValSetWithPowers(testCase.powers) + testProposerFreq(t, caseNum, valSet) + } + } + + // some random test cases with up to 300 validators + maxVals := 100 + maxPower := 1000 + nTestCases := 5 + for i := 0; i < nTestCases; i++ { + N := cmn.RandInt() % maxVals + vals := make([]*types.Validator, N) + totalVotePower := int64(0) + for j := 0; j < N; j++ { + // make sure votePower > 0 + votePower := int64(cmn.RandInt()%maxPower) + 1 + totalVotePower += votePower + privVal := types.NewMockPV() + pubKey := privVal.GetPubKey() + val := types.NewValidator(pubKey, votePower) + val.ProposerPriority = cmn.RandInt64() + vals[j] = val + } + valSet := types.NewValidatorSet(vals) + valSet.RescalePriorities(totalVotePower) + testProposerFreq(t, i, valSet) + } +} + +// new val set with given powers and random initial priorities +func genValSetWithPowers(powers []int64) *types.ValidatorSet { + size := len(powers) + vals := make([]*types.Validator, size) + totalVotePower := int64(0) + for i := 0; i < size; i++ { + totalVotePower += powers[i] + val := types.NewValidator(ed25519.GenPrivKey().PubKey(), powers[i]) + val.ProposerPriority = cmn.RandInt64() + vals[i] = val + } + valSet := types.NewValidatorSet(vals) + valSet.RescalePriorities(totalVotePower) + return valSet +} + +// test a proposer appears as frequently as expected +func testProposerFreq(t *testing.T, caseNum int, valSet *types.ValidatorSet) { + N := valSet.Size() + totalPower := valSet.TotalVotingPower() + + // run the proposer selection and track frequencies + runMult := 1 + runs := int(totalPower) * runMult + freqs := make([]int, N) + for i := 0; i < runs; i++ { + prop := valSet.GetProposer() + idx, _ := valSet.GetByAddress(prop.Address) + freqs[idx] += 1 + valSet.IncrementProposerPriority(1) + } + + // assert frequencies match expected (max off by 1) + for i, freq := range freqs { + _, val := valSet.GetByIndex(i) + expectFreq := int(val.VotingPower) * runMult + gotFreq := freq + abs := int(math.Abs(float64(expectFreq - gotFreq))) + + // max bound on expected vs seen freq was proven + // to be 1 for the 2 validator case in + // https://github.com/cwgoes/tm-proposer-idris + // and inferred to generalize to N-1 + bound := N - 1 + require.True(t, abs <= bound, fmt.Sprintf("Case %d val %d (%d): got %d, expected %d", caseNum, i, N, gotFreq, expectFreq)) + } +} + +// TestProposerPriorityDoesNotGetResetToZero assert that we preserve accum when calling updateState +// see https://github.com/tendermint/tendermint/issues/2718 +func TestProposerPriorityDoesNotGetResetToZero(t *testing.T) { + tearDown, _, state := setupTestCase(t) + defer tearDown(t) + val1VotingPower := int64(10) + val1PubKey := ed25519.GenPrivKey().PubKey() + val1 := &types.Validator{Address: val1PubKey.Address(), PubKey: val1PubKey, VotingPower: val1VotingPower} + + state.Validators = types.NewValidatorSet([]*types.Validator{val1}) + state.NextValidators = state.Validators + + // NewValidatorSet calls IncrementProposerPriority but uses on a copy of val1 + assert.EqualValues(t, 0, val1.ProposerPriority) + + block := makeBlock(state, state.LastBlockHeight+1) + blockID := types.BlockID{block.Hash(), block.MakePartSet(testPartSize).Header()} + abciResponses := &ABCIResponses{ + EndBlock: &abci.ResponseEndBlock{ValidatorUpdates: nil}, + } + validatorUpdates, err := types.PB2TM.ValidatorUpdates(abciResponses.EndBlock.ValidatorUpdates) + require.NoError(t, err) + updatedState, err := updateState(state, blockID, &block.Header, abciResponses, validatorUpdates) + assert.NoError(t, err) + curTotal := val1VotingPower + // one increment step and one validator: 0 + power - total_power == 0 + assert.Equal(t, 0+val1VotingPower-curTotal, updatedState.NextValidators.Validators[0].ProposerPriority) + + // add a validator + val2PubKey := ed25519.GenPrivKey().PubKey() + val2VotingPower := int64(100) + updateAddVal := abci.ValidatorUpdate{PubKey: types.TM2PB.PubKey(val2PubKey), Power: val2VotingPower} + validatorUpdates, err = types.PB2TM.ValidatorUpdates([]abci.ValidatorUpdate{updateAddVal}) + assert.NoError(t, err) + updatedState2, err := updateState(updatedState, blockID, &block.Header, abciResponses, validatorUpdates) + assert.NoError(t, err) + + require.Equal(t, len(updatedState2.NextValidators.Validators), 2) + _, updatedVal1 := updatedState2.NextValidators.GetByAddress(val1PubKey.Address()) + _, addedVal2 := updatedState2.NextValidators.GetByAddress(val2PubKey.Address()) + + // adding a validator should not lead to a ProposerPriority equal to zero (unless the combination of averaging and + // incrementing would cause so; which is not the case here) + // Steps from adding new validator: + // 0 - val1 prio is 0, TVP after add: + wantVal1Prio := int64(0) + totalPowerAfter := val1VotingPower + val2VotingPower + // 1. Add - Val2 should be initially added with (-123) => + wantVal2Prio := -(totalPowerAfter + (totalPowerAfter >> 3)) + // 2. Scale - noop + // 3. Center - with avg, resulting val2:-61, val1:62 + avg := big.NewInt(0).Add(big.NewInt(wantVal1Prio), big.NewInt(wantVal2Prio)) + avg.Div(avg, big.NewInt(2)) + wantVal2Prio = wantVal2Prio - avg.Int64() // -61 + wantVal1Prio = wantVal1Prio - avg.Int64() // 62 + + // 4. Steps from IncrementProposerPriority + wantVal1Prio = wantVal1Prio + val1VotingPower // 72 + wantVal2Prio = wantVal2Prio + val2VotingPower // 39 + wantVal1Prio = wantVal1Prio - totalPowerAfter // -38 as val1 is proposer + + assert.Equal(t, wantVal1Prio, updatedVal1.ProposerPriority) + assert.Equal(t, wantVal2Prio, addedVal2.ProposerPriority) + + // Updating a validator does not reset the ProposerPriority to zero: + // 1. Add - Val2 VotingPower change to 1 => + updatedVotingPowVal2 := int64(1) + updateVal := abci.ValidatorUpdate{PubKey: types.TM2PB.PubKey(val2PubKey), Power: updatedVotingPowVal2} + validatorUpdates, err = types.PB2TM.ValidatorUpdates([]abci.ValidatorUpdate{updateVal}) + assert.NoError(t, err) + + // this will cause the diff of priorities (77) + // to be larger than threshold == 2*totalVotingPower (22): + updatedState3, err := updateState(updatedState2, blockID, &block.Header, abciResponses, validatorUpdates) + assert.NoError(t, err) + + require.Equal(t, len(updatedState3.NextValidators.Validators), 2) + _, prevVal1 := updatedState3.Validators.GetByAddress(val1PubKey.Address()) + _, prevVal2 := updatedState3.Validators.GetByAddress(val2PubKey.Address()) + _, updatedVal1 = updatedState3.NextValidators.GetByAddress(val1PubKey.Address()) + _, updatedVal2 := updatedState3.NextValidators.GetByAddress(val2PubKey.Address()) + + // 2. Scale + // old prios: v1(10):-38, v2(1):39 + wantVal1Prio = prevVal1.ProposerPriority + wantVal2Prio = prevVal2.ProposerPriority + // scale to diffMax = 22 = 2 * tvp, diff=39-(-38)=77 + // new totalPower + totalPower := updatedVal1.VotingPower + updatedVal2.VotingPower + dist := wantVal2Prio - wantVal1Prio + // ratio := (dist + 2*totalPower - 1) / 2*totalPower = 98/22 = 4 + ratio := (dist + 2*totalPower - 1) / (2 * totalPower) + // v1(10):-38/4, v2(1):39/4 + wantVal1Prio /= ratio // -9 + wantVal2Prio /= ratio // 9 + + // 3. Center - noop + // 4. IncrementProposerPriority() -> + // v1(10):-9+10, v2(1):9+1 -> v2 proposer so subsract tvp(11) + // v1(10):1, v2(1):-1 + wantVal2Prio += updatedVal2.VotingPower // 10 -> prop + wantVal1Prio += updatedVal1.VotingPower // 1 + wantVal2Prio -= totalPower // -1 + + assert.Equal(t, wantVal2Prio, updatedVal2.ProposerPriority) + assert.Equal(t, wantVal1Prio, updatedVal1.ProposerPriority) +} + +func TestProposerPriorityProposerAlternates(t *testing.T) { + // Regression test that would fail if the inner workings of + // IncrementProposerPriority change. + // Additionally, make sure that same power validators alternate if both + // have the same voting power (and the 2nd was added later). + tearDown, _, state := setupTestCase(t) + defer tearDown(t) + val1VotingPower := int64(10) + val1PubKey := ed25519.GenPrivKey().PubKey() + val1 := &types.Validator{Address: val1PubKey.Address(), PubKey: val1PubKey, VotingPower: val1VotingPower} + + // reset state validators to above validator + state.Validators = types.NewValidatorSet([]*types.Validator{val1}) + state.NextValidators = state.Validators + // we only have one validator: + assert.Equal(t, val1PubKey.Address(), state.Validators.Proposer.Address) + + block := makeBlock(state, state.LastBlockHeight+1) + blockID := types.BlockID{block.Hash(), block.MakePartSet(testPartSize).Header()} + // no updates: + abciResponses := &ABCIResponses{ + EndBlock: &abci.ResponseEndBlock{ValidatorUpdates: nil}, + } + validatorUpdates, err := types.PB2TM.ValidatorUpdates(abciResponses.EndBlock.ValidatorUpdates) + require.NoError(t, err) + + updatedState, err := updateState(state, blockID, &block.Header, abciResponses, validatorUpdates) + assert.NoError(t, err) + + // 0 + 10 (initial prio) - 10 (avg) - 10 (mostest - total) = -10 + totalPower := val1VotingPower + wantVal1Prio := 0 + val1VotingPower - totalPower + assert.Equal(t, wantVal1Prio, updatedState.NextValidators.Validators[0].ProposerPriority) + assert.Equal(t, val1PubKey.Address(), updatedState.NextValidators.Proposer.Address) + + // add a validator with the same voting power as the first + val2PubKey := ed25519.GenPrivKey().PubKey() + updateAddVal := abci.ValidatorUpdate{PubKey: types.TM2PB.PubKey(val2PubKey), Power: val1VotingPower} + validatorUpdates, err = types.PB2TM.ValidatorUpdates([]abci.ValidatorUpdate{updateAddVal}) + assert.NoError(t, err) + + updatedState2, err := updateState(updatedState, blockID, &block.Header, abciResponses, validatorUpdates) + assert.NoError(t, err) + + require.Equal(t, len(updatedState2.NextValidators.Validators), 2) + assert.Equal(t, updatedState2.Validators, updatedState.NextValidators) + + // val1 will still be proposer as val2 just got added: + assert.Equal(t, val1PubKey.Address(), updatedState.NextValidators.Proposer.Address) + assert.Equal(t, updatedState2.Validators.Proposer.Address, updatedState2.NextValidators.Proposer.Address) + assert.Equal(t, updatedState2.Validators.Proposer.Address, val1PubKey.Address()) + assert.Equal(t, updatedState2.NextValidators.Proposer.Address, val1PubKey.Address()) + + _, updatedVal1 := updatedState2.NextValidators.GetByAddress(val1PubKey.Address()) + _, oldVal1 := updatedState2.Validators.GetByAddress(val1PubKey.Address()) + _, updatedVal2 := updatedState2.NextValidators.GetByAddress(val2PubKey.Address()) + + // 1. Add + val2VotingPower := val1VotingPower + totalPower = val1VotingPower + val2VotingPower // 20 + v2PrioWhenAddedVal2 := -(totalPower + (totalPower >> 3)) // -22 + // 2. Scale - noop + // 3. Center + avgSum := big.NewInt(0).Add(big.NewInt(v2PrioWhenAddedVal2), big.NewInt(oldVal1.ProposerPriority)) + avg := avgSum.Div(avgSum, big.NewInt(2)) // -11 + expectedVal2Prio := v2PrioWhenAddedVal2 - avg.Int64() // -11 + expectedVal1Prio := oldVal1.ProposerPriority - avg.Int64() // 11 + // 4. Increment + expectedVal2Prio = expectedVal2Prio + val2VotingPower // -11 + 10 = -1 + expectedVal1Prio = expectedVal1Prio + val1VotingPower // 11 + 10 == 21 + expectedVal1Prio = expectedVal1Prio - totalPower // 1, val1 proposer + + assert.EqualValues(t, expectedVal1Prio, updatedVal1.ProposerPriority) + assert.EqualValues(t, expectedVal2Prio, updatedVal2.ProposerPriority, "unexpected proposer priority for validator: %v", updatedVal2) + + validatorUpdates, err = types.PB2TM.ValidatorUpdates(abciResponses.EndBlock.ValidatorUpdates) + require.NoError(t, err) + + updatedState3, err := updateState(updatedState2, blockID, &block.Header, abciResponses, validatorUpdates) + assert.NoError(t, err) + + assert.Equal(t, updatedState3.Validators.Proposer.Address, updatedState3.NextValidators.Proposer.Address) + + assert.Equal(t, updatedState3.Validators, updatedState2.NextValidators) + _, updatedVal1 = updatedState3.NextValidators.GetByAddress(val1PubKey.Address()) + _, updatedVal2 = updatedState3.NextValidators.GetByAddress(val2PubKey.Address()) + + // val1 will still be proposer: + assert.Equal(t, val1PubKey.Address(), updatedState3.NextValidators.Proposer.Address) + + // check if expected proposer prio is matched: + // Increment + expectedVal2Prio2 := expectedVal2Prio + val2VotingPower // -1 + 10 = 9 + expectedVal1Prio2 := expectedVal1Prio + val1VotingPower // 1 + 10 == 11 + expectedVal1Prio2 = expectedVal1Prio2 - totalPower // -9, val1 proposer + + assert.EqualValues(t, expectedVal1Prio2, updatedVal1.ProposerPriority, "unexpected proposer priority for validator: %v", updatedVal2) + assert.EqualValues(t, expectedVal2Prio2, updatedVal2.ProposerPriority, "unexpected proposer priority for validator: %v", updatedVal2) + + // no changes in voting power and both validators have same voting power + // -> proposers should alternate: + oldState := updatedState3 + abciResponses = &ABCIResponses{ + EndBlock: &abci.ResponseEndBlock{ValidatorUpdates: nil}, + } + validatorUpdates, err = types.PB2TM.ValidatorUpdates(abciResponses.EndBlock.ValidatorUpdates) + require.NoError(t, err) + + oldState, err = updateState(oldState, blockID, &block.Header, abciResponses, validatorUpdates) + assert.NoError(t, err) + expectedVal1Prio2 = 1 + expectedVal2Prio2 = -1 + expectedVal1Prio = -9 + expectedVal2Prio = 9 + + for i := 0; i < 1000; i++ { + // no validator updates: + abciResponses := &ABCIResponses{ + EndBlock: &abci.ResponseEndBlock{ValidatorUpdates: nil}, + } + validatorUpdates, err = types.PB2TM.ValidatorUpdates(abciResponses.EndBlock.ValidatorUpdates) + require.NoError(t, err) + + updatedState, err := updateState(oldState, blockID, &block.Header, abciResponses, validatorUpdates) + assert.NoError(t, err) + // alternate (and cyclic priorities): + assert.NotEqual(t, updatedState.Validators.Proposer.Address, updatedState.NextValidators.Proposer.Address, "iter: %v", i) + assert.Equal(t, oldState.Validators.Proposer.Address, updatedState.NextValidators.Proposer.Address, "iter: %v", i) + + _, updatedVal1 = updatedState.NextValidators.GetByAddress(val1PubKey.Address()) + _, updatedVal2 = updatedState.NextValidators.GetByAddress(val2PubKey.Address()) + + if i%2 == 0 { + assert.Equal(t, updatedState.Validators.Proposer.Address, val2PubKey.Address()) + assert.Equal(t, expectedVal1Prio, updatedVal1.ProposerPriority) // -19 + assert.Equal(t, expectedVal2Prio, updatedVal2.ProposerPriority) // 0 + } else { + assert.Equal(t, updatedState.Validators.Proposer.Address, val1PubKey.Address()) + assert.Equal(t, expectedVal1Prio2, updatedVal1.ProposerPriority) // -9 + assert.Equal(t, expectedVal2Prio2, updatedVal2.ProposerPriority) // -10 + } + // update for next iteration: + oldState = updatedState + } +} + +func TestLargeGenesisValidator(t *testing.T) { + tearDown, _, state := setupTestCase(t) + defer tearDown(t) + + genesisVotingPower := int64(types.MaxTotalVotingPower / 1000) + genesisPubKey := ed25519.GenPrivKey().PubKey() + // fmt.Println("genesis addr: ", genesisPubKey.Address()) + genesisVal := &types.Validator{Address: genesisPubKey.Address(), PubKey: genesisPubKey, VotingPower: genesisVotingPower} + // reset state validators to above validator + state.Validators = types.NewValidatorSet([]*types.Validator{genesisVal}) + state.NextValidators = state.Validators + require.True(t, len(state.Validators.Validators) == 1) + + // update state a few times with no validator updates + // asserts that the single validator's ProposerPrio stays the same + oldState := state + for i := 0; i < 10; i++ { + // no updates: + abciResponses := &ABCIResponses{ + EndBlock: &abci.ResponseEndBlock{ValidatorUpdates: nil}, + } + validatorUpdates, err := types.PB2TM.ValidatorUpdates(abciResponses.EndBlock.ValidatorUpdates) + require.NoError(t, err) + + block := makeBlock(oldState, oldState.LastBlockHeight+1) + blockID := types.BlockID{block.Hash(), block.MakePartSet(testPartSize).Header()} + + updatedState, err := updateState(oldState, blockID, &block.Header, abciResponses, validatorUpdates) + // no changes in voting power (ProposerPrio += VotingPower == Voting in 1st round; than shiftByAvg == 0, + // than -Total == -Voting) + // -> no change in ProposerPrio (stays zero): + assert.EqualValues(t, oldState.NextValidators, updatedState.NextValidators) + assert.EqualValues(t, 0, updatedState.NextValidators.Proposer.ProposerPriority) + + oldState = updatedState + } + // add another validator, do a few iterations (create blocks), + // add more validators with same voting power as the 2nd + // let the genesis validator "unbond", + // see how long it takes until the effect wears off and both begin to alternate + // see: https://github.com/tendermint/tendermint/issues/2960 + firstAddedValPubKey := ed25519.GenPrivKey().PubKey() + firstAddedValVotingPower := int64(10) + firstAddedVal := abci.ValidatorUpdate{PubKey: types.TM2PB.PubKey(firstAddedValPubKey), Power: firstAddedValVotingPower} + validatorUpdates, err := types.PB2TM.ValidatorUpdates([]abci.ValidatorUpdate{firstAddedVal}) + assert.NoError(t, err) + abciResponses := &ABCIResponses{ + EndBlock: &abci.ResponseEndBlock{ValidatorUpdates: []abci.ValidatorUpdate{firstAddedVal}}, + } + block := makeBlock(oldState, oldState.LastBlockHeight+1) + blockID := types.BlockID{block.Hash(), block.MakePartSet(testPartSize).Header()} + updatedState, err := updateState(oldState, blockID, &block.Header, abciResponses, validatorUpdates) + + lastState := updatedState + for i := 0; i < 200; i++ { + // no updates: + abciResponses := &ABCIResponses{ + EndBlock: &abci.ResponseEndBlock{ValidatorUpdates: nil}, + } + validatorUpdates, err := types.PB2TM.ValidatorUpdates(abciResponses.EndBlock.ValidatorUpdates) + require.NoError(t, err) + + block := makeBlock(lastState, lastState.LastBlockHeight+1) + blockID := types.BlockID{block.Hash(), block.MakePartSet(testPartSize).Header()} + + updatedStateInner, err := updateState(lastState, blockID, &block.Header, abciResponses, validatorUpdates) + lastState = updatedStateInner + } + // set state to last state of above iteration + state = lastState + + // set oldState to state before above iteration + oldState = updatedState + _, oldGenesisVal := oldState.NextValidators.GetByAddress(genesisVal.Address) + _, newGenesisVal := state.NextValidators.GetByAddress(genesisVal.Address) + _, addedOldVal := oldState.NextValidators.GetByAddress(firstAddedValPubKey.Address()) + _, addedNewVal := state.NextValidators.GetByAddress(firstAddedValPubKey.Address()) + // expect large negative proposer priority for both (genesis validator decreased, 2nd validator increased): + assert.True(t, oldGenesisVal.ProposerPriority > newGenesisVal.ProposerPriority) + assert.True(t, addedOldVal.ProposerPriority < addedNewVal.ProposerPriority) + + // add 10 validators with the same voting power as the one added directly after genesis: + for i := 0; i < 10; i++ { + addedPubKey := ed25519.GenPrivKey().PubKey() + + addedVal := abci.ValidatorUpdate{PubKey: types.TM2PB.PubKey(addedPubKey), Power: firstAddedValVotingPower} + validatorUpdates, err := types.PB2TM.ValidatorUpdates([]abci.ValidatorUpdate{addedVal}) + assert.NoError(t, err) + + abciResponses := &ABCIResponses{ + EndBlock: &abci.ResponseEndBlock{ValidatorUpdates: []abci.ValidatorUpdate{addedVal}}, + } + block := makeBlock(oldState, oldState.LastBlockHeight+1) + blockID := types.BlockID{block.Hash(), block.MakePartSet(testPartSize).Header()} + state, err = updateState(state, blockID, &block.Header, abciResponses, validatorUpdates) + } + require.Equal(t, 10+2, len(state.NextValidators.Validators)) + + // remove genesis validator: + removeGenesisVal := abci.ValidatorUpdate{PubKey: types.TM2PB.PubKey(genesisPubKey), Power: 0} + abciResponses = &ABCIResponses{ + EndBlock: &abci.ResponseEndBlock{ValidatorUpdates: []abci.ValidatorUpdate{removeGenesisVal}}, + } + block = makeBlock(oldState, oldState.LastBlockHeight+1) + blockID = types.BlockID{block.Hash(), block.MakePartSet(testPartSize).Header()} + validatorUpdates, err = types.PB2TM.ValidatorUpdates(abciResponses.EndBlock.ValidatorUpdates) + require.NoError(t, err) + updatedState, err = updateState(state, blockID, &block.Header, abciResponses, validatorUpdates) + require.NoError(t, err) + // only the first added val (not the genesis val) should be left + assert.Equal(t, 11, len(updatedState.NextValidators.Validators)) + + // call update state until the effect for the 3rd added validator + // being proposer for a long time after the genesis validator left wears off: + curState := updatedState + count := 0 + isProposerUnchanged := true + for isProposerUnchanged { + abciResponses := &ABCIResponses{ + EndBlock: &abci.ResponseEndBlock{ValidatorUpdates: nil}, + } + validatorUpdates, err = types.PB2TM.ValidatorUpdates(abciResponses.EndBlock.ValidatorUpdates) + require.NoError(t, err) + block = makeBlock(curState, curState.LastBlockHeight+1) + blockID = types.BlockID{block.Hash(), block.MakePartSet(testPartSize).Header()} + curState, err = updateState(curState, blockID, &block.Header, abciResponses, validatorUpdates) + if !bytes.Equal(curState.Validators.Proposer.Address, curState.NextValidators.Proposer.Address) { + isProposerUnchanged = false + } + count++ + } + updatedState = curState + // the proposer changes after this number of blocks + firstProposerChangeExpectedAfter := 1 + assert.Equal(t, firstProposerChangeExpectedAfter, count) + // store proposers here to see if we see them again in the same order: + numVals := len(updatedState.Validators.Validators) + proposers := make([]*types.Validator, numVals) + for i := 0; i < 100; i++ { + // no updates: + abciResponses := &ABCIResponses{ + EndBlock: &abci.ResponseEndBlock{ValidatorUpdates: nil}, + } + validatorUpdates, err := types.PB2TM.ValidatorUpdates(abciResponses.EndBlock.ValidatorUpdates) + require.NoError(t, err) + + block := makeBlock(updatedState, updatedState.LastBlockHeight+1) + blockID := types.BlockID{block.Hash(), block.MakePartSet(testPartSize).Header()} + + updatedState, err = updateState(updatedState, blockID, &block.Header, abciResponses, validatorUpdates) + if i > numVals { // expect proposers to cycle through after the first iteration (of numVals blocks): + if proposers[i%numVals] == nil { + proposers[i%numVals] = updatedState.NextValidators.Proposer + } else { + assert.Equal(t, proposers[i%numVals], updatedState.NextValidators.Proposer) + } + } + } +} + +func TestStoreLoadValidatorsIncrementsProposerPriority(t *testing.T) { + const valSetSize = 2 + tearDown, stateDB, state := setupTestCase(t) + state.Validators = genValSet(valSetSize) + state.NextValidators = state.Validators.CopyIncrementProposerPriority(1) + SaveState(stateDB, state) + defer tearDown(t) + + nextHeight := state.LastBlockHeight + 1 + + v0, err := LoadValidators(stateDB, nextHeight) + assert.Nil(t, err) + acc0 := v0.Validators[0].ProposerPriority + + v1, err := LoadValidators(stateDB, nextHeight+1) + assert.Nil(t, err) + acc1 := v1.Validators[0].ProposerPriority + + assert.NotEqual(t, acc1, acc0, "expected ProposerPriority value to change between heights") +} + // TestValidatorChangesSaveLoad tests saving and loading a validator set with // changes. func TestManyValidatorChangesSaveLoad(t *testing.T) { @@ -253,7 +827,7 @@ func TestManyValidatorChangesSaveLoad(t *testing.T) { tearDown, stateDB, state := setupTestCase(t) require.Equal(t, int64(0), state.LastBlockHeight) state.Validators = genValSet(valSetSize) - state.NextValidators = state.Validators.CopyIncrementAccum(1) + state.NextValidators = state.Validators.CopyIncrementProposerPriority(1) SaveState(stateDB, state) defer tearDown(t) @@ -267,7 +841,10 @@ func TestManyValidatorChangesSaveLoad(t *testing.T) { // Save state etc. var err error - state, err = updateState(state, blockID, &header, responses) + var validatorUpdates []*types.Validator + validatorUpdates, err = types.PB2TM.ValidatorUpdates(responses.EndBlock.ValidatorUpdates) + require.NoError(t, err) + state, err = updateState(state, blockID, &header, responses, validatorUpdates) require.Nil(t, err) nextHeight := state.LastBlockHeight + 1 saveValidatorsInfo(stateDB, nextHeight+1, state.LastHeightValidatorsChanged, state.NextValidators) @@ -306,9 +883,11 @@ func TestStateMakeBlock(t *testing.T) { defer tearDown(t) proposerAddress := state.Validators.GetProposer().Address + stateVersion := state.Version.Consensus block := makeBlock(state, 2) - // test we set proposer address + // test we set some fields + assert.Equal(t, stateVersion, block.Version) assert.Equal(t, proposerAddress, block.ProposerAddress) } @@ -337,6 +916,7 @@ func TestConsensusParamsChangesSaveLoad(t *testing.T) { changeIndex := 0 cp := params[changeIndex] var err error + var validatorUpdates []*types.Validator for i := int64(1); i < highestHeight; i++ { // When we get to a change height, use the next params. if changeIndex < len(changeHeights) && i == changeHeights[changeIndex] { @@ -344,7 +924,9 @@ func TestConsensusParamsChangesSaveLoad(t *testing.T) { cp = params[changeIndex] } header, blockID, responses := makeHeaderPartsResponsesParams(state, i, cp) - state, err = updateState(state, blockID, &header, responses) + validatorUpdates, err = types.PB2TM.ValidatorUpdates(responses.EndBlock.ValidatorUpdates) + require.NoError(t, err) + state, err = updateState(state, blockID, &header, responses, validatorUpdates) require.Nil(t, err) nextHeight := state.LastBlockHeight + 1 @@ -375,20 +957,16 @@ func TestConsensusParamsChangesSaveLoad(t *testing.T) { func makeParams(blockBytes, blockGas, evidenceAge int64) types.ConsensusParams { return types.ConsensusParams{ - BlockSize: types.BlockSize{ + BlockSize: types.BlockSizeParams{ MaxBytes: blockBytes, MaxGas: blockGas, }, - EvidenceParams: types.EvidenceParams{ + Evidence: types.EvidenceParams{ MaxAge: evidenceAge, }, } } -func pk() []byte { - return ed25519.GenPrivKey().PubKey().Bytes() -} - func TestApplyUpdates(t *testing.T) { initParams := makeParams(1, 2, 3) @@ -401,7 +979,7 @@ func TestApplyUpdates(t *testing.T) { 1: {initParams, abci.ConsensusParams{}, initParams}, 2: {initParams, abci.ConsensusParams{ - BlockSize: &abci.BlockSize{ + BlockSize: &abci.BlockSizeParams{ MaxBytes: 44, MaxGas: 55, }, @@ -409,7 +987,7 @@ func TestApplyUpdates(t *testing.T) { makeParams(44, 55, 3)}, 3: {initParams, abci.ConsensusParams{ - EvidenceParams: &abci.EvidenceParams{ + Evidence: &abci.EvidenceParams{ MaxAge: 66, }, }, diff --git a/state/store.go b/state/store.go index 2f90c747..6b01a829 100644 --- a/state/store.go +++ b/state/store.go @@ -89,14 +89,16 @@ func saveState(db dbm.DB, state State, key []byte) { nextHeight := state.LastBlockHeight + 1 // If first block, save validators for block 1. if nextHeight == 1 { - lastHeightVoteChanged := int64(1) // Due to Tendermint validator set changes being delayed 1 block. + // This extra logic due to Tendermint validator set changes being delayed 1 block. + // It may get overwritten due to InitChain validator updates. + lastHeightVoteChanged := int64(1) saveValidatorsInfo(db, nextHeight, lastHeightVoteChanged, state.Validators) } // Save next validators. saveValidatorsInfo(db, nextHeight+1, state.LastHeightValidatorsChanged, state.NextValidators) // Save next consensus params. saveConsensusParamsInfo(db, nextHeight, state.LastHeightConsensusParamsChanged, state.ConsensusParams) - db.SetSync(stateKey, state.Bytes()) + db.SetSync(key, state.Bytes()) } //------------------------------------------------------------------------ @@ -105,8 +107,9 @@ func saveState(db dbm.DB, state State, key []byte) { // of the various ABCI calls during block processing. // It is persisted to disk for each height before calling Commit. type ABCIResponses struct { - DeliverTx []*abci.ResponseDeliverTx - EndBlock *abci.ResponseEndBlock + DeliverTx []*abci.ResponseDeliverTx + EndBlock *abci.ResponseEndBlock + BeginBlock *abci.ResponseBeginBlock } // NewABCIResponses returns a new ABCIResponses @@ -191,12 +194,14 @@ func LoadValidators(db dbm.DB, height int64) (*types.ValidatorSet, error) { ), ) } + valInfo2.ValidatorSet.IncrementProposerPriority(int(height - valInfo.LastHeightChanged)) // mutate valInfo = valInfo2 } return valInfo.ValidatorSet, nil } +// CONTRACT: Returned ValidatorsInfo can be mutated. func loadValidatorsInfo(db dbm.DB, height int64) *ValidatorsInfo { buf := db.Get(calcValidatorsKey(height)) if len(buf) == 0 { @@ -215,18 +220,22 @@ func loadValidatorsInfo(db dbm.DB, height int64) *ValidatorsInfo { return v } -// saveValidatorsInfo persists the validator set for the next block to disk. +// saveValidatorsInfo persists the validator set. +// `height` is the effective height for which the validator is responsible for signing. // It should be called from s.Save(), right before the state itself is persisted. // If the validator set did not change after processing the latest block, // only the last height for which the validators changed is persisted. -func saveValidatorsInfo(db dbm.DB, nextHeight, changeHeight int64, valSet *types.ValidatorSet) { - valInfo := &ValidatorsInfo{ - LastHeightChanged: changeHeight, +func saveValidatorsInfo(db dbm.DB, height, lastHeightChanged int64, valSet *types.ValidatorSet) { + if lastHeightChanged > height { + panic("LastHeightChanged cannot be greater than ValidatorsInfo height") } - if changeHeight == nextHeight { + valInfo := &ValidatorsInfo{ + LastHeightChanged: lastHeightChanged, + } + if lastHeightChanged == height { valInfo.ValidatorSet = valSet } - db.Set(calcValidatorsKey(nextHeight), valInfo.Bytes()) + db.Set(calcValidatorsKey(height), valInfo.Bytes()) } //----------------------------------------------------------------------------- @@ -251,7 +260,7 @@ func LoadConsensusParams(db dbm.DB, height int64) (types.ConsensusParams, error) return empty, ErrNoConsensusParamsForHeight{height} } - if paramsInfo.ConsensusParams == empty { + if paramsInfo.ConsensusParams.Equals(&empty) { paramsInfo2 := loadConsensusParamsInfo(db, paramsInfo.LastHeightChanged) if paramsInfo2 == nil { panic( diff --git a/state/tx_filter.go b/state/tx_filter.go index b8882d8e..518eb187 100644 --- a/state/tx_filter.go +++ b/state/tx_filter.go @@ -1,15 +1,22 @@ package state import ( + mempl "github.com/tendermint/tendermint/mempool" "github.com/tendermint/tendermint/types" ) -// TxFilter returns a function to filter transactions. The function limits the -// size of a transaction to the maximum block's data size. -func TxFilter(state State) func(tx types.Tx) bool { +// TxPreCheck returns a function to filter transactions before processing. +// The function limits the size of a transaction to the block's maximum data size. +func TxPreCheck(state State) mempl.PreCheckFunc { maxDataBytes := types.MaxDataBytesUnknownEvidence( state.ConsensusParams.BlockSize.MaxBytes, state.Validators.Size(), ) - return func(tx types.Tx) bool { return int64(len(tx)) <= maxDataBytes } + return mempl.PreCheckAminoMaxBytes(maxDataBytes) +} + +// TxPostCheck returns a function to filter transactions after processing. +// The function limits the gas wanted by a transaction to the block's maximum total gas. +func TxPostCheck(state State) mempl.PostCheckFunc { + return mempl.PostCheckMaxGas(state.ConsensusParams.BlockSize.MaxGas) } diff --git a/state/tx_filter_test.go b/state/tx_filter_test.go index e6b8999f..52ae396b 100644 --- a/state/tx_filter_test.go +++ b/state/tx_filter_test.go @@ -18,12 +18,18 @@ func TestTxFilter(t *testing.T) { genDoc := randomGenesisDoc() genDoc.ConsensusParams.BlockSize.MaxBytes = 3000 + // Max size of Txs is much smaller than size of block, + // since we need to account for commits and evidence. testCases := []struct { - tx types.Tx - isTxValid bool + tx types.Tx + isErr bool }{ - {types.Tx(cmn.RandBytes(250)), true}, - {types.Tx(cmn.RandBytes(3001)), false}, + {types.Tx(cmn.RandBytes(250)), false}, + {types.Tx(cmn.RandBytes(1809)), false}, + {types.Tx(cmn.RandBytes(1810)), false}, + {types.Tx(cmn.RandBytes(1811)), true}, + {types.Tx(cmn.RandBytes(1812)), true}, + {types.Tx(cmn.RandBytes(3000)), true}, } for i, tc := range testCases { @@ -31,8 +37,12 @@ func TestTxFilter(t *testing.T) { state, err := LoadStateFromDBOrGenesisDoc(stateDB, genDoc) require.NoError(t, err) - f := TxFilter(state) - assert.Equal(t, tc.isTxValid, f(tc.tx), "#%v", i) + f := TxPreCheck(state) + if tc.isErr { + assert.NotNil(t, f(tc.tx), "#%v", i) + } else { + assert.Nil(t, f(tc.tx), "#%v", i) + } } } diff --git a/state/txindex/kv/kv.go b/state/txindex/kv/kv.go index 363ab119..93249b7f 100644 --- a/state/txindex/kv/kv.go +++ b/state/txindex/kv/kv.go @@ -172,10 +172,10 @@ func (txi *TxIndex) Search(q *query.Query) ([]*types.TxResult, error) { for _, r := range ranges { if !hashesInitialized { - hashes = txi.matchRange(r, []byte(r.key)) + hashes = txi.matchRange(r, startKey(r.key)) hashesInitialized = true } else { - hashes = intersect(hashes, txi.matchRange(r, []byte(r.key))) + hashes = intersect(hashes, txi.matchRange(r, startKey(r.key))) } } } @@ -190,10 +190,10 @@ func (txi *TxIndex) Search(q *query.Query) ([]*types.TxResult, error) { } if !hashesInitialized { - hashes = txi.match(c, startKey(c, height)) + hashes = txi.match(c, startKeyForCondition(c, height)) hashesInitialized = true } else { - hashes = intersect(hashes, txi.match(c, startKey(c, height))) + hashes = intersect(hashes, txi.match(c, startKeyForCondition(c, height))) } } @@ -207,8 +207,11 @@ func (txi *TxIndex) Search(q *query.Query) ([]*types.TxResult, error) { i++ } - // sort by height by default + // sort by height & index by default sort.Slice(results, func(i, j int) bool { + if results[i].Height == results[j].Height { + return results[i].Index < results[j].Index + } return results[i].Height < results[j].Height }) @@ -225,9 +228,10 @@ func lookForHash(conditions []query.Condition) (hash []byte, err error, ok bool) return } +// lookForHeight returns a height if there is an "height=X" condition. func lookForHeight(conditions []query.Condition) (height int64) { for _, c := range conditions { - if c.Tag == types.TxHeightKey { + if c.Tag == types.TxHeightKey && c.Op == query.OpEqual { return c.Operand.(int64) } } @@ -328,18 +332,18 @@ func isRangeOperation(op query.Operator) bool { } } -func (txi *TxIndex) match(c query.Condition, startKey []byte) (hashes [][]byte) { +func (txi *TxIndex) match(c query.Condition, startKeyBz []byte) (hashes [][]byte) { if c.Op == query.OpEqual { - it := dbm.IteratePrefix(txi.store, startKey) + it := dbm.IteratePrefix(txi.store, startKeyBz) defer it.Close() for ; it.Valid(); it.Next() { hashes = append(hashes, it.Value()) } } else if c.Op == query.OpContains { - // XXX: doing full scan because startKey does not apply here - // For example, if startKey = "account.owner=an" and search query = "accoutn.owner CONSISTS an" - // we can't iterate with prefix "account.owner=an" because we might miss keys like "account.owner=Ulan" - it := txi.store.Iterator(nil, nil) + // XXX: startKey does not apply here. + // For example, if startKey = "account.owner/an/" and search query = "accoutn.owner CONTAINS an" + // we can't iterate with prefix "account.owner/an/" because we might miss keys like "account.owner/Ulan/" + it := dbm.IteratePrefix(txi.store, startKey(c.Tag)) defer it.Close() for ; it.Valid(); it.Next() { if !isTagKey(it.Key()) { @@ -355,14 +359,14 @@ func (txi *TxIndex) match(c query.Condition, startKey []byte) (hashes [][]byte) return } -func (txi *TxIndex) matchRange(r queryRange, prefix []byte) (hashes [][]byte) { +func (txi *TxIndex) matchRange(r queryRange, startKey []byte) (hashes [][]byte) { // create a map to prevent duplicates hashesMap := make(map[string][]byte) lowerBound := r.lowerBoundValue() upperBound := r.upperBoundValue() - it := dbm.IteratePrefix(txi.store, prefix) + it := dbm.IteratePrefix(txi.store, startKey) defer it.Close() LOOP: for ; it.Valid(); it.Next() { @@ -405,16 +409,6 @@ LOOP: /////////////////////////////////////////////////////////////////////////////// // Keys -func startKey(c query.Condition, height int64) []byte { - var key string - if height > 0 { - key = fmt.Sprintf("%s/%v/%d", c.Tag, c.Operand, height) - } else { - key = fmt.Sprintf("%s/%v", c.Tag, c.Operand) - } - return []byte(key) -} - func isTagKey(key []byte) bool { return strings.Count(string(key), tagKeySeparator) == 3 } @@ -425,11 +419,36 @@ func extractValueFromKey(key []byte) string { } func keyForTag(tag cmn.KVPair, result *types.TxResult) []byte { - return []byte(fmt.Sprintf("%s/%s/%d/%d", tag.Key, tag.Value, result.Height, result.Index)) + return []byte(fmt.Sprintf("%s/%s/%d/%d", + tag.Key, + tag.Value, + result.Height, + result.Index, + )) } func keyForHeight(result *types.TxResult) []byte { - return []byte(fmt.Sprintf("%s/%d/%d/%d", types.TxHeightKey, result.Height, result.Height, result.Index)) + return []byte(fmt.Sprintf("%s/%d/%d/%d", + types.TxHeightKey, + result.Height, + result.Height, + result.Index, + )) +} + +func startKeyForCondition(c query.Condition, height int64) []byte { + if height > 0 { + return startKey(c.Tag, c.Operand, height) + } + return startKey(c.Tag, c.Operand) +} + +func startKey(fields ...interface{}) []byte { + var b bytes.Buffer + for _, f := range fields { + b.Write([]byte(fmt.Sprintf("%v", f) + tagKeySeparator)) + } + return b.Bytes() } /////////////////////////////////////////////////////////////////////////////// diff --git a/state/txindex/kv/kv_test.go b/state/txindex/kv/kv_test.go index 78a76168..b726a423 100644 --- a/state/txindex/kv/kv_test.go +++ b/state/txindex/kv/kv_test.go @@ -73,6 +73,8 @@ func TestTxSearch(t *testing.T) { {"account.number = 1 AND account.owner = 'Ivan'", 1}, // search by exact match (two tags) {"account.number = 1 AND account.owner = 'Vlad'", 0}, + // search using a prefix of the stored value + {"account.owner = 'Iv'", 0}, // search by range {"account.number >= 1 AND account.number <= 5", 1}, // search by range (lower bound) @@ -87,8 +89,10 @@ func TestTxSearch(t *testing.T) { {"account.date >= TIME 2013-05-03T14:45:00Z", 0}, // search using CONTAINS {"account.owner CONTAINS 'an'", 1}, - // search using CONTAINS + // search for non existing value using CONTAINS {"account.owner CONTAINS 'Vlad'", 0}, + // search using the wrong tag (of numeric type) using CONTAINS + {"account.number CONTAINS 'Iv'", 0}, } for _, tc := range testCases { @@ -124,7 +128,7 @@ func TestTxSearchOneTxWithMultipleSameTagsButDifferentValues(t *testing.T) { } func TestTxSearchMultipleTxs(t *testing.T) { - allowedTags := []string{"account.number"} + allowedTags := []string{"account.number", "account.number.id"} indexer := NewTxIndex(db.NewMemDB(), IndexTags(allowedTags)) // indexed first, but bigger height (to test the order of transactions) @@ -133,6 +137,7 @@ func TestTxSearchMultipleTxs(t *testing.T) { }) txResult.Tx = types.Tx("Bob's account") txResult.Height = 2 + txResult.Index = 1 err := indexer.Index(txResult) require.NoError(t, err) @@ -142,22 +147,45 @@ func TestTxSearchMultipleTxs(t *testing.T) { }) txResult2.Tx = types.Tx("Alice's account") txResult2.Height = 1 + txResult2.Index = 2 + err = indexer.Index(txResult2) require.NoError(t, err) + // indexed third (to test the order of transactions) + txResult3 := txResultWithTags([]cmn.KVPair{ + {Key: []byte("account.number"), Value: []byte("3")}, + }) + txResult3.Tx = types.Tx("Jack's account") + txResult3.Height = 1 + txResult3.Index = 1 + err = indexer.Index(txResult3) + require.NoError(t, err) + + // indexed fourth (to test we don't include txs with similar tags) + // https://github.com/tendermint/tendermint/issues/2908 + txResult4 := txResultWithTags([]cmn.KVPair{ + {Key: []byte("account.number.id"), Value: []byte("1")}, + }) + txResult4.Tx = types.Tx("Mike's account") + txResult4.Height = 2 + txResult4.Index = 2 + err = indexer.Index(txResult4) + require.NoError(t, err) + results, err := indexer.Search(query.MustParse("account.number >= 1")) assert.NoError(t, err) - require.Len(t, results, 2) - assert.Equal(t, []*types.TxResult{txResult2, txResult}, results) + require.Len(t, results, 3) + assert.Equal(t, []*types.TxResult{txResult3, txResult2, txResult}, results) } func TestIndexAllTags(t *testing.T) { indexer := NewTxIndex(db.NewMemDB(), IndexAllTags()) txResult := txResultWithTags([]cmn.KVPair{ - cmn.KVPair{Key: []byte("account.owner"), Value: []byte("Ivan")}, - cmn.KVPair{Key: []byte("account.number"), Value: []byte("1")}, + {Key: []byte("account.owner"), Value: []byte("Ivan")}, + {Key: []byte("account.number"), Value: []byte("1")}, }) err := indexer.Index(txResult) diff --git a/state/txindex/kv/wire.go b/state/txindex/kv/wire.go index ccca7525..de168b22 100644 --- a/state/txindex/kv/wire.go +++ b/state/txindex/kv/wire.go @@ -1,7 +1,7 @@ package kv import ( - "github.com/tendermint/go-amino" + amino "github.com/tendermint/go-amino" ) var cdc = amino.NewCodec() diff --git a/state/validation.go b/state/validation.go index 9d8ef97a..cd571e34 100644 --- a/state/validation.go +++ b/state/validation.go @@ -5,7 +5,7 @@ import ( "errors" "fmt" - "github.com/tendermint/tendermint/crypto/tmhash" + "github.com/tendermint/tendermint/crypto" dbm "github.com/tendermint/tendermint/libs/db" "github.com/tendermint/tendermint/types" ) @@ -20,16 +20,20 @@ func validateBlock(stateDB dbm.DB, state State, block *types.Block) error { } // Validate basic info. + if block.Version != state.Version.Consensus { + return fmt.Errorf("Wrong Block.Header.Version. Expected %v, got %v", + state.Version.Consensus, + block.Version, + ) + } if block.ChainID != state.ChainID { - return fmt.Errorf( - "Wrong Block.Header.ChainID. Expected %v, got %v", + return fmt.Errorf("Wrong Block.Header.ChainID. Expected %v, got %v", state.ChainID, block.ChainID, ) } if block.Height != state.LastBlockHeight+1 { - return fmt.Errorf( - "Wrong Block.Header.Height. Expected %v, got %v", + return fmt.Errorf("Wrong Block.Header.Height. Expected %v, got %v", state.LastBlockHeight+1, block.Height, ) @@ -37,16 +41,15 @@ func validateBlock(stateDB dbm.DB, state State, block *types.Block) error { // Validate prev block info. if !block.LastBlockID.Equals(state.LastBlockID) { - return fmt.Errorf( - "Wrong Block.Header.LastBlockID. Expected %v, got %v", + return fmt.Errorf("Wrong Block.Header.LastBlockID. Expected %v, got %v", state.LastBlockID, block.LastBlockID, ) } + newTxs := int64(len(block.Data.Txs)) if block.TotalTxs != state.LastBlockTotalTx+newTxs { - return fmt.Errorf( - "Wrong Block.Header.TotalTxs. Expected %v, got %v", + return fmt.Errorf("Wrong Block.Header.TotalTxs. Expected %v, got %v", state.LastBlockTotalTx+newTxs, block.TotalTxs, ) @@ -54,46 +57,44 @@ func validateBlock(stateDB dbm.DB, state State, block *types.Block) error { // Validate app info if !bytes.Equal(block.AppHash, state.AppHash) { - return fmt.Errorf( - "Wrong Block.Header.AppHash. Expected %X, got %v", + return fmt.Errorf("Wrong Block.Header.AppHash. Expected %X, got %v", state.AppHash, block.AppHash, ) } if !bytes.Equal(block.ConsensusHash, state.ConsensusParams.Hash()) { - return fmt.Errorf( - "Wrong Block.Header.ConsensusHash. Expected %X, got %v", + return fmt.Errorf("Wrong Block.Header.ConsensusHash. Expected %X, got %v", state.ConsensusParams.Hash(), block.ConsensusHash, ) } if !bytes.Equal(block.LastResultsHash, state.LastResultsHash) { - return fmt.Errorf( - "Wrong Block.Header.LastResultsHash. Expected %X, got %v", + return fmt.Errorf("Wrong Block.Header.LastResultsHash. Expected %X, got %v", state.LastResultsHash, block.LastResultsHash, ) } if !bytes.Equal(block.ValidatorsHash, state.Validators.Hash()) { - return fmt.Errorf( - "Wrong Block.Header.ValidatorsHash. Expected %X, got %v", + return fmt.Errorf("Wrong Block.Header.ValidatorsHash. Expected %X, got %v", state.Validators.Hash(), block.ValidatorsHash, ) } if !bytes.Equal(block.NextValidatorsHash, state.NextValidators.Hash()) { - return fmt.Errorf("Wrong Block.Header.NextValidatorsHash. Expected %X, got %v", state.NextValidators.Hash(), block.NextValidatorsHash) + return fmt.Errorf("Wrong Block.Header.NextValidatorsHash. Expected %X, got %v", + state.NextValidators.Hash(), + block.NextValidatorsHash, + ) } // Validate block LastCommit. if block.Height == 1 { if len(block.LastCommit.Precommits) != 0 { - return errors.New("Block at height 1 (first block) should have no LastCommit precommits") + return errors.New("Block at height 1 can't have LastCommit precommits") } } else { if len(block.LastCommit.Precommits) != state.LastValidators.Size() { - return fmt.Errorf( - "Invalid block commit size. Expected %v, got %v", + return fmt.Errorf("Invalid block commit size. Expected %v, got %v", state.LastValidators.Size(), len(block.LastCommit.Precommits), ) @@ -108,8 +109,7 @@ func validateBlock(stateDB dbm.DB, state State, block *types.Block) error { // Validate block Time if block.Height > 1 { if !block.Time.After(state.LastBlockTime) { - return fmt.Errorf( - "Block time %v not greater than last block time %v", + return fmt.Errorf("Block time %v not greater than last block time %v", block.Time, state.LastBlockTime, ) @@ -117,19 +117,27 @@ func validateBlock(stateDB dbm.DB, state State, block *types.Block) error { medianTime := MedianTime(block.LastCommit, state.LastValidators) if !block.Time.Equal(medianTime) { - return fmt.Errorf( - "Invalid block time. Expected %v, got %v", + return fmt.Errorf("Invalid block time. Expected %v, got %v", medianTime, block.Time, ) } + } else if block.Height == 1 { + genesisTime := state.LastBlockTime + if !block.Time.Equal(genesisTime) { + return fmt.Errorf("Block time %v is not equal to genesis time %v", + block.Time, + genesisTime, + ) + } } // Limit the amount of evidence - maxEvidenceBytes := types.MaxEvidenceBytesPerBlock(state.ConsensusParams.BlockSize.MaxBytes) - evidenceBytes := int64(len(block.Evidence.Evidence)) * types.MaxEvidenceBytes - if evidenceBytes > maxEvidenceBytes { - return types.NewErrEvidenceOverflow(maxEvidenceBytes, evidenceBytes) + maxNumEvidence, _ := types.MaxEvidencePerBlock(state.ConsensusParams.BlockSize.MaxBytes) + numEvidence := int64(len(block.Evidence.Evidence)) + if numEvidence > maxNumEvidence { + return types.NewErrEvidenceOverflow(maxNumEvidence, numEvidence) + } // Validate all evidence. @@ -142,10 +150,9 @@ func validateBlock(stateDB dbm.DB, state State, block *types.Block) error { // NOTE: We can't actually verify it's the right proposer because we dont // know what round the block was first proposed. So just check that it's // a legit address and a known validator. - if len(block.ProposerAddress) != tmhash.Size || + if len(block.ProposerAddress) != crypto.AddressSize || !state.Validators.HasAddress(block.ProposerAddress) { - return fmt.Errorf( - "Block.Header.ProposerAddress, %X, is not a validator", + return fmt.Errorf("Block.Header.ProposerAddress, %X, is not a validator", block.ProposerAddress, ) } @@ -162,7 +169,7 @@ func VerifyEvidence(stateDB dbm.DB, state State, evidence types.Evidence) error height := state.LastBlockHeight evidenceAge := height - evidence.Height() - maxAge := state.ConsensusParams.EvidenceParams.MaxAge + maxAge := state.ConsensusParams.Evidence.MaxAge if evidenceAge > maxAge { return fmt.Errorf("Evidence from height %d is too old. Min height is %d", evidence.Height(), height-maxAge) diff --git a/state/validation_test.go b/state/validation_test.go index e5f45166..12aaf636 100644 --- a/state/validation_test.go +++ b/state/validation_test.go @@ -2,8 +2,10 @@ package state import ( "testing" + "time" "github.com/stretchr/testify/require" + "github.com/tendermint/tendermint/crypto/ed25519" "github.com/tendermint/tendermint/crypto/tmhash" "github.com/tendermint/tendermint/libs/log" @@ -25,18 +27,25 @@ func TestValidateBlockHeader(t *testing.T) { err := blockExec.ValidateBlock(state, block) require.NoError(t, err) + // some bad values wrongHash := tmhash.Sum([]byte("this hash is wrong")) + wrongVersion1 := state.Version.Consensus + wrongVersion1.Block += 1 + wrongVersion2 := state.Version.Consensus + wrongVersion2.App += 1 // Manipulation of any header field causes failure. testCases := []struct { name string malleateBlock func(block *types.Block) }{ - {"ChainID wrong", func(block *types.Block) { block.ChainID = "not-the-real-one" }}, // wrong chain id - {"Height wrong", func(block *types.Block) { block.Height += 10 }}, // wrong height - // TODO(#2589) (#2587) : {"Time", func(block *types.Block) { block.Time.Add(-time.Second * 3600 * 24) }}, // wrong time - {"NumTxs wrong", func(block *types.Block) { block.NumTxs += 10 }}, // wrong num txs - {"TotalTxs wrong", func(block *types.Block) { block.TotalTxs += 10 }}, // wrong total txs + {"Version wrong1", func(block *types.Block) { block.Version = wrongVersion1 }}, + {"Version wrong2", func(block *types.Block) { block.Version = wrongVersion2 }}, + {"ChainID wrong", func(block *types.Block) { block.ChainID = "not-the-real-one" }}, + {"Height wrong", func(block *types.Block) { block.Height += 10 }}, + {"Time wrong", func(block *types.Block) { block.Time = block.Time.Add(-time.Second * 3600 * 24) }}, + {"NumTxs wrong", func(block *types.Block) { block.NumTxs += 10 }}, + {"TotalTxs wrong", func(block *types.Block) { block.TotalTxs += 10 }}, {"LastBlockID wrong", func(block *types.Block) { block.LastBlockID.PartsHeader.Total += 10 }}, {"LastCommitHash wrong", func(block *types.Block) { block.LastCommitHash = wrongHash }}, @@ -100,10 +109,9 @@ func TestValidateBlockEvidence(t *testing.T) { // A block with too much evidence fails. maxBlockSize := state.ConsensusParams.BlockSize.MaxBytes - maxEvidenceBytes := types.MaxEvidenceBytesPerBlock(maxBlockSize) - maxEvidence := maxEvidenceBytes / types.MaxEvidenceBytes - require.True(t, maxEvidence > 2) - for i := int64(0); i < maxEvidence; i++ { + maxNumEvidence, _ := types.MaxEvidencePerBlock(maxBlockSize) + require.True(t, maxNumEvidence > 2) + for i := int64(0); i < maxNumEvidence; i++ { block.Evidence.Evidence = append(block.Evidence.Evidence, goodEvidence) } block.EvidenceHash = block.Evidence.Hash() diff --git a/state/wire.go b/state/wire.go index eeb156d6..f7a61129 100644 --- a/state/wire.go +++ b/state/wire.go @@ -1,7 +1,7 @@ package state import ( - "github.com/tendermint/go-amino" + amino "github.com/tendermint/go-amino" cryptoAmino "github.com/tendermint/tendermint/crypto/encoding/amino" ) diff --git a/test/docker/Dockerfile b/test/docker/Dockerfile index 6bb320be..1a64d417 100644 --- a/test/docker/Dockerfile +++ b/test/docker/Dockerfile @@ -14,6 +14,7 @@ ENV GOBIN $GOPATH/bin WORKDIR $REPO # Copy in the code +# TODO: rewrite to only copy Makefile & other files? COPY . $REPO # Install the vendored dependencies @@ -21,16 +22,18 @@ COPY . $REPO RUN make get_tools RUN make get_vendor_deps -# Now copy in the code -# NOTE: this will overwrite whatever is in vendor/ -COPY . $REPO - # install ABCI CLI RUN make install_abci # install Tendermint RUN make install +RUN tendermint testnet --node-dir-prefix="mach" --v=4 --populate-persistent-peers=false --o=$REPO/test/p2p/data + +# Now copy in the code +# NOTE: this will overwrite whatever is in vendor/ +COPY . $REPO + # expose the volume for debugging VOLUME $REPO diff --git a/test/p2p/README.md b/test/p2p/README.md index 4ee3690a..956ce906 100644 --- a/test/p2p/README.md +++ b/test/p2p/README.md @@ -19,7 +19,7 @@ docker network create --driver bridge --subnet 172.57.0.0/16 my_testnet ``` This gives us a new network with IP addresses in the rage `172.57.0.0 - 172.57.255.255`. -Peers on the network can have any IP address in this range. +Peers on the network can have any IP address in this range. For our four node network, let's pick `172.57.0.101 - 172.57.0.104`. Since we use Tendermint's default listening port of 26656, our list of seed nodes will look like: @@ -37,7 +37,7 @@ for i in $(seq 1 4); do --ip="172.57.0.$((100 + $i))" \ --name local_testnet_$i \ --entrypoint tendermint \ - -e TMHOME=/go/src/github.com/tendermint/tendermint/test/p2p/data/mach$i/core \ + -e TMHOME=/go/src/github.com/tendermint/tendermint/test/p2p/data/mach$((i-1)) \ tendermint_tester node --p2p.persistent_peers 172.57.0.101:26656,172.57.0.102:26656,172.57.0.103:26656,172.57.0.104:26656 --proxy_app=kvstore done ``` @@ -47,8 +47,5 @@ If you now run `docker ps`, you'll see your containers! We can confirm they are making blocks by checking the `/status` message using `curl` and `jq` to pretty print the output json: ``` -curl 172.57.0.101:26657/status | jq . +curl 172.57.0.101:26657/status | jq . ``` - - - diff --git a/test/p2p/data/mach1/core/config/genesis.json b/test/p2p/data/mach1/core/config/genesis.json deleted file mode 100644 index 515c1071..00000000 --- a/test/p2p/data/mach1/core/config/genesis.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "genesis_time": "2016-06-24T20:01:19.322Z", - "chain_id": "chain-9ujDWI", - "validators": [ - { - "pub_key": { - "type": "tendermint/PubKeyEd25519", - "value": "vokz3/FgDAJuNHGPF4Wkzeq5DDVpizlOOLaUeukd4RY=" - }, - "power": "1", - "name": "mach1" - }, - { - "pub_key": { - "type": "tendermint/PubKeyEd25519", - "value": "bcU0RlMjEmWH0qKpO1nWibcXBzsd6WiiWm7xPVlTGK0=" - }, - "power": "1", - "name": "mach2" - }, - { - "pub_key": { - "type": "tendermint/PubKeyEd25519", - "value": "rmesaX0TWqC0YB6lfqqz/r9Lqk8inEWlmMKYWxL80aE=" - }, - "power": "1", - "name": "mach3" - }, - { - "pub_key": { - "type": "tendermint/PubKeyEd25519", - "value": "nryPWM7UtG3NWrirpZHdJTzXy1A3Jz/aMrwLZGHE79k=" - }, - "power": "1", - "name": "mach4" - } - ], - "app_hash": "" -} diff --git a/test/p2p/data/mach1/core/config/node_key.json b/test/p2p/data/mach1/core/config/node_key.json deleted file mode 100644 index 4fa96085..00000000 --- a/test/p2p/data/mach1/core/config/node_key.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "priv_key": { - "type": "tendermint/PrivKeyEd25519", - "value": "BpYtFp8xSrudBa5aBLRuSPD72PGDAUm0dJORDL3Kd5YJbluUzRefVFrjwoHZv1yeDj2P9xkEi2L3hJCUz/qFkQ==" - } -} diff --git a/test/p2p/data/mach1/core/config/priv_validator.json b/test/p2p/data/mach1/core/config/priv_validator.json deleted file mode 100644 index ea2a01f5..00000000 --- a/test/p2p/data/mach1/core/config/priv_validator.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "address": "AE47BBD4B3ACD80BFE17F6E0C66C5B8492A81AE4", - "pub_key": { - "type": "tendermint/PubKeyEd25519", - "value": "vokz3/FgDAJuNHGPF4Wkzeq5DDVpizlOOLaUeukd4RY=" - }, - "last_height": "0", - "last_round": "0", - "last_step": 0, - "priv_key": { - "type": "tendermint/PrivKeyEd25519", - "value": "VHqgfHqM4WxcsqQMbCbRWwoylgQQqfHqblC2NvGrOJq+iTPf8WAMAm40cY8XhaTN6rkMNWmLOU44tpR66R3hFg==" - } -} diff --git a/test/p2p/data/mach2/core/config/genesis.json b/test/p2p/data/mach2/core/config/genesis.json deleted file mode 100644 index 515c1071..00000000 --- a/test/p2p/data/mach2/core/config/genesis.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "genesis_time": "2016-06-24T20:01:19.322Z", - "chain_id": "chain-9ujDWI", - "validators": [ - { - "pub_key": { - "type": "tendermint/PubKeyEd25519", - "value": "vokz3/FgDAJuNHGPF4Wkzeq5DDVpizlOOLaUeukd4RY=" - }, - "power": "1", - "name": "mach1" - }, - { - "pub_key": { - "type": "tendermint/PubKeyEd25519", - "value": "bcU0RlMjEmWH0qKpO1nWibcXBzsd6WiiWm7xPVlTGK0=" - }, - "power": "1", - "name": "mach2" - }, - { - "pub_key": { - "type": "tendermint/PubKeyEd25519", - "value": "rmesaX0TWqC0YB6lfqqz/r9Lqk8inEWlmMKYWxL80aE=" - }, - "power": "1", - "name": "mach3" - }, - { - "pub_key": { - "type": "tendermint/PubKeyEd25519", - "value": "nryPWM7UtG3NWrirpZHdJTzXy1A3Jz/aMrwLZGHE79k=" - }, - "power": "1", - "name": "mach4" - } - ], - "app_hash": "" -} diff --git a/test/p2p/data/mach2/core/config/node_key.json b/test/p2p/data/mach2/core/config/node_key.json deleted file mode 100644 index 6eb15110..00000000 --- a/test/p2p/data/mach2/core/config/node_key.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "priv_key": { - "type": "tendermint/PrivKeyEd25519", - "value": "uM6LDVE4wQIIUmq9rc6RxzX8zEGG4G4Jcuw15klzQopF68YfJM4bkbPSavurEcJ4nvBMusKBg2GcARFrZqnFKA==" - } -} diff --git a/test/p2p/data/mach2/core/config/priv_validator.json b/test/p2p/data/mach2/core/config/priv_validator.json deleted file mode 100644 index 6e0cd7f8..00000000 --- a/test/p2p/data/mach2/core/config/priv_validator.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "address": "5D61EE46CCE91F579086522D7FD8CEC3F854E946", - "pub_key": { - "type": "tendermint/PubKeyEd25519", - "value": "bcU0RlMjEmWH0qKpO1nWibcXBzsd6WiiWm7xPVlTGK0=" - }, - "last_height": "0", - "last_round": "0", - "last_step": 0, - "priv_key": { - "type": "tendermint/PrivKeyEd25519", - "value": "0EeInmBQL8MSnQq38zSxg47Z7R7Nmcu5a3GtWr9agUNtxTRGUyMSZYfSoqk7WdaJtxcHOx3paKJabvE9WVMYrQ==" - } -} diff --git a/test/p2p/data/mach3/core/config/genesis.json b/test/p2p/data/mach3/core/config/genesis.json deleted file mode 100644 index 515c1071..00000000 --- a/test/p2p/data/mach3/core/config/genesis.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "genesis_time": "2016-06-24T20:01:19.322Z", - "chain_id": "chain-9ujDWI", - "validators": [ - { - "pub_key": { - "type": "tendermint/PubKeyEd25519", - "value": "vokz3/FgDAJuNHGPF4Wkzeq5DDVpizlOOLaUeukd4RY=" - }, - "power": "1", - "name": "mach1" - }, - { - "pub_key": { - "type": "tendermint/PubKeyEd25519", - "value": "bcU0RlMjEmWH0qKpO1nWibcXBzsd6WiiWm7xPVlTGK0=" - }, - "power": "1", - "name": "mach2" - }, - { - "pub_key": { - "type": "tendermint/PubKeyEd25519", - "value": "rmesaX0TWqC0YB6lfqqz/r9Lqk8inEWlmMKYWxL80aE=" - }, - "power": "1", - "name": "mach3" - }, - { - "pub_key": { - "type": "tendermint/PubKeyEd25519", - "value": "nryPWM7UtG3NWrirpZHdJTzXy1A3Jz/aMrwLZGHE79k=" - }, - "power": "1", - "name": "mach4" - } - ], - "app_hash": "" -} diff --git a/test/p2p/data/mach3/core/config/node_key.json b/test/p2p/data/mach3/core/config/node_key.json deleted file mode 100644 index 0885bcf9..00000000 --- a/test/p2p/data/mach3/core/config/node_key.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "priv_key": { - "type": "tendermint/PrivKeyEd25519", - "value": "kT3orG0YkipT9rAZbvAjtGk/7Pu1ZeCE8LSUF2jz2uiSs1rdlUVi/gccRlvCRLKvrtSicOyEkmk0FHPOGS3mgg==" - } -} diff --git a/test/p2p/data/mach3/core/config/priv_validator.json b/test/p2p/data/mach3/core/config/priv_validator.json deleted file mode 100644 index ec68ca7b..00000000 --- a/test/p2p/data/mach3/core/config/priv_validator.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "address": "705F9DA2CC7D7AF5F4519455ED99622E40E439A1", - "pub_key": { - "type": "tendermint/PubKeyEd25519", - "value": "rmesaX0TWqC0YB6lfqqz/r9Lqk8inEWlmMKYWxL80aE=" - }, - "last_height": "0", - "last_round": "0", - "last_step": 0, - "priv_key": { - "type": "tendermint/PrivKeyEd25519", - "value": "waTkfzSfxfVW9Kmie6d2uUQkwxK6ps9u5EuGc0jXw/KuZ6xpfRNaoLRgHqV+qrP+v0uqTyKcRaWYwphbEvzRoQ==" - } -} diff --git a/test/p2p/data/mach4/core/config/genesis.json b/test/p2p/data/mach4/core/config/genesis.json deleted file mode 100644 index 515c1071..00000000 --- a/test/p2p/data/mach4/core/config/genesis.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "genesis_time": "2016-06-24T20:01:19.322Z", - "chain_id": "chain-9ujDWI", - "validators": [ - { - "pub_key": { - "type": "tendermint/PubKeyEd25519", - "value": "vokz3/FgDAJuNHGPF4Wkzeq5DDVpizlOOLaUeukd4RY=" - }, - "power": "1", - "name": "mach1" - }, - { - "pub_key": { - "type": "tendermint/PubKeyEd25519", - "value": "bcU0RlMjEmWH0qKpO1nWibcXBzsd6WiiWm7xPVlTGK0=" - }, - "power": "1", - "name": "mach2" - }, - { - "pub_key": { - "type": "tendermint/PubKeyEd25519", - "value": "rmesaX0TWqC0YB6lfqqz/r9Lqk8inEWlmMKYWxL80aE=" - }, - "power": "1", - "name": "mach3" - }, - { - "pub_key": { - "type": "tendermint/PubKeyEd25519", - "value": "nryPWM7UtG3NWrirpZHdJTzXy1A3Jz/aMrwLZGHE79k=" - }, - "power": "1", - "name": "mach4" - } - ], - "app_hash": "" -} diff --git a/test/p2p/data/mach4/core/config/node_key.json b/test/p2p/data/mach4/core/config/node_key.json deleted file mode 100644 index d6a5d79c..00000000 --- a/test/p2p/data/mach4/core/config/node_key.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "priv_key": { - "type": "tendermint/PrivKeyEd25519", - "value": "QIIm8/QEEawiJi3Zozv+J9b+1CufCEkGs3lxGMlRy4L4FVIXCoXJTwYIrotZtwoMqLYEqQV1hbKKJmFA3GFelw==" - } -} diff --git a/test/p2p/data/mach4/core/config/priv_validator.json b/test/p2p/data/mach4/core/config/priv_validator.json deleted file mode 100644 index 468550ea..00000000 --- a/test/p2p/data/mach4/core/config/priv_validator.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "address": "D1054266EC9EEA511ED9A76DEFD520BBE1B5E850", - "pub_key": { - "type": "tendermint/PubKeyEd25519", - "value": "nryPWM7UtG3NWrirpZHdJTzXy1A3Jz/aMrwLZGHE79k=" - }, - "last_height": "0", - "last_round": "0", - "last_step": 0, - "priv_key": { - "type": "tendermint/PrivKeyEd25519", - "value": "xMw+0o8CDC29qYvNvwjDztNwRw508l6TjV0pXo49KwyevI9YztS0bc1auKulkd0lPNfLUDcnP9oyvAtkYcTv2Q==" - } -} diff --git a/test/p2p/ip_plus_id.sh b/test/p2p/ip_plus_id.sh index 0d2248fe..95871d3f 100755 --- a/test/p2p/ip_plus_id.sh +++ b/test/p2p/ip_plus_id.sh @@ -3,5 +3,5 @@ set -eu ID=$1 DOCKER_IMAGE=$2 -NODEID="$(docker run --rm -e TMHOME=/go/src/github.com/tendermint/tendermint/test/p2p/data/mach$ID/core $DOCKER_IMAGE tendermint show_node_id)" +NODEID="$(docker run --rm -e TMHOME=/go/src/github.com/tendermint/tendermint/test/p2p/data/mach$((ID-1)) $DOCKER_IMAGE tendermint show_node_id)" echo "$NODEID@172.57.0.$((100+$ID))" diff --git a/test/p2p/peer.sh b/test/p2p/peer.sh index ad04d000..63d46f8d 100644 --- a/test/p2p/peer.sh +++ b/test/p2p/peer.sh @@ -15,13 +15,15 @@ echo "starting tendermint peer ID=$ID" # NOTE: $NODE_FLAGS should be unescaped (no quotes). otherwise it will be # treated as one flag. +# test/p2p/data/mach$((ID-1)) data is generated in test/docker/Dockerfile using +# the tendermint testnet command. if [[ "$ID" == "x" ]]; then # Set "x" to "1" to print to console. docker run \ --net="$NETWORK_NAME" \ --ip=$(test/p2p/ip.sh "$ID") \ --name "local_testnet_$ID" \ --entrypoint tendermint \ - -e TMHOME="/go/src/github.com/tendermint/tendermint/test/p2p/data/mach$ID/core" \ + -e TMHOME="/go/src/github.com/tendermint/tendermint/test/p2p/data/mach$((ID-1))" \ -e GOMAXPROCS=1 \ --log-driver=syslog \ --log-opt syslog-address=udp://127.0.0.1:5514 \ @@ -34,7 +36,7 @@ else --ip=$(test/p2p/ip.sh "$ID") \ --name "local_testnet_$ID" \ --entrypoint tendermint \ - -e TMHOME="/go/src/github.com/tendermint/tendermint/test/p2p/data/mach$ID/core" \ + -e TMHOME="/go/src/github.com/tendermint/tendermint/test/p2p/data/mach$((ID-1))" \ -e GOMAXPROCS=1 \ --log-driver=syslog \ --log-opt syslog-address=udp://127.0.0.1:5514 \ diff --git a/test/p2p/pex/test_addrbook.sh b/test/p2p/pex/test_addrbook.sh index 9c58db30..06f9212f 100644 --- a/test/p2p/pex/test_addrbook.sh +++ b/test/p2p/pex/test_addrbook.sh @@ -18,7 +18,7 @@ echo "1. restart peer $ID" docker stop "local_testnet_$ID" echo "stopped local_testnet_$ID" # preserve addrbook.json -docker cp "local_testnet_$ID:/go/src/github.com/tendermint/tendermint/test/p2p/data/mach1/core/config/addrbook.json" "/tmp/addrbook.json" +docker cp "local_testnet_$ID:/go/src/github.com/tendermint/tendermint/test/p2p/data/mach0/config/addrbook.json" "/tmp/addrbook.json" set +e #CIRCLE docker rm -vf "local_testnet_$ID" set -e @@ -32,11 +32,11 @@ bash test/p2p/client.sh "$DOCKER_IMAGE" "$NETWORK_NAME" "$CLIENT_NAME" "test/p2p # Now we know that the node is up. -docker cp "/tmp/addrbook.json" "local_testnet_$ID:/go/src/github.com/tendermint/tendermint/test/p2p/data/mach1/core/config/addrbook.json" +docker cp "/tmp/addrbook.json" "local_testnet_$ID:/go/src/github.com/tendermint/tendermint/test/p2p/data/mach0/config/addrbook.json" echo "with the following addrbook:" cat /tmp/addrbook.json # exec doesn't work on circle -# docker exec "local_testnet_$ID" cat "/go/src/github.com/tendermint/tendermint/test/p2p/data/mach1/core/config/addrbook.json" +# docker exec "local_testnet_$ID" cat "/go/src/github.com/tendermint/tendermint/test/p2p/data/mach0/config/addrbook.json" echo "" echo "----------------------------------------------------------------------" diff --git a/tools/README.md b/tools/README.md index aeb41141..041067e7 100644 --- a/tools/README.md +++ b/tools/README.md @@ -1,3 +1,5 @@ # tools -Tools for working with tendermint and associated technologies. Documentation can be found in the `README.md` of each the `tm-bench/` and `tm-monitor/` directories. +Tools for working with Tendermint and associated technologies. Documentation for +these tools can be found online in the [Tendermint tools +documentation](https://tendermint.com/docs/tools/). diff --git a/tools/tm-bench/README.md b/tools/tm-bench/README.md index 9159a754..b4e8cec5 100644 --- a/tools/tm-bench/README.md +++ b/tools/tm-bench/README.md @@ -2,7 +2,7 @@ Tendermint blockchain benchmarking tool: -- https://github.com/tendermint/tools/tree/master/tm-bench +- [https://github.com/tendermint/tendermint/tree/master/tools/tm-bench](https://github.com/tendermint/tendermint/tree/master/tools/tm-bench) For example, the following: `tm-bench -T 30 -r 10000 localhost:26657` diff --git a/tools/tm-bench/transacter.go b/tools/tm-bench/transacter.go index 36cc761e..c20aa5b5 100644 --- a/tools/tm-bench/transacter.go +++ b/tools/tm-bench/transacter.go @@ -191,7 +191,7 @@ func (t *transacter) sendLoop(connIndex int) { c.SetWriteDeadline(now.Add(sendTimeout)) err = c.WriteJSON(rpctypes.RPCRequest{ JSONRPC: "2.0", - ID: "tm-bench", + ID: rpctypes.JSONRPCStringID("tm-bench"), Method: t.BroadcastTxMethod, Params: rawParamsJSON, }) diff --git a/tools/tm-monitor/README.md b/tools/tm-monitor/README.md index cf421684..374a56b0 100644 --- a/tools/tm-monitor/README.md +++ b/tools/tm-monitor/README.md @@ -3,7 +3,7 @@ Tendermint blockchain monitoring tool; watches over one or more nodes, collecting and providing various statistics to the user: -- https://github.com/tendermint/tools/tree/master/tm-monitor +- [https://github.com/tendermint/tendermint/tree/master/tools/tm-monitor](https://github.com/tendermint/tendermint/tree/master/tools/tm-monitor) ## Quick Start diff --git a/tools/tm-monitor/eventmeter/eventmeter.go b/tools/tm-monitor/eventmeter/eventmeter.go index 185f3774..63d58b96 100644 --- a/tools/tm-monitor/eventmeter/eventmeter.go +++ b/tools/tm-monitor/eventmeter/eventmeter.go @@ -196,7 +196,7 @@ func (em *EventMeter) RegisterDisconnectCallback(f DisconnectCallbackFunc) { // Private func (em *EventMeter) subscribe() error { - for query, _ := range em.queryToMetricMap { + for query := range em.queryToMetricMap { if err := em.wsc.Subscribe(context.TODO(), query); err != nil { return err } diff --git a/tools/tm-monitor/main.go b/tools/tm-monitor/main.go index 32897b97..6e4aea5f 100644 --- a/tools/tm-monitor/main.go +++ b/tools/tm-monitor/main.go @@ -48,13 +48,13 @@ Examples: logger = log.NewTMLogger(log.NewSyncWriter(os.Stdout)) } - m := startMonitor(flag.Arg(0)) + monitor := startMonitor(flag.Arg(0)) - startRPC(listenAddr, m, logger) + listener := startRPC(listenAddr, monitor, logger) var ton *Ton if !noton { - ton = NewTon(m) + ton = NewTon(monitor) ton.Start() } @@ -62,7 +62,8 @@ Examples: if !noton { ton.Stop() } - m.Stop() + monitor.Stop() + listener.Close() }) } diff --git a/tools/tm-monitor/mock/eventmeter.go b/tools/tm-monitor/mock/eventmeter.go index 27129758..7bbedc7f 100644 --- a/tools/tm-monitor/mock/eventmeter.go +++ b/tools/tm-monitor/mock/eventmeter.go @@ -4,7 +4,7 @@ import ( stdlog "log" "reflect" - "github.com/tendermint/go-amino" + amino "github.com/tendermint/go-amino" "github.com/tendermint/tendermint/libs/log" em "github.com/tendermint/tendermint/tools/tm-monitor/eventmeter" ) diff --git a/tools/tm-monitor/monitor/monitor.go b/tools/tm-monitor/monitor/monitor.go index 764f281f..86022c31 100644 --- a/tools/tm-monitor/monitor/monitor.go +++ b/tools/tm-monitor/monitor/monitor.go @@ -46,7 +46,7 @@ func NewMonitor(options ...func(*Monitor)) *Monitor { nodeQuit: make(map[string]chan struct{}), recalculateNetworkUptimeEvery: 10 * time.Second, numValidatorsUpdateInterval: 5 * time.Second, - logger: log.NewNopLogger(), + logger: log.NewNopLogger(), } for _, option := range options { diff --git a/tools/tm-monitor/monitor/monitor_test.go b/tools/tm-monitor/monitor/monitor_test.go index 9694e577..324f43f7 100644 --- a/tools/tm-monitor/monitor/monitor_test.go +++ b/tools/tm-monitor/monitor/monitor_test.go @@ -7,7 +7,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/tendermint/go-amino" + amino "github.com/tendermint/go-amino" "github.com/tendermint/tendermint/crypto/ed25519" ctypes "github.com/tendermint/tendermint/rpc/core/types" mock "github.com/tendermint/tendermint/tools/tm-monitor/mock" diff --git a/tools/tm-monitor/monitor/network.go b/tools/tm-monitor/monitor/network.go index 9b147c06..bb5dd0ba 100644 --- a/tools/tm-monitor/monitor/network.go +++ b/tools/tm-monitor/monitor/network.go @@ -140,14 +140,22 @@ func (n *Network) NodeIsOnline(name string) { // NewNode is called when the new node is added to the monitor. func (n *Network) NewNode(name string) { + n.mu.Lock() + defer n.mu.Unlock() + n.NumNodesMonitored++ n.NumNodesMonitoredOnline++ + n.updateHealth() } // NodeDeleted is called when the node is deleted from under the monitor. func (n *Network) NodeDeleted(name string) { + n.mu.Lock() + defer n.mu.Unlock() + n.NumNodesMonitored-- n.NumNodesMonitoredOnline-- + n.updateHealth() } func (n *Network) updateHealth() { diff --git a/tools/tm-monitor/monitor/node.go b/tools/tm-monitor/monitor/node.go index 8bc15a15..6f705145 100644 --- a/tools/tm-monitor/monitor/node.go +++ b/tools/tm-monitor/monitor/node.go @@ -7,7 +7,7 @@ import ( "github.com/pkg/errors" - crypto "github.com/tendermint/tendermint/crypto" + "github.com/tendermint/tendermint/crypto" "github.com/tendermint/tendermint/libs/events" "github.com/tendermint/tendermint/libs/log" ctypes "github.com/tendermint/tendermint/rpc/core/types" @@ -55,13 +55,13 @@ func NewNode(rpcAddr string, options ...func(*Node)) *Node { func NewNodeWithEventMeterAndRpcClient(rpcAddr string, em eventMeter, rpcClient rpc_client.HTTPClient, options ...func(*Node)) *Node { n := &Node{ - rpcAddr: rpcAddr, - em: em, - rpcClient: rpcClient, - Name: rpcAddr, - quit: make(chan struct{}), + rpcAddr: rpcAddr, + em: em, + rpcClient: rpcClient, + Name: rpcAddr, + quit: make(chan struct{}), checkIsValidatorInterval: 5 * time.Second, - logger: log.NewNopLogger(), + logger: log.NewNopLogger(), } for _, option := range options { diff --git a/tools/tm-monitor/monitor/node_test.go b/tools/tm-monitor/monitor/node_test.go index 10c2a13f..0048e48f 100644 --- a/tools/tm-monitor/monitor/node_test.go +++ b/tools/tm-monitor/monitor/node_test.go @@ -34,7 +34,7 @@ func TestNodeNewBlockReceived(t *testing.T) { n.SendBlocksTo(blockCh) blockHeader := tmtypes.Header{Height: 5} - emMock.Call("eventCallback", &em.EventMetric{}, tmtypes.EventDataNewBlockHeader{blockHeader}) + emMock.Call("eventCallback", &em.EventMetric{}, tmtypes.EventDataNewBlockHeader{Header: blockHeader}) assert.Equal(t, int64(5), n.Height) assert.Equal(t, blockHeader, <-blockCh) diff --git a/tools/tm-monitor/rpc.go b/tools/tm-monitor/rpc.go index ab62e046..1a08a9ec 100644 --- a/tools/tm-monitor/rpc.go +++ b/tools/tm-monitor/rpc.go @@ -2,6 +2,7 @@ package main import ( "errors" + "net" "net/http" "github.com/tendermint/tendermint/libs/log" @@ -9,16 +10,19 @@ import ( monitor "github.com/tendermint/tendermint/tools/tm-monitor/monitor" ) -func startRPC(listenAddr string, m *monitor.Monitor, logger log.Logger) { +func startRPC(listenAddr string, m *monitor.Monitor, logger log.Logger) net.Listener { routes := routes(m) mux := http.NewServeMux() wm := rpc.NewWebsocketManager(routes, nil) mux.HandleFunc("/websocket", wm.WebsocketHandler) rpc.RegisterRPCFuncs(mux, routes, cdc, logger) - if _, err := rpc.StartHTTPServer(listenAddr, mux, logger, rpc.Config{}); err != nil { + listener, err := rpc.Listen(listenAddr, rpc.Config{}) + if err != nil { panic(err) } + go rpc.StartHTTPServer(listener, mux, logger) + return listener } func routes(m *monitor.Monitor) map[string]*rpc.RPCFunc { diff --git a/tools/tm-signer-harness/Dockerfile b/tools/tm-signer-harness/Dockerfile new file mode 100644 index 00000000..83f57a3d --- /dev/null +++ b/tools/tm-signer-harness/Dockerfile @@ -0,0 +1,4 @@ +ARG TENDERMINT_VERSION=latest +FROM tendermint/tendermint:${TENDERMINT_VERSION} + +COPY tm-signer-harness /usr/bin/tm-signer-harness diff --git a/tools/tm-signer-harness/Makefile b/tools/tm-signer-harness/Makefile new file mode 100644 index 00000000..47cd0365 --- /dev/null +++ b/tools/tm-signer-harness/Makefile @@ -0,0 +1,20 @@ +.PHONY: build install docker-image + +TENDERMINT_VERSION?=latest +BUILD_TAGS?='tendermint' +BUILD_FLAGS = -ldflags "-X github.com/tendermint/tendermint/version.GitCommit=`git rev-parse --short=8 HEAD`" + +.DEFAULT_GOAL := build + +build: + CGO_ENABLED=0 go build $(BUILD_FLAGS) -tags $(BUILD_TAGS) -o ../../build/tm-signer-harness main.go + +install: + CGO_ENABLED=0 go install $(BUILD_FLAGS) -tags $(BUILD_TAGS) . + +docker-image: + GOOS=linux GOARCH=amd64 go build $(BUILD_FLAGS) -tags $(BUILD_TAGS) -o tm-signer-harness main.go + docker build \ + --build-arg TENDERMINT_VERSION=$(TENDERMINT_VERSION) \ + -t tendermint/tm-signer-harness:$(TENDERMINT_VERSION) . + rm -rf tm-signer-harness diff --git a/tools/tm-signer-harness/README.md b/tools/tm-signer-harness/README.md new file mode 100644 index 00000000..7add3a99 --- /dev/null +++ b/tools/tm-signer-harness/README.md @@ -0,0 +1,5 @@ +# tm-signer-harness + +See the [`tm-signer-harness` +documentation](https://tendermint.com/docs/tools/remote-signer-validation.html) +for more details. diff --git a/tools/tm-signer-harness/internal/test_harness.go b/tools/tm-signer-harness/internal/test_harness.go new file mode 100644 index 00000000..b961f238 --- /dev/null +++ b/tools/tm-signer-harness/internal/test_harness.go @@ -0,0 +1,392 @@ +package internal + +import ( + "fmt" + "net" + "os" + "os/signal" + "time" + + "github.com/tendermint/tendermint/crypto/tmhash" + + "github.com/tendermint/tendermint/crypto/ed25519" + "github.com/tendermint/tendermint/privval" + "github.com/tendermint/tendermint/state" + + cmn "github.com/tendermint/tendermint/libs/common" + "github.com/tendermint/tendermint/libs/log" + "github.com/tendermint/tendermint/types" +) + +// Test harness error codes (which act as exit codes when the test harness fails). +const ( + NoError int = iota // 0 + ErrInvalidParameters // 1 + ErrMaxAcceptRetriesReached // 2 + ErrFailedToLoadGenesisFile // 3 + ErrFailedToCreateListener // 4 + ErrFailedToStartListener // 5 + ErrInterrupted // 6 + ErrOther // 7 + ErrTestPublicKeyFailed // 8 + ErrTestSignProposalFailed // 9 + ErrTestSignVoteFailed // 10 +) + +var voteTypes = []types.SignedMsgType{types.PrevoteType, types.PrecommitType} + +// TestHarnessError allows us to keep track of which exit code should be used +// when exiting the main program. +type TestHarnessError struct { + Code int // The exit code to return + Err error // The original error + Info string // Any additional information +} + +var _ error = (*TestHarnessError)(nil) + +// TestHarness allows for testing of a remote signer to ensure compatibility +// with this version of Tendermint. +type TestHarness struct { + addr string + spv *privval.SocketVal + fpv *privval.FilePV + chainID string + acceptRetries int + logger log.Logger + exitWhenComplete bool + exitCode int +} + +// TestHarnessConfig provides configuration to set up a remote signer test +// harness. +type TestHarnessConfig struct { + BindAddr string + + KeyFile string + StateFile string + GenesisFile string + + AcceptDeadline time.Duration + ConnDeadline time.Duration + AcceptRetries int + + SecretConnKey ed25519.PrivKeyEd25519 + + ExitWhenComplete bool // Whether or not to call os.Exit when the harness has completed. +} + +// timeoutError can be used to check if an error returned from the netp package +// was due to a timeout. +type timeoutError interface { + Timeout() bool +} + +// NewTestHarness will load Tendermint data from the given files (including +// validator public/private keypairs and chain details) and create a new +// harness. +func NewTestHarness(logger log.Logger, cfg TestHarnessConfig) (*TestHarness, error) { + keyFile := ExpandPath(cfg.KeyFile) + stateFile := ExpandPath(cfg.StateFile) + logger.Info("Loading private validator configuration", "keyFile", keyFile, "stateFile", stateFile) + // NOTE: LoadFilePV ultimately calls os.Exit on failure. No error will be + // returned if this call fails. + fpv := privval.LoadFilePV(keyFile, stateFile) + + genesisFile := ExpandPath(cfg.GenesisFile) + logger.Info("Loading chain ID from genesis file", "genesisFile", genesisFile) + st, err := state.MakeGenesisDocFromFile(genesisFile) + if err != nil { + return nil, newTestHarnessError(ErrFailedToLoadGenesisFile, err, genesisFile) + } + logger.Info("Loaded genesis file", "chainID", st.ChainID) + + spv, err := newTestHarnessSocketVal(logger, cfg) + if err != nil { + return nil, newTestHarnessError(ErrFailedToCreateListener, err, "") + } + + return &TestHarness{ + addr: cfg.BindAddr, + spv: spv, + fpv: fpv, + chainID: st.ChainID, + acceptRetries: cfg.AcceptRetries, + logger: logger, + exitWhenComplete: cfg.ExitWhenComplete, + exitCode: 0, + }, nil +} + +// Run will execute the tests associated with this test harness. The intention +// here is to call this from one's `main` function, as the way it succeeds or +// fails at present is to call os.Exit() with an exit code related to the error +// that caused the tests to fail, or exit code 0 on success. +func (th *TestHarness) Run() { + c := make(chan os.Signal, 1) + signal.Notify(c, os.Interrupt) + go func() { + for sig := range c { + th.logger.Info("Caught interrupt, terminating...", "sig", sig) + th.Shutdown(newTestHarnessError(ErrInterrupted, nil, "")) + } + }() + + th.logger.Info("Starting test harness") + accepted := false + var startErr error + for acceptRetries := th.acceptRetries; acceptRetries > 0; acceptRetries-- { + th.logger.Info("Attempting to accept incoming connection", "acceptRetries", acceptRetries) + if err := th.spv.Start(); err != nil { + // if it wasn't a timeout error + if _, ok := err.(timeoutError); !ok { + th.logger.Error("Failed to start listener", "err", err) + th.Shutdown(newTestHarnessError(ErrFailedToStartListener, err, "")) + // we need the return statements in case this is being run + // from a unit test - otherwise this function will just die + // when os.Exit is called + return + } + startErr = err + } else { + accepted = true + break + } + } + if !accepted { + th.logger.Error("Maximum accept retries reached", "acceptRetries", th.acceptRetries) + th.Shutdown(newTestHarnessError(ErrMaxAcceptRetriesReached, startErr, "")) + return + } + + // Run the tests + if err := th.TestPublicKey(); err != nil { + th.Shutdown(err) + return + } + if err := th.TestSignProposal(); err != nil { + th.Shutdown(err) + return + } + if err := th.TestSignVote(); err != nil { + th.Shutdown(err) + return + } + th.logger.Info("SUCCESS! All tests passed.") + th.Shutdown(nil) +} + +// TestPublicKey just validates that we can (1) fetch the public key from the +// remote signer, and (2) it matches the public key we've configured for our +// local Tendermint version. +func (th *TestHarness) TestPublicKey() error { + th.logger.Info("TEST: Public key of remote signer") + th.logger.Info("Local", "pubKey", th.fpv.GetPubKey()) + th.logger.Info("Remote", "pubKey", th.spv.GetPubKey()) + if th.fpv.GetPubKey() != th.spv.GetPubKey() { + th.logger.Error("FAILED: Local and remote public keys do not match") + return newTestHarnessError(ErrTestPublicKeyFailed, nil, "") + } + return nil +} + +// TestSignProposal makes sure the remote signer can successfully sign +// proposals. +func (th *TestHarness) TestSignProposal() error { + th.logger.Info("TEST: Signing of proposals") + // sha256 hash of "hash" + hash := tmhash.Sum([]byte("hash")) + prop := &types.Proposal{ + Type: types.ProposalType, + Height: 12345, + Round: 23456, + POLRound: -1, + BlockID: types.BlockID{ + Hash: hash, + PartsHeader: types.PartSetHeader{ + Hash: hash, + Total: 1000000, + }, + }, + Timestamp: time.Now(), + } + propBytes := prop.SignBytes(th.chainID) + if err := th.spv.SignProposal(th.chainID, prop); err != nil { + th.logger.Error("FAILED: Signing of proposal", "err", err) + return newTestHarnessError(ErrTestSignProposalFailed, err, "") + } + th.logger.Debug("Signed proposal", "prop", prop) + // first check that it's a basically valid proposal + if err := prop.ValidateBasic(); err != nil { + th.logger.Error("FAILED: Signed proposal is invalid", "err", err) + return newTestHarnessError(ErrTestSignProposalFailed, err, "") + } + // now validate the signature on the proposal + if th.spv.GetPubKey().VerifyBytes(propBytes, prop.Signature) { + th.logger.Info("Successfully validated proposal signature") + } else { + th.logger.Error("FAILED: Proposal signature validation failed") + return newTestHarnessError(ErrTestSignProposalFailed, nil, "signature validation failed") + } + return nil +} + +// TestSignVote makes sure the remote signer can successfully sign all kinds of +// votes. +func (th *TestHarness) TestSignVote() error { + th.logger.Info("TEST: Signing of votes") + for _, voteType := range voteTypes { + th.logger.Info("Testing vote type", "type", voteType) + hash := tmhash.Sum([]byte("hash")) + vote := &types.Vote{ + Type: voteType, + Height: 12345, + Round: 23456, + BlockID: types.BlockID{ + Hash: hash, + PartsHeader: types.PartSetHeader{ + Hash: hash, + Total: 1000000, + }, + }, + ValidatorIndex: 0, + ValidatorAddress: tmhash.SumTruncated([]byte("addr")), + Timestamp: time.Now(), + } + voteBytes := vote.SignBytes(th.chainID) + // sign the vote + if err := th.spv.SignVote(th.chainID, vote); err != nil { + th.logger.Error("FAILED: Signing of vote", "err", err) + return newTestHarnessError(ErrTestSignVoteFailed, err, fmt.Sprintf("voteType=%d", voteType)) + } + th.logger.Debug("Signed vote", "vote", vote) + // validate the contents of the vote + if err := vote.ValidateBasic(); err != nil { + th.logger.Error("FAILED: Signed vote is invalid", "err", err) + return newTestHarnessError(ErrTestSignVoteFailed, err, fmt.Sprintf("voteType=%d", voteType)) + } + // now validate the signature on the proposal + if th.spv.GetPubKey().VerifyBytes(voteBytes, vote.Signature) { + th.logger.Info("Successfully validated vote signature", "type", voteType) + } else { + th.logger.Error("FAILED: Vote signature validation failed", "type", voteType) + return newTestHarnessError(ErrTestSignVoteFailed, nil, "signature validation failed") + } + } + return nil +} + +// Shutdown will kill the test harness and attempt to close all open sockets +// gracefully. If the supplied error is nil, it is assumed that the exit code +// should be 0. If err is not nil, it will exit with an exit code related to the +// error. +func (th *TestHarness) Shutdown(err error) { + var exitCode int + + if err == nil { + exitCode = NoError + } else if therr, ok := err.(*TestHarnessError); ok { + exitCode = therr.Code + } else { + exitCode = ErrOther + } + th.exitCode = exitCode + + // in case sc.Stop() takes too long + if th.exitWhenComplete { + go func() { + time.Sleep(time.Duration(5) * time.Second) + th.logger.Error("Forcibly exiting program after timeout") + os.Exit(exitCode) + }() + } + + if th.spv.IsRunning() { + if err := th.spv.Stop(); err != nil { + th.logger.Error("Failed to cleanly stop listener: %s", err.Error()) + } + } + + if th.exitWhenComplete { + os.Exit(exitCode) + } +} + +// newTestHarnessSocketVal creates our client instance which we will use for +// testing. +func newTestHarnessSocketVal(logger log.Logger, cfg TestHarnessConfig) (*privval.SocketVal, error) { + proto, addr := cmn.ProtocolAndAddress(cfg.BindAddr) + if proto == "unix" { + // make sure the socket doesn't exist - if so, try to delete it + if cmn.FileExists(addr) { + if err := os.Remove(addr); err != nil { + logger.Error("Failed to remove existing Unix domain socket", "addr", addr) + return nil, err + } + } + } + ln, err := net.Listen(proto, addr) + if err != nil { + return nil, err + } + logger.Info("Listening at", "proto", proto, "addr", addr) + var svln net.Listener + switch proto { + case "unix": + unixLn := privval.NewUnixListener(ln) + privval.UnixListenerAcceptDeadline(cfg.AcceptDeadline)(unixLn) + privval.UnixListenerConnDeadline(cfg.ConnDeadline)(unixLn) + svln = unixLn + case "tcp": + tcpLn := privval.NewTCPListener(ln, cfg.SecretConnKey) + privval.TCPListenerAcceptDeadline(cfg.AcceptDeadline)(tcpLn) + privval.TCPListenerConnDeadline(cfg.ConnDeadline)(tcpLn) + logger.Info("Resolved TCP address for listener", "addr", tcpLn.Addr()) + svln = tcpLn + default: + logger.Error("Unsupported protocol (must be unix:// or tcp://)", "proto", proto) + return nil, newTestHarnessError(ErrInvalidParameters, nil, fmt.Sprintf("Unsupported protocol: %s", proto)) + } + return privval.NewSocketVal(logger, svln), nil +} + +func newTestHarnessError(code int, err error, info string) *TestHarnessError { + return &TestHarnessError{ + Code: code, + Err: err, + Info: info, + } +} + +func (e *TestHarnessError) Error() string { + var msg string + switch e.Code { + case ErrInvalidParameters: + msg = "Invalid parameters supplied to application" + case ErrMaxAcceptRetriesReached: + msg = "Maximum accept retries reached" + case ErrFailedToLoadGenesisFile: + msg = "Failed to load genesis file" + case ErrFailedToCreateListener: + msg = "Failed to create listener" + case ErrFailedToStartListener: + msg = "Failed to start listener" + case ErrInterrupted: + msg = "Interrupted" + case ErrTestPublicKeyFailed: + msg = "Public key validation test failed" + case ErrTestSignProposalFailed: + msg = "Proposal signing validation test failed" + case ErrTestSignVoteFailed: + msg = "Vote signing validation test failed" + default: + msg = "Unknown error" + } + if len(e.Info) > 0 { + msg = fmt.Sprintf("%s: %s", msg, e.Info) + } + if e.Err != nil { + msg = fmt.Sprintf("%s (original error: %s)", msg, e.Err.Error()) + } + return msg +} diff --git a/tools/tm-signer-harness/internal/test_harness_test.go b/tools/tm-signer-harness/internal/test_harness_test.go new file mode 100644 index 00000000..804aca45 --- /dev/null +++ b/tools/tm-signer-harness/internal/test_harness_test.go @@ -0,0 +1,201 @@ +package internal + +import ( + "fmt" + "io/ioutil" + "net" + "os" + "testing" + "time" + + "github.com/tendermint/tendermint/crypto" + "github.com/tendermint/tendermint/privval" + "github.com/tendermint/tendermint/types" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/tendermint/tendermint/crypto/ed25519" + "github.com/tendermint/tendermint/libs/log" +) + +const ( + keyFileContents = `{ + "address": "D08FCA3BA74CF17CBFC15E64F9505302BB0E2748", + "pub_key": { + "type": "tendermint/PubKeyEd25519", + "value": "ZCsuTjaczEyon70nmKxwvwu+jqrbq5OH3yQjcK0SFxc=" + }, + "priv_key": { + "type": "tendermint/PrivKeyEd25519", + "value": "8O39AkQsoe1sBQwud/Kdul8lg8K9SFsql9aZvwXQSt1kKy5ONpzMTKifvSeYrHC/C76Oqturk4ffJCNwrRIXFw==" + } +}` + + stateFileContents = `{ + "height": "0", + "round": "0", + "step": 0 +}` + + genesisFileContents = `{ + "genesis_time": "2019-01-15T11:56:34.8963Z", + "chain_id": "test-chain-0XwP5E", + "consensus_params": { + "block_size": { + "max_bytes": "22020096", + "max_gas": "-1" + }, + "evidence": { + "max_age": "100000" + }, + "validator": { + "pub_key_types": [ + "ed25519" + ] + } + }, + "validators": [ + { + "address": "D08FCA3BA74CF17CBFC15E64F9505302BB0E2748", + "pub_key": { + "type": "tendermint/PubKeyEd25519", + "value": "ZCsuTjaczEyon70nmKxwvwu+jqrbq5OH3yQjcK0SFxc=" + }, + "power": "10", + "name": "" + } + ], + "app_hash": "" +}` + + defaultConnDeadline = 100 +) + +func TestRemoteSignerTestHarnessMaxAcceptRetriesReached(t *testing.T) { + cfg := makeConfig(t, 1, 2) + defer cleanup(cfg) + + th, err := NewTestHarness(log.TestingLogger(), cfg) + require.NoError(t, err) + th.Run() + assert.Equal(t, ErrMaxAcceptRetriesReached, th.exitCode) +} + +func TestRemoteSignerTestHarnessSuccessfulRun(t *testing.T) { + harnessTest( + t, + func(th *TestHarness) *privval.RemoteSigner { + return newMockRemoteSigner(t, th, th.fpv.Key.PrivKey, false, false) + }, + NoError, + ) +} + +func TestRemoteSignerPublicKeyCheckFailed(t *testing.T) { + harnessTest( + t, + func(th *TestHarness) *privval.RemoteSigner { + return newMockRemoteSigner(t, th, ed25519.GenPrivKey(), false, false) + }, + ErrTestPublicKeyFailed, + ) +} + +func TestRemoteSignerProposalSigningFailed(t *testing.T) { + harnessTest( + t, + func(th *TestHarness) *privval.RemoteSigner { + return newMockRemoteSigner(t, th, th.fpv.Key.PrivKey, true, false) + }, + ErrTestSignProposalFailed, + ) +} + +func TestRemoteSignerVoteSigningFailed(t *testing.T) { + harnessTest( + t, + func(th *TestHarness) *privval.RemoteSigner { + return newMockRemoteSigner(t, th, th.fpv.Key.PrivKey, false, true) + }, + ErrTestSignVoteFailed, + ) +} + +func newMockRemoteSigner(t *testing.T, th *TestHarness, privKey crypto.PrivKey, breakProposalSigning bool, breakVoteSigning bool) *privval.RemoteSigner { + return privval.NewRemoteSigner( + th.logger, + th.chainID, + types.NewMockPVWithParams(privKey, breakProposalSigning, breakVoteSigning), + privval.DialTCPFn( + th.addr, + time.Duration(defaultConnDeadline)*time.Millisecond, + ed25519.GenPrivKey(), + ), + ) +} + +// For running relatively standard tests. +func harnessTest(t *testing.T, rsMaker func(th *TestHarness) *privval.RemoteSigner, expectedExitCode int) { + cfg := makeConfig(t, 100, 3) + defer cleanup(cfg) + + th, err := NewTestHarness(log.TestingLogger(), cfg) + require.NoError(t, err) + donec := make(chan struct{}) + go func() { + defer close(donec) + th.Run() + }() + + rs := rsMaker(th) + require.NoError(t, rs.Start()) + assert.True(t, rs.IsRunning()) + defer rs.Stop() + + <-donec + assert.Equal(t, expectedExitCode, th.exitCode) +} + +func makeConfig(t *testing.T, acceptDeadline, acceptRetries int) TestHarnessConfig { + return TestHarnessConfig{ + BindAddr: testFreeTCPAddr(t), + KeyFile: makeTempFile("tm-testharness-keyfile", keyFileContents), + StateFile: makeTempFile("tm-testharness-statefile", stateFileContents), + GenesisFile: makeTempFile("tm-testharness-genesisfile", genesisFileContents), + AcceptDeadline: time.Duration(acceptDeadline) * time.Millisecond, + ConnDeadline: time.Duration(defaultConnDeadline) * time.Millisecond, + AcceptRetries: acceptRetries, + SecretConnKey: ed25519.GenPrivKey(), + ExitWhenComplete: false, + } +} + +func cleanup(cfg TestHarnessConfig) { + os.Remove(cfg.KeyFile) + os.Remove(cfg.StateFile) + os.Remove(cfg.GenesisFile) +} + +func makeTempFile(name, content string) string { + tempFile, err := ioutil.TempFile("", fmt.Sprintf("%s-*", name)) + if err != nil { + panic(err) + } + if _, err := tempFile.Write([]byte(content)); err != nil { + tempFile.Close() + panic(err) + } + if err := tempFile.Close(); err != nil { + panic(err) + } + return tempFile.Name() +} + +// testFreeTCPAddr claims a free port so we don't block on listener being ready. +func testFreeTCPAddr(t *testing.T) string { + ln, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + defer ln.Close() + + return fmt.Sprintf("127.0.0.1:%d", ln.Addr().(*net.TCPAddr).Port) +} diff --git a/tools/tm-signer-harness/internal/utils.go b/tools/tm-signer-harness/internal/utils.go new file mode 100644 index 00000000..9783ca95 --- /dev/null +++ b/tools/tm-signer-harness/internal/utils.go @@ -0,0 +1,25 @@ +package internal + +import ( + "os/user" + "path/filepath" + "strings" +) + +// ExpandPath will check if the given path begins with a "~" symbol, and if so, +// will expand it to become the user's home directory. If it fails to expand the +// path it will automatically return the original path itself. +func ExpandPath(path string) string { + usr, err := user.Current() + if err != nil { + return path + } + + if path == "~" { + return usr.HomeDir + } else if strings.HasPrefix(path, "~/") { + return filepath.Join(usr.HomeDir, path[2:]) + } + + return path +} diff --git a/tools/tm-signer-harness/main.go b/tools/tm-signer-harness/main.go new file mode 100644 index 00000000..13aaf03a --- /dev/null +++ b/tools/tm-signer-harness/main.go @@ -0,0 +1,174 @@ +package main + +import ( + "flag" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "time" + + "github.com/tendermint/tendermint/crypto/ed25519" + "github.com/tendermint/tendermint/libs/log" + "github.com/tendermint/tendermint/privval" + "github.com/tendermint/tendermint/tools/tm-signer-harness/internal" + "github.com/tendermint/tendermint/version" +) + +const ( + defaultAcceptRetries = 100 + defaultBindAddr = "tcp://127.0.0.1:0" + defaultTMHome = "~/.tendermint" + defaultAcceptDeadline = 1 + defaultConnDeadline = 3 + defaultExtractKeyOutput = "./signing.key" +) + +var logger = log.NewTMLogger(log.NewSyncWriter(os.Stdout)) + +// Command line flags +var ( + flagAcceptRetries int + flagBindAddr string + flagTMHome string + flagKeyOutputPath string +) + +// Command line commands +var ( + rootCmd *flag.FlagSet + runCmd *flag.FlagSet + extractKeyCmd *flag.FlagSet + versionCmd *flag.FlagSet +) + +func init() { + rootCmd = flag.NewFlagSet("root", flag.ExitOnError) + rootCmd.Usage = func() { + fmt.Println(`Remote signer test harness for Tendermint. + +Usage: + tm-signer-harness [flags] + +Available Commands: + extract_key Extracts a signing key from a local Tendermint instance + help Help on the available commands + run Runs the test harness + version Display version information and exit + +Use "tm-signer-harness help " for more information about that command.`) + fmt.Println("") + } + + runCmd = flag.NewFlagSet("run", flag.ExitOnError) + runCmd.IntVar(&flagAcceptRetries, "accept-retries", defaultAcceptRetries, "The number of attempts to listen for incoming connections") + runCmd.StringVar(&flagBindAddr, "addr", defaultBindAddr, "Bind to this address for the testing") + runCmd.StringVar(&flagTMHome, "tmhome", defaultTMHome, "Path to the Tendermint home directory") + runCmd.Usage = func() { + fmt.Println(`Runs the remote signer test harness for Tendermint. + +Usage: + tm-signer-harness run [flags] + +Flags:`) + runCmd.PrintDefaults() + fmt.Println("") + } + + extractKeyCmd = flag.NewFlagSet("extract_key", flag.ExitOnError) + extractKeyCmd.StringVar(&flagKeyOutputPath, "output", defaultExtractKeyOutput, "Path to which signing key should be written") + extractKeyCmd.StringVar(&flagTMHome, "tmhome", defaultTMHome, "Path to the Tendermint home directory") + extractKeyCmd.Usage = func() { + fmt.Println(`Extracts a signing key from a local Tendermint instance for use in the remote +signer under test. + +Usage: + tm-signer-harness extract_key [flags] + +Flags:`) + extractKeyCmd.PrintDefaults() + fmt.Println("") + } + + versionCmd = flag.NewFlagSet("version", flag.ExitOnError) + versionCmd.Usage = func() { + fmt.Println(` +Prints the Tendermint version for which this remote signer harness was built. + +Usage: + tm-signer-harness version`) + fmt.Println("") + } +} + +func runTestHarness(acceptRetries int, bindAddr, tmhome string) { + tmhome = internal.ExpandPath(tmhome) + cfg := internal.TestHarnessConfig{ + BindAddr: bindAddr, + KeyFile: filepath.Join(tmhome, "config", "priv_validator_key.json"), + StateFile: filepath.Join(tmhome, "data", "priv_validator_state.json"), + GenesisFile: filepath.Join(tmhome, "config", "genesis.json"), + AcceptDeadline: time.Duration(defaultAcceptDeadline) * time.Second, + AcceptRetries: acceptRetries, + ConnDeadline: time.Duration(defaultConnDeadline) * time.Second, + SecretConnKey: ed25519.GenPrivKey(), + ExitWhenComplete: true, + } + harness, err := internal.NewTestHarness(logger, cfg) + if err != nil { + logger.Error(err.Error()) + if therr, ok := err.(*internal.TestHarnessError); ok { + os.Exit(therr.Code) + } + os.Exit(internal.ErrOther) + } + harness.Run() +} + +func extractKey(tmhome, outputPath string) { + keyFile := filepath.Join(internal.ExpandPath(tmhome), "config", "priv_validator_key.json") + stateFile := filepath.Join(internal.ExpandPath(tmhome), "data", "priv_validator_state.json") + fpv := privval.LoadFilePV(keyFile, stateFile) + pkb := [64]byte(fpv.Key.PrivKey.(ed25519.PrivKeyEd25519)) + if err := ioutil.WriteFile(internal.ExpandPath(outputPath), pkb[:32], 0644); err != nil { + logger.Info("Failed to write private key", "output", outputPath, "err", err) + os.Exit(1) + } + logger.Info("Successfully wrote private key", "output", outputPath) +} + +func main() { + rootCmd.Parse(os.Args[1:]) + if rootCmd.NArg() == 0 || (rootCmd.NArg() == 1 && rootCmd.Arg(0) == "help") { + rootCmd.Usage() + os.Exit(0) + } + + logger = log.NewFilter(logger, log.AllowInfo()) + + switch rootCmd.Arg(0) { + case "help": + switch rootCmd.Arg(1) { + case "run": + runCmd.Usage() + case "extract_key": + extractKeyCmd.Usage() + case "version": + versionCmd.Usage() + default: + fmt.Printf("Unrecognized command: %s\n", rootCmd.Arg(1)) + os.Exit(1) + } + case "run": + runCmd.Parse(os.Args[2:]) + runTestHarness(flagAcceptRetries, flagBindAddr, flagTMHome) + case "extract_key": + extractKeyCmd.Parse(os.Args[2:]) + extractKey(flagTMHome, flagKeyOutputPath) + case "version": + fmt.Println(version.Version) + default: + fmt.Printf("Unrecognized command: %s\n", flag.Arg(0)) + os.Exit(1) + } +} diff --git a/types/block.go b/types/block.go index 07a71ca8..ec09fd44 100644 --- a/types/block.go +++ b/types/block.go @@ -2,22 +2,28 @@ package types import ( "bytes" - "errors" "fmt" "strings" "sync" "time" + "github.com/pkg/errors" + + "github.com/tendermint/tendermint/crypto" "github.com/tendermint/tendermint/crypto/merkle" + "github.com/tendermint/tendermint/crypto/tmhash" cmn "github.com/tendermint/tendermint/libs/common" + "github.com/tendermint/tendermint/version" ) const ( // MaxHeaderBytes is a maximum header size (including amino overhead). - MaxHeaderBytes int64 = 511 + MaxHeaderBytes int64 = 653 // MaxAminoOverheadForBlock - maximum amino overhead to encode a block (up to // MaxBlockSizeBytes in size) not including it's parts except Data. + // This means it also excludes the overhead for individual transactions. + // To compute individual transactions' overhead use types.ComputeAminoOverhead(tx types.Tx, fieldNum int). // // Uvarint length of MaxBlockSizeBytes: 4 bytes // 2 fields (2 embedded): 2 bytes @@ -27,7 +33,6 @@ const ( ) // Block defines the atomic unit of a Tendermint blockchain. -// TODO: add Version byte type Block struct { mtx sync.Mutex Header `json:"header"` @@ -57,47 +62,117 @@ func MakeBlock(height int64, txs []Tx, lastCommit *Commit, evidence []Evidence) // ValidateBasic performs basic validation that doesn't involve state data. // It checks the internal consistency of the block. +// Further validation is done using state#ValidateBlock. func (b *Block) ValidateBasic() error { if b == nil { - return errors.New("Nil blocks are invalid") + return errors.New("nil block") } b.mtx.Lock() defer b.mtx.Unlock() + if len(b.ChainID) > MaxChainIDLen { + return fmt.Errorf("ChainID is too long. Max is %d, got %d", MaxChainIDLen, len(b.ChainID)) + } + + if b.Height < 0 { + return errors.New("Negative Header.Height") + } else if b.Height == 0 { + return errors.New("Zero Header.Height") + } + + // NOTE: Timestamp validation is subtle and handled elsewhere. + newTxs := int64(len(b.Data.Txs)) if b.NumTxs != newTxs { - return fmt.Errorf( - "Wrong Block.Header.NumTxs. Expected %v, got %v", + return fmt.Errorf("Wrong Header.NumTxs. Expected %v, got %v", newTxs, b.NumTxs, ) } + + // TODO: fix tests so we can do this + /*if b.TotalTxs < b.NumTxs { + return fmt.Errorf("Header.TotalTxs (%d) is less than Header.NumTxs (%d)", b.TotalTxs, b.NumTxs) + }*/ + if b.TotalTxs < 0 { + return errors.New("Negative Header.TotalTxs") + } + + if err := b.LastBlockID.ValidateBasic(); err != nil { + return fmt.Errorf("Wrong Header.LastBlockID: %v", err) + } + + // Validate the last commit and its hash. + if b.Header.Height > 1 { + if b.LastCommit == nil { + return errors.New("nil LastCommit") + } + if err := b.LastCommit.ValidateBasic(); err != nil { + return fmt.Errorf("Wrong LastCommit") + } + } + if err := ValidateHash(b.LastCommitHash); err != nil { + return fmt.Errorf("Wrong Header.LastCommitHash: %v", err) + } if !bytes.Equal(b.LastCommitHash, b.LastCommit.Hash()) { - return fmt.Errorf( - "Wrong Block.Header.LastCommitHash. Expected %v, got %v", - b.LastCommitHash, + return fmt.Errorf("Wrong Header.LastCommitHash. Expected %v, got %v", b.LastCommit.Hash(), + b.LastCommitHash, ) } - if b.Header.Height != 1 { - if err := b.LastCommit.ValidateBasic(); err != nil { - return err - } + + // Validate the hash of the transactions. + // NOTE: b.Data.Txs may be nil, but b.Data.Hash() + // still works fine + if err := ValidateHash(b.DataHash); err != nil { + return fmt.Errorf("Wrong Header.DataHash: %v", err) } if !bytes.Equal(b.DataHash, b.Data.Hash()) { return fmt.Errorf( - "Wrong Block.Header.DataHash. Expected %v, got %v", - b.DataHash, + "Wrong Header.DataHash. Expected %v, got %v", b.Data.Hash(), + b.DataHash, ) } + + // Basic validation of hashes related to application data. + // Will validate fully against state in state#ValidateBlock. + if err := ValidateHash(b.ValidatorsHash); err != nil { + return fmt.Errorf("Wrong Header.ValidatorsHash: %v", err) + } + if err := ValidateHash(b.NextValidatorsHash); err != nil { + return fmt.Errorf("Wrong Header.NextValidatorsHash: %v", err) + } + if err := ValidateHash(b.ConsensusHash); err != nil { + return fmt.Errorf("Wrong Header.ConsensusHash: %v", err) + } + // NOTE: AppHash is arbitrary length + if err := ValidateHash(b.LastResultsHash); err != nil { + return fmt.Errorf("Wrong Header.LastResultsHash: %v", err) + } + + // Validate evidence and its hash. + if err := ValidateHash(b.EvidenceHash); err != nil { + return fmt.Errorf("Wrong Header.EvidenceHash: %v", err) + } + // NOTE: b.Evidence.Evidence may be nil, but we're just looping. + for i, ev := range b.Evidence.Evidence { + if err := ev.ValidateBasic(); err != nil { + return fmt.Errorf("Invalid evidence (#%d): %v", i, err) + } + } if !bytes.Equal(b.EvidenceHash, b.Evidence.Hash()) { - return fmt.Errorf( - "Wrong Block.Header.EvidenceHash. Expected %v, got %v", + return fmt.Errorf("Wrong Header.EvidenceHash. Expected %v, got %v", b.EvidenceHash, b.Evidence.Hash(), ) } + + if len(b.ProposerAddress) != crypto.AddressSize { + return fmt.Errorf("Expected len(Header.ProposerAddress) to be %d, got %d", + crypto.AddressSize, len(b.ProposerAddress)) + } + return nil } @@ -142,7 +217,7 @@ func (b *Block) MakePartSet(partSize int) *PartSet { // We prefix the byte length, so that unmarshaling // can easily happen via a reader. - bz, err := cdc.MarshalBinary(b) + bz, err := cdc.MarshalBinaryLengthPrefixed(b) if err != nil { panic(err) } @@ -201,6 +276,28 @@ func (b *Block) StringShort() string { return fmt.Sprintf("Block#%v", b.Hash()) } +//----------------------------------------------------------- +// These methods are for Protobuf Compatibility + +// Marshal returns the amino encoding. +func (b *Block) Marshal() ([]byte, error) { + return cdc.MarshalBinaryBare(b) +} + +// MarshalTo calls Marshal and copies to the given buffer. +func (b *Block) MarshalTo(data []byte) (int, error) { + bs, err := b.Marshal() + if err != nil { + return -1, err + } + return copy(data, bs), nil +} + +// Unmarshal deserializes from amino encoded form. +func (b *Block) Unmarshal(bs []byte) error { + return cdc.UnmarshalBinaryBare(bs, b) +} + //----------------------------------------------------------------------------- // MaxDataBytes returns the maximum size of block's data. @@ -226,16 +323,17 @@ func MaxDataBytes(maxBytes int64, valsCount, evidenceCount int) int64 { } // MaxDataBytesUnknownEvidence returns the maximum size of block's data when -// evidence count is unknown. MaxEvidenceBytesPerBlock will be used as the size +// evidence count is unknown. MaxEvidencePerBlock will be used for the size // of evidence. // // XXX: Panics on negative result. func MaxDataBytesUnknownEvidence(maxBytes int64, valsCount int) int64 { + _, maxEvidenceBytes := MaxEvidencePerBlock(maxBytes) maxDataBytes := maxBytes - MaxAminoOverheadForBlock - MaxHeaderBytes - int64(valsCount)*MaxVoteBytes - - MaxEvidenceBytesPerBlock(maxBytes) + maxEvidenceBytes if maxDataBytes < 0 { panic(fmt.Sprintf( @@ -250,17 +348,19 @@ func MaxDataBytesUnknownEvidence(maxBytes int64, valsCount int) int64 { //----------------------------------------------------------------------------- -// Header defines the structure of a Tendermint block header -// TODO: limit header size -// NOTE: changes to the Header should be duplicated in the abci Header -// and in /docs/spec/blockchain/blockchain.md +// Header defines the structure of a Tendermint block header. +// NOTE: changes to the Header should be duplicated in: +// - header.Hash() +// - abci.Header +// - /docs/spec/blockchain/blockchain.md type Header struct { // basic block info - ChainID string `json:"chain_id"` - Height int64 `json:"height"` - Time time.Time `json:"time"` - NumTxs int64 `json:"num_txs"` - TotalTxs int64 `json:"total_txs"` + Version version.Consensus `json:"version"` + ChainID string `json:"chain_id"` + Height int64 `json:"height"` + Time time.Time `json:"time"` + NumTxs int64 `json:"num_txs"` + TotalTxs int64 `json:"total_txs"` // prev block info LastBlockID BlockID `json:"last_block_id"` @@ -281,7 +381,31 @@ type Header struct { ProposerAddress Address `json:"proposer_address"` // original proposer of the block } +// Populate the Header with state-derived data. +// Call this after MakeBlock to complete the Header. +func (h *Header) Populate( + version version.Consensus, chainID string, + timestamp time.Time, lastBlockID BlockID, totalTxs int64, + valHash, nextValHash []byte, + consensusHash, appHash, lastResultsHash []byte, + proposerAddress Address, +) { + h.Version = version + h.ChainID = chainID + h.Time = timestamp + h.LastBlockID = lastBlockID + h.TotalTxs = totalTxs + h.ValidatorsHash = valHash + h.NextValidatorsHash = nextValHash + h.ConsensusHash = consensusHash + h.AppHash = appHash + h.LastResultsHash = lastResultsHash + h.ProposerAddress = proposerAddress +} + // Hash returns the hash of the header. +// It computes a Merkle tree from the header fields +// ordered as they appear in the Header. // Returns nil if ValidatorHash is missing, // since a Header is not valid unless there is // a ValidatorsHash (corresponding to the validator set). @@ -289,22 +413,23 @@ func (h *Header) Hash() cmn.HexBytes { if h == nil || len(h.ValidatorsHash) == 0 { return nil } - return merkle.SimpleHashFromMap(map[string][]byte{ - "ChainID": cdcEncode(h.ChainID), - "Height": cdcEncode(h.Height), - "Time": cdcEncode(h.Time), - "NumTxs": cdcEncode(h.NumTxs), - "TotalTxs": cdcEncode(h.TotalTxs), - "LastBlockID": cdcEncode(h.LastBlockID), - "LastCommit": cdcEncode(h.LastCommitHash), - "Data": cdcEncode(h.DataHash), - "Validators": cdcEncode(h.ValidatorsHash), - "NextValidators": cdcEncode(h.NextValidatorsHash), - "App": cdcEncode(h.AppHash), - "Consensus": cdcEncode(h.ConsensusHash), - "Results": cdcEncode(h.LastResultsHash), - "Evidence": cdcEncode(h.EvidenceHash), - "Proposer": cdcEncode(h.ProposerAddress), + return merkle.SimpleHashFromByteSlices([][]byte{ + cdcEncode(h.Version), + cdcEncode(h.ChainID), + cdcEncode(h.Height), + cdcEncode(h.Time), + cdcEncode(h.NumTxs), + cdcEncode(h.TotalTxs), + cdcEncode(h.LastBlockID), + cdcEncode(h.LastCommitHash), + cdcEncode(h.DataHash), + cdcEncode(h.ValidatorsHash), + cdcEncode(h.NextValidatorsHash), + cdcEncode(h.ConsensusHash), + cdcEncode(h.AppHash), + cdcEncode(h.LastResultsHash), + cdcEncode(h.EvidenceHash), + cdcEncode(h.ProposerAddress), }) } @@ -314,6 +439,7 @@ func (h *Header) StringIndented(indent string) string { return "nil-Header" } return fmt.Sprintf(`Header{ +%s Version: %v %s ChainID: %v %s Height: %v %s Time: %v @@ -330,6 +456,7 @@ func (h *Header) StringIndented(indent string) string { %s Evidence: %v %s Proposer: %v %s}#%v`, + indent, h.Version, indent, h.ChainID, indent, h.Height, indent, h.Time, @@ -350,39 +477,77 @@ func (h *Header) StringIndented(indent string) string { //------------------------------------- +// CommitSig is a vote included in a Commit. +// For now, it is identical to a vote, +// but in the future it will contain fewer fields +// to eliminate the redundancy in commits. +// See https://github.com/tendermint/tendermint/issues/1648. +type CommitSig Vote + +// String returns the underlying Vote.String() +func (cs *CommitSig) String() string { + return cs.toVote().String() +} + +// toVote converts the CommitSig to a vote. +// Once CommitSig has fewer fields than vote, +// converting to a Vote will require more information. +func (cs *CommitSig) toVote() *Vote { + if cs == nil { + return nil + } + v := Vote(*cs) + return &v +} + // Commit contains the evidence that a block was committed by a set of validators. // NOTE: Commit is empty for height 1, but never nil. type Commit struct { // NOTE: The Precommits are in order of address to preserve the bonded ValidatorSet order. // Any peer with a block can gossip precommits by index with a peer without recalculating the // active ValidatorSet. - BlockID BlockID `json:"block_id"` - Precommits []*Vote `json:"precommits"` + BlockID BlockID `json:"block_id"` + Precommits []*CommitSig `json:"precommits"` // Volatile - firstPrecommit *Vote - hash cmn.HexBytes - bitArray *cmn.BitArray + height int64 + round int + hash cmn.HexBytes + bitArray *cmn.BitArray } -// FirstPrecommit returns the first non-nil precommit in the commit. -// If all precommits are nil, it returns an empty precommit with height 0. -func (commit *Commit) FirstPrecommit() *Vote { +// VoteSignBytes constructs the SignBytes for the given CommitSig. +// The only unique part of the SignBytes is the Timestamp - all other fields +// signed over are otherwise the same for all validators. +func (commit *Commit) VoteSignBytes(chainID string, cs *CommitSig) []byte { + return cs.toVote().SignBytes(chainID) +} + +// memoizeHeightRound memoizes the height and round of the commit using +// the first non-nil vote. +func (commit *Commit) memoizeHeightRound() { if len(commit.Precommits) == 0 { - return nil + return } - if commit.firstPrecommit != nil { - return commit.firstPrecommit + if commit.height > 0 { + return } for _, precommit := range commit.Precommits { if precommit != nil { - commit.firstPrecommit = precommit - return precommit + commit.height = precommit.Height + commit.round = precommit.Round + return } } - return &Vote{ - Type: VoteTypePrecommit, - } +} + +// ToVote converts a CommitSig to a Vote. +// If the CommitSig is nil, the Vote will be nil. +// When CommitSig is reduced to contain fewer fields, +// this will need access to the ValidatorSet to properly +// reconstruct the vote. +func (commit *Commit) ToVote(cs *CommitSig) *Vote { + return cs.toVote() } // Height returns the height of the commit @@ -390,7 +555,8 @@ func (commit *Commit) Height() int64 { if len(commit.Precommits) == 0 { return 0 } - return commit.FirstPrecommit().Height + commit.memoizeHeightRound() + return commit.height } // Round returns the round of the commit @@ -398,12 +564,13 @@ func (commit *Commit) Round() int { if len(commit.Precommits) == 0 { return 0 } - return commit.FirstPrecommit().Round + commit.memoizeHeightRound() + return commit.round } // Type returns the vote type of the commit, which is always VoteTypePrecommit func (commit *Commit) Type() byte { - return VoteTypePrecommit + return byte(PrecommitType) } // Size returns the number of votes in the commit @@ -427,12 +594,13 @@ func (commit *Commit) BitArray() *cmn.BitArray { return commit.bitArray } -// GetByIndex returns the vote corresponding to a given validator index +// GetByIndex returns the vote corresponding to a given validator index. +// Implements VoteSetReader. func (commit *Commit) GetByIndex(index int) *Vote { - return commit.Precommits[index] + return commit.Precommits[index].toVote() } -// IsCommit returns true if there is at least one vote +// IsCommit returns true if there is at least one vote. func (commit *Commit) IsCommit() bool { return len(commit.Precommits) != 0 } @@ -455,7 +623,7 @@ func (commit *Commit) ValidateBasic() error { continue } // Ensure that all votes are precommits. - if precommit.Type != VoteTypePrecommit { + if precommit.Type != PrecommitType { return fmt.Errorf("Invalid commit vote. Expected precommit, got %v", precommit.Type) } @@ -511,6 +679,7 @@ func (commit *Commit) StringIndented(indent string) string { //----------------------------------------------------------------------------- // SignedHeader is a header along with the commits that prove it. +// It is the basis of the lite client. type SignedHeader struct { *Header `json:"header"` Commit *Commit `json:"commit"` @@ -531,6 +700,7 @@ func (sh SignedHeader) ValidateBasic(chainID string) error { if sh.Commit == nil { return errors.New("SignedHeader missing commit (precommit votes).") } + // Check ChainID. if sh.ChainID != chainID { return fmt.Errorf("Header belongs to another chain '%s' not '%s'", @@ -569,7 +739,6 @@ func (sh SignedHeader) StringIndented(indent string) string { indent, sh.Header.StringIndented(indent+" "), indent, sh.Commit.StringIndented(indent+" "), indent) - return "" } //----------------------------------------------------------------------------- @@ -653,7 +822,6 @@ func (data *EvidenceData) StringIndented(indent string) string { %s}#%v`, indent, strings.Join(evStrings, "\n"+indent+" "), indent, data.hash) - return "" } //-------------------------------------------------------------------------------- @@ -664,11 +832,6 @@ type BlockID struct { PartsHeader PartSetHeader `json:"parts"` } -// IsZero returns true if this is the BlockID for a nil-block -func (blockID BlockID) IsZero() bool { - return len(blockID.Hash) == 0 && blockID.PartsHeader.IsZero() -} - // Equals returns true if the BlockID matches the given BlockID func (blockID BlockID) Equals(other BlockID) bool { return bytes.Equal(blockID.Hash, other.Hash) && @@ -684,6 +847,31 @@ func (blockID BlockID) Key() string { return string(blockID.Hash) + string(bz) } +// ValidateBasic performs basic validation. +func (blockID BlockID) ValidateBasic() error { + // Hash can be empty in case of POLBlockID in Proposal. + if err := ValidateHash(blockID.Hash); err != nil { + return fmt.Errorf("Wrong Hash") + } + if err := blockID.PartsHeader.ValidateBasic(); err != nil { + return fmt.Errorf("Wrong PartsHeader: %v", err) + } + return nil +} + +// IsZero returns true if this is the BlockID of a nil block. +func (blockID BlockID) IsZero() bool { + return len(blockID.Hash) == 0 && + blockID.PartsHeader.IsZero() +} + +// IsComplete returns true if this is a valid BlockID of a non-nil block. +func (blockID BlockID) IsComplete() bool { + return len(blockID.Hash) == tmhash.Size && + blockID.PartsHeader.Total > 0 && + len(blockID.PartsHeader.Hash) == tmhash.Size +} + // String returns a human readable string representation of the BlockID func (blockID BlockID) String() string { return fmt.Sprintf(`%v:%v`, blockID.Hash, blockID.PartsHeader) diff --git a/types/block_meta.go b/types/block_meta.go index d8926af0..8297446a 100644 --- a/types/block_meta.go +++ b/types/block_meta.go @@ -13,3 +13,31 @@ func NewBlockMeta(block *Block, blockParts *PartSet) *BlockMeta { Header: block.Header, } } + +//----------------------------------------------------------- +// These methods are for Protobuf Compatibility + +// Size returns the size of the amino encoding, in bytes. +func (bm *BlockMeta) Size() int { + bs, _ := bm.Marshal() + return len(bs) +} + +// Marshal returns the amino encoding. +func (bm *BlockMeta) Marshal() ([]byte, error) { + return cdc.MarshalBinaryBare(bm) +} + +// MarshalTo calls Marshal and copies to the given buffer. +func (bm *BlockMeta) MarshalTo(data []byte) (int, error) { + bs, err := bm.Marshal() + if err != nil { + return -1, err + } + return copy(data, bs), nil +} + +// Unmarshal deserializes from amino encoded form. +func (bm *BlockMeta) Unmarshal(bs []byte) error { + return cdc.UnmarshalBinaryBare(bs, bm) +} diff --git a/types/block_test.go b/types/block_test.go index 887f35a1..31e7983f 100644 --- a/types/block_test.go +++ b/types/block_test.go @@ -10,8 +10,10 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/tendermint/tendermint/crypto" "github.com/tendermint/tendermint/crypto/tmhash" cmn "github.com/tendermint/tendermint/libs/common" + "github.com/tendermint/tendermint/version" ) func TestMain(m *testing.M) { @@ -26,7 +28,7 @@ func TestBlockAddEvidence(t *testing.T) { lastID := makeBlockIDRandom() h := int64(3) - voteSet, valSet, vals := randVoteSet(h-1, 1, VoteTypePrecommit, 10, 1) + voteSet, valSet, vals := randVoteSet(h-1, 1, PrecommitType, 10, 1) commit, err := MakeCommit(lastID, h-1, 1, voteSet, vals) require.NoError(t, err) @@ -46,7 +48,7 @@ func TestBlockValidateBasic(t *testing.T) { lastID := makeBlockIDRandom() h := int64(3) - voteSet, valSet, vals := randVoteSet(h-1, 1, VoteTypePrecommit, 10, 1) + voteSet, valSet, vals := randVoteSet(h-1, 1, PrecommitType, 10, 1) commit, err := MakeCommit(lastID, h-1, 1, voteSet, vals) require.NoError(t, err) @@ -60,6 +62,7 @@ func TestBlockValidateBasic(t *testing.T) { }{ {"Make Block", func(blk *Block) {}, false}, {"Make Block w/ proposer Addr", func(blk *Block) { blk.ProposerAddress = valSet.GetProposer().Address }, false}, + {"Negative Height", func(blk *Block) { blk.Height = -1 }, true}, {"Increase NumTxs", func(blk *Block) { blk.NumTxs++ }, true}, {"Remove 1/2 the commits", func(blk *Block) { blk.LastCommit.Precommits = commit.Precommits[:commit.Size()/2] @@ -77,11 +80,13 @@ func TestBlockValidateBasic(t *testing.T) { blk.EvidenceHash = []byte("something else") }, true}, } - for _, tc := range testCases { + for i, tc := range testCases { t.Run(tc.testName, func(t *testing.T) { block := MakeBlock(h, txs, commit, evList) + block.ProposerAddress = valSet.GetProposer().Address tc.malleateBlock(block) - assert.Equal(t, tc.expErr, block.ValidateBasic() != nil, "Validate Basic had an unexpected result") + err = block.ValidateBasic() + assert.Equal(t, tc.expErr, err != nil, "#%d: %v", i, err) }) } } @@ -105,7 +110,7 @@ func TestBlockMakePartSetWithEvidence(t *testing.T) { lastID := makeBlockIDRandom() h := int64(3) - voteSet, valSet, vals := randVoteSet(h-1, 1, VoteTypePrecommit, 10, 1) + voteSet, valSet, vals := randVoteSet(h-1, 1, PrecommitType, 10, 1) commit, err := MakeCommit(lastID, h-1, 1, voteSet, vals) require.NoError(t, err) @@ -114,7 +119,7 @@ func TestBlockMakePartSetWithEvidence(t *testing.T) { partSet := MakeBlock(h, []Tx{Tx("Hello World")}, commit, evList).MakePartSet(1024) assert.NotNil(t, partSet) - assert.Equal(t, 2, partSet.Total()) + assert.Equal(t, 3, partSet.Total()) } func TestBlockHashesTo(t *testing.T) { @@ -122,7 +127,7 @@ func TestBlockHashesTo(t *testing.T) { lastID := makeBlockIDRandom() h := int64(3) - voteSet, valSet, vals := randVoteSet(h-1, 1, VoteTypePrecommit, 10, 1) + voteSet, valSet, vals := randVoteSet(h-1, 1, PrecommitType, 10, 1) commit, err := MakeCommit(lastID, h-1, 1, voteSet, vals) require.NoError(t, err) @@ -189,14 +194,13 @@ func TestNilDataHashDoesntCrash(t *testing.T) { func TestCommit(t *testing.T) { lastID := makeBlockIDRandom() h := int64(3) - voteSet, _, vals := randVoteSet(h-1, 1, VoteTypePrecommit, 10, 1) + voteSet, _, vals := randVoteSet(h-1, 1, PrecommitType, 10, 1) commit, err := MakeCommit(lastID, h-1, 1, voteSet, vals) require.NoError(t, err) - assert.NotNil(t, commit.FirstPrecommit()) assert.Equal(t, h-1, commit.Height()) assert.Equal(t, 1, commit.Round()) - assert.Equal(t, VoteTypePrecommit, commit.Type()) + assert.Equal(t, PrecommitType, SignedMsgType(commit.Type())) if commit.Size() <= 0 { t.Fatalf("commit %v has a zero or negative size: %d", commit, commit.Size()) } @@ -217,7 +221,7 @@ func TestCommitValidateBasic(t *testing.T) { {"Random Commit", func(com *Commit) {}, false}, {"Nil precommit", func(com *Commit) { com.Precommits[0] = nil }, false}, {"Incorrect signature", func(com *Commit) { com.Precommits[0].Signature = []byte{0} }, false}, - {"Incorrect type", func(com *Commit) { com.Precommits[0].Type = VoteTypePrevote }, true}, + {"Incorrect type", func(com *Commit) { com.Precommits[0].Type = PrevoteType }, true}, {"Incorrect height", func(com *Commit) { com.Precommits[0].Height = int64(100) }, true}, {"Incorrect round", func(com *Commit) { com.Precommits[0].Round = 100 }, true}, } @@ -240,10 +244,15 @@ func TestMaxHeaderBytes(t *testing.T) { maxChainID += "𠜎" } + // time is varint encoded so need to pick the max. + // year int, month Month, day, hour, min, sec, nsec int, loc *Location + timestamp := time.Date(math.MaxInt64, 0, 0, 0, 0, 0, math.MaxInt64, time.UTC) + h := Header{ + Version: version.Consensus{Block: math.MaxInt64, App: math.MaxInt64}, ChainID: maxChainID, Height: math.MaxInt64, - Time: time.Now().UTC(), + Time: timestamp, NumTxs: math.MaxInt64, TotalTxs: math.MaxInt64, LastBlockID: makeBlockID(make([]byte, tmhash.Size), math.MaxInt64, make([]byte, tmhash.Size)), @@ -255,10 +264,10 @@ func TestMaxHeaderBytes(t *testing.T) { AppHash: tmhash.Sum([]byte("app_hash")), LastResultsHash: tmhash.Sum([]byte("last_results_hash")), EvidenceHash: tmhash.Sum([]byte("evidence_hash")), - ProposerAddress: tmhash.Sum([]byte("proposer_address")), + ProposerAddress: crypto.AddressHash([]byte("proposer_address")), } - bz, err := cdc.MarshalBinary(h) + bz, err := cdc.MarshalBinaryLengthPrefixed(h) require.NoError(t, err) assert.EqualValues(t, MaxHeaderBytes, len(bz)) @@ -267,7 +276,7 @@ func TestMaxHeaderBytes(t *testing.T) { func randCommit() *Commit { lastID := makeBlockIDRandom() h := int64(3) - voteSet, _, vals := randVoteSet(h-1, 1, VoteTypePrecommit, 10, 1) + voteSet, _, vals := randVoteSet(h-1, 1, PrecommitType, 10, 1) commit, err := MakeCommit(lastID, h-1, 1, voteSet, vals) if err != nil { panic(err) @@ -285,9 +294,9 @@ func TestBlockMaxDataBytes(t *testing.T) { }{ 0: {-10, 1, 0, true, 0}, 1: {10, 1, 0, true, 0}, - 2: {721, 1, 0, true, 0}, - 3: {722, 1, 0, false, 0}, - 4: {723, 1, 0, false, 1}, + 2: {886, 1, 0, true, 0}, + 3: {887, 1, 0, false, 0}, + 4: {888, 1, 0, false, 1}, } for i, tc := range testCases { @@ -313,9 +322,9 @@ func TestBlockMaxDataBytesUnknownEvidence(t *testing.T) { }{ 0: {-10, 1, true, 0}, 1: {10, 1, true, 0}, - 2: {801, 1, true, 0}, - 3: {802, 1, false, 0}, - 4: {803, 1, false, 1}, + 2: {984, 1, true, 0}, + 3: {985, 1, false, 0}, + 4: {986, 1, false, 1}, } for i, tc := range testCases { diff --git a/types/canonical.go b/types/canonical.go index cdf0bd7b..47a8c817 100644 --- a/types/canonical.go +++ b/types/canonical.go @@ -13,44 +13,32 @@ import ( const TimeFormat = time.RFC3339Nano type CanonicalBlockID struct { - Hash cmn.HexBytes `json:"hash,omitempty"` - PartsHeader CanonicalPartSetHeader `json:"parts,omitempty"` + Hash cmn.HexBytes + PartsHeader CanonicalPartSetHeader } type CanonicalPartSetHeader struct { - Hash cmn.HexBytes `json:"hash,omitempty"` - Total int `json:"total,omitempty"` + Hash cmn.HexBytes + Total int } type CanonicalProposal struct { - ChainID string `json:"@chain_id"` - Type string `json:"@type"` - BlockPartsHeader CanonicalPartSetHeader `json:"block_parts_header"` - Height int64 `json:"height"` - POLBlockID CanonicalBlockID `json:"pol_block_id"` - POLRound int `json:"pol_round"` - Round int `json:"round"` - Timestamp time.Time `json:"timestamp"` + Type SignedMsgType // type alias for byte + Height int64 `binary:"fixed64"` + Round int64 `binary:"fixed64"` + POLRound int64 `binary:"fixed64"` + BlockID CanonicalBlockID + Timestamp time.Time + ChainID string } type CanonicalVote struct { - ChainID string `json:"@chain_id"` - Type string `json:"@type"` - BlockID CanonicalBlockID `json:"block_id"` - Height int64 `json:"height"` - Round int `json:"round"` - Timestamp time.Time `json:"timestamp"` - VoteType byte `json:"type"` -} - -type CanonicalHeartbeat struct { - ChainID string `json:"@chain_id"` - Type string `json:"@type"` - Height int64 `json:"height"` - Round int `json:"round"` - Sequence int `json:"sequence"` - ValidatorAddress Address `json:"validator_address"` - ValidatorIndex int `json:"validator_index"` + Type SignedMsgType // type alias for byte + Height int64 `binary:"fixed64"` + Round int64 `binary:"fixed64"` + BlockID CanonicalBlockID + Timestamp time.Time + ChainID string } //----------------------------------- @@ -72,38 +60,24 @@ func CanonicalizePartSetHeader(psh PartSetHeader) CanonicalPartSetHeader { func CanonicalizeProposal(chainID string, proposal *Proposal) CanonicalProposal { return CanonicalProposal{ - ChainID: chainID, - Type: "proposal", - BlockPartsHeader: CanonicalizePartSetHeader(proposal.BlockPartsHeader), - Height: proposal.Height, - Timestamp: proposal.Timestamp, - POLBlockID: CanonicalizeBlockID(proposal.POLBlockID), - POLRound: proposal.POLRound, - Round: proposal.Round, + Type: ProposalType, + Height: proposal.Height, + Round: int64(proposal.Round), // cast int->int64 to make amino encode it fixed64 (does not work for int) + POLRound: int64(proposal.POLRound), + BlockID: CanonicalizeBlockID(proposal.BlockID), + Timestamp: proposal.Timestamp, + ChainID: chainID, } } func CanonicalizeVote(chainID string, vote *Vote) CanonicalVote { return CanonicalVote{ - ChainID: chainID, - Type: "vote", - BlockID: CanonicalizeBlockID(vote.BlockID), + Type: vote.Type, Height: vote.Height, - Round: vote.Round, + Round: int64(vote.Round), // cast int->int64 to make amino encode it fixed64 (does not work for int) + BlockID: CanonicalizeBlockID(vote.BlockID), Timestamp: vote.Timestamp, - VoteType: vote.Type, - } -} - -func CanonicalizeHeartbeat(chainID string, heartbeat *Heartbeat) CanonicalHeartbeat { - return CanonicalHeartbeat{ - ChainID: chainID, - Type: "heartbeat", - Height: heartbeat.Height, - Round: heartbeat.Round, - Sequence: heartbeat.Sequence, - ValidatorAddress: heartbeat.ValidatorAddress, - ValidatorIndex: heartbeat.ValidatorIndex, + ChainID: chainID, } } diff --git a/types/event_bus.go b/types/event_bus.go index 269d5ab1..055cbd3f 100644 --- a/types/event_bus.go +++ b/types/event_bus.go @@ -71,18 +71,58 @@ func (b *EventBus) Publish(eventType string, eventData TMEventData) error { return nil } +func (b *EventBus) validateAndStringifyTags(tags []cmn.KVPair, logger log.Logger) map[string]string { + result := make(map[string]string) + for _, tag := range tags { + // basic validation + if len(tag.Key) == 0 { + logger.Debug("Got tag with an empty key (skipping)", "tag", tag) + continue + } + result[string(tag.Key)] = string(tag.Value) + } + return result +} + func (b *EventBus) PublishEventNewBlock(data EventDataNewBlock) error { - return b.Publish(EventNewBlock, data) + // no explicit deadline for publishing events + ctx := context.Background() + + resultTags := append(data.ResultBeginBlock.Tags, data.ResultEndBlock.Tags...) + tags := b.validateAndStringifyTags(resultTags, b.Logger.With("block", data.Block.StringShort())) + + // add predefined tags + logIfTagExists(EventTypeKey, tags, b.Logger) + tags[EventTypeKey] = EventNewBlock + + b.pubsub.PublishWithTags(ctx, data, tmpubsub.NewTagMap(tags)) + return nil } func (b *EventBus) PublishEventNewBlockHeader(data EventDataNewBlockHeader) error { - return b.Publish(EventNewBlockHeader, data) + // no explicit deadline for publishing events + ctx := context.Background() + + resultTags := append(data.ResultBeginBlock.Tags, data.ResultEndBlock.Tags...) + // TODO: Create StringShort method for Header and use it in logger. + tags := b.validateAndStringifyTags(resultTags, b.Logger.With("header", data.Header)) + + // add predefined tags + logIfTagExists(EventTypeKey, tags, b.Logger) + tags[EventTypeKey] = EventNewBlockHeader + + b.pubsub.PublishWithTags(ctx, data, tmpubsub.NewTagMap(tags)) + return nil } func (b *EventBus) PublishEventVote(data EventDataVote) error { return b.Publish(EventVote, data) } +func (b *EventBus) PublishEventValidBlock(data EventDataRoundState) error { + return b.Publish(EventValidBlock, data) +} + // PublishEventTx publishes tx event with tags from Result. Note it will add // predefined tags (EventTypeKey, TxHashKey). Existing tags with the same names // will be overwritten. @@ -90,17 +130,7 @@ func (b *EventBus) PublishEventTx(data EventDataTx) error { // no explicit deadline for publishing events ctx := context.Background() - tags := make(map[string]string) - - // validate and fill tags from tx result - for _, tag := range data.Result.Tags { - // basic validation - if len(tag.Key) == 0 { - b.Logger.Info("Got tag with an empty key (skipping)", "tag", tag, "tx", data.Tx) - continue - } - tags[string(tag.Key)] = string(tag.Value) - } + tags := b.validateAndStringifyTags(data.Result.Tags, b.Logger.With("tx", data.Tx)) // add predefined tags logIfTagExists(EventTypeKey, tags, b.Logger) @@ -116,10 +146,6 @@ func (b *EventBus) PublishEventTx(data EventDataTx) error { return nil } -func (b *EventBus) PublishEventProposalHeartbeat(data EventDataProposalHeartbeat) error { - return b.Publish(EventProposalHeartbeat, data) -} - func (b *EventBus) PublishEventNewRoundStep(data EventDataRoundState) error { return b.Publish(EventNewRoundStep, data) } @@ -132,11 +158,11 @@ func (b *EventBus) PublishEventTimeoutWait(data EventDataRoundState) error { return b.Publish(EventTimeoutWait, data) } -func (b *EventBus) PublishEventNewRound(data EventDataRoundState) error { +func (b *EventBus) PublishEventNewRound(data EventDataNewRound) error { return b.Publish(EventNewRound, data) } -func (b *EventBus) PublishEventCompleteProposal(data EventDataRoundState) error { +func (b *EventBus) PublishEventCompleteProposal(data EventDataCompleteProposal) error { return b.Publish(EventCompleteProposal, data) } diff --git a/types/event_bus_test.go b/types/event_bus_test.go index f0e825d5..6845927b 100644 --- a/types/event_bus_test.go +++ b/types/event_bus_test.go @@ -58,6 +58,90 @@ func TestEventBusPublishEventTx(t *testing.T) { } } +func TestEventBusPublishEventNewBlock(t *testing.T) { + eventBus := NewEventBus() + err := eventBus.Start() + require.NoError(t, err) + defer eventBus.Stop() + + block := MakeBlock(0, []Tx{}, nil, []Evidence{}) + resultBeginBlock := abci.ResponseBeginBlock{Tags: []cmn.KVPair{{Key: []byte("baz"), Value: []byte("1")}}} + resultEndBlock := abci.ResponseEndBlock{Tags: []cmn.KVPair{{Key: []byte("foz"), Value: []byte("2")}}} + + txEventsCh := make(chan interface{}) + + // PublishEventNewBlock adds the tm.event tag, so the query below should work + query := "tm.event='NewBlock' AND baz=1 AND foz=2" + err = eventBus.Subscribe(context.Background(), "test", tmquery.MustParse(query), txEventsCh) + require.NoError(t, err) + + done := make(chan struct{}) + go func() { + for e := range txEventsCh { + edt := e.(EventDataNewBlock) + assert.Equal(t, block, edt.Block) + assert.Equal(t, resultBeginBlock, edt.ResultBeginBlock) + assert.Equal(t, resultEndBlock, edt.ResultEndBlock) + close(done) + } + }() + + err = eventBus.PublishEventNewBlock(EventDataNewBlock{ + Block: block, + ResultBeginBlock: resultBeginBlock, + ResultEndBlock: resultEndBlock, + }) + assert.NoError(t, err) + + select { + case <-done: + case <-time.After(1 * time.Second): + t.Fatal("did not receive a block after 1 sec.") + } +} + +func TestEventBusPublishEventNewBlockHeader(t *testing.T) { + eventBus := NewEventBus() + err := eventBus.Start() + require.NoError(t, err) + defer eventBus.Stop() + + block := MakeBlock(0, []Tx{}, nil, []Evidence{}) + resultBeginBlock := abci.ResponseBeginBlock{Tags: []cmn.KVPair{{Key: []byte("baz"), Value: []byte("1")}}} + resultEndBlock := abci.ResponseEndBlock{Tags: []cmn.KVPair{{Key: []byte("foz"), Value: []byte("2")}}} + + txEventsCh := make(chan interface{}) + + // PublishEventNewBlockHeader adds the tm.event tag, so the query below should work + query := "tm.event='NewBlockHeader' AND baz=1 AND foz=2" + err = eventBus.Subscribe(context.Background(), "test", tmquery.MustParse(query), txEventsCh) + require.NoError(t, err) + + done := make(chan struct{}) + go func() { + for e := range txEventsCh { + edt := e.(EventDataNewBlockHeader) + assert.Equal(t, block.Header, edt.Header) + assert.Equal(t, resultBeginBlock, edt.ResultBeginBlock) + assert.Equal(t, resultEndBlock, edt.ResultEndBlock) + close(done) + } + }() + + err = eventBus.PublishEventNewBlockHeader(EventDataNewBlockHeader{ + Header: block.Header, + ResultBeginBlock: resultBeginBlock, + ResultEndBlock: resultEndBlock, + }) + assert.NoError(t, err) + + select { + case <-done: + case <-time.After(1 * time.Second): + t.Fatal("did not receive a block header after 1 sec.") + } +} + func TestEventBusPublish(t *testing.T) { eventBus := NewEventBus() err := eventBus.Start() @@ -68,7 +152,7 @@ func TestEventBusPublish(t *testing.T) { err = eventBus.Subscribe(context.Background(), "test", tmquery.Empty{}, eventsCh) require.NoError(t, err) - const numEventsExpected = 15 + const numEventsExpected = 14 done := make(chan struct{}) go func() { numEvents := 0 @@ -88,17 +172,15 @@ func TestEventBusPublish(t *testing.T) { require.NoError(t, err) err = eventBus.PublishEventVote(EventDataVote{}) require.NoError(t, err) - err = eventBus.PublishEventProposalHeartbeat(EventDataProposalHeartbeat{}) - require.NoError(t, err) err = eventBus.PublishEventNewRoundStep(EventDataRoundState{}) require.NoError(t, err) err = eventBus.PublishEventTimeoutPropose(EventDataRoundState{}) require.NoError(t, err) err = eventBus.PublishEventTimeoutWait(EventDataRoundState{}) require.NoError(t, err) - err = eventBus.PublishEventNewRound(EventDataRoundState{}) + err = eventBus.PublishEventNewRound(EventDataNewRound{}) require.NoError(t, err) - err = eventBus.PublishEventCompleteProposal(EventDataRoundState{}) + err = eventBus.PublishEventCompleteProposal(EventDataCompleteProposal{}) require.NoError(t, err) err = eventBus.PublishEventPolka(EventDataRoundState{}) require.NoError(t, err) diff --git a/types/events.go b/types/events.go index 09f7216e..b70bc9dc 100644 --- a/types/events.go +++ b/types/events.go @@ -4,27 +4,37 @@ import ( "fmt" amino "github.com/tendermint/go-amino" + abci "github.com/tendermint/tendermint/abci/types" tmpubsub "github.com/tendermint/tendermint/libs/pubsub" tmquery "github.com/tendermint/tendermint/libs/pubsub/query" ) // Reserved event types (alphabetically sorted). const ( - EventCompleteProposal = "CompleteProposal" - EventLock = "Lock" + // Block level events for mass consumption by users. + // These events are triggered from the state package, + // after a block has been committed. + // These are also used by the tx indexer for async indexing. + // All of this data can be fetched through the rpc. EventNewBlock = "NewBlock" EventNewBlockHeader = "NewBlockHeader" - EventNewRound = "NewRound" - EventNewRoundStep = "NewRoundStep" - EventPolka = "Polka" - EventProposalHeartbeat = "ProposalHeartbeat" - EventRelock = "Relock" - EventTimeoutPropose = "TimeoutPropose" - EventTimeoutWait = "TimeoutWait" EventTx = "Tx" - EventUnlock = "Unlock" EventValidatorSetUpdates = "ValidatorSetUpdates" - EventVote = "Vote" + + // Internal consensus events. + // These are used for testing the consensus state machine. + // They can also be used to build real-time consensus visualizers. + EventCompleteProposal = "CompleteProposal" + EventLock = "Lock" + EventNewRound = "NewRound" + EventNewRoundStep = "NewRoundStep" + EventPolka = "Polka" + EventRelock = "Relock" + EventTimeoutPropose = "TimeoutPropose" + EventTimeoutWait = "TimeoutWait" + EventUnlock = "Unlock" + EventValidBlock = "ValidBlock" + EventVote = "Vote" ) /////////////////////////////////////////////////////////////////////////////// @@ -42,8 +52,9 @@ func RegisterEventDatas(cdc *amino.Codec) { cdc.RegisterConcrete(EventDataNewBlockHeader{}, "tendermint/event/NewBlockHeader", nil) cdc.RegisterConcrete(EventDataTx{}, "tendermint/event/Tx", nil) cdc.RegisterConcrete(EventDataRoundState{}, "tendermint/event/RoundState", nil) + cdc.RegisterConcrete(EventDataNewRound{}, "tendermint/event/NewRound", nil) + cdc.RegisterConcrete(EventDataCompleteProposal{}, "tendermint/event/CompleteProposal", nil) cdc.RegisterConcrete(EventDataVote{}, "tendermint/event/Vote", nil) - cdc.RegisterConcrete(EventDataProposalHeartbeat{}, "tendermint/event/ProposalHeartbeat", nil) cdc.RegisterConcrete(EventDataValidatorSetUpdates{}, "tendermint/event/ValidatorSetUpdates", nil) cdc.RegisterConcrete(EventDataString(""), "tendermint/event/ProposalString", nil) } @@ -53,11 +64,17 @@ func RegisterEventDatas(cdc *amino.Codec) { type EventDataNewBlock struct { Block *Block `json:"block"` + + ResultBeginBlock abci.ResponseBeginBlock `json:"result_begin_block"` + ResultEndBlock abci.ResponseEndBlock `json:"result_end_block"` } // light weight event for benchmarking type EventDataNewBlockHeader struct { Header Header `json:"header"` + + ResultBeginBlock abci.ResponseBeginBlock `json:"result_begin_block"` + ResultEndBlock abci.ResponseEndBlock `json:"result_end_block"` } // All txs fire EventDataTx @@ -65,10 +82,6 @@ type EventDataTx struct { TxResult } -type EventDataProposalHeartbeat struct { - Heartbeat *Heartbeat -} - // NOTE: This goes into the replay WAL type EventDataRoundState struct { Height int64 `json:"height"` @@ -79,6 +92,27 @@ type EventDataRoundState struct { RoundState interface{} `json:"-"` } +type ValidatorInfo struct { + Address Address `json:"address"` + Index int `json:"index"` +} + +type EventDataNewRound struct { + Height int64 `json:"height"` + Round int `json:"round"` + Step string `json:"step"` + + Proposer ValidatorInfo `json:"proposer"` +} + +type EventDataCompleteProposal struct { + Height int64 `json:"height"` + Round int `json:"round"` + Step string `json:"step"` + + BlockID BlockID `json:"block_id"` +} + type EventDataVote struct { Vote *Vote } @@ -112,13 +146,13 @@ var ( EventQueryNewRound = QueryForEvent(EventNewRound) EventQueryNewRoundStep = QueryForEvent(EventNewRoundStep) EventQueryPolka = QueryForEvent(EventPolka) - EventQueryProposalHeartbeat = QueryForEvent(EventProposalHeartbeat) EventQueryRelock = QueryForEvent(EventRelock) EventQueryTimeoutPropose = QueryForEvent(EventTimeoutPropose) EventQueryTimeoutWait = QueryForEvent(EventTimeoutWait) EventQueryTx = QueryForEvent(EventTx) EventQueryUnlock = QueryForEvent(EventUnlock) EventQueryValidatorSetUpdates = QueryForEvent(EventValidatorSetUpdates) + EventQueryValidBlock = QueryForEvent(EventValidBlock) EventQueryVote = QueryForEvent(EventVote) ) diff --git a/types/evidence.go b/types/evidence.go index 00c46c59..f04b7e43 100644 --- a/types/evidence.go +++ b/types/evidence.go @@ -4,6 +4,7 @@ import ( "bytes" "fmt" + "github.com/pkg/errors" "github.com/tendermint/tendermint/crypto/tmhash" amino "github.com/tendermint/go-amino" @@ -14,7 +15,7 @@ import ( const ( // MaxEvidenceBytes is a maximum size of any evidence (including amino overhead). - MaxEvidenceBytes int64 = 440 + MaxEvidenceBytes int64 = 484 ) // ErrEvidenceInvalid wraps a piece of evidence and the error denoting how or why it is invalid. @@ -35,8 +36,8 @@ func (err *ErrEvidenceInvalid) Error() string { // ErrEvidenceOverflow is for when there is too much evidence in a block. type ErrEvidenceOverflow struct { - MaxBytes int64 - GotBytes int64 + MaxNum int64 + GotNum int64 } // NewErrEvidenceOverflow returns a new ErrEvidenceOverflow where got > max. @@ -46,7 +47,7 @@ func NewErrEvidenceOverflow(max, got int64) *ErrEvidenceOverflow { // Error returns a string representation of the error. func (err *ErrEvidenceOverflow) Error() string { - return fmt.Sprintf("Too much evidence: Max %d bytes, got %d bytes", err.MaxBytes, err.GotBytes) + return fmt.Sprintf("Too much evidence: Max %d, got %d", err.MaxNum, err.GotNum) } //------------------------------------------- @@ -55,10 +56,12 @@ func (err *ErrEvidenceOverflow) Error() string { type Evidence interface { Height() int64 // height of the equivocation Address() []byte // address of the equivocating validator + Bytes() []byte // bytes which compromise the evidence Hash() []byte // hash of the evidence Verify(chainID string, pubKey crypto.PubKey) error // verify the evidence Equal(Evidence) bool // check equality of evidence + ValidateBasic() error String() string } @@ -69,13 +72,23 @@ func RegisterEvidences(cdc *amino.Codec) { func RegisterMockEvidences(cdc *amino.Codec) { cdc.RegisterConcrete(MockGoodEvidence{}, "tendermint/MockGoodEvidence", nil) + cdc.RegisterConcrete(MockRandomGoodEvidence{}, "tendermint/MockRandomGoodEvidence", nil) cdc.RegisterConcrete(MockBadEvidence{}, "tendermint/MockBadEvidence", nil) } -// MaxEvidenceBytesPerBlock returns the maximum evidence size per block - -// 1/10th of the maximum block size. -func MaxEvidenceBytesPerBlock(blockMaxBytes int64) int64 { - return blockMaxBytes / 10 +const ( + MaxEvidenceBytesDenominator = 10 +) + +// MaxEvidencePerBlock returns the maximum number of evidences +// allowed in the block and their maximum total size (limitted to 1/10th +// of the maximum block size). +// TODO: change to a constant, or to a fraction of the validator set size. +// See https://github.com/tendermint/tendermint/issues/2590 +func MaxEvidencePerBlock(blockMaxBytes int64) (int64, int64) { + maxBytes := blockMaxBytes / MaxEvidenceBytesDenominator + maxNum := maxBytes / MaxEvidenceBytes + return maxNum, maxBytes } //------------------------------------------- @@ -88,6 +101,8 @@ type DuplicateVoteEvidence struct { VoteB *Vote } +var _ Evidence = &DuplicateVoteEvidence{} + // String returns a string representation of the evidence. func (dve *DuplicateVoteEvidence) String() string { return fmt.Sprintf("VoteA: %v; VoteB: %v", dve.VoteA, dve.VoteB) @@ -104,6 +119,11 @@ func (dve *DuplicateVoteEvidence) Address() []byte { return dve.PubKey.Address() } +// Hash returns the hash of the evidence. +func (dve *DuplicateVoteEvidence) Bytes() []byte { + return cdcEncode(dve) +} + // Hash returns the hash of the evidence. func (dve *DuplicateVoteEvidence) Hash() []byte { return tmhash.Sum(cdcEncode(dve)) @@ -164,14 +184,52 @@ func (dve *DuplicateVoteEvidence) Equal(ev Evidence) bool { return bytes.Equal(dveHash, evHash) } +// ValidateBasic performs basic validation. +func (dve *DuplicateVoteEvidence) ValidateBasic() error { + if len(dve.PubKey.Bytes()) == 0 { + return errors.New("Empty PubKey") + } + if dve.VoteA == nil || dve.VoteB == nil { + return fmt.Errorf("One or both of the votes are empty %v, %v", dve.VoteA, dve.VoteB) + } + if err := dve.VoteA.ValidateBasic(); err != nil { + return fmt.Errorf("Invalid VoteA: %v", err) + } + if err := dve.VoteB.ValidateBasic(); err != nil { + return fmt.Errorf("Invalid VoteB: %v", err) + } + return nil +} + //----------------------------------------------------------------- +// UNSTABLE +type MockRandomGoodEvidence struct { + MockGoodEvidence + randBytes []byte +} + +var _ Evidence = &MockRandomGoodEvidence{} + +// UNSTABLE +func NewMockRandomGoodEvidence(height int64, address []byte, randBytes []byte) MockRandomGoodEvidence { + return MockRandomGoodEvidence{ + MockGoodEvidence{height, address}, randBytes, + } +} + +func (e MockRandomGoodEvidence) Hash() []byte { + return []byte(fmt.Sprintf("%d-%x", e.Height_, e.randBytes)) +} + // UNSTABLE type MockGoodEvidence struct { Height_ int64 Address_ []byte } +var _ Evidence = &MockGoodEvidence{} + // UNSTABLE func NewMockGoodEvidence(height int64, idx int, address []byte) MockGoodEvidence { return MockGoodEvidence{height, address} @@ -182,12 +240,16 @@ func (e MockGoodEvidence) Address() []byte { return e.Address_ } func (e MockGoodEvidence) Hash() []byte { return []byte(fmt.Sprintf("%d-%x", e.Height_, e.Address_)) } +func (e MockGoodEvidence) Bytes() []byte { + return []byte(fmt.Sprintf("%d-%x", e.Height_, e.Address_)) +} func (e MockGoodEvidence) Verify(chainID string, pubKey crypto.PubKey) error { return nil } func (e MockGoodEvidence) Equal(ev Evidence) bool { e2 := ev.(MockGoodEvidence) return e.Height_ == e2.Height_ && bytes.Equal(e.Address_, e2.Address_) } +func (e MockGoodEvidence) ValidateBasic() error { return nil } func (e MockGoodEvidence) String() string { return fmt.Sprintf("GoodEvidence: %d/%s", e.Height_, e.Address_) } @@ -205,6 +267,7 @@ func (e MockBadEvidence) Equal(ev Evidence) bool { return e.Height_ == e2.Height_ && bytes.Equal(e.Address_, e2.Address_) } +func (e MockBadEvidence) ValidateBasic() error { return nil } func (e MockBadEvidence) String() string { return fmt.Sprintf("BadEvidence: %d/%s", e.Height_, e.Address_) } @@ -216,18 +279,14 @@ type EvidenceList []Evidence // Hash returns the simple merkle root hash of the EvidenceList. func (evl EvidenceList) Hash() []byte { - // Recursive impl. - // Copied from crypto/merkle to avoid allocations - switch len(evl) { - case 0: - return nil - case 1: - return evl[0].Hash() - default: - left := EvidenceList(evl[:(len(evl)+1)/2]).Hash() - right := EvidenceList(evl[(len(evl)+1)/2:]).Hash() - return merkle.SimpleHashFromTwoHashes(left, right) + // These allocations are required because Evidence is not of type Bytes, and + // golang slices can't be typed cast. This shouldn't be a performance problem since + // the Evidence size is capped. + evidenceBzs := make([][]byte, len(evl)) + for i := 0; i < len(evl); i++ { + evidenceBzs[i] = evl[i].Bytes() } + return merkle.SimpleHashFromByteSlices(evidenceBzs) } func (evl EvidenceList) String() string { diff --git a/types/evidence_test.go b/types/evidence_test.go index 1a7e9ea5..1f1338ca 100644 --- a/types/evidence_test.go +++ b/types/evidence_test.go @@ -17,12 +17,13 @@ type voteData struct { } func makeVote(val PrivValidator, chainID string, valIndex int, height int64, round, step int, blockID BlockID) *Vote { + addr := val.GetPubKey().Address() v := &Vote{ - ValidatorAddress: val.GetAddress(), + ValidatorAddress: addr, ValidatorIndex: valIndex, Height: height, Round: round, - Type: byte(step), + Type: SignedMsgType(step), BlockID: blockID, } err := val.SignVote(chainID, v) @@ -61,7 +62,7 @@ func TestEvidence(t *testing.T) { {vote1, makeVote(val, chainID, 0, 10, 3, 1, blockID2), false}, // wrong round {vote1, makeVote(val, chainID, 0, 10, 2, 2, blockID2), false}, // wrong step {vote1, makeVote(val2, chainID, 0, 10, 2, 1, blockID), false}, // wrong validator - {vote1, badVote, false}, // signed by wrong key + {vote1, badVote, false}, // signed by wrong key } pubKey := val.GetPubKey() @@ -105,7 +106,7 @@ func TestMaxEvidenceBytes(t *testing.T) { VoteB: makeVote(val, chainID, math.MaxInt64, math.MaxInt64, math.MaxInt64, math.MaxInt64, blockID2), } - bz, err := cdc.MarshalBinary(ev) + bz, err := cdc.MarshalBinaryLengthPrefixed(ev) require.NoError(t, err) assert.EqualValues(t, MaxEvidenceBytes, len(bz)) @@ -121,3 +122,38 @@ func randomDuplicatedVoteEvidence() *DuplicateVoteEvidence { VoteB: makeVote(val, chainID, 0, 10, 2, 1, blockID2), } } + +func TestDuplicateVoteEvidenceValidation(t *testing.T) { + val := NewMockPV() + blockID := makeBlockID(tmhash.Sum([]byte("blockhash")), math.MaxInt64, tmhash.Sum([]byte("partshash"))) + blockID2 := makeBlockID(tmhash.Sum([]byte("blockhash2")), math.MaxInt64, tmhash.Sum([]byte("partshash"))) + const chainID = "mychain" + + testCases := []struct { + testName string + malleateEvidence func(*DuplicateVoteEvidence) + expectErr bool + }{ + {"Good DuplicateVoteEvidence", func(ev *DuplicateVoteEvidence) {}, false}, + {"Nil vote A", func(ev *DuplicateVoteEvidence) { ev.VoteA = nil }, true}, + {"Nil vote B", func(ev *DuplicateVoteEvidence) { ev.VoteB = nil }, true}, + {"Nil votes", func(ev *DuplicateVoteEvidence) { + ev.VoteA = nil + ev.VoteB = nil + }, true}, + {"Invalid vote type", func(ev *DuplicateVoteEvidence) { + ev.VoteA = makeVote(val, chainID, math.MaxInt64, math.MaxInt64, math.MaxInt64, 0, blockID2) + }, true}, + } + for _, tc := range testCases { + t.Run(tc.testName, func(t *testing.T) { + ev := &DuplicateVoteEvidence{ + PubKey: secp256k1.GenPrivKey().PubKey(), + VoteA: makeVote(val, chainID, math.MaxInt64, math.MaxInt64, math.MaxInt64, 0x02, blockID), + VoteB: makeVote(val, chainID, math.MaxInt64, math.MaxInt64, math.MaxInt64, 0x02, blockID2), + } + tc.malleateEvidence(ev) + assert.Equal(t, tc.expectErr, ev.ValidateBasic() != nil, "Validate Basic had an unexpected result") + }) + } +} diff --git a/types/genesis.go b/types/genesis.go index 8684eb33..54b81e9e 100644 --- a/types/genesis.go +++ b/types/genesis.go @@ -19,6 +19,9 @@ const ( //------------------------------------------------------------ // core types for a genesis definition +// NOTE: any changes to the genesis definition should +// be reflected in the documentation: +// docs/tendermint-core/using-tendermint.md // GenesisValidator is an initial validator. type GenesisValidator struct { diff --git a/types/genesis_test.go b/types/genesis_test.go index e7f041a8..0e81187e 100644 --- a/types/genesis_test.go +++ b/types/genesis_test.go @@ -15,9 +15,9 @@ import ( func TestGenesisBad(t *testing.T) { // test some bad ones from raw json testCases := [][]byte{ - []byte{}, // empty - []byte{1, 1, 1, 1, 1}, // junk - []byte(`{}`), // empty + {}, // empty + {1, 1, 1, 1, 1}, // junk + []byte(`{}`), // empty []byte(`{"chain_id":"mychain","validators":[{}]}`), // invalid validator // missing pub_key type []byte(`{"validators":[{"pub_key":{"value":"AT/+aaL1eB0477Mud9JMm8Sh8BIvOYlPGC9KkIUmFaE="},"power":"10","name":""}]}`), diff --git a/types/heartbeat.go b/types/heartbeat.go deleted file mode 100644 index de03d5cc..00000000 --- a/types/heartbeat.go +++ /dev/null @@ -1,52 +0,0 @@ -package types - -import ( - "fmt" - - cmn "github.com/tendermint/tendermint/libs/common" -) - -// Heartbeat is a simple vote-like structure so validators can -// alert others that they are alive and waiting for transactions. -// Note: We aren't adding ",omitempty" to Heartbeat's -// json field tags because we always want the JSON -// representation to be in its canonical form. -type Heartbeat struct { - ValidatorAddress Address `json:"validator_address"` - ValidatorIndex int `json:"validator_index"` - Height int64 `json:"height"` - Round int `json:"round"` - Sequence int `json:"sequence"` - Signature []byte `json:"signature"` -} - -// SignBytes returns the Heartbeat bytes for signing. -// It panics if the Heartbeat is nil. -func (heartbeat *Heartbeat) SignBytes(chainID string) []byte { - bz, err := cdc.MarshalBinary(CanonicalizeHeartbeat(chainID, heartbeat)) - if err != nil { - panic(err) - } - return bz -} - -// Copy makes a copy of the Heartbeat. -func (heartbeat *Heartbeat) Copy() *Heartbeat { - if heartbeat == nil { - return nil - } - heartbeatCopy := *heartbeat - return &heartbeatCopy -} - -// String returns a string representation of the Heartbeat. -func (heartbeat *Heartbeat) String() string { - if heartbeat == nil { - return "nil-heartbeat" - } - - return fmt.Sprintf("Heartbeat{%v:%X %v/%02d (%v) %v}", - heartbeat.ValidatorIndex, cmn.Fingerprint(heartbeat.ValidatorAddress), - heartbeat.Height, heartbeat.Round, heartbeat.Sequence, - fmt.Sprintf("/%X.../", cmn.Fingerprint(heartbeat.Signature[:]))) -} diff --git a/types/heartbeat_test.go b/types/heartbeat_test.go deleted file mode 100644 index 550bcc73..00000000 --- a/types/heartbeat_test.go +++ /dev/null @@ -1,60 +0,0 @@ -package types - -import ( - "testing" - - "github.com/stretchr/testify/require" - "github.com/tendermint/tendermint/crypto/ed25519" -) - -func TestHeartbeatCopy(t *testing.T) { - hb := &Heartbeat{ValidatorIndex: 1, Height: 10, Round: 1} - hbCopy := hb.Copy() - require.Equal(t, hbCopy, hb, "heartbeat copy should be the same") - hbCopy.Round = hb.Round + 10 - require.NotEqual(t, hbCopy, hb, "heartbeat copy mutation should not change original") - - var nilHb *Heartbeat - nilHbCopy := nilHb.Copy() - require.Nil(t, nilHbCopy, "copy of nil should also return nil") -} - -func TestHeartbeatString(t *testing.T) { - var nilHb *Heartbeat - require.Contains(t, nilHb.String(), "nil", "expecting a string and no panic") - - hb := &Heartbeat{ValidatorIndex: 1, Height: 11, Round: 2} - require.Equal(t, "Heartbeat{1:000000000000 11/02 (0) /000000000000.../}", hb.String()) - - var key ed25519.PrivKeyEd25519 - sig, err := key.Sign([]byte("Tendermint")) - require.NoError(t, err) - hb.Signature = sig - require.Equal(t, "Heartbeat{1:000000000000 11/02 (0) /FF41E371B9BF.../}", hb.String()) -} - -func TestHeartbeatWriteSignBytes(t *testing.T) { - chainID := "test_chain_id" - - { - testHeartbeat := &Heartbeat{ValidatorIndex: 1, Height: 10, Round: 1} - signBytes := testHeartbeat.SignBytes(chainID) - expected, err := cdc.MarshalBinary(CanonicalizeHeartbeat(chainID, testHeartbeat)) - require.NoError(t, err) - require.Equal(t, expected, signBytes, "Got unexpected sign bytes for Heartbeat") - } - - { - testHeartbeat := &Heartbeat{} - signBytes := testHeartbeat.SignBytes(chainID) - expected, err := cdc.MarshalBinary(CanonicalizeHeartbeat(chainID, testHeartbeat)) - require.NoError(t, err) - require.Equal(t, expected, signBytes, "Got unexpected sign bytes for Heartbeat") - } - - require.Panics(t, func() { - var nilHb *Heartbeat - signBytes := nilHb.SignBytes(chainID) - require.Equal(t, string(signBytes), "null") - }) -} diff --git a/types/params.go b/types/params.go index 129d4762..03e43c19 100644 --- a/types/params.go +++ b/types/params.go @@ -2,7 +2,7 @@ package types import ( abci "github.com/tendermint/tendermint/abci/types" - "github.com/tendermint/tendermint/crypto/merkle" + "github.com/tendermint/tendermint/crypto/tmhash" cmn "github.com/tendermint/tendermint/libs/common" ) @@ -17,12 +17,21 @@ const ( // ConsensusParams contains consensus critical parameters that determine the // validity of blocks. type ConsensusParams struct { - BlockSize `json:"block_size_params"` - EvidenceParams `json:"evidence_params"` + BlockSize BlockSizeParams `json:"block_size"` + Evidence EvidenceParams `json:"evidence"` + Validator ValidatorParams `json:"validator"` } -// BlockSize contain limits on the block size. -type BlockSize struct { +// HashedParams is a subset of ConsensusParams. +// It is amino encoded and hashed into +// the Header.ConsensusHash. +type HashedParams struct { + BlockMaxBytes int64 + BlockMaxGas int64 +} + +// BlockSizeParams define limits on the block size. +type BlockSizeParams struct { MaxBytes int64 `json:"max_bytes"` MaxGas int64 `json:"max_gas"` } @@ -32,17 +41,24 @@ type EvidenceParams struct { MaxAge int64 `json:"max_age"` // only accept new evidence more recent than this } +// ValidatorParams restrict the public key types validators can use. +// NOTE: uses ABCI pubkey naming, not Amino names. +type ValidatorParams struct { + PubKeyTypes []string `json:"pub_key_types"` +} + // DefaultConsensusParams returns a default ConsensusParams. func DefaultConsensusParams() *ConsensusParams { return &ConsensusParams{ - DefaultBlockSize(), + DefaultBlockSizeParams(), DefaultEvidenceParams(), + DefaultValidatorParams(), } } -// DefaultBlockSize returns a default BlockSize. -func DefaultBlockSize() BlockSize { - return BlockSize{ +// DefaultBlockSizeParams returns a default BlockSizeParams. +func DefaultBlockSizeParams() BlockSizeParams { + return BlockSizeParams{ MaxBytes: 22020096, // 21MB MaxGas: -1, } @@ -55,6 +71,21 @@ func DefaultEvidenceParams() EvidenceParams { } } +// DefaultValidatorParams returns a default ValidatorParams, which allows +// only ed25519 pubkeys. +func DefaultValidatorParams() ValidatorParams { + return ValidatorParams{[]string{ABCIPubKeyTypeEd25519}} +} + +func (params *ValidatorParams) IsValidPubkeyType(pubkeyType string) bool { + for i := 0; i < len(params.PubKeyTypes); i++ { + if params.PubKeyTypes[i] == pubkeyType { + return true + } + } + return false +} + // Validate validates the ConsensusParams to ensure all values are within their // allowed limits, and returns an error if they are not. func (params *ConsensusParams) Validate() error { @@ -72,21 +103,48 @@ func (params *ConsensusParams) Validate() error { params.BlockSize.MaxGas) } - if params.EvidenceParams.MaxAge <= 0 { + if params.Evidence.MaxAge <= 0 { return cmn.NewError("EvidenceParams.MaxAge must be greater than 0. Got %d", - params.EvidenceParams.MaxAge) + params.Evidence.MaxAge) + } + + if len(params.Validator.PubKeyTypes) == 0 { + return cmn.NewError("len(Validator.PubKeyTypes) must be greater than 0") + } + + // Check if keyType is a known ABCIPubKeyType + for i := 0; i < len(params.Validator.PubKeyTypes); i++ { + keyType := params.Validator.PubKeyTypes[i] + if _, ok := ABCIPubKeyTypesToAminoNames[keyType]; !ok { + return cmn.NewError("params.Validator.PubKeyTypes[%d], %s, is an unknown pubkey type", + i, keyType) + } } return nil } -// Hash returns a merkle hash of the parameters to store in the block header +// Hash returns a hash of a subset of the parameters to store in the block header. +// Only the Block.MaxBytes and Block.MaxGas are included in the hash. +// This allows the ConsensusParams to evolve more without breaking the block +// protocol. No need for a Merkle tree here, just a small struct to hash. func (params *ConsensusParams) Hash() []byte { - return merkle.SimpleHashFromMap(map[string][]byte{ - "block_size_max_bytes": cdcEncode(params.BlockSize.MaxBytes), - "block_size_max_gas": cdcEncode(params.BlockSize.MaxGas), - "evidence_params_max_age": cdcEncode(params.EvidenceParams.MaxAge), + hasher := tmhash.New() + bz := cdcEncode(HashedParams{ + params.BlockSize.MaxBytes, + params.BlockSize.MaxGas, }) + if bz == nil { + panic("cannot fail to encode ConsensusParams") + } + hasher.Write(bz) + return hasher.Sum(nil) +} + +func (params *ConsensusParams) Equals(params2 *ConsensusParams) bool { + return params.BlockSize == params2.BlockSize && + params.Evidence == params2.Evidence && + cmn.StringSliceEqual(params.Validator.PubKeyTypes, params2.Validator.PubKeyTypes) } // Update returns a copy of the params with updates from the non-zero fields of p2. @@ -103,8 +161,13 @@ func (params ConsensusParams) Update(params2 *abci.ConsensusParams) ConsensusPar res.BlockSize.MaxBytes = params2.BlockSize.MaxBytes res.BlockSize.MaxGas = params2.BlockSize.MaxGas } - if params2.EvidenceParams != nil { - res.EvidenceParams.MaxAge = params2.EvidenceParams.MaxAge + if params2.Evidence != nil { + res.Evidence.MaxAge = params2.Evidence.MaxAge + } + if params2.Validator != nil { + // Copy params2.Validator.PubkeyTypes, and set result's value to the copy. + // This avoids having to initialize the slice to 0 values, and then write to it again. + res.Validator.PubKeyTypes = append([]string{}, params2.Validator.PubKeyTypes...) } return res } diff --git a/types/params_test.go b/types/params_test.go index 888b678b..dc1936fb 100644 --- a/types/params_test.go +++ b/types/params_test.go @@ -9,23 +9,32 @@ import ( abci "github.com/tendermint/tendermint/abci/types" ) +var ( + valEd25519 = []string{ABCIPubKeyTypeEd25519} + valSecp256k1 = []string{ABCIPubKeyTypeSecp256k1} +) + func TestConsensusParamsValidation(t *testing.T) { testCases := []struct { params ConsensusParams valid bool }{ // test block size - 0: {makeParams(1, 0, 1), true}, - 1: {makeParams(0, 0, 1), false}, - 2: {makeParams(47*1024*1024, 0, 1), true}, - 3: {makeParams(10, 0, 1), true}, - 4: {makeParams(100*1024*1024, 0, 1), true}, - 5: {makeParams(101*1024*1024, 0, 1), false}, - 6: {makeParams(1024*1024*1024, 0, 1), false}, - 7: {makeParams(1024*1024*1024, 0, -1), false}, + 0: {makeParams(1, 0, 1, valEd25519), true}, + 1: {makeParams(0, 0, 1, valEd25519), false}, + 2: {makeParams(47*1024*1024, 0, 1, valEd25519), true}, + 3: {makeParams(10, 0, 1, valEd25519), true}, + 4: {makeParams(100*1024*1024, 0, 1, valEd25519), true}, + 5: {makeParams(101*1024*1024, 0, 1, valEd25519), false}, + 6: {makeParams(1024*1024*1024, 0, 1, valEd25519), false}, + 7: {makeParams(1024*1024*1024, 0, -1, valEd25519), false}, // test evidence age - 8: {makeParams(1, 0, 0), false}, - 9: {makeParams(1, 0, -1), false}, + 8: {makeParams(1, 0, 0, valEd25519), false}, + 9: {makeParams(1, 0, -1, valEd25519), false}, + // test no pubkey type provided + 10: {makeParams(1, 0, 1, []string{}), false}, + // test invalid pubkey type provided + 11: {makeParams(1, 0, 1, []string{"potatoes make good pubkeys"}), false}, } for i, tc := range testCases { if tc.valid { @@ -36,23 +45,31 @@ func TestConsensusParamsValidation(t *testing.T) { } } -func makeParams(blockBytes, blockGas, evidenceAge int64) ConsensusParams { +func makeParams(blockBytes, blockGas, evidenceAge int64, pubkeyTypes []string) ConsensusParams { return ConsensusParams{ - BlockSize: BlockSize{ + BlockSize: BlockSizeParams{ MaxBytes: blockBytes, MaxGas: blockGas, }, - EvidenceParams: EvidenceParams{ + Evidence: EvidenceParams{ MaxAge: evidenceAge, }, + Validator: ValidatorParams{ + PubKeyTypes: pubkeyTypes, + }, } } func TestConsensusParamsHash(t *testing.T) { params := []ConsensusParams{ - makeParams(4, 2, 3), - makeParams(1, 4, 3), - makeParams(1, 2, 4), + makeParams(4, 2, 3, valEd25519), + makeParams(1, 4, 3, valEd25519), + makeParams(1, 2, 4, valEd25519), + makeParams(2, 5, 7, valEd25519), + makeParams(1, 7, 6, valEd25519), + makeParams(9, 5, 4, valEd25519), + makeParams(7, 8, 9, valEd25519), + makeParams(4, 6, 5, valEd25519), } hashes := make([][]byte, len(params)) @@ -78,23 +95,26 @@ func TestConsensusParamsUpdate(t *testing.T) { }{ // empty updates { - makeParams(1, 2, 3), + makeParams(1, 2, 3, valEd25519), &abci.ConsensusParams{}, - makeParams(1, 2, 3), + makeParams(1, 2, 3, valEd25519), }, // fine updates { - makeParams(1, 2, 3), + makeParams(1, 2, 3, valEd25519), &abci.ConsensusParams{ - BlockSize: &abci.BlockSize{ + BlockSize: &abci.BlockSizeParams{ MaxBytes: 100, MaxGas: 200, }, - EvidenceParams: &abci.EvidenceParams{ + Evidence: &abci.EvidenceParams{ MaxAge: 300, }, + Validator: &abci.ValidatorParams{ + PubKeyTypes: valSecp256k1, + }, }, - makeParams(100, 200, 300), + makeParams(100, 200, 300, valSecp256k1), }, } for _, tc := range testCases { diff --git a/types/part_set.go b/types/part_set.go index 812b1c2f..4533fb75 100644 --- a/types/part_set.go +++ b/types/part_set.go @@ -2,13 +2,13 @@ package types import ( "bytes" - "errors" "fmt" "io" "sync" + "github.com/pkg/errors" + "github.com/tendermint/tendermint/crypto/merkle" - "github.com/tendermint/tendermint/crypto/tmhash" cmn "github.com/tendermint/tendermint/libs/common" ) @@ -21,19 +21,17 @@ type Part struct { Index int `json:"index"` Bytes cmn.HexBytes `json:"bytes"` Proof merkle.SimpleProof `json:"proof"` - - // Cache - hash []byte } -func (part *Part) Hash() []byte { - if part.hash != nil { - return part.hash +// ValidateBasic performs basic validation. +func (part *Part) ValidateBasic() error { + if part.Index < 0 { + return errors.New("Negative Index") } - hasher := tmhash.New() - hasher.Write(part.Bytes) // nolint: errcheck, gas - part.hash = hasher.Sum(nil) - return part.hash + if len(part.Bytes) > BlockPartSizeBytes { + return fmt.Errorf("Too big (max: %d)", BlockPartSizeBytes) + } + return nil } func (part *Part) String() string { @@ -63,13 +61,25 @@ func (psh PartSetHeader) String() string { } func (psh PartSetHeader) IsZero() bool { - return psh.Total == 0 + return psh.Total == 0 && len(psh.Hash) == 0 } func (psh PartSetHeader) Equals(other PartSetHeader) bool { return psh.Total == other.Total && bytes.Equal(psh.Hash, other.Hash) } +// ValidateBasic performs basic validation. +func (psh PartSetHeader) ValidateBasic() error { + if psh.Total < 0 { + return errors.New("Negative Total") + } + // Hash can be empty in case of POLBlockID.PartsHeader in Proposal. + if err := ValidateHash(psh.Hash); err != nil { + return errors.Wrap(err, "Wrong Hash") + } + return nil +} + //------------------------------------- type PartSet struct { @@ -176,6 +186,9 @@ func (ps *PartSet) Total() int { } func (ps *PartSet) AddPart(part *Part) (bool, error) { + if ps == nil { + return false, nil + } ps.mtx.Lock() defer ps.mtx.Unlock() @@ -190,7 +203,7 @@ func (ps *PartSet) AddPart(part *Part) (bool, error) { } // Check hash proof - if part.Proof.Verify(ps.Hash(), part.Hash()) != nil { + if part.Proof.Verify(ps.Hash(), part.Bytes) != nil { return false, ErrPartSetInvalidProof } diff --git a/types/part_set_test.go b/types/part_set_test.go index 3576e747..daa2fa5c 100644 --- a/types/part_set_test.go +++ b/types/part_set_test.go @@ -83,3 +83,46 @@ func TestWrongProof(t *testing.T) { t.Errorf("Expected to fail adding a part with bad bytes.") } } + +func TestPartSetHeaderValidateBasic(t *testing.T) { + testCases := []struct { + testName string + malleatePartSetHeader func(*PartSetHeader) + expectErr bool + }{ + {"Good PartSet", func(psHeader *PartSetHeader) {}, false}, + {"Negative Total", func(psHeader *PartSetHeader) { psHeader.Total = -2 }, true}, + {"Invalid Hash", func(psHeader *PartSetHeader) { psHeader.Hash = make([]byte, 1) }, true}, + } + for _, tc := range testCases { + t.Run(tc.testName, func(t *testing.T) { + data := cmn.RandBytes(testPartSize * 100) + ps := NewPartSetFromData(data, testPartSize) + psHeader := ps.Header() + tc.malleatePartSetHeader(&psHeader) + assert.Equal(t, tc.expectErr, psHeader.ValidateBasic() != nil, "Validate Basic had an unexpected result") + }) + } +} + +func TestPartValidateBasic(t *testing.T) { + testCases := []struct { + testName string + malleatePart func(*Part) + expectErr bool + }{ + {"Good Part", func(pt *Part) {}, false}, + {"Negative index", func(pt *Part) { pt.Index = -1 }, true}, + {"Too big part", func(pt *Part) { pt.Bytes = make([]byte, BlockPartSizeBytes+1) }, true}, + } + + for _, tc := range testCases { + t.Run(tc.testName, func(t *testing.T) { + data := cmn.RandBytes(testPartSize * 100) + ps := NewPartSetFromData(data, testPartSize) + part := ps.GetPart(0) + tc.malleatePart(part) + assert.Equal(t, tc.expectErr, part.ValidateBasic() != nil, "Validate Basic had an unexpected result") + }) + } +} diff --git a/types/priv_validator.go b/types/priv_validator.go index 25be5220..8acab243 100644 --- a/types/priv_validator.go +++ b/types/priv_validator.go @@ -10,14 +10,12 @@ import ( ) // PrivValidator defines the functionality of a local Tendermint validator -// that signs votes, proposals, and heartbeats, and never double signs. +// that signs votes and proposals, and never double signs. type PrivValidator interface { - GetAddress() Address // redundant since .PubKey().Address() GetPubKey() crypto.PubKey SignVote(chainID string, vote *Vote) error SignProposal(chainID string, proposal *Proposal) error - SignHeartbeat(chainID string, heartbeat *Heartbeat) error } //---------------------------------------- @@ -30,7 +28,7 @@ func (pvs PrivValidatorsByAddress) Len() int { } func (pvs PrivValidatorsByAddress) Less(i, j int) bool { - return bytes.Compare(pvs[i].GetAddress(), pvs[j].GetAddress()) == -1 + return bytes.Compare(pvs[i].GetPubKey().Address(), pvs[j].GetPubKey().Address()) == -1 } func (pvs PrivValidatorsByAddress) Swap(i, j int) { @@ -45,16 +43,20 @@ func (pvs PrivValidatorsByAddress) Swap(i, j int) { // MockPV implements PrivValidator without any safety or persistence. // Only use it for testing. type MockPV struct { - privKey crypto.PrivKey + privKey crypto.PrivKey + breakProposalSigning bool + breakVoteSigning bool } func NewMockPV() *MockPV { - return &MockPV{ed25519.GenPrivKey()} + return &MockPV{ed25519.GenPrivKey(), false, false} } -// Implements PrivValidator. -func (pv *MockPV) GetAddress() Address { - return pv.privKey.PubKey().Address() +// NewMockPVWithParams allows one to create a MockPV instance, but with finer +// grained control over the operation of the mock validator. This is useful for +// mocking test failures. +func NewMockPVWithParams(privKey crypto.PrivKey, breakProposalSigning, breakVoteSigning bool) *MockPV { + return &MockPV{privKey, breakProposalSigning, breakVoteSigning} } // Implements PrivValidator. @@ -64,7 +66,11 @@ func (pv *MockPV) GetPubKey() crypto.PubKey { // Implements PrivValidator. func (pv *MockPV) SignVote(chainID string, vote *Vote) error { - signBytes := vote.SignBytes(chainID) + useChainID := chainID + if pv.breakVoteSigning { + useChainID = "incorrect-chain-id" + } + signBytes := vote.SignBytes(useChainID) sig, err := pv.privKey.Sign(signBytes) if err != nil { return err @@ -75,7 +81,11 @@ func (pv *MockPV) SignVote(chainID string, vote *Vote) error { // Implements PrivValidator. func (pv *MockPV) SignProposal(chainID string, proposal *Proposal) error { - signBytes := proposal.SignBytes(chainID) + useChainID := chainID + if pv.breakProposalSigning { + useChainID = "incorrect-chain-id" + } + signBytes := proposal.SignBytes(useChainID) sig, err := pv.privKey.Sign(signBytes) if err != nil { return err @@ -84,19 +94,10 @@ func (pv *MockPV) SignProposal(chainID string, proposal *Proposal) error { return nil } -// signHeartbeat signs the heartbeat without any checking. -func (pv *MockPV) SignHeartbeat(chainID string, heartbeat *Heartbeat) error { - sig, err := pv.privKey.Sign(heartbeat.SignBytes(chainID)) - if err != nil { - return err - } - heartbeat.Signature = sig - return nil -} - // String returns a string representation of the MockPV. func (pv *MockPV) String() string { - return fmt.Sprintf("MockPV{%v}", pv.GetAddress()) + addr := pv.GetPubKey().Address() + return fmt.Sprintf("MockPV{%v}", addr) } // XXX: Implement. @@ -121,12 +122,7 @@ func (pv *erroringMockPV) SignProposal(chainID string, proposal *Proposal) error return ErroringMockPVErr } -// signHeartbeat signs the heartbeat without any checking. -func (pv *erroringMockPV) SignHeartbeat(chainID string, heartbeat *Heartbeat) error { - return ErroringMockPVErr -} - // NewErroringMockPV returns a MockPV that fails on each signing request. Again, for testing only. func NewErroringMockPV() *erroringMockPV { - return &erroringMockPV{&MockPV{ed25519.GenPrivKey()}} + return &erroringMockPV{&MockPV{ed25519.GenPrivKey(), false, false}} } diff --git a/types/proposal.go b/types/proposal.go index a2bc8e36..97c06596 100644 --- a/types/proposal.go +++ b/types/proposal.go @@ -15,44 +15,81 @@ var ( ) // Proposal defines a block proposal for the consensus. -// It refers to the block only by its PartSetHeader. +// It refers to the block by BlockID field. // It must be signed by the correct proposer for the given Height/Round // to be considered valid. It may depend on votes from a previous round, -// a so-called Proof-of-Lock (POL) round, as noted in the POLRound and POLBlockID. +// a so-called Proof-of-Lock (POL) round, as noted in the POLRound. +// If POLRound >= 0, then BlockID corresponds to the block that is locked in POLRound. type Proposal struct { - Height int64 `json:"height"` - Round int `json:"round"` - Timestamp time.Time `json:"timestamp"` - BlockPartsHeader PartSetHeader `json:"block_parts_header"` - POLRound int `json:"pol_round"` // -1 if null. - POLBlockID BlockID `json:"pol_block_id"` // zero if null. - Signature []byte `json:"signature"` + Type SignedMsgType + Height int64 `json:"height"` + Round int `json:"round"` + POLRound int `json:"pol_round"` // -1 if null. + BlockID BlockID `json:"block_id"` + Timestamp time.Time `json:"timestamp"` + Signature []byte `json:"signature"` } // NewProposal returns a new Proposal. // If there is no POLRound, polRound should be -1. -func NewProposal(height int64, round int, blockPartsHeader PartSetHeader, polRound int, polBlockID BlockID) *Proposal { +func NewProposal(height int64, round int, polRound int, blockID BlockID) *Proposal { return &Proposal{ - Height: height, - Round: round, - Timestamp: tmtime.Now(), - BlockPartsHeader: blockPartsHeader, - POLRound: polRound, - POLBlockID: polBlockID, + Type: ProposalType, + Height: height, + Round: round, + BlockID: blockID, + POLRound: polRound, + Timestamp: tmtime.Now(), } } +// ValidateBasic performs basic validation. +func (p *Proposal) ValidateBasic() error { + if p.Type != ProposalType { + return errors.New("Invalid Type") + } + if p.Height < 0 { + return errors.New("Negative Height") + } + if p.Round < 0 { + return errors.New("Negative Round") + } + if p.POLRound < -1 { + return errors.New("Negative POLRound (exception: -1)") + } + if err := p.BlockID.ValidateBasic(); err != nil { + return fmt.Errorf("Wrong BlockID: %v", err) + } + // ValidateBasic above would pass even if the BlockID was empty: + if !p.BlockID.IsComplete() { + return fmt.Errorf("Expected a complete, non-empty BlockID, got: %v", p.BlockID) + } + + // NOTE: Timestamp validation is subtle and handled elsewhere. + + if len(p.Signature) == 0 { + return errors.New("Signature is missing") + } + if len(p.Signature) > MaxSignatureSize { + return fmt.Errorf("Signature is too big (max: %d)", MaxSignatureSize) + } + return nil +} + // String returns a string representation of the Proposal. func (p *Proposal) String() string { - return fmt.Sprintf("Proposal{%v/%v %v (%v,%v) %X @ %s}", - p.Height, p.Round, p.BlockPartsHeader, p.POLRound, - p.POLBlockID, - cmn.Fingerprint(p.Signature), CanonicalTime(p.Timestamp)) + return fmt.Sprintf("Proposal{%v/%v (%v, %v) %X @ %s}", + p.Height, + p.Round, + p.BlockID, + p.POLRound, + cmn.Fingerprint(p.Signature), + CanonicalTime(p.Timestamp)) } // SignBytes returns the Proposal bytes for signing func (p *Proposal) SignBytes(chainID string) []byte { - bz, err := cdc.MarshalBinary(CanonicalizeProposal(chainID, p)) + bz, err := cdc.MarshalBinaryLengthPrefixed(CanonicalizeProposal(chainID, p)) if err != nil { panic(err) } diff --git a/types/proposal_test.go b/types/proposal_test.go index 5f943308..f1c048e1 100644 --- a/types/proposal_test.go +++ b/types/proposal_test.go @@ -1,10 +1,13 @@ package types import ( + "math" "testing" "time" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/tendermint/tendermint/crypto/tmhash" ) var testProposal *Proposal @@ -15,11 +18,11 @@ func init() { panic(err) } testProposal = &Proposal{ - Height: 12345, - Round: 23456, - BlockPartsHeader: PartSetHeader{111, []byte("blockparts")}, - POLRound: -1, - Timestamp: stamp, + Height: 12345, + Round: 23456, + BlockID: BlockID{[]byte{1, 2, 3}, PartSetHeader{111, []byte("blockparts")}}, + POLRound: -1, + Timestamp: stamp, } } @@ -27,14 +30,14 @@ func TestProposalSignable(t *testing.T) { chainID := "test_chain_id" signBytes := testProposal.SignBytes(chainID) - expected, err := cdc.MarshalBinary(CanonicalizeProposal(chainID, testProposal)) + expected, err := cdc.MarshalBinaryLengthPrefixed(CanonicalizeProposal(chainID, testProposal)) require.NoError(t, err) require.Equal(t, expected, signBytes, "Got unexpected sign bytes for Proposal") } func TestProposalString(t *testing.T) { str := testProposal.String() - expected := `Proposal{12345/23456 111:626C6F636B70 (-1,:0:000000000000) 000000000000 @ 2018-02-11T07:09:22.765Z}` + expected := `Proposal{12345/23456 (010203:111:626C6F636B70, -1) 000000000000 @ 2018-02-11T07:09:22.765Z}` if str != expected { t.Errorf("Got unexpected string for Proposal. Expected:\n%v\nGot:\n%v", expected, str) } @@ -44,7 +47,9 @@ func TestProposalVerifySignature(t *testing.T) { privVal := NewMockPV() pubKey := privVal.GetPubKey() - prop := NewProposal(4, 2, PartSetHeader{777, []byte("proper")}, 2, BlockID{}) + prop := NewProposal( + 4, 2, 2, + BlockID{[]byte{1, 2, 3}, PartSetHeader{777, []byte("proper")}}) signBytes := prop.SignBytes("test_chain_id") // sign it @@ -57,9 +62,9 @@ func TestProposalVerifySignature(t *testing.T) { // serialize, deserialize and verify again.... newProp := new(Proposal) - bs, err := cdc.MarshalBinary(prop) + bs, err := cdc.MarshalBinaryLengthPrefixed(prop) require.NoError(t, err) - err = cdc.UnmarshalBinary(bs, &newProp) + err = cdc.UnmarshalBinaryLengthPrefixed(bs, &newProp) require.NoError(t, err) // verify the transmitted proposal @@ -95,3 +100,41 @@ func BenchmarkProposalVerifySignature(b *testing.B) { pubKey.VerifyBytes(testProposal.SignBytes("test_chain_id"), testProposal.Signature) } } + +func TestProposalValidateBasic(t *testing.T) { + + privVal := NewMockPV() + testCases := []struct { + testName string + malleateProposal func(*Proposal) + expectErr bool + }{ + {"Good Proposal", func(p *Proposal) {}, false}, + {"Invalid Type", func(p *Proposal) { p.Type = PrecommitType }, true}, + {"Invalid Height", func(p *Proposal) { p.Height = -1 }, true}, + {"Invalid Round", func(p *Proposal) { p.Round = -1 }, true}, + {"Invalid POLRound", func(p *Proposal) { p.POLRound = -2 }, true}, + {"Invalid BlockId", func(p *Proposal) { + p.BlockID = BlockID{[]byte{1, 2, 3}, PartSetHeader{111, []byte("blockparts")}} + }, true}, + {"Invalid Signature", func(p *Proposal) { + p.Signature = make([]byte, 0) + }, true}, + {"Too big Signature", func(p *Proposal) { + p.Signature = make([]byte, MaxSignatureSize+1) + }, true}, + } + blockID := makeBlockID(tmhash.Sum([]byte("blockhash")), math.MaxInt64, tmhash.Sum([]byte("partshash"))) + + for _, tc := range testCases { + t.Run(tc.testName, func(t *testing.T) { + prop := NewProposal( + 4, 2, 2, + blockID) + err := privVal.SignProposal("test_chain_id", prop) + require.NoError(t, err) + tc.malleateProposal(prop) + assert.Equal(t, tc.expectErr, prop.ValidateBasic() != nil, "Validate Basic had an unexpected result") + }) + } +} diff --git a/types/proto3/block.pb.go b/types/proto3/block.pb.go index ab1c66cf..99dadac1 100644 --- a/types/proto3/block.pb.go +++ b/types/proto3/block.pb.go @@ -1,10 +1,9 @@ -// Code generated by protoc-gen-go. DO NOT EDIT. +// Code generated by protoc-gen-gogo. DO NOT EDIT. // source: types/proto3/block.proto -//nolint package proto3 -import proto "github.com/golang/protobuf/proto" +import proto "github.com/gogo/protobuf/proto" import fmt "fmt" import math "math" @@ -17,10 +16,10 @@ var _ = math.Inf // is compatible with the proto package it is being compiled against. // A compilation error at this line likely means your copy of the // proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package +const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package type PartSetHeader struct { - Total int32 `protobuf:"zigzag32,1,opt,name=Total,proto3" json:"Total,omitempty"` + Total int32 `protobuf:"varint,1,opt,name=Total,proto3" json:"Total,omitempty"` Hash []byte `protobuf:"bytes,2,opt,name=Hash,proto3" json:"Hash,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` @@ -31,7 +30,7 @@ func (m *PartSetHeader) Reset() { *m = PartSetHeader{} } func (m *PartSetHeader) String() string { return proto.CompactTextString(m) } func (*PartSetHeader) ProtoMessage() {} func (*PartSetHeader) Descriptor() ([]byte, []int) { - return fileDescriptor_block_c8c1dcbe91697ccd, []int{0} + return fileDescriptor_block_57c41dfc0fc285b3, []int{0} } func (m *PartSetHeader) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_PartSetHeader.Unmarshal(m, b) @@ -67,7 +66,7 @@ func (m *PartSetHeader) GetHash() []byte { type BlockID struct { Hash []byte `protobuf:"bytes,1,opt,name=Hash,proto3" json:"Hash,omitempty"` - PartsHeader *PartSetHeader `protobuf:"bytes,2,opt,name=PartsHeader,proto3" json:"PartsHeader,omitempty"` + PartsHeader *PartSetHeader `protobuf:"bytes,2,opt,name=PartsHeader" json:"PartsHeader,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -77,7 +76,7 @@ func (m *BlockID) Reset() { *m = BlockID{} } func (m *BlockID) String() string { return proto.CompactTextString(m) } func (*BlockID) ProtoMessage() {} func (*BlockID) Descriptor() ([]byte, []int) { - return fileDescriptor_block_c8c1dcbe91697ccd, []int{1} + return fileDescriptor_block_57c41dfc0fc285b3, []int{1} } func (m *BlockID) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_BlockID.Unmarshal(m, b) @@ -113,24 +112,26 @@ func (m *BlockID) GetPartsHeader() *PartSetHeader { type Header struct { // basic block info - ChainID string `protobuf:"bytes,1,opt,name=ChainID,proto3" json:"ChainID,omitempty"` - Height int64 `protobuf:"zigzag64,2,opt,name=Height,proto3" json:"Height,omitempty"` - Time *Timestamp `protobuf:"bytes,3,opt,name=Time,proto3" json:"Time,omitempty"` - NumTxs int64 `protobuf:"zigzag64,4,opt,name=NumTxs,proto3" json:"NumTxs,omitempty"` - TotalTxs int64 `protobuf:"zigzag64,5,opt,name=TotalTxs,proto3" json:"TotalTxs,omitempty"` + Version *Version `protobuf:"bytes,1,opt,name=Version" json:"Version,omitempty"` + ChainID string `protobuf:"bytes,2,opt,name=ChainID,proto3" json:"ChainID,omitempty"` + Height int64 `protobuf:"varint,3,opt,name=Height,proto3" json:"Height,omitempty"` + Time *Timestamp `protobuf:"bytes,4,opt,name=Time" json:"Time,omitempty"` + NumTxs int64 `protobuf:"varint,5,opt,name=NumTxs,proto3" json:"NumTxs,omitempty"` + TotalTxs int64 `protobuf:"varint,6,opt,name=TotalTxs,proto3" json:"TotalTxs,omitempty"` // prev block info - LastBlockID *BlockID `protobuf:"bytes,6,opt,name=LastBlockID,proto3" json:"LastBlockID,omitempty"` + LastBlockID *BlockID `protobuf:"bytes,7,opt,name=LastBlockID" json:"LastBlockID,omitempty"` // hashes of block data - LastCommitHash []byte `protobuf:"bytes,7,opt,name=LastCommitHash,proto3" json:"LastCommitHash,omitempty"` - DataHash []byte `protobuf:"bytes,8,opt,name=DataHash,proto3" json:"DataHash,omitempty"` + LastCommitHash []byte `protobuf:"bytes,8,opt,name=LastCommitHash,proto3" json:"LastCommitHash,omitempty"` + DataHash []byte `protobuf:"bytes,9,opt,name=DataHash,proto3" json:"DataHash,omitempty"` // hashes from the app output from the prev block - ValidatorsHash []byte `protobuf:"bytes,9,opt,name=ValidatorsHash,proto3" json:"ValidatorsHash,omitempty"` - ConsensusHash []byte `protobuf:"bytes,10,opt,name=ConsensusHash,proto3" json:"ConsensusHash,omitempty"` - AppHash []byte `protobuf:"bytes,11,opt,name=AppHash,proto3" json:"AppHash,omitempty"` - LastResultsHash []byte `protobuf:"bytes,12,opt,name=LastResultsHash,proto3" json:"LastResultsHash,omitempty"` + ValidatorsHash []byte `protobuf:"bytes,10,opt,name=ValidatorsHash,proto3" json:"ValidatorsHash,omitempty"` + NextValidatorsHash []byte `protobuf:"bytes,11,opt,name=NextValidatorsHash,proto3" json:"NextValidatorsHash,omitempty"` + ConsensusHash []byte `protobuf:"bytes,12,opt,name=ConsensusHash,proto3" json:"ConsensusHash,omitempty"` + AppHash []byte `protobuf:"bytes,13,opt,name=AppHash,proto3" json:"AppHash,omitempty"` + LastResultsHash []byte `protobuf:"bytes,14,opt,name=LastResultsHash,proto3" json:"LastResultsHash,omitempty"` // consensus info - EvidenceHash []byte `protobuf:"bytes,13,opt,name=EvidenceHash,proto3" json:"EvidenceHash,omitempty"` - ProposerAddress []byte `protobuf:"bytes,14,opt,name=ProposerAddress,proto3" json:"ProposerAddress,omitempty"` + EvidenceHash []byte `protobuf:"bytes,15,opt,name=EvidenceHash,proto3" json:"EvidenceHash,omitempty"` + ProposerAddress []byte `protobuf:"bytes,16,opt,name=ProposerAddress,proto3" json:"ProposerAddress,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -140,7 +141,7 @@ func (m *Header) Reset() { *m = Header{} } func (m *Header) String() string { return proto.CompactTextString(m) } func (*Header) ProtoMessage() {} func (*Header) Descriptor() ([]byte, []int) { - return fileDescriptor_block_c8c1dcbe91697ccd, []int{2} + return fileDescriptor_block_57c41dfc0fc285b3, []int{2} } func (m *Header) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Header.Unmarshal(m, b) @@ -160,6 +161,13 @@ func (m *Header) XXX_DiscardUnknown() { var xxx_messageInfo_Header proto.InternalMessageInfo +func (m *Header) GetVersion() *Version { + if m != nil { + return m.Version + } + return nil +} + func (m *Header) GetChainID() string { if m != nil { return m.ChainID @@ -223,6 +231,13 @@ func (m *Header) GetValidatorsHash() []byte { return nil } +func (m *Header) GetNextValidatorsHash() []byte { + if m != nil { + return m.NextValidatorsHash + } + return nil +} + func (m *Header) GetConsensusHash() []byte { if m != nil { return m.ConsensusHash @@ -258,13 +273,60 @@ func (m *Header) GetProposerAddress() []byte { return nil } -// Timestamp wraps how amino encodes time. Note that this is different from the protobuf well-known type -// protobuf/timestamp.proto in the sense that there seconds and nanos are varint encoded. See: +type Version struct { + Block uint64 `protobuf:"varint,1,opt,name=Block,proto3" json:"Block,omitempty"` + App uint64 `protobuf:"varint,2,opt,name=App,proto3" json:"App,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Version) Reset() { *m = Version{} } +func (m *Version) String() string { return proto.CompactTextString(m) } +func (*Version) ProtoMessage() {} +func (*Version) Descriptor() ([]byte, []int) { + return fileDescriptor_block_57c41dfc0fc285b3, []int{3} +} +func (m *Version) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_Version.Unmarshal(m, b) +} +func (m *Version) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_Version.Marshal(b, m, deterministic) +} +func (dst *Version) XXX_Merge(src proto.Message) { + xxx_messageInfo_Version.Merge(dst, src) +} +func (m *Version) XXX_Size() int { + return xxx_messageInfo_Version.Size(m) +} +func (m *Version) XXX_DiscardUnknown() { + xxx_messageInfo_Version.DiscardUnknown(m) +} + +var xxx_messageInfo_Version proto.InternalMessageInfo + +func (m *Version) GetBlock() uint64 { + if m != nil { + return m.Block + } + return 0 +} + +func (m *Version) GetApp() uint64 { + if m != nil { + return m.App + } + return 0 +} + +// Timestamp wraps how amino encodes time. +// This is the protobuf well-known type protobuf/timestamp.proto +// See: // https://github.com/google/protobuf/blob/d2980062c859649523d5fd51d6b55ab310e47482/src/google/protobuf/timestamp.proto#L123-L135 -// Also nanos do not get skipped if they are zero in amino. +// NOTE/XXX: nanos do not get skipped if they are zero in amino. type Timestamp struct { - Seconds int64 `protobuf:"fixed64,1,opt,name=seconds,proto3" json:"seconds,omitempty"` - Nanos int32 `protobuf:"fixed32,2,opt,name=nanos,proto3" json:"nanos,omitempty"` + Seconds int64 `protobuf:"varint,1,opt,name=seconds,proto3" json:"seconds,omitempty"` + Nanos int32 `protobuf:"varint,2,opt,name=nanos,proto3" json:"nanos,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -274,7 +336,7 @@ func (m *Timestamp) Reset() { *m = Timestamp{} } func (m *Timestamp) String() string { return proto.CompactTextString(m) } func (*Timestamp) ProtoMessage() {} func (*Timestamp) Descriptor() ([]byte, []int) { - return fileDescriptor_block_c8c1dcbe91697ccd, []int{3} + return fileDescriptor_block_57c41dfc0fc285b3, []int{4} } func (m *Timestamp) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Timestamp.Unmarshal(m, b) @@ -312,36 +374,41 @@ func init() { proto.RegisterType((*PartSetHeader)(nil), "proto3.PartSetHeader") proto.RegisterType((*BlockID)(nil), "proto3.BlockID") proto.RegisterType((*Header)(nil), "proto3.Header") + proto.RegisterType((*Version)(nil), "proto3.Version") proto.RegisterType((*Timestamp)(nil), "proto3.Timestamp") } -func init() { proto.RegisterFile("types/proto3/block.proto", fileDescriptor_block_c8c1dcbe91697ccd) } +func init() { proto.RegisterFile("types/proto3/block.proto", fileDescriptor_block_57c41dfc0fc285b3) } -var fileDescriptor_block_c8c1dcbe91697ccd = []byte{ - // 395 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x5c, 0x52, 0x4b, 0x8b, 0xdb, 0x30, - 0x10, 0xc6, 0xcd, 0x7b, 0x9c, 0x47, 0x23, 0xda, 0x22, 0x7a, 0x0a, 0xa6, 0x2d, 0x39, 0x25, 0xb4, - 0x39, 0x94, 0xd2, 0x53, 0x9a, 0x14, 0x12, 0x28, 0x25, 0x68, 0x43, 0xee, 0x4a, 0x2c, 0x36, 0x66, - 0x6d, 0xcb, 0x78, 0x94, 0x65, 0xf7, 0x3f, 0xef, 0x8f, 0x58, 0x34, 0xb2, 0xbd, 0x71, 0x6e, 0xfe, - 0x1e, 0xfa, 0x3e, 0x79, 0x46, 0xc0, 0xcd, 0x73, 0xa6, 0x70, 0x9e, 0xe5, 0xda, 0xe8, 0xc5, 0xfc, - 0x18, 0xeb, 0xd3, 0xc3, 0x8c, 0x00, 0x6b, 0x3b, 0x2e, 0xf8, 0x05, 0x83, 0x9d, 0xcc, 0xcd, 0x9d, - 0x32, 0x1b, 0x25, 0x43, 0x95, 0xb3, 0x0f, 0xd0, 0xda, 0x6b, 0x23, 0x63, 0xee, 0x4d, 0xbc, 0xe9, - 0x58, 0x38, 0xc0, 0x18, 0x34, 0x37, 0x12, 0xcf, 0xfc, 0xdd, 0xc4, 0x9b, 0xf6, 0x05, 0x7d, 0x07, - 0x07, 0xe8, 0xfc, 0xb1, 0x89, 0xdb, 0x75, 0x25, 0x7b, 0x6f, 0x32, 0xfb, 0x09, 0xbe, 0x4d, 0x46, - 0x97, 0x4b, 0x27, 0xfd, 0x1f, 0x1f, 0x5d, 0xfd, 0x62, 0x56, 0x2b, 0x15, 0xd7, 0xce, 0xe0, 0xa5, - 0x01, 0xed, 0xe2, 0x32, 0x1c, 0x3a, 0xab, 0xb3, 0x8c, 0xd2, 0xed, 0x9a, 0xa2, 0x7b, 0xa2, 0x84, - 0xec, 0x93, 0xf5, 0x44, 0xf7, 0x67, 0x43, 0xc1, 0x4c, 0x14, 0x88, 0x7d, 0x85, 0xe6, 0x3e, 0x4a, - 0x14, 0x6f, 0x50, 0xdd, 0xb8, 0xac, 0xb3, 0x1c, 0x1a, 0x99, 0x64, 0x82, 0x64, 0x7b, 0xfc, 0xff, - 0x25, 0xd9, 0x3f, 0x21, 0x6f, 0xba, 0xe3, 0x0e, 0xb1, 0xcf, 0xd0, 0xa5, 0x1f, 0xb6, 0x4a, 0x8b, - 0x94, 0x0a, 0xb3, 0xef, 0xe0, 0xff, 0x93, 0x68, 0x8a, 0x7f, 0xe6, 0x6d, 0x6a, 0x18, 0x95, 0x0d, - 0x05, 0x2d, 0xae, 0x3d, 0xec, 0x1b, 0x0c, 0x2d, 0x5c, 0xe9, 0x24, 0x89, 0x0c, 0x4d, 0xa8, 0x43, - 0x13, 0xba, 0x61, 0x6d, 0xed, 0x5a, 0x1a, 0x49, 0x8e, 0x2e, 0x39, 0x2a, 0x6c, 0x33, 0x0e, 0x32, - 0x8e, 0x42, 0x69, 0x74, 0x8e, 0xe4, 0xe8, 0xb9, 0x8c, 0x3a, 0xcb, 0xbe, 0xc0, 0x60, 0xa5, 0x53, - 0x54, 0x29, 0x5e, 0x9c, 0x0d, 0xc8, 0x56, 0x27, 0xed, 0x44, 0x97, 0x59, 0x46, 0xba, 0x4f, 0x7a, - 0x09, 0xd9, 0x14, 0x46, 0xf6, 0x56, 0x42, 0xe1, 0x25, 0x36, 0x2e, 0xa1, 0x4f, 0x8e, 0x5b, 0x9a, - 0x05, 0xd0, 0xff, 0xfb, 0x18, 0x85, 0x2a, 0x3d, 0x29, 0xb2, 0x0d, 0xc8, 0x56, 0xe3, 0x6c, 0xda, - 0x2e, 0xd7, 0x99, 0x46, 0x95, 0x2f, 0xc3, 0x30, 0x57, 0x88, 0x7c, 0xe8, 0xd2, 0x6e, 0xe8, 0xe0, - 0x37, 0xf4, 0xaa, 0xed, 0xd8, 0xeb, 0xa1, 0x3a, 0xe9, 0x34, 0x44, 0x5a, 0xf8, 0x7b, 0x51, 0x42, - 0xfb, 0x2e, 0x53, 0x99, 0x6a, 0xa4, 0x7d, 0x8f, 0x84, 0x03, 0xc7, 0xe2, 0x19, 0xbf, 0x06, 0x00, - 0x00, 0xff, 0xff, 0xde, 0x29, 0x34, 0x75, 0xe9, 0x02, 0x00, 0x00, +var fileDescriptor_block_57c41dfc0fc285b3 = []byte{ + // 451 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x53, 0x5f, 0x6f, 0xd3, 0x30, + 0x10, 0x57, 0x68, 0xda, 0xae, 0x97, 0x76, 0x1d, 0x27, 0x40, 0x16, 0x4f, 0x55, 0x04, 0xa8, 0xbc, + 0x74, 0xda, 0xf6, 0x80, 0x10, 0x4f, 0xa5, 0x45, 0xda, 0x24, 0x34, 0x4d, 0xa6, 0xea, 0xbb, 0xd7, + 0x58, 0x34, 0xa2, 0x89, 0xa3, 0x9c, 0x8b, 0xc6, 0x27, 0xe4, 0x6b, 0x21, 0x9f, 0x93, 0xd0, 0x44, + 0x7b, 0xf3, 0xef, 0xcf, 0xfd, 0xce, 0xbe, 0x5c, 0x40, 0xd8, 0x3f, 0x85, 0xa6, 0xcb, 0xa2, 0x34, + 0xd6, 0xdc, 0x5c, 0x3e, 0x1e, 0xcc, 0xee, 0xd7, 0x82, 0x01, 0x0e, 0x3c, 0x17, 0x7f, 0x86, 0xc9, + 0x83, 0x2a, 0xed, 0x0f, 0x6d, 0x6f, 0xb5, 0x4a, 0x74, 0x89, 0xaf, 0xa0, 0xbf, 0x31, 0x56, 0x1d, + 0x44, 0x30, 0x0b, 0xe6, 0x7d, 0xe9, 0x01, 0x22, 0x84, 0xb7, 0x8a, 0xf6, 0xe2, 0xc5, 0x2c, 0x98, + 0x8f, 0x25, 0x9f, 0xe3, 0x2d, 0x0c, 0xbf, 0xba, 0xc4, 0xbb, 0x75, 0x23, 0x07, 0xff, 0x65, 0xfc, + 0x04, 0x91, 0x4b, 0x26, 0x9f, 0xcb, 0x95, 0xd1, 0xf5, 0x6b, 0xdf, 0xfe, 0x66, 0xd1, 0x6a, 0x2a, + 0x4f, 0x9d, 0xf1, 0xdf, 0x10, 0x06, 0xd5, 0x65, 0x3e, 0xc2, 0x70, 0xab, 0x4b, 0x4a, 0x4d, 0xce, + 0xd1, 0xd1, 0xf5, 0xb4, 0xae, 0xaf, 0x68, 0x59, 0xeb, 0x28, 0x60, 0xb8, 0xda, 0xab, 0x34, 0xbf, + 0x5b, 0x73, 0xab, 0x91, 0xac, 0x21, 0xbe, 0x71, 0x71, 0xe9, 0xcf, 0xbd, 0x15, 0xbd, 0x59, 0x30, + 0xef, 0xc9, 0x0a, 0xe1, 0x7b, 0x08, 0x37, 0x69, 0xa6, 0x45, 0xc8, 0xc9, 0x2f, 0xeb, 0x64, 0xc7, + 0x91, 0x55, 0x59, 0x21, 0x59, 0x76, 0xe5, 0xf7, 0xc7, 0x6c, 0xf3, 0x44, 0xa2, 0xef, 0xcb, 0x3d, + 0xc2, 0xb7, 0x70, 0xc6, 0xb3, 0x71, 0xca, 0x80, 0x95, 0x06, 0xe3, 0x15, 0x44, 0xdf, 0x15, 0xd9, + 0x6a, 0x3c, 0x62, 0xd8, 0xbe, 0x7b, 0x45, 0xcb, 0x53, 0x0f, 0x7e, 0x80, 0x73, 0x07, 0x57, 0x26, + 0xcb, 0x52, 0xcb, 0xc3, 0x3c, 0xe3, 0x61, 0x76, 0x58, 0xd7, 0x76, 0xad, 0xac, 0x62, 0xc7, 0x88, + 0x1d, 0x0d, 0x76, 0x19, 0x5b, 0x75, 0x48, 0x13, 0x65, 0x4d, 0x49, 0xec, 0x00, 0x9f, 0xd1, 0x66, + 0x71, 0x01, 0x78, 0xaf, 0x9f, 0x6c, 0xc7, 0x1b, 0xb1, 0xf7, 0x19, 0x05, 0xdf, 0xc1, 0x64, 0x65, + 0x72, 0xd2, 0x39, 0x1d, 0xbd, 0x75, 0xcc, 0xd6, 0x36, 0xe9, 0xbe, 0xc0, 0xb2, 0x28, 0x58, 0x9f, + 0xb0, 0x5e, 0x43, 0x9c, 0xc3, 0xd4, 0xbd, 0x42, 0x6a, 0x3a, 0x1e, 0xac, 0x4f, 0x38, 0x67, 0x47, + 0x97, 0xc6, 0x18, 0xc6, 0xdf, 0x7e, 0xa7, 0x89, 0xce, 0x77, 0x9a, 0x6d, 0x53, 0xb6, 0xb5, 0x38, + 0x97, 0xf6, 0x50, 0x9a, 0xc2, 0x90, 0x2e, 0x97, 0x49, 0x52, 0x6a, 0x22, 0x71, 0xe1, 0xd3, 0x3a, + 0x74, 0x7c, 0xd5, 0xac, 0x8f, 0x5b, 0x6b, 0x9e, 0x34, 0xef, 0x51, 0x28, 0x3d, 0xc0, 0x0b, 0xe8, + 0x2d, 0x8b, 0x82, 0x17, 0x26, 0x94, 0xee, 0x18, 0x7f, 0x81, 0x51, 0xb3, 0x00, 0xee, 0x45, 0xa4, + 0x77, 0x26, 0x4f, 0x88, 0xcb, 0x7a, 0xb2, 0x86, 0x2e, 0x2e, 0x57, 0xb9, 0x21, 0x2e, 0xed, 0x4b, + 0x0f, 0x1e, 0xab, 0x9f, 0xea, 0x5f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x4f, 0x84, 0xb5, 0xf8, 0x77, + 0x03, 0x00, 0x00, } diff --git a/types/proto3/block.proto b/types/proto3/block.proto index 835d6b74..93cf1bc7 100644 --- a/types/proto3/block.proto +++ b/types/proto3/block.proto @@ -4,7 +4,7 @@ package proto3; message PartSetHeader { - sint32 Total = 1; + int32 Total = 1; bytes Hash = 2; } @@ -15,36 +15,43 @@ message BlockID { message Header { // basic block info - string ChainID = 1; - sint64 Height = 2; - Timestamp Time = 3; - sint64 NumTxs = 4; - sint64 TotalTxs = 5; + Version Version = 1; + string ChainID = 2; + int64 Height = 3; + Timestamp Time = 4; + int64 NumTxs = 5; + int64 TotalTxs = 6; // prev block info - BlockID LastBlockID = 6; + BlockID LastBlockID = 7; // hashes of block data - bytes LastCommitHash = 7; // commit from validators from the last block - bytes DataHash = 8; // transactions + bytes LastCommitHash = 8; // commit from validators from the last block + bytes DataHash = 9; // transactions // hashes from the app output from the prev block - bytes ValidatorsHash = 9; // validators for the current block - bytes NextValidatorsHash = 10; // validators for the next block - bytes ConsensusHash = 11; // consensus params for current block - bytes AppHash = 12; // state after txs from the previous block - bytes LastResultsHash = 13; // root hash of all results from the txs from the previous block + bytes ValidatorsHash = 10; // validators for the current block + bytes NextValidatorsHash = 11; // validators for the next block + bytes ConsensusHash = 12; // consensus params for current block + bytes AppHash = 13; // state after txs from the previous block + bytes LastResultsHash = 14; // root hash of all results from the txs from the previous block // consensus info - bytes EvidenceHash = 14; // evidence included in the block - bytes ProposerAddress = 15; // original proposer of the block + bytes EvidenceHash = 15; // evidence included in the block + bytes ProposerAddress = 16; // original proposer of the block } -// Timestamp wraps how amino encodes time. Note that this is different from the protobuf well-known type -// protobuf/timestamp.proto in the sense that there seconds and nanos are varint encoded. See: -// https://github.com/google/protobuf/blob/d2980062c859649523d5fd51d6b55ab310e47482/src/google/protobuf/timestamp.proto#L123-L135 -// Also nanos do not get skipped if they are zero in amino. -message Timestamp { - sfixed64 seconds = 1; - sfixed32 nanos = 2; +message Version { + uint64 Block = 1; + uint64 App = 2; +} + +// Timestamp wraps how amino encodes time. +// This is the protobuf well-known type protobuf/timestamp.proto +// See: +// https://github.com/google/protobuf/blob/d2980062c859649523d5fd51d6b55ab310e47482/src/google/protobuf/timestamp.proto#L123-L135 +// NOTE/XXX: nanos do not get skipped if they are zero in amino. +message Timestamp { + int64 seconds = 1; + int32 nanos = 2; } diff --git a/types/protobuf.go b/types/protobuf.go index c9c429c8..eed73b56 100644 --- a/types/protobuf.go +++ b/types/protobuf.go @@ -24,6 +24,12 @@ const ( ABCIPubKeyTypeSecp256k1 = "secp256k1" ) +// TODO: Make non-global by allowing for registration of more pubkey types +var ABCIPubKeyTypesToAminoNames = map[string]string{ + ABCIPubKeyTypeEd25519: ed25519.PubKeyAminoName, + ABCIPubKeyTypeSecp256k1: secp256k1.PubKeyAminoName, +} + //------------------------------------------------------- // TM2PB is used for converting Tendermint ABCI to protobuf ABCI. @@ -34,6 +40,10 @@ type tm2pb struct{} func (tm2pb) Header(header *Header) abci.Header { return abci.Header{ + Version: abci.Version{ + Block: header.Version.Block.Uint64(), + App: header.Version.App.Uint64(), + }, ChainID: header.ChainID, Height: header.Height, Time: header.Time, @@ -45,10 +55,11 @@ func (tm2pb) Header(header *Header) abci.Header { LastCommitHash: header.LastCommitHash, DataHash: header.DataHash, - ValidatorsHash: header.ValidatorsHash, - ConsensusHash: header.ConsensusHash, - AppHash: header.AppHash, - LastResultsHash: header.LastResultsHash, + ValidatorsHash: header.ValidatorsHash, + NextValidatorsHash: header.NextValidatorsHash, + ConsensusHash: header.ConsensusHash, + AppHash: header.AppHash, + LastResultsHash: header.LastResultsHash, EvidenceHash: header.EvidenceHash, ProposerAddress: header.ProposerAddress, @@ -114,12 +125,15 @@ func (tm2pb) ValidatorUpdates(vals *ValidatorSet) []abci.ValidatorUpdate { func (tm2pb) ConsensusParams(params *ConsensusParams) *abci.ConsensusParams { return &abci.ConsensusParams{ - BlockSize: &abci.BlockSize{ + BlockSize: &abci.BlockSizeParams{ MaxBytes: params.BlockSize.MaxBytes, MaxGas: params.BlockSize.MaxGas, }, - EvidenceParams: &abci.EvidenceParams{ - MaxAge: params.EvidenceParams.MaxAge, + Evidence: &abci.EvidenceParams{ + MaxAge: params.Evidence.MaxAge, + }, + Validator: &abci.ValidatorParams{ + PubKeyTypes: params.Validator.PubKeyTypes, }, } } @@ -173,20 +187,19 @@ var PB2TM = pb2tm{} type pb2tm struct{} func (pb2tm) PubKey(pubKey abci.PubKey) (crypto.PubKey, error) { - // TODO: define these in crypto and use them - sizeEd := 32 - sizeSecp := 33 switch pubKey.Type { case ABCIPubKeyTypeEd25519: - if len(pubKey.Data) != sizeEd { - return nil, fmt.Errorf("Invalid size for PubKeyEd25519. Got %d, expected %d", len(pubKey.Data), sizeEd) + if len(pubKey.Data) != ed25519.PubKeyEd25519Size { + return nil, fmt.Errorf("Invalid size for PubKeyEd25519. Got %d, expected %d", + len(pubKey.Data), ed25519.PubKeyEd25519Size) } var pk ed25519.PubKeyEd25519 copy(pk[:], pubKey.Data) return pk, nil case ABCIPubKeyTypeSecp256k1: - if len(pubKey.Data) != sizeSecp { - return nil, fmt.Errorf("Invalid size for PubKeyEd25519. Got %d, expected %d", len(pubKey.Data), sizeSecp) + if len(pubKey.Data) != secp256k1.PubKeySecp256k1Size { + return nil, fmt.Errorf("Invalid size for PubKeySecp256k1. Got %d, expected %d", + len(pubKey.Data), secp256k1.PubKeySecp256k1Size) } var pk secp256k1.PubKeySecp256k1 copy(pk[:], pubKey.Data) @@ -210,12 +223,15 @@ func (pb2tm) ValidatorUpdates(vals []abci.ValidatorUpdate) ([]*Validator, error) func (pb2tm) ConsensusParams(csp *abci.ConsensusParams) ConsensusParams { return ConsensusParams{ - BlockSize: BlockSize{ + BlockSize: BlockSizeParams{ MaxBytes: csp.BlockSize.MaxBytes, MaxGas: csp.BlockSize.MaxGas, }, - EvidenceParams: EvidenceParams{ - MaxAge: csp.EvidenceParams.MaxAge, + Evidence: EvidenceParams{ + MaxAge: csp.Evidence.MaxAge, + }, + Validator: ValidatorParams{ + PubKeyTypes: csp.Validator.PubKeyTypes, }, } } diff --git a/types/protobuf_test.go b/types/protobuf_test.go index f8682abf..40859d9e 100644 --- a/types/protobuf_test.go +++ b/types/protobuf_test.go @@ -4,12 +4,15 @@ import ( "testing" "time" + "github.com/golang/protobuf/proto" "github.com/stretchr/testify/assert" + + amino "github.com/tendermint/go-amino" abci "github.com/tendermint/tendermint/abci/types" "github.com/tendermint/tendermint/crypto" "github.com/tendermint/tendermint/crypto/ed25519" "github.com/tendermint/tendermint/crypto/secp256k1" - tmtime "github.com/tendermint/tendermint/types/time" + "github.com/tendermint/tendermint/version" ) func TestABCIPubKey(t *testing.T) { @@ -30,17 +33,9 @@ func TestABCIValidators(t *testing.T) { pkEd := ed25519.GenPrivKey().PubKey() // correct validator - tmValExpected := &Validator{ - Address: pkEd.Address(), - PubKey: pkEd, - VotingPower: 10, - } + tmValExpected := NewValidator(pkEd, 10) - tmVal := &Validator{ - Address: pkEd.Address(), - PubKey: pkEd, - VotingPower: 10, - } + tmVal := NewValidator(pkEd, 10) abciVal := TM2PB.ValidatorUpdate(tmVal) tmVals, err := PB2TM.ValidatorUpdates([]abci.ValidatorUpdate{abciVal}) @@ -68,24 +63,77 @@ func TestABCIValidators(t *testing.T) { func TestABCIConsensusParams(t *testing.T) { cp := DefaultConsensusParams() - cp.EvidenceParams.MaxAge = 0 // TODO add this to ABCI abciCP := TM2PB.ConsensusParams(cp) cp2 := PB2TM.ConsensusParams(abciCP) assert.Equal(t, *cp, cp2) } -func TestABCIHeader(t *testing.T) { - header := &Header{ - Height: int64(3), - Time: tmtime.Now(), - NumTxs: int64(10), - ProposerAddress: []byte("cloak"), +func newHeader( + height, numTxs int64, + commitHash, dataHash, evidenceHash []byte, +) *Header { + return &Header{ + Height: height, + NumTxs: numTxs, + LastCommitHash: commitHash, + DataHash: dataHash, + EvidenceHash: evidenceHash, } - abciHeader := TM2PB.Header(header) +} + +func TestABCIHeader(t *testing.T) { + // build a full header + var height int64 = 5 + var numTxs int64 = 3 + header := newHeader( + height, numTxs, + []byte("lastCommitHash"), []byte("dataHash"), []byte("evidenceHash"), + ) + protocolVersion := version.Consensus{7, 8} + timestamp := time.Now() + lastBlockID := BlockID{ + Hash: []byte("hash"), + PartsHeader: PartSetHeader{ + Total: 10, + Hash: []byte("hash"), + }, + } + var totalTxs int64 = 100 + header.Populate( + protocolVersion, "chainID", + timestamp, lastBlockID, totalTxs, + []byte("valHash"), []byte("nextValHash"), + []byte("consHash"), []byte("appHash"), []byte("lastResultsHash"), + []byte("proposerAddress"), + ) + + cdc := amino.NewCodec() + headerBz := cdc.MustMarshalBinaryBare(header) + + pbHeader := TM2PB.Header(header) + pbHeaderBz, err := proto.Marshal(&pbHeader) + assert.NoError(t, err) + + // assert some fields match + assert.EqualValues(t, protocolVersion.Block, pbHeader.Version.Block) + assert.EqualValues(t, protocolVersion.App, pbHeader.Version.App) + assert.EqualValues(t, "chainID", pbHeader.ChainID) + assert.EqualValues(t, height, pbHeader.Height) + assert.EqualValues(t, timestamp, pbHeader.Time) + assert.EqualValues(t, numTxs, pbHeader.NumTxs) + assert.EqualValues(t, totalTxs, pbHeader.TotalTxs) + assert.EqualValues(t, lastBlockID.Hash, pbHeader.LastBlockId.Hash) + assert.EqualValues(t, []byte("lastCommitHash"), pbHeader.LastCommitHash) + assert.Equal(t, []byte("proposerAddress"), pbHeader.ProposerAddress) + + // assert the encodings match + // NOTE: they don't yet because Amino encodes + // int64 as zig-zag and we're using non-zigzag in the protobuf. + // See https://github.com/tendermint/tendermint/issues/2682 + _, _ = headerBz, pbHeaderBz + // assert.EqualValues(t, headerBz, pbHeaderBz) - assert.Equal(t, int64(3), abciHeader.Height) - assert.Equal(t, []byte("cloak"), abciHeader.ProposerAddress) } func TestABCIEvidence(t *testing.T) { @@ -93,14 +141,15 @@ func TestABCIEvidence(t *testing.T) { blockID := makeBlockID([]byte("blockhash"), 1000, []byte("partshash")) blockID2 := makeBlockID([]byte("blockhash2"), 1000, []byte("partshash")) const chainID = "mychain" + pubKey := val.GetPubKey() ev := &DuplicateVoteEvidence{ - PubKey: val.GetPubKey(), + PubKey: pubKey, VoteA: makeVote(val, chainID, 0, 10, 2, 1, blockID), VoteB: makeVote(val, chainID, 0, 10, 2, 1, blockID2), } abciEv := TM2PB.Evidence( ev, - NewValidatorSet([]*Validator{NewValidator(val.GetPubKey(), 10)}), + NewValidatorSet([]*Validator{NewValidator(pubKey, 10)}), time.Now(), ) @@ -127,11 +176,7 @@ func TestABCIValidatorFromPubKeyAndPower(t *testing.T) { func TestABCIValidatorWithoutPubKey(t *testing.T) { pkEd := ed25519.GenPrivKey().PubKey() - abciVal := TM2PB.Validator(&Validator{ - Address: pkEd.Address(), - PubKey: pkEd, - VotingPower: 10, - }) + abciVal := TM2PB.Validator(NewValidator(pkEd, 10)) // pubkey must be nil tmValExpected := abci.Validator{ diff --git a/types/results.go b/types/results.go index 6b5b82d2..d7d82d89 100644 --- a/types/results.go +++ b/types/results.go @@ -3,25 +3,20 @@ package types import ( abci "github.com/tendermint/tendermint/abci/types" "github.com/tendermint/tendermint/crypto/merkle" - "github.com/tendermint/tendermint/crypto/tmhash" cmn "github.com/tendermint/tendermint/libs/common" ) //----------------------------------------------------------------------------- // ABCIResult is the deterministic component of a ResponseDeliverTx. -// TODO: add Tags +// TODO: add tags and other fields +// https://github.com/tendermint/tendermint/issues/1007 type ABCIResult struct { Code uint32 `json:"code"` Data cmn.HexBytes `json:"data"` } -// Hash returns the canonical hash of the ABCIResult -func (a ABCIResult) Hash() []byte { - bz := tmhash.Sum(cdcEncode(a)) - return bz -} - +// Bytes returns the amino encoded ABCIResult func (a ABCIResult) Bytes() []byte { return cdcEncode(a) } @@ -48,7 +43,7 @@ func NewResultFromResponse(response *abci.ResponseDeliverTx) ABCIResult { // Bytes serializes the ABCIResponse using wire func (a ABCIResults) Bytes() []byte { - bz, err := cdc.MarshalBinary(a) + bz, err := cdc.MarshalBinaryLengthPrefixed(a) if err != nil { panic(err) } diff --git a/types/results_test.go b/types/results_test.go index 80803385..a37de9ec 100644 --- a/types/results_test.go +++ b/types/results_test.go @@ -16,20 +16,21 @@ func TestABCIResults(t *testing.T) { e := ABCIResult{Code: 14, Data: []byte("foo")} f := ABCIResult{Code: 14, Data: []byte("bar")} - // Nil and []byte{} should produce the same hash. - require.Equal(t, a.Hash(), a.Hash()) - require.Equal(t, b.Hash(), b.Hash()) - require.Equal(t, a.Hash(), b.Hash()) + // Nil and []byte{} should produce the same bytes + require.Equal(t, a.Bytes(), a.Bytes()) + require.Equal(t, b.Bytes(), b.Bytes()) + require.Equal(t, a.Bytes(), b.Bytes()) // a and b should be the same, don't go in results. results := ABCIResults{a, c, d, e, f} - // Make sure each result hashes properly. + // Make sure each result serializes differently var last []byte - for i, res := range results { - h := res.Hash() - assert.NotEqual(t, last, h, "%d", i) - last = h + assert.Equal(t, last, a.Bytes()) // first one is empty + for i, res := range results[1:] { + bz := res.Bytes() + assert.NotEqual(t, last, bz, "%d", i) + last = bz } // Make sure that we can get a root hash from results and verify proofs. @@ -38,12 +39,12 @@ func TestABCIResults(t *testing.T) { for i, res := range results { proof := results.ProveResult(i) - valid := proof.Verify(root, res.Hash()) + valid := proof.Verify(root, res.Bytes()) assert.NoError(t, valid, "%d", i) } } -func TestABCIBytes(t *testing.T) { +func TestABCIResultsBytes(t *testing.T) { results := NewResults([]*abci.ResponseDeliverTx{ {Code: 0, Data: []byte{}}, {Code: 0, Data: []byte("one")}, diff --git a/types/signable.go b/types/signable.go index cc649888..72d2ea9a 100644 --- a/types/signable.go +++ b/types/signable.go @@ -1,5 +1,17 @@ package types +import ( + "github.com/tendermint/tendermint/crypto/ed25519" + cmn "github.com/tendermint/tendermint/libs/common" +) + +var ( + // MaxSignatureSize is a maximum allowed signature size for the Proposal + // and Vote. + // XXX: secp256k1 does not have Size nor MaxSize defined. + MaxSignatureSize = cmn.MaxInt(ed25519.SignatureSize, 64) +) + // Signable is an interface for all signable things. // It typically removes signatures before serializing. // SignBytes returns the bytes to be signed diff --git a/types/signed_msg_type.go b/types/signed_msg_type.go new file mode 100644 index 00000000..6bd5f057 --- /dev/null +++ b/types/signed_msg_type.go @@ -0,0 +1,23 @@ +package types + +// SignedMsgType is a type of signed message in the consensus. +type SignedMsgType byte + +const ( + // Votes + PrevoteType SignedMsgType = 0x01 + PrecommitType SignedMsgType = 0x02 + + // Proposals + ProposalType SignedMsgType = 0x20 +) + +// IsVoteTypeValid returns true if t is a valid vote type. +func IsVoteTypeValid(t SignedMsgType) bool { + switch t { + case PrevoteType, PrecommitType: + return true + default: + return false + } +} diff --git a/types/test_util.go b/types/test_util.go index e20ea212..18e47214 100644 --- a/types/test_util.go +++ b/types/test_util.go @@ -10,13 +10,13 @@ func MakeCommit(blockID BlockID, height int64, round int, // all sign for i := 0; i < len(validators); i++ { - + addr := validators[i].GetPubKey().Address() vote := &Vote{ - ValidatorAddress: validators[i].GetAddress(), + ValidatorAddress: addr, ValidatorIndex: i, Height: height, Round: round, - Type: VoteTypePrecommit, + Type: PrecommitType, BlockID: blockID, Timestamp: tmtime.Now(), } diff --git a/types/tx.go b/types/tx.go index ec42f3f1..0c6845a7 100644 --- a/types/tx.go +++ b/types/tx.go @@ -5,6 +5,8 @@ import ( "errors" "fmt" + amino "github.com/tendermint/go-amino" + abci "github.com/tendermint/tendermint/abci/types" "github.com/tendermint/tendermint/crypto/merkle" "github.com/tendermint/tendermint/crypto/tmhash" @@ -29,20 +31,16 @@ func (tx Tx) String() string { // Txs is a slice of Tx. type Txs []Tx -// Hash returns the simple Merkle root hash of the transactions. +// Hash returns the Merkle root hash of the transaction hashes. +// i.e. the leaves of the tree are the hashes of the txs. func (txs Txs) Hash() []byte { - // Recursive impl. - // Copied from tendermint/crypto/merkle to avoid allocations - switch len(txs) { - case 0: - return nil - case 1: - return txs[0].Hash() - default: - left := Txs(txs[:(len(txs)+1)/2]).Hash() - right := Txs(txs[(len(txs)+1)/2:]).Hash() - return merkle.SimpleHashFromTwoHashes(left, right) + // These allocations will be removed once Txs is switched to [][]byte, + // ref #2603. This is because golang does not allow type casting slices without unsafe + txBzs := make([][]byte, len(txs)) + for i := 0; i < len(txs); i++ { + txBzs[i] = txs[i].Hash() } + return merkle.SimpleHashFromByteSlices(txBzs) } // Index returns the index of this transaction in the list, or -1 if not found @@ -72,7 +70,7 @@ func (txs Txs) Proof(i int) TxProof { l := len(txs) bzs := make([][]byte, l) for i := 0; i < l; i++ { - bzs[i] = txs[i] + bzs[i] = txs[i].Hash() } root, proofs := merkle.SimpleProofsFromByteSlices(bzs) @@ -90,8 +88,8 @@ type TxProof struct { Proof merkle.SimpleProof } -// LeadHash returns the hash of the this proof refers to. -func (tp TxProof) LeafHash() []byte { +// Leaf returns the hash(tx), which is the leaf in the merkle tree which this proof refers to. +func (tp TxProof) Leaf() []byte { return tp.Data.Hash() } @@ -107,7 +105,7 @@ func (tp TxProof) Validate(dataHash []byte) error { if tp.Proof.Total <= 0 { return errors.New("Proof total must be positive") } - valid := tp.Proof.Verify(tp.RootHash, tp.LeafHash()) + valid := tp.Proof.Verify(tp.RootHash, tp.Leaf()) if valid != nil { return errors.New("Proof is not internally consistent") } @@ -123,3 +121,18 @@ type TxResult struct { Tx Tx `json:"tx"` Result abci.ResponseDeliverTx `json:"result"` } + +// ComputeAminoOverhead calculates the overhead for amino encoding a transaction. +// The overhead consists of varint encoding the field number and the wire type +// (= length-delimited = 2), and another varint encoding the length of the +// transaction. +// The field number can be the field number of the particular transaction, or +// the field number of the parenting struct that contains the transactions []Tx +// as a field (this field number is repeated for each contained Tx). +// If some []Tx are encoded directly (without a parenting struct), the default +// fieldNum is also 1 (see BinFieldNum in amino.MarshalBinaryBare). +func ComputeAminoOverhead(tx Tx, fieldNum int) int64 { + fnum := uint64(fieldNum) + typ3AndFieldNum := (uint64(fnum) << 3) | uint64(amino.Typ3_ByteLength) + return int64(amino.UvarintSize(typ3AndFieldNum)) + int64(amino.UvarintSize(uint64(len(tx)))) +} diff --git a/types/tx_test.go b/types/tx_test.go index 9fb8ff34..511f4c3a 100644 --- a/types/tx_test.go +++ b/types/tx_test.go @@ -66,22 +66,21 @@ func TestValidTxProof(t *testing.T) { root := txs.Hash() // make sure valid proof for every tx for i := range txs { - leaf := txs[i] - leafHash := leaf.Hash() + tx := []byte(txs[i]) proof := txs.Proof(i) assert.Equal(t, i, proof.Proof.Index, "%d: %d", h, i) assert.Equal(t, len(txs), proof.Proof.Total, "%d: %d", h, i) assert.EqualValues(t, root, proof.RootHash, "%d: %d", h, i) - assert.EqualValues(t, leaf, proof.Data, "%d: %d", h, i) - assert.EqualValues(t, leafHash, proof.LeafHash(), "%d: %d", h, i) + assert.EqualValues(t, tx, proof.Data, "%d: %d", h, i) + assert.EqualValues(t, txs[i].Hash(), proof.Leaf(), "%d: %d", h, i) assert.Nil(t, proof.Validate(root), "%d: %d", h, i) assert.NotNil(t, proof.Validate([]byte("foobar")), "%d: %d", h, i) // read-write must also work var p2 TxProof - bin, err := cdc.MarshalBinary(proof) + bin, err := cdc.MarshalBinaryLengthPrefixed(proof) assert.Nil(t, err) - err = cdc.UnmarshalBinary(bin, &p2) + err = cdc.UnmarshalBinaryLengthPrefixed(bin, &p2) if assert.Nil(t, err, "%d: %d: %+v", h, i, err) { assert.Nil(t, p2.Validate(root), "%d: %d", h, i) } @@ -96,6 +95,63 @@ func TestTxProofUnchangable(t *testing.T) { } } +func TestComputeTxsOverhead(t *testing.T) { + cases := []struct { + txs Txs + wantOverhead int + }{ + {Txs{[]byte{6, 6, 6, 6, 6, 6}}, 2}, + // one 21 Mb transaction: + {Txs{make([]byte, 22020096)}, 5}, + // two 21Mb/2 sized transactions: + {Txs{make([]byte, 11010048), make([]byte, 11010048)}, 10}, + {Txs{[]byte{1, 2, 3}, []byte{1, 2, 3}, []byte{4, 5, 6}}, 6}, + {Txs{[]byte{100, 5, 64}, []byte{42, 116, 118}, []byte{6, 6, 6}, []byte{6, 6, 6}}, 8}, + } + + for _, tc := range cases { + totalBytes := int64(0) + totalOverhead := int64(0) + for _, tx := range tc.txs { + aminoOverhead := ComputeAminoOverhead(tx, 1) + totalOverhead += aminoOverhead + totalBytes += aminoOverhead + int64(len(tx)) + } + bz, err := cdc.MarshalBinaryBare(tc.txs) + assert.EqualValues(t, tc.wantOverhead, totalOverhead) + assert.NoError(t, err) + assert.EqualValues(t, len(bz), totalBytes) + } +} + +func TestComputeAminoOverhead(t *testing.T) { + cases := []struct { + tx Tx + fieldNum int + want int + }{ + {[]byte{6, 6, 6}, 1, 2}, + {[]byte{6, 6, 6}, 16, 3}, + {[]byte{6, 6, 6}, 32, 3}, + {[]byte{6, 6, 6}, 64, 3}, + {[]byte{6, 6, 6}, 512, 3}, + {[]byte{6, 6, 6}, 1024, 3}, + {[]byte{6, 6, 6}, 2048, 4}, + {make([]byte, 64), 1, 2}, + {make([]byte, 65), 1, 2}, + {make([]byte, 127), 1, 2}, + {make([]byte, 128), 1, 3}, + {make([]byte, 256), 1, 3}, + {make([]byte, 512), 1, 3}, + {make([]byte, 1024), 1, 3}, + {make([]byte, 128), 16, 4}, + } + for _, tc := range cases { + got := ComputeAminoOverhead(tc.tx, tc.fieldNum) + assert.EqualValues(t, tc.want, got) + } +} + func testTxProofUnchangable(t *testing.T) { // make some proof txs := makeTxs(randInt(2, 100), randInt(16, 128)) @@ -105,7 +161,7 @@ func testTxProofUnchangable(t *testing.T) { // make sure it is valid to start with assert.Nil(t, proof.Validate(root)) - bin, err := cdc.MarshalBinary(proof) + bin, err := cdc.MarshalBinaryLengthPrefixed(proof) assert.Nil(t, err) // try mutating the data and make sure nothing breaks @@ -120,7 +176,7 @@ func testTxProofUnchangable(t *testing.T) { // This makes sure that the proof doesn't deserialize into something valid. func assertBadProof(t *testing.T, root []byte, bad []byte, good TxProof) { var proof TxProof - err := cdc.UnmarshalBinary(bad, &proof) + err := cdc.UnmarshalBinaryLengthPrefixed(bad, &proof) if err == nil { err = proof.Validate(root) if err == nil { diff --git a/types/validation.go b/types/validation.go new file mode 100644 index 00000000..1271cfd9 --- /dev/null +++ b/types/validation.go @@ -0,0 +1,40 @@ +package types + +import ( + "fmt" + "time" + + "github.com/tendermint/tendermint/crypto/tmhash" + tmtime "github.com/tendermint/tendermint/types/time" +) + +// ValidateTime does a basic time validation ensuring time does not drift too +// much: +/- one year. +// TODO: reduce this to eg 1 day +// NOTE: DO NOT USE in ValidateBasic methods in this package. This function +// can only be used for real time validation, like on proposals and votes +// in the consensus. If consensus is stuck, and rounds increase for more than a day, +// having only a 1-day band here could break things... +// Can't use for validating blocks because we may be syncing years worth of history. +func ValidateTime(t time.Time) error { + var ( + now = tmtime.Now() + oneYear = 8766 * time.Hour + ) + if t.Before(now.Add(-oneYear)) || t.After(now.Add(oneYear)) { + return fmt.Errorf("Time drifted too much. Expected: -1 < %v < 1 year", now) + } + return nil +} + +// ValidateHash returns an error if the hash is not empty, but its +// size != tmhash.Size. +func ValidateHash(h []byte) error { + if len(h) > 0 && len(h) != tmhash.Size { + return fmt.Errorf("Expected size to be %d bytes, got %d bytes", + tmhash.Size, + len(h), + ) + } + return nil +} diff --git a/types/validator.go b/types/validator.go index af347184..325d20f5 100644 --- a/types/validator.go +++ b/types/validator.go @@ -3,48 +3,47 @@ package types import ( "bytes" "fmt" - - "github.com/tendermint/tendermint/crypto/tmhash" + "strings" "github.com/tendermint/tendermint/crypto" cmn "github.com/tendermint/tendermint/libs/common" ) // Volatile state for each Validator -// NOTE: The Accum is not included in Validator.Hash(); +// NOTE: The ProposerPriority is not included in Validator.Hash(); // make sure to update that method if changes are made here type Validator struct { Address Address `json:"address"` PubKey crypto.PubKey `json:"pub_key"` VotingPower int64 `json:"voting_power"` - Accum int64 `json:"accum"` + ProposerPriority int64 `json:"proposer_priority"` } func NewValidator(pubKey crypto.PubKey, votingPower int64) *Validator { return &Validator{ - Address: pubKey.Address(), - PubKey: pubKey, - VotingPower: votingPower, - Accum: 0, + Address: pubKey.Address(), + PubKey: pubKey, + VotingPower: votingPower, + ProposerPriority: 0, } } -// Creates a new copy of the validator so we can mutate accum. +// Creates a new copy of the validator so we can mutate ProposerPriority. // Panics if the validator is nil. func (v *Validator) Copy() *Validator { vCopy := *v return &vCopy } -// Returns the one with higher Accum. -func (v *Validator) CompareAccum(other *Validator) *Validator { +// Returns the one with higher ProposerPriority. +func (v *Validator) CompareProposerPriority(other *Validator) *Validator { if v == nil { return other } - if v.Accum > other.Accum { + if v.ProposerPriority > other.ProposerPriority { return v - } else if v.Accum < other.Accum { + } else if v.ProposerPriority < other.ProposerPriority { return other } else { result := bytes.Compare(v.Address, other.Address) @@ -67,27 +66,31 @@ func (v *Validator) String() string { v.Address, v.PubKey, v.VotingPower, - v.Accum) + v.ProposerPriority) } -// Hash computes the unique ID of a validator with a given voting power. -// It excludes the Accum value, which changes with every round. -func (v *Validator) Hash() []byte { - return tmhash.Sum(v.Bytes()) +// ValidatorListString returns a prettified validator list for logging purposes. +func ValidatorListString(vals []*Validator) string { + chunks := make([]string, len(vals)) + for i, val := range vals { + chunks[i] = fmt.Sprintf("%s:%d", val.Address, val.VotingPower) + } + + return strings.Join(chunks, ",") } // Bytes computes the unique encoding of a validator with a given voting power. -// These are the bytes that gets hashed in consensus. It excludes pubkey -// as its redundant with the address. This also excludes accum which changes -// every round. +// These are the bytes that gets hashed in consensus. It excludes address +// as its redundant with the pubkey. This also excludes ProposerPriority +// which changes every round. func (v *Validator) Bytes() []byte { - return cdcEncode((struct { - Address Address + return cdcEncode(struct { + PubKey crypto.PubKey VotingPower int64 }{ - v.Address, + v.PubKey, v.VotingPower, - })) + }) } //---------------------------------------- @@ -101,6 +104,7 @@ func RandValidator(randPower bool, minPower int64) (*Validator, PrivValidator) { if randPower { votePower += int64(cmn.RandUint32()) } - val := NewValidator(privVal.GetPubKey(), votePower) + pubKey := privVal.GetPubKey() + val := NewValidator(pubKey, votePower) return val, privVal } diff --git a/types/validator_set.go b/types/validator_set.go index 72ab68c0..c70f3396 100644 --- a/types/validator_set.go +++ b/types/validator_set.go @@ -4,6 +4,7 @@ import ( "bytes" "fmt" "math" + "math/big" "sort" "strings" @@ -11,13 +12,28 @@ import ( cmn "github.com/tendermint/tendermint/libs/common" ) +// MaxTotalVotingPower - the maximum allowed total voting power. +// It needs to be sufficiently small to, in all cases: +// 1. prevent clipping in incrementProposerPriority() +// 2. let (diff+diffMax-1) not overflow in IncrementProposerPriority() +// (Proof of 1 is tricky, left to the reader). +// It could be higher, but this is sufficiently large for our purposes, +// and leaves room for defensive purposes. +// PriorityWindowSizeFactor - is a constant that when multiplied with the total voting power gives +// the maximum allowed distance between validator priorities. + +const ( + MaxTotalVotingPower = int64(math.MaxInt64) / 8 + PriorityWindowSizeFactor = 2 +) + // ValidatorSet represent a set of *Validator at a given height. // The validators can be fetched by address or index. // The index is in order of .Address, so the indices are fixed // for all rounds of a given blockchain height. -// On the other hand, the .AccumPower of each validator and +// On the other hand, the .ProposerPriority of each validator and // the designated .GetProposer() of a set changes every round, -// upon calling .IncrementAccum(). +// upon calling .IncrementProposerPriority(). // NOTE: Not goroutine-safe. // NOTE: All get/set to validators should copy the value for safety. type ValidatorSet struct { @@ -29,22 +45,20 @@ type ValidatorSet struct { totalVotingPower int64 } +// NewValidatorSet initializes a ValidatorSet by copying over the +// values from `valz`, a list of Validators. If valz is nil or empty, +// the new ValidatorSet will have an empty list of Validators. +// The addresses of validators in `valz` must be unique otherwise the +// function panics. func NewValidatorSet(valz []*Validator) *ValidatorSet { - if valz != nil && len(valz) == 0 { - panic("validator set initialization slice cannot be an empty slice (but it can be nil)") - } - validators := make([]*Validator, len(valz)) - for i, val := range valz { - validators[i] = val.Copy() - } - sort.Sort(ValidatorsByAddress(validators)) - vals := &ValidatorSet{ - Validators: validators, + vals := &ValidatorSet{} + err := vals.updateWithChangeSet(valz, false) + if err != nil { + panic(fmt.Sprintf("cannot create validator set: %s", err)) } if len(valz) > 0 { - vals.IncrementAccum(1) + vals.IncrementProposerPriority(1) } - return vals } @@ -53,48 +67,150 @@ func (vals *ValidatorSet) IsNilOrEmpty() bool { return vals == nil || len(vals.Validators) == 0 } -// Increment Accum and update the proposer on a copy, and return it. -func (vals *ValidatorSet) CopyIncrementAccum(times int) *ValidatorSet { +// Increment ProposerPriority and update the proposer on a copy, and return it. +func (vals *ValidatorSet) CopyIncrementProposerPriority(times int) *ValidatorSet { copy := vals.Copy() - copy.IncrementAccum(times) + copy.IncrementProposerPriority(times) return copy } -// IncrementAccum increments accum of each validator and updates the +// IncrementProposerPriority increments ProposerPriority of each validator and updates the // proposer. Panics if validator set is empty. -func (vals *ValidatorSet) IncrementAccum(times int) { - - // Add VotingPower * times to each validator and order into heap. - validatorsHeap := cmn.NewHeap() - for _, val := range vals.Validators { - // Check for overflow both multiplication and sum. - val.Accum = safeAddClip(val.Accum, safeMulClip(val.VotingPower, int64(times))) - validatorsHeap.PushComparable(val, accumComparable{val}) +// `times` must be positive. +func (vals *ValidatorSet) IncrementProposerPriority(times int) { + if vals.IsNilOrEmpty() { + panic("empty validator set") + } + if times <= 0 { + panic("Cannot call IncrementProposerPriority with non-positive times") } - // Decrement the validator with most accum times times. - for i := 0; i < times; i++ { - mostest := validatorsHeap.Peek().(*Validator) - // mind underflow - mostest.Accum = safeSubClip(mostest.Accum, vals.TotalVotingPower()) + // Cap the difference between priorities to be proportional to 2*totalPower by + // re-normalizing priorities, i.e., rescale all priorities by multiplying with: + // 2*totalVotingPower/(maxPriority - minPriority) + diffMax := PriorityWindowSizeFactor * vals.TotalVotingPower() + vals.RescalePriorities(diffMax) + vals.shiftByAvgProposerPriority() - if i == times-1 { - vals.Proposer = mostest - } else { - validatorsHeap.Update(mostest, accumComparable{mostest}) + var proposer *Validator + // call IncrementProposerPriority(1) times times: + for i := 0; i < times; i++ { + proposer = vals.incrementProposerPriority() + } + + vals.Proposer = proposer +} + +func (vals *ValidatorSet) RescalePriorities(diffMax int64) { + if vals.IsNilOrEmpty() { + panic("empty validator set") + } + // NOTE: This check is merely a sanity check which could be + // removed if all tests would init. voting power appropriately; + // i.e. diffMax should always be > 0 + if diffMax <= 0 { + return + } + + // Calculating ceil(diff/diffMax): + // Re-normalization is performed by dividing by an integer for simplicity. + // NOTE: This may make debugging priority issues easier as well. + diff := computeMaxMinPriorityDiff(vals) + ratio := (diff + diffMax - 1) / diffMax + if ratio > 1 { + for _, val := range vals.Validators { + val.ProposerPriority /= ratio } } } +func (vals *ValidatorSet) incrementProposerPriority() *Validator { + for _, val := range vals.Validators { + // Check for overflow for sum. + newPrio := safeAddClip(val.ProposerPriority, val.VotingPower) + val.ProposerPriority = newPrio + } + // Decrement the validator with most ProposerPriority: + mostest := vals.getValWithMostPriority() + // mind underflow + mostest.ProposerPriority = safeSubClip(mostest.ProposerPriority, vals.TotalVotingPower()) + + return mostest +} + +// should not be called on an empty validator set +func (vals *ValidatorSet) computeAvgProposerPriority() int64 { + n := int64(len(vals.Validators)) + sum := big.NewInt(0) + for _, val := range vals.Validators { + sum.Add(sum, big.NewInt(val.ProposerPriority)) + } + avg := sum.Div(sum, big.NewInt(n)) + if avg.IsInt64() { + return avg.Int64() + } + + // this should never happen: each val.ProposerPriority is in bounds of int64 + panic(fmt.Sprintf("Cannot represent avg ProposerPriority as an int64 %v", avg)) +} + +// compute the difference between the max and min ProposerPriority of that set +func computeMaxMinPriorityDiff(vals *ValidatorSet) int64 { + if vals.IsNilOrEmpty() { + panic("empty validator set") + } + max := int64(math.MinInt64) + min := int64(math.MaxInt64) + for _, v := range vals.Validators { + if v.ProposerPriority < min { + min = v.ProposerPriority + } + if v.ProposerPriority > max { + max = v.ProposerPriority + } + } + diff := max - min + if diff < 0 { + return -1 * diff + } else { + return diff + } +} + +func (vals *ValidatorSet) getValWithMostPriority() *Validator { + var res *Validator + for _, val := range vals.Validators { + res = res.CompareProposerPriority(val) + } + return res +} + +func (vals *ValidatorSet) shiftByAvgProposerPriority() { + if vals.IsNilOrEmpty() { + panic("empty validator set") + } + avgProposerPriority := vals.computeAvgProposerPriority() + for _, val := range vals.Validators { + val.ProposerPriority = safeSubClip(val.ProposerPriority, avgProposerPriority) + } +} + +// Makes a copy of the validator list +func validatorListCopy(valsList []*Validator) []*Validator { + if valsList == nil { + return nil + } + valsCopy := make([]*Validator, len(valsList)) + for i, val := range valsList { + valsCopy[i] = val.Copy() + } + return valsCopy +} + // Copy each validator into a new ValidatorSet func (vals *ValidatorSet) Copy() *ValidatorSet { - validators := make([]*Validator, len(vals.Validators)) - for i, val := range vals.Validators { - // NOTE: must copy, since IncrementAccum updates in place. - validators[i] = val.Copy() - } return &ValidatorSet{ - Validators: validators, + Validators: validatorListCopy(vals.Validators), Proposer: vals.Proposer, totalVotingPower: vals.totalVotingPower, } @@ -140,10 +256,18 @@ func (vals *ValidatorSet) Size() int { // TotalVotingPower returns the sum of the voting powers of all validators. func (vals *ValidatorSet) TotalVotingPower() int64 { if vals.totalVotingPower == 0 { + sum := int64(0) for _, val := range vals.Validators { // mind overflow - vals.totalVotingPower = safeAddClip(vals.totalVotingPower, val.VotingPower) + sum = safeAddClip(sum, val.VotingPower) } + if sum > MaxTotalVotingPower { + panic(fmt.Sprintf( + "Total voting power should be guarded to not exceed %v; got: %v", + MaxTotalVotingPower, + sum)) + } + vals.totalVotingPower = sum } return vals.totalVotingPower } @@ -164,7 +288,7 @@ func (vals *ValidatorSet) findProposer() *Validator { var proposer *Validator for _, val := range vals.Validators { if proposer == nil || !bytes.Equal(val.Address, proposer.Address) { - proposer = proposer.CompareAccum(val) + proposer = proposer.CompareProposerPriority(val) } } return proposer @@ -183,48 +307,6 @@ func (vals *ValidatorSet) Hash() []byte { return merkle.SimpleHashFromByteSlices(bzs) } -// Add adds val to the validator set and returns true. It returns false if val -// is already in the set. -func (vals *ValidatorSet) Add(val *Validator) (added bool) { - val = val.Copy() - idx := sort.Search(len(vals.Validators), func(i int) bool { - return bytes.Compare(val.Address, vals.Validators[i].Address) <= 0 - }) - if idx >= len(vals.Validators) { - vals.Validators = append(vals.Validators, val) - // Invalidate cache - vals.Proposer = nil - vals.totalVotingPower = 0 - return true - } else if bytes.Equal(vals.Validators[idx].Address, val.Address) { - return false - } else { - newValidators := make([]*Validator, len(vals.Validators)+1) - copy(newValidators[:idx], vals.Validators[:idx]) - newValidators[idx] = val - copy(newValidators[idx+1:], vals.Validators[idx:]) - vals.Validators = newValidators - // Invalidate cache - vals.Proposer = nil - vals.totalVotingPower = 0 - return true - } -} - -// Update updates val and returns true. It returns false if val is not present -// in the set. -func (vals *ValidatorSet) Update(val *Validator) (updated bool) { - index, sameVal := vals.GetByAddress(val.Address) - if sameVal == nil { - return false - } - vals.Validators[index] = val.Copy() - // Invalidate cache - vals.Proposer = nil - vals.totalVotingPower = 0 - return true -} - // Remove deletes the validator with address. It returns the validator removed // and true. If returns nil and false if validator is not present in the set. func (vals *ValidatorSet) Remove(address []byte) (val *Validator, removed bool) { @@ -256,8 +338,259 @@ func (vals *ValidatorSet) Iterate(fn func(index int, val *Validator) bool) { } } +// Checks changes against duplicates, splits the changes in updates and removals, sorts them by address +// +// Returns: +// updates, removals - the sorted lists of updates and removals +// err - non-nil if duplicate entries or entries with negative voting power are seen +// +// No changes are made to 'origChanges' +func processChanges(origChanges []*Validator) (updates, removals []*Validator, err error) { + // Make a deep copy of the changes and sort by address + changes := validatorListCopy(origChanges) + sort.Sort(ValidatorsByAddress(changes)) + + removals = make([]*Validator, 0, len(changes)) + updates = make([]*Validator, 0, len(changes)) + var prevAddr Address + + // Scan changes by address and append valid validators to updates or removals lists + for _, valUpdate := range changes { + if bytes.Equal(valUpdate.Address, prevAddr) { + err = fmt.Errorf("duplicate entry %v in %v", valUpdate, changes) + return nil, nil, err + } + if valUpdate.VotingPower < 0 { + err = fmt.Errorf("voting power can't be negative %v", valUpdate) + return nil, nil, err + } + if valUpdate.VotingPower == 0 { + removals = append(removals, valUpdate) + } else { + updates = append(updates, valUpdate) + } + prevAddr = valUpdate.Address + } + return updates, removals, err +} + +// Verifies a list of updates against a validator set, making sure the allowed +// total voting power would not be exceeded if these updates would be applied to the set. +// It also computes the total voting power of the set that would result after the updates but +// before the removals. +// +// Returns: +// updatedTotalVotingPower - the new total voting power if these updates would be applied +// err - non-nil if the maximum allowed total voting power would be exceeded +// +// 'updates' should be a list of proper validator changes, i.e. they have been scanned +// by processChanges for duplicates and invalid values. +// No changes are made to the validator set 'vals'. +func verifyUpdates(updates []*Validator, vals *ValidatorSet) (updatedTotalVotingPower int64, err error) { + + // Scan the updates, compute new total voting power, check for overflow + updatedTotalVotingPower = vals.TotalVotingPower() + + for _, valUpdate := range updates { + address := valUpdate.Address + _, val := vals.GetByAddress(address) + if val == nil { + // new validator, add its voting power the the total + updatedTotalVotingPower += valUpdate.VotingPower + } else { + // updated validator, add the difference in power to the total + updatedTotalVotingPower += valUpdate.VotingPower - val.VotingPower + } + + if updatedTotalVotingPower < 0 { + err = fmt.Errorf( + "failed to add/update validator with negative voting power %v", + valUpdate) + return 0, err + } + overflow := updatedTotalVotingPower > MaxTotalVotingPower + if overflow { + err = fmt.Errorf( + "failed to add/update validator %v, total voting power would exceed the max allowed %v", + valUpdate, MaxTotalVotingPower) + return 0, err + } + } + + return updatedTotalVotingPower, nil +} + +// Computes the proposer priority for the validators not present in the set based on 'updatedTotalVotingPower' +// Leaves unchanged the priorities of validators that are changed. +// +// 'updates' parameter must be a list of unique validators to be added or updated. +// No changes are made to the validator set 'vals'. +func computeNewPriorities(updates []*Validator, vals *ValidatorSet, updatedTotalVotingPower int64) int { + + numNew := 0 + // Scan and update the proposerPriority for newly added and updated validators + for _, valUpdate := range updates { + address := valUpdate.Address + _, val := vals.GetByAddress(address) + if val == nil { + // add val + // Set ProposerPriority to -C*totalVotingPower (with C ~= 1.125) to make sure validators can't + // un-bond and then re-bond to reset their (potentially previously negative) ProposerPriority to zero. + // + // Contract: updatedVotingPower < MaxTotalVotingPower to ensure ProposerPriority does + // not exceed the bounds of int64. + // + // Compute ProposerPriority = -1.125*totalVotingPower == -(updatedVotingPower + (updatedVotingPower >> 3)). + valUpdate.ProposerPriority = -(updatedTotalVotingPower + (updatedTotalVotingPower >> 3)) + numNew++ + } else { + valUpdate.ProposerPriority = val.ProposerPriority + } + } + + return numNew +} + +// Merges the vals' validator list with the updates list. +// When two elements with same address are seen, the one from updates is selected. +// Expects updates to be a list of updates sorted by address with no duplicates or errors, +// must have been validated with verifyUpdates() and priorities computed with computeNewPriorities(). +func (vals *ValidatorSet) applyUpdates(updates []*Validator) { + + existing := make([]*Validator, len(vals.Validators)) + copy(existing, vals.Validators) + + merged := make([]*Validator, len(existing)+len(updates)) + i := 0 + + for len(existing) > 0 && len(updates) > 0 { + if bytes.Compare(existing[0].Address, updates[0].Address) < 0 { + merged[i] = existing[0] + existing = existing[1:] + } else { + merged[i] = updates[0] + if bytes.Equal(existing[0].Address, updates[0].Address) { + // validator present in both, advance existing + existing = existing[1:] + } + updates = updates[1:] + } + i++ + } + + for j := 0; j < len(existing); j++ { + merged[i] = existing[j] + i++ + } + + for j := 0; j < len(updates); j++ { + merged[i] = updates[j] + i++ + } + + vals.Validators = merged[:i] + vals.totalVotingPower = 0 +} + +// Checks that the validators to be removed are part of the validator set. +// No changes are made to the validator set 'vals'. +func verifyRemovals(deletes []*Validator, vals *ValidatorSet) error { + + for _, valUpdate := range deletes { + address := valUpdate.Address + _, val := vals.GetByAddress(address) + if val == nil { + return fmt.Errorf("failed to find validator %X to remove", address) + } + } + return nil +} + +// Removes the validators specified in 'deletes' from validator set 'vals'. +// Should not fail as verification has been done before. +func (vals *ValidatorSet) applyRemovals(deletes []*Validator) { + + for _, valUpdate := range deletes { + address := valUpdate.Address + _, removed := vals.Remove(address) + if !removed { + // Should never happen + panic(fmt.Sprintf("failed to remove validator %X", address)) + } + } +} + +// UpdateWithChangeSet attempts to update the validator set with 'changes' +// It performs the following steps: +// - validates the changes making sure there are no duplicates and splits them in updates and deletes +// - verifies that applying the changes will not result in errors +// - computes the total voting power BEFORE removals to ensure that in the next steps the relative priorities +// across old and newly added validators is fair +// - computes the priorities of new validators against the final set +// - applies the updates against the validator set +// - applies the removals against the validator set +// - performs scaling and centering of priority values +// If error is detected during verification steps it is returned and the validator set +// is not changed. +func (vals *ValidatorSet) UpdateWithChangeSet(changes []*Validator) error { + return vals.updateWithChangeSet(changes, true) +} + +// main function used by UpdateWithChangeSet() and NewValidatorSet() +// If 'allowDeletes' is false then delete operations are not allowed and must be reported if +// present in 'changes' +func (vals *ValidatorSet) updateWithChangeSet(changes []*Validator, allowDeletes bool) error { + + if len(changes) <= 0 { + return nil + } + + // Check for duplicates within changes, split in 'updates' and 'deletes' lists (sorted) + updates, deletes, err := processChanges(changes) + if err != nil { + return err + } + + if !allowDeletes && len(deletes) != 0 { + err = fmt.Errorf("cannot process validators with voting power 0: %v", deletes) + return err + } + + // Verify that applying the 'deletes' against 'vals' will not result in error. + if err := verifyRemovals(deletes, vals); err != nil { + return err + } + + // Verify that applying the 'updates' against 'vals' will not result in error. + updatedTotalVotingPower, err := verifyUpdates(updates, vals) + if err != nil { + return err + } + + // Compute the priorities for updates + numNewValidators := computeNewPriorities(updates, vals, updatedTotalVotingPower) + if len(vals.Validators)+numNewValidators <= len(deletes) { + err = fmt.Errorf("applying the validator changes would result in empty set") + return err + } + + // Apply updates and removals + vals.applyUpdates(updates) + vals.applyRemovals(deletes) + + // Scale and center + vals.RescalePriorities(PriorityWindowSizeFactor * vals.TotalVotingPower()) + vals.shiftByAvgProposerPriority() + + return nil +} + // Verify that +2/3 of the set had signed the given signBytes. func (vals *ValidatorSet) VerifyCommit(chainID string, blockID BlockID, height int64, commit *Commit) error { + + if err := commit.ValidateBasic(); err != nil { + return err + } if vals.Size() != len(commit.Precommits) { return fmt.Errorf("Invalid commit -- wrong set size: %v vs %v", vals.Size(), len(commit.Precommits)) } @@ -270,24 +603,14 @@ func (vals *ValidatorSet) VerifyCommit(chainID string, blockID BlockID, height i } talliedVotingPower := int64(0) - round := commit.Round() for idx, precommit := range commit.Precommits { if precommit == nil { continue // OK, some precommits can be missing. } - if precommit.Height != height { - return fmt.Errorf("Invalid commit -- wrong height: want %v got %v", height, precommit.Height) - } - if precommit.Round != round { - return fmt.Errorf("Invalid commit -- wrong round: want %v got %v", round, precommit.Round) - } - if precommit.Type != VoteTypePrecommit { - return fmt.Errorf("Invalid commit -- not precommit @ index %v", idx) - } _, val := vals.GetByIndex(idx) // Validate signature. - precommitSignBytes := precommit.SignBytes(chainID) + precommitSignBytes := commit.VoteSignBytes(chainID, precommit) if !val.PubKey.VerifyBytes(precommitSignBytes, precommit.Signature) { return fmt.Errorf("Invalid commit -- invalid signature: %v", precommit) } @@ -303,8 +626,7 @@ func (vals *ValidatorSet) VerifyCommit(chainID string, blockID BlockID, height i if talliedVotingPower > vals.TotalVotingPower()*2/3 { return nil } - return fmt.Errorf("Invalid commit -- insufficient voting power: got %v, needed %v", - talliedVotingPower, (vals.TotalVotingPower()*2/3 + 1)) + return errTooMuchChange{talliedVotingPower, vals.TotalVotingPower()*2/3 + 1} } // VerifyFutureCommit will check to see if the set would be valid with a different @@ -361,7 +683,7 @@ func (vals *ValidatorSet) VerifyFutureCommit(newSet *ValidatorSet, chainID strin if precommit.Round != round { return cmn.NewError("Invalid commit -- wrong round: %v vs %v", round, precommit.Round) } - if precommit.Type != VoteTypePrecommit { + if precommit.Type != PrecommitType { return cmn.NewError("Invalid commit -- not precommit @ index %v", idx) } // See if this validator is in oldVals. @@ -372,7 +694,7 @@ func (vals *ValidatorSet) VerifyFutureCommit(newSet *ValidatorSet, chainID strin seen[idx] = true // Validate signature. - precommitSignBytes := precommit.SignBytes(chainID) + precommitSignBytes := commit.VoteSignBytes(chainID, precommit) if !val.PubKey.VerifyBytes(precommitSignBytes, precommit.Signature) { return cmn.NewError("Invalid commit -- invalid signature: %v", precommit) } @@ -386,12 +708,37 @@ func (vals *ValidatorSet) VerifyFutureCommit(newSet *ValidatorSet, chainID strin } if oldVotingPower <= oldVals.TotalVotingPower()*2/3 { - return cmn.NewError("Invalid commit -- insufficient old voting power: got %v, needed %v", - oldVotingPower, (oldVals.TotalVotingPower()*2/3 + 1)) + return errTooMuchChange{oldVotingPower, oldVals.TotalVotingPower()*2/3 + 1} } return nil } +//----------------- +// ErrTooMuchChange + +func IsErrTooMuchChange(err error) bool { + switch err_ := err.(type) { + case cmn.Error: + _, ok := err_.Data().(errTooMuchChange) + return ok + case errTooMuchChange: + return true + default: + return false + } +} + +type errTooMuchChange struct { + got int64 + needed int64 +} + +func (e errTooMuchChange) Error() string { + return fmt.Sprintf("Invalid commit -- insufficient old voting power: got %v, needed %v", e.got, e.needed) +} + +//---------------- + func (vals *ValidatorSet) String() string { return vals.StringIndented("") } @@ -401,7 +748,7 @@ func (vals *ValidatorSet) StringIndented(indent string) string { if vals == nil { return "nil-ValidatorSet" } - valStrings := []string{} + var valStrings []string vals.Iterate(func(index int, val *Validator) bool { valStrings = append(valStrings, val.String()) return false @@ -438,20 +785,6 @@ func (valz ValidatorsByAddress) Swap(i, j int) { valz[j] = it } -//------------------------------------- -// Use with Heap for sorting validators by accum - -type accumComparable struct { - *Validator -} - -// We want to find the validator with the greatest accum. -func (ac accumComparable) Less(o interface{}) bool { - other := o.(accumComparable).Validator - larger := ac.CompareAccum(other) - return bytes.Equal(larger.Address, ac.Address) -} - //---------------------------------------- // For testing @@ -472,24 +805,7 @@ func RandValidatorSet(numValidators int, votingPower int64) (*ValidatorSet, []Pr } /////////////////////////////////////////////////////////////////////////////// -// Safe multiplication and addition/subtraction - -func safeMul(a, b int64) (int64, bool) { - if a == 0 || b == 0 { - return 0, false - } - if a == 1 { - return b, false - } - if b == 1 { - return a, false - } - if a == math.MinInt64 || b == math.MinInt64 { - return -1, true - } - c := a * b - return c, c/b != a -} +// Safe addition/subtraction func safeAdd(a, b int64) (int64, bool) { if b > 0 && a > math.MaxInt64-b { @@ -509,17 +825,6 @@ func safeSub(a, b int64) (int64, bool) { return a - b, false } -func safeMulClip(a, b int64) int64 { - c, overflow := safeMul(a, b) - if overflow { - if (a < 0 || b < 0) && !(a < 0 && b < 0) { - return math.MinInt64 - } - return math.MaxInt64 - } - return c -} - func safeAddClip(a, b int64) int64 { c, overflow := safeAdd(a, b) if overflow { diff --git a/types/validator_set_test.go b/types/validator_set_test.go index e4111707..04874c19 100644 --- a/types/validator_set_test.go +++ b/types/validator_set_test.go @@ -4,6 +4,7 @@ import ( "bytes" "fmt" "math" + "math/rand" "strings" "testing" "testing/quick" @@ -17,10 +18,13 @@ import ( ) func TestValidatorSetBasic(t *testing.T) { - assert.Panics(t, func() { NewValidatorSet([]*Validator{}) }) + // empty or nil validator lists are allowed, + // but attempting to IncrementProposerPriority on them will panic. + vset := NewValidatorSet([]*Validator{}) + assert.Panics(t, func() { vset.IncrementProposerPriority(1) }) - vset := NewValidatorSet(nil) - assert.Panics(t, func() { vset.IncrementAccum(1) }) + vset = NewValidatorSet(nil) + assert.Panics(t, func() { vset.IncrementProposerPriority(1) }) assert.EqualValues(t, vset, vset.Copy()) assert.False(t, vset.HasAddress([]byte("some val"))) @@ -42,28 +46,34 @@ func TestValidatorSetBasic(t *testing.T) { assert.Nil(t, vset.Hash()) // add - val = randValidator_() - assert.True(t, vset.Add(val)) + val = randValidator_(vset.TotalVotingPower()) + assert.NoError(t, vset.UpdateWithChangeSet([]*Validator{val})) + assert.True(t, vset.HasAddress(val.Address)) - idx, val2 := vset.GetByAddress(val.Address) + idx, _ = vset.GetByAddress(val.Address) assert.Equal(t, 0, idx) - assert.Equal(t, val, val2) - addr, val2 = vset.GetByIndex(0) + addr, _ = vset.GetByIndex(0) assert.Equal(t, []byte(val.Address), addr) - assert.Equal(t, val, val2) assert.Equal(t, 1, vset.Size()) assert.Equal(t, val.VotingPower, vset.TotalVotingPower()) - assert.Equal(t, val, vset.GetProposer()) assert.NotNil(t, vset.Hash()) - assert.NotPanics(t, func() { vset.IncrementAccum(1) }) + assert.NotPanics(t, func() { vset.IncrementProposerPriority(1) }) + assert.Equal(t, val.Address, vset.GetProposer().Address) // update - assert.False(t, vset.Update(randValidator_())) - val.VotingPower = 100 - assert.True(t, vset.Update(val)) + val = randValidator_(vset.TotalVotingPower()) + assert.NoError(t, vset.UpdateWithChangeSet([]*Validator{val})) + _, val = vset.GetByAddress(val.Address) + val.VotingPower += 100 + proposerPriority := val.ProposerPriority + + val.ProposerPriority = 0 + assert.NoError(t, vset.UpdateWithChangeSet([]*Validator{val})) + _, val = vset.GetByAddress(val.Address) + assert.Equal(t, proposerPriority, val.ProposerPriority) // remove - val2, removed := vset.Remove(randValidator_().Address) + val2, removed := vset.Remove(randValidator_(vset.TotalVotingPower()).Address) assert.Nil(t, val2) assert.False(t, removed) val2, removed = vset.Remove(val.Address) @@ -86,14 +96,28 @@ func TestCopy(t *testing.T) { } } +// Test that IncrementProposerPriority requires positive times. +func TestIncrementProposerPriorityPositiveTimes(t *testing.T) { + vset := NewValidatorSet([]*Validator{ + newValidator([]byte("foo"), 1000), + newValidator([]byte("bar"), 300), + newValidator([]byte("baz"), 330), + }) + + assert.Panics(t, func() { vset.IncrementProposerPriority(-1) }) + assert.Panics(t, func() { vset.IncrementProposerPriority(0) }) + vset.IncrementProposerPriority(1) +} + func BenchmarkValidatorSetCopy(b *testing.B) { b.StopTimer() vset := NewValidatorSet([]*Validator{}) for i := 0; i < 1000; i++ { privKey := ed25519.GenPrivKey() pubKey := privKey.PubKey() - val := NewValidator(pubKey, 0) - if !vset.Add(val) { + val := NewValidator(pubKey, 10) + err := vset.UpdateWithChangeSet([]*Validator{val}) + if err != nil { panic("Failed to add validator") } } @@ -112,11 +136,11 @@ func TestProposerSelection1(t *testing.T) { newValidator([]byte("bar"), 300), newValidator([]byte("baz"), 330), }) - proposers := []string{} + var proposers []string for i := 0; i < 99; i++ { val := vset.GetProposer() proposers = append(proposers, string(val.Address)) - vset.IncrementAccum(1) + vset.IncrementProposerPriority(1) } expected := `foo baz foo bar foo foo baz foo bar foo foo baz foo foo bar foo baz foo foo bar foo foo baz foo bar foo foo baz foo bar foo foo baz foo foo bar foo baz foo foo bar foo baz foo foo bar foo baz foo foo bar foo baz foo foo foo baz bar foo foo foo baz foo bar foo foo baz foo bar foo foo baz foo bar foo foo baz foo bar foo foo baz foo foo bar foo baz foo foo bar foo baz foo foo bar foo baz foo foo` if expected != strings.Join(proposers, " ") { @@ -139,18 +163,18 @@ func TestProposerSelection2(t *testing.T) { if !bytes.Equal(prop.Address, valList[ii].Address) { t.Fatalf("(%d): Expected %X. Got %X", i, valList[ii].Address, prop.Address) } - vals.IncrementAccum(1) + vals.IncrementProposerPriority(1) } // One validator has more than the others, but not enough to propose twice in a row *val2 = *newValidator(addr2, 400) vals = NewValidatorSet(valList) - // vals.IncrementAccum(1) + // vals.IncrementProposerPriority(1) prop := vals.GetProposer() if !bytes.Equal(prop.Address, addr2) { t.Fatalf("Expected address with highest voting power to be first proposer. Got %X", prop.Address) } - vals.IncrementAccum(1) + vals.IncrementProposerPriority(1) prop = vals.GetProposer() if !bytes.Equal(prop.Address, addr0) { t.Fatalf("Expected smallest address to be validator. Got %X", prop.Address) @@ -163,12 +187,12 @@ func TestProposerSelection2(t *testing.T) { if !bytes.Equal(prop.Address, addr2) { t.Fatalf("Expected address with highest voting power to be first proposer. Got %X", prop.Address) } - vals.IncrementAccum(1) + vals.IncrementProposerPriority(1) prop = vals.GetProposer() if !bytes.Equal(prop.Address, addr2) { t.Fatalf("Expected address with highest voting power to be second proposer. Got %X", prop.Address) } - vals.IncrementAccum(1) + vals.IncrementProposerPriority(1) prop = vals.GetProposer() if !bytes.Equal(prop.Address, addr0) { t.Fatalf("Expected smallest address to be validator. Got %X", prop.Address) @@ -184,7 +208,7 @@ func TestProposerSelection2(t *testing.T) { prop := vals.GetProposer() ii := prop.Address[19] propCount[ii]++ - vals.IncrementAccum(1) + vals.IncrementProposerPriority(1) } if propCount[0] != 40*N { @@ -209,12 +233,12 @@ func TestProposerSelection3(t *testing.T) { proposerOrder := make([]*Validator, 4) for i := 0; i < 4; i++ { proposerOrder[i] = vset.GetProposer() - vset.IncrementAccum(1) + vset.IncrementProposerPriority(1) } // i for the loop // j for the times - // we should go in order for ever, despite some IncrementAccums with times > 1 + // we should go in order for ever, despite some IncrementProposerPriority with times > 1 var i, j int for ; i < 10000; i++ { got := vset.GetProposer().Address @@ -239,9 +263,9 @@ func TestProposerSelection3(t *testing.T) { mod := (cmn.RandInt() % 5) + 1 if cmn.RandInt()%mod > 0 { // sometimes its up to 5 - times = cmn.RandInt() % 5 + times = (cmn.RandInt() % 4) + 1 } - vset.IncrementAccum(times) + vset.IncrementProposerPriority(times) j += times } @@ -257,22 +281,26 @@ func randPubKey() crypto.PubKey { return ed25519.PubKeyEd25519(pubKey) } -func randValidator_() *Validator { - val := NewValidator(randPubKey(), cmn.RandInt64()) - val.Accum = cmn.RandInt64() +func randValidator_(totalVotingPower int64) *Validator { + // this modulo limits the ProposerPriority/VotingPower to stay in the + // bounds of MaxTotalVotingPower minus the already existing voting power: + val := NewValidator(randPubKey(), int64(cmn.RandUint64()%uint64((MaxTotalVotingPower-totalVotingPower)))) + val.ProposerPriority = cmn.RandInt64() % (MaxTotalVotingPower - totalVotingPower) return val } func randValidatorSet(numValidators int) *ValidatorSet { validators := make([]*Validator, numValidators) + totalVotingPower := int64(0) for i := 0; i < numValidators; i++ { - validators[i] = randValidator_() + validators[i] = randValidator_(totalVotingPower) + totalVotingPower += validators[i].VotingPower } return NewValidatorSet(validators) } func (valSet *ValidatorSet) toBytes() []byte { - bz, err := cdc.MarshalBinary(valSet) + bz, err := cdc.MarshalBinaryLengthPrefixed(valSet) if err != nil { panic(err) } @@ -280,7 +308,7 @@ func (valSet *ValidatorSet) toBytes() []byte { } func (valSet *ValidatorSet) fromBytes(b []byte) { - err := cdc.UnmarshalBinary(b, &valSet) + err := cdc.UnmarshalBinaryLengthPrefixed(b, &valSet) if err != nil { // DATA HAS BEEN CORRUPTED OR THE SPEC HAS CHANGED panic(err) @@ -289,52 +317,203 @@ func (valSet *ValidatorSet) fromBytes(b []byte) { //------------------------------------------------------------------- -func TestValidatorSetTotalVotingPowerOverflows(t *testing.T) { - vset := NewValidatorSet([]*Validator{ - {Address: []byte("a"), VotingPower: math.MaxInt64, Accum: 0}, - {Address: []byte("b"), VotingPower: math.MaxInt64, Accum: 0}, - {Address: []byte("c"), VotingPower: math.MaxInt64, Accum: 0}, - }) - - assert.EqualValues(t, math.MaxInt64, vset.TotalVotingPower()) -} - -func TestValidatorSetIncrementAccumOverflows(t *testing.T) { - // NewValidatorSet calls IncrementAccum(1) - vset := NewValidatorSet([]*Validator{ - // too much voting power - 0: {Address: []byte("a"), VotingPower: math.MaxInt64, Accum: 0}, - // too big accum - 1: {Address: []byte("b"), VotingPower: 10, Accum: math.MaxInt64}, - // almost too big accum - 2: {Address: []byte("c"), VotingPower: 10, Accum: math.MaxInt64 - 5}, - }) - - assert.Equal(t, int64(0), vset.Validators[0].Accum, "0") // because we decrement val with most voting power - assert.EqualValues(t, math.MaxInt64, vset.Validators[1].Accum, "1") - assert.EqualValues(t, math.MaxInt64, vset.Validators[2].Accum, "2") -} - -func TestValidatorSetIncrementAccumUnderflows(t *testing.T) { - // NewValidatorSet calls IncrementAccum(1) - vset := NewValidatorSet([]*Validator{ - 0: {Address: []byte("a"), VotingPower: math.MaxInt64, Accum: math.MinInt64}, - 1: {Address: []byte("b"), VotingPower: 1, Accum: math.MinInt64}, - }) - - vset.IncrementAccum(5) - - assert.EqualValues(t, math.MinInt64, vset.Validators[0].Accum, "0") - assert.EqualValues(t, math.MinInt64, vset.Validators[1].Accum, "1") -} - -func TestSafeMul(t *testing.T) { - f := func(a, b int64) bool { - c, overflow := safeMul(a, b) - return overflow || (!overflow && c == a*b) +func TestValidatorSetTotalVotingPowerPanicsOnOverflow(t *testing.T) { + // NewValidatorSet calls IncrementProposerPriority which calls TotalVotingPower() + // which should panic on overflows: + shouldPanic := func() { + NewValidatorSet([]*Validator{ + {Address: []byte("a"), VotingPower: math.MaxInt64, ProposerPriority: 0}, + {Address: []byte("b"), VotingPower: math.MaxInt64, ProposerPriority: 0}, + {Address: []byte("c"), VotingPower: math.MaxInt64, ProposerPriority: 0}, + }) } - if err := quick.Check(f, nil); err != nil { - t.Error(err) + + assert.Panics(t, shouldPanic) +} + +func TestAvgProposerPriority(t *testing.T) { + // Create Validator set without calling IncrementProposerPriority: + tcs := []struct { + vs ValidatorSet + want int64 + }{ + 0: {ValidatorSet{Validators: []*Validator{{ProposerPriority: 0}, {ProposerPriority: 0}, {ProposerPriority: 0}}}, 0}, + 1: {ValidatorSet{Validators: []*Validator{{ProposerPriority: math.MaxInt64}, {ProposerPriority: 0}, {ProposerPriority: 0}}}, math.MaxInt64 / 3}, + 2: {ValidatorSet{Validators: []*Validator{{ProposerPriority: math.MaxInt64}, {ProposerPriority: 0}}}, math.MaxInt64 / 2}, + 3: {ValidatorSet{Validators: []*Validator{{ProposerPriority: math.MaxInt64}, {ProposerPriority: math.MaxInt64}}}, math.MaxInt64}, + 4: {ValidatorSet{Validators: []*Validator{{ProposerPriority: math.MinInt64}, {ProposerPriority: math.MinInt64}}}, math.MinInt64}, + } + for i, tc := range tcs { + got := tc.vs.computeAvgProposerPriority() + assert.Equal(t, tc.want, got, "test case: %v", i) + } +} + +func TestAveragingInIncrementProposerPriority(t *testing.T) { + // Test that the averaging works as expected inside of IncrementProposerPriority. + // Each validator comes with zero voting power which simplifies reasoning about + // the expected ProposerPriority. + tcs := []struct { + vs ValidatorSet + times int + avg int64 + }{ + 0: {ValidatorSet{ + Validators: []*Validator{ + {Address: []byte("a"), ProposerPriority: 1}, + {Address: []byte("b"), ProposerPriority: 2}, + {Address: []byte("c"), ProposerPriority: 3}}}, + 1, 2}, + 1: {ValidatorSet{ + Validators: []*Validator{ + {Address: []byte("a"), ProposerPriority: 10}, + {Address: []byte("b"), ProposerPriority: -10}, + {Address: []byte("c"), ProposerPriority: 1}}}, + // this should average twice but the average should be 0 after the first iteration + // (voting power is 0 -> no changes) + 11, 1 / 3}, + 2: {ValidatorSet{ + Validators: []*Validator{ + {Address: []byte("a"), ProposerPriority: 100}, + {Address: []byte("b"), ProposerPriority: -10}, + {Address: []byte("c"), ProposerPriority: 1}}}, + 1, 91 / 3}, + } + for i, tc := range tcs { + // work on copy to have the old ProposerPriorities: + newVset := tc.vs.CopyIncrementProposerPriority(tc.times) + for _, val := range tc.vs.Validators { + _, updatedVal := newVset.GetByAddress(val.Address) + assert.Equal(t, updatedVal.ProposerPriority, val.ProposerPriority-tc.avg, "test case: %v", i) + } + } +} + +func TestAveragingInIncrementProposerPriorityWithVotingPower(t *testing.T) { + // Other than TestAveragingInIncrementProposerPriority this is a more complete test showing + // how each ProposerPriority changes in relation to the validator's voting power respectively. + // average is zero in each round: + vp0 := int64(10) + vp1 := int64(1) + vp2 := int64(1) + total := vp0 + vp1 + vp2 + avg := (vp0 + vp1 + vp2 - total) / 3 + vals := ValidatorSet{Validators: []*Validator{ + {Address: []byte{0}, ProposerPriority: 0, VotingPower: vp0}, + {Address: []byte{1}, ProposerPriority: 0, VotingPower: vp1}, + {Address: []byte{2}, ProposerPriority: 0, VotingPower: vp2}}} + tcs := []struct { + vals *ValidatorSet + wantProposerPrioritys []int64 + times int + wantProposer *Validator + }{ + + 0: { + vals.Copy(), + []int64{ + // Acumm+VotingPower-Avg: + 0 + vp0 - total - avg, // mostest will be subtracted by total voting power (12) + 0 + vp1, + 0 + vp2}, + 1, + vals.Validators[0]}, + 1: { + vals.Copy(), + []int64{ + (0 + vp0 - total) + vp0 - total - avg, // this will be mostest on 2nd iter, too + (0 + vp1) + vp1, + (0 + vp2) + vp2}, + 2, + vals.Validators[0]}, // increment twice -> expect average to be subtracted twice + 2: { + vals.Copy(), + []int64{ + 0 + 3*(vp0-total) - avg, // still mostest + 0 + 3*vp1, + 0 + 3*vp2}, + 3, + vals.Validators[0]}, + 3: { + vals.Copy(), + []int64{ + 0 + 4*(vp0-total), // still mostest + 0 + 4*vp1, + 0 + 4*vp2}, + 4, + vals.Validators[0]}, + 4: { + vals.Copy(), + []int64{ + 0 + 4*(vp0-total) + vp0, // 4 iters was mostest + 0 + 5*vp1 - total, // now this val is mostest for the 1st time (hence -12==totalVotingPower) + 0 + 5*vp2}, + 5, + vals.Validators[1]}, + 5: { + vals.Copy(), + []int64{ + 0 + 6*vp0 - 5*total, // mostest again + 0 + 6*vp1 - total, // mostest once up to here + 0 + 6*vp2}, + 6, + vals.Validators[0]}, + 6: { + vals.Copy(), + []int64{ + 0 + 7*vp0 - 6*total, // in 7 iters this val is mostest 6 times + 0 + 7*vp1 - total, // in 7 iters this val is mostest 1 time + 0 + 7*vp2}, + 7, + vals.Validators[0]}, + 7: { + vals.Copy(), + []int64{ + 0 + 8*vp0 - 7*total, // mostest again + 0 + 8*vp1 - total, + 0 + 8*vp2}, + 8, + vals.Validators[0]}, + 8: { + vals.Copy(), + []int64{ + 0 + 9*vp0 - 7*total, + 0 + 9*vp1 - total, + 0 + 9*vp2 - total}, // mostest + 9, + vals.Validators[2]}, + 9: { + vals.Copy(), + []int64{ + 0 + 10*vp0 - 8*total, // after 10 iters this is mostest again + 0 + 10*vp1 - total, // after 6 iters this val is "mostest" once and not in between + 0 + 10*vp2 - total}, // in between 10 iters this val is "mostest" once + 10, + vals.Validators[0]}, + 10: { + vals.Copy(), + []int64{ + 0 + 11*vp0 - 9*total, + 0 + 11*vp1 - total, // after 6 iters this val is "mostest" once and not in between + 0 + 11*vp2 - total}, // after 10 iters this val is "mostest" once + 11, + vals.Validators[0]}, + } + for i, tc := range tcs { + tc.vals.IncrementProposerPriority(tc.times) + + assert.Equal(t, tc.wantProposer.Address, tc.vals.GetProposer().Address, + "test case: %v", + i) + + for valIdx, val := range tc.vals.Validators { + assert.Equal(t, + tc.wantProposerPrioritys[valIdx], + val.ProposerPriority, + "test case: %v, validator: %v", + i, + valIdx) + } } } @@ -348,13 +527,6 @@ func TestSafeAdd(t *testing.T) { } } -func TestSafeMulClip(t *testing.T) { - assert.EqualValues(t, math.MaxInt64, safeMulClip(math.MinInt64, math.MinInt64)) - assert.EqualValues(t, math.MinInt64, safeMulClip(math.MaxInt64, math.MinInt64)) - assert.EqualValues(t, math.MinInt64, safeMulClip(math.MinInt64, math.MaxInt64)) - assert.EqualValues(t, math.MaxInt64, safeMulClip(math.MaxInt64, 2)) -} - func TestSafeAddClip(t *testing.T) { assert.EqualValues(t, math.MaxInt64, safeAddClip(math.MaxInt64, 10)) assert.EqualValues(t, math.MaxInt64, safeAddClip(math.MaxInt64, math.MaxInt64)) @@ -385,7 +557,7 @@ func TestValidatorSetVerifyCommit(t *testing.T) { Height: height, Round: 0, Timestamp: tmtime.Now(), - Type: VoteTypePrecommit, + Type: PrecommitType, BlockID: blockID, } sig, err := privKey.Sign(vote.SignBytes(chainID)) @@ -393,7 +565,7 @@ func TestValidatorSetVerifyCommit(t *testing.T) { vote.Signature = sig commit := &Commit{ BlockID: blockID, - Precommits: []*Vote{vote}, + Precommits: []*CommitSig{vote.CommitSig()}, } badChainID := "notmychainID" @@ -401,7 +573,7 @@ func TestValidatorSetVerifyCommit(t *testing.T) { badHeight := height + 1 badCommit := &Commit{ BlockID: blockID, - Precommits: []*Vote{nil}, + Precommits: []*CommitSig{nil}, } // test some error cases @@ -427,3 +599,357 @@ func TestValidatorSetVerifyCommit(t *testing.T) { err = vset.VerifyCommit(chainID, blockID, height, commit) assert.Nil(t, err) } + +func TestEmptySet(t *testing.T) { + + var valList []*Validator + valSet := NewValidatorSet(valList) + assert.Panics(t, func() { valSet.IncrementProposerPriority(1) }) + assert.Panics(t, func() { valSet.RescalePriorities(100) }) + assert.Panics(t, func() { valSet.shiftByAvgProposerPriority() }) + assert.Panics(t, func() { assert.Zero(t, computeMaxMinPriorityDiff(valSet)) }) + valSet.GetProposer() + + // Add to empty set + v1 := newValidator([]byte("v1"), 100) + v2 := newValidator([]byte("v2"), 100) + valList = []*Validator{v1, v2} + assert.NoError(t, valSet.UpdateWithChangeSet(valList)) + verifyValidatorSet(t, valSet) + + // Delete all validators from set + v1 = newValidator([]byte("v1"), 0) + v2 = newValidator([]byte("v2"), 0) + delList := []*Validator{v1, v2} + assert.Error(t, valSet.UpdateWithChangeSet(delList)) + + // Attempt delete from empty set + assert.Error(t, valSet.UpdateWithChangeSet(delList)) + +} + +func TestUpdatesForNewValidatorSet(t *testing.T) { + + v1 := newValidator([]byte("v1"), 100) + v2 := newValidator([]byte("v2"), 100) + valList := []*Validator{v1, v2} + valSet := NewValidatorSet(valList) + verifyValidatorSet(t, valSet) + + // Verify duplicates are caught in NewValidatorSet() and it panics + v111 := newValidator([]byte("v1"), 100) + v112 := newValidator([]byte("v1"), 123) + v113 := newValidator([]byte("v1"), 234) + valList = []*Validator{v111, v112, v113} + assert.Panics(t, func() { NewValidatorSet(valList) }) + + // Verify set including validator with voting power 0 cannot be created + v1 = newValidator([]byte("v1"), 0) + v2 = newValidator([]byte("v2"), 22) + v3 := newValidator([]byte("v3"), 33) + valList = []*Validator{v1, v2, v3} + assert.Panics(t, func() { NewValidatorSet(valList) }) + + // Verify set including validator with negative voting power cannot be created + v1 = newValidator([]byte("v1"), 10) + v2 = newValidator([]byte("v2"), -20) + v3 = newValidator([]byte("v3"), 30) + valList = []*Validator{v1, v2, v3} + assert.Panics(t, func() { NewValidatorSet(valList) }) + +} + +type testVal struct { + name string + power int64 +} + +func TestValSetUpdatesBasicTestsExecute(t *testing.T) { + valSetUpdatesBasicTests := []struct { + startVals []testVal + updateVals []testVal + expectedVals []testVal + expError bool + }{ + // Operations that should result in error + 0: { // updates leading to overflows + []testVal{{"v1", 10}, {"v2", 10}}, + []testVal{{"v1", math.MaxInt64}}, + []testVal{{"v1", 10}, {"v2", 10}}, + true}, + 1: { // duplicate entries in changes + []testVal{{"v1", 10}, {"v2", 10}}, + []testVal{{"v1", 11}, {"v1", 22}}, + []testVal{{"v1", 10}, {"v2", 10}}, + true}, + 2: { // duplicate entries in removes + []testVal{{"v1", 10}, {"v2", 10}}, + []testVal{{"v1", 0}, {"v1", 0}}, + []testVal{{"v1", 10}, {"v2", 10}}, + true}, + 3: { // duplicate entries in removes + changes + []testVal{{"v1", 10}, {"v2", 10}}, + []testVal{{"v1", 0}, {"v2", 20}, {"v2", 30}, {"v1", 0}}, + []testVal{{"v1", 10}, {"v2", 10}}, + true}, + 4: { // update with negative voting power + []testVal{{"v1", 10}, {"v2", 10}}, + []testVal{{"v1", -123}}, + []testVal{{"v1", 10}, {"v2", 10}}, + true}, + 5: { // delete non existing validator + []testVal{{"v1", 10}, {"v2", 10}}, + []testVal{{"v3", 0}}, + []testVal{{"v1", 10}, {"v2", 10}}, + true}, + + // Operations that should be successful + 6: { // no changes + []testVal{{"v1", 10}, {"v2", 10}}, + []testVal{}, + []testVal{{"v1", 10}, {"v2", 10}}, + false}, + 7: { // voting power changes + []testVal{{"v1", 10}, {"v2", 10}}, + []testVal{{"v1", 11}, {"v2", 22}}, + []testVal{{"v1", 11}, {"v2", 22}}, + false}, + 8: { // add new validators + []testVal{{"v1", 10}, {"v2", 20}}, + []testVal{{"v3", 30}, {"v4", 40}}, + []testVal{{"v1", 10}, {"v2", 20}, {"v3", 30}, {"v4", 40}}, + false}, + 9: { // delete validators + []testVal{{"v1", 10}, {"v2", 20}, {"v3", 30}}, + []testVal{{"v2", 0}}, + []testVal{{"v1", 10}, {"v3", 30}}, + false}, + 10: { // delete all validators + []testVal{{"v1", 10}, {"v2", 20}, {"v3", 30}}, + []testVal{{"v1", 0}, {"v2", 0}, {"v3", 0}}, + []testVal{{"v1", 10}, {"v2", 20}, {"v3", 30}}, + true}, + } + + for i, tt := range valSetUpdatesBasicTests { + // create a new set and apply updates, keeping copies for the checks + valSet := createNewValidatorSet(tt.startVals) + valSetCopy := valSet.Copy() + valList := createNewValidatorList(tt.updateVals) + valListCopy := validatorListCopy(valList) + err := valSet.UpdateWithChangeSet(valList) + + if tt.expError { + // for errors check the validator set has not been changed + assert.Error(t, err, "test %d", i) + assert.Equal(t, valSet, valSetCopy, "test %v", i) + } else { + assert.NoError(t, err, "test %d", i) + } + // check the parameter list has not changed + assert.Equal(t, valList, valListCopy, "test %v", i) + + // check the final validator list is as expected and the set is properly scaled and centered. + assert.Equal(t, getValidatorResults(valSet.Validators), tt.expectedVals, "test %v", i) + verifyValidatorSet(t, valSet) + } +} + +func getValidatorResults(valList []*Validator) []testVal { + testList := make([]testVal, len(valList)) + for i, val := range valList { + testList[i].name = string(val.Address) + testList[i].power = val.VotingPower + } + return testList +} + +// Test that different permutations of an update give the same result. +func TestValSetUpdatesOrderTestsExecute(t *testing.T) { + // startVals - initial validators to create the set with + // updateVals - a sequence of updates to be applied to the set. + // updateVals is shuffled a number of times during testing to check for same resulting validator set. + valSetUpdatesOrderTests := []struct { + startVals []testVal + updateVals []testVal + }{ + 0: { // order of changes should not matter, the final validator sets should be the same + []testVal{{"v1", 10}, {"v2", 10}, {"v3", 30}, {"v4", 40}}, + []testVal{{"v1", 11}, {"v2", 22}, {"v3", 33}, {"v4", 44}}}, + + 1: { // order of additions should not matter + []testVal{{"v1", 10}, {"v2", 20}}, + []testVal{{"v3", 30}, {"v4", 40}, {"v5", 50}, {"v6", 60}}}, + + 2: { // order of removals should not matter + []testVal{{"v1", 10}, {"v2", 20}, {"v3", 30}, {"v4", 40}}, + []testVal{{"v1", 0}, {"v3", 0}, {"v4", 0}}}, + + 3: { // order of mixed operations should not matter + []testVal{{"v1", 10}, {"v2", 20}, {"v3", 30}, {"v4", 40}}, + []testVal{{"v1", 0}, {"v3", 0}, {"v2", 22}, {"v5", 50}, {"v4", 44}}}, + } + + for i, tt := range valSetUpdatesOrderTests { + // create a new set and apply updates + valSet := createNewValidatorSet(tt.startVals) + valSetCopy := valSet.Copy() + valList := createNewValidatorList(tt.updateVals) + assert.NoError(t, valSetCopy.UpdateWithChangeSet(valList)) + + // save the result as expected for next updates + valSetExp := valSetCopy.Copy() + + // perform at most 20 permutations on the updates and call UpdateWithChangeSet() + n := len(tt.updateVals) + maxNumPerms := cmn.MinInt(20, n*n) + for j := 0; j < maxNumPerms; j++ { + // create a copy of original set and apply a random permutation of updates + valSetCopy := valSet.Copy() + valList := createNewValidatorList(permutation(tt.updateVals)) + + // check there was no error and the set is properly scaled and centered. + assert.NoError(t, valSetCopy.UpdateWithChangeSet(valList), + "test %v failed for permutation %v", i, valList) + verifyValidatorSet(t, valSetCopy) + + // verify the resulting test is same as the expected + assert.Equal(t, valSetCopy, valSetExp, + "test %v failed for permutation %v", i, valList) + } + } +} + +// This tests the private function validator_set.go:applyUpdates() function, used only for additions and changes. +// Should perform a proper merge of updatedVals and startVals +func TestValSetApplyUpdatesTestsExecute(t *testing.T) { + valSetUpdatesBasicTests := []struct { + startVals []testVal + updateVals []testVal + expectedVals []testVal + }{ + // additions + 0: { // prepend + []testVal{{"v4", 44}, {"v5", 55}}, + []testVal{{"v1", 11}}, + []testVal{{"v1", 11}, {"v4", 44}, {"v5", 55}}}, + 1: { // append + []testVal{{"v4", 44}, {"v5", 55}}, + []testVal{{"v6", 66}}, + []testVal{{"v4", 44}, {"v5", 55}, {"v6", 66}}}, + 2: { // insert + []testVal{{"v4", 44}, {"v6", 66}}, + []testVal{{"v5", 55}}, + []testVal{{"v4", 44}, {"v5", 55}, {"v6", 66}}}, + 3: { // insert multi + []testVal{{"v4", 44}, {"v6", 66}, {"v9", 99}}, + []testVal{{"v5", 55}, {"v7", 77}, {"v8", 88}}, + []testVal{{"v4", 44}, {"v5", 55}, {"v6", 66}, {"v7", 77}, {"v8", 88}, {"v9", 99}}}, + // changes + 4: { // head + []testVal{{"v1", 111}, {"v2", 22}}, + []testVal{{"v1", 11}}, + []testVal{{"v1", 11}, {"v2", 22}}}, + 5: { // tail + []testVal{{"v1", 11}, {"v2", 222}}, + []testVal{{"v2", 22}}, + []testVal{{"v1", 11}, {"v2", 22}}}, + 6: { // middle + []testVal{{"v1", 11}, {"v2", 222}, {"v3", 33}}, + []testVal{{"v2", 22}}, + []testVal{{"v1", 11}, {"v2", 22}, {"v3", 33}}}, + 7: { // multi + []testVal{{"v1", 111}, {"v2", 222}, {"v3", 333}}, + []testVal{{"v1", 11}, {"v2", 22}, {"v3", 33}}, + []testVal{{"v1", 11}, {"v2", 22}, {"v3", 33}}}, + // additions and changes + 8: { + []testVal{{"v1", 111}, {"v2", 22}}, + []testVal{{"v1", 11}, {"v3", 33}, {"v4", 44}}, + []testVal{{"v1", 11}, {"v2", 22}, {"v3", 33}, {"v4", 44}}}, + } + + for i, tt := range valSetUpdatesBasicTests { + // create a new validator set with the start values + valSet := createNewValidatorSet(tt.startVals) + + // applyUpdates() with the update values + valList := createNewValidatorList(tt.updateVals) + valSet.applyUpdates(valList) + + // check the new list of validators for proper merge + assert.Equal(t, getValidatorResults(valSet.Validators), tt.expectedVals, "test %v", i) + verifyValidatorSet(t, valSet) + } +} + +func permutation(valList []testVal) []testVal { + if len(valList) == 0 { + return nil + } + permList := make([]testVal, len(valList)) + perm := rand.Perm(len(valList)) + for i, v := range perm { + permList[v] = valList[i] + } + return permList +} + +func createNewValidatorList(testValList []testVal) []*Validator { + valList := make([]*Validator, 0, len(testValList)) + for _, val := range testValList { + valList = append(valList, newValidator([]byte(val.name), val.power)) + } + return valList +} + +func createNewValidatorSet(testValList []testVal) *ValidatorSet { + valList := createNewValidatorList(testValList) + valSet := NewValidatorSet(valList) + return valSet +} + +func verifyValidatorSet(t *testing.T, valSet *ValidatorSet) { + // verify that the vals' tvp is set to the sum of the all vals voting powers + tvp := valSet.TotalVotingPower() + assert.Equal(t, valSet.totalVotingPower, tvp, + "expected TVP %d. Got %d, valSet=%s", tvp, valSet.totalVotingPower, valSet) + + // verify that validator priorities are centered + l := int64(len(valSet.Validators)) + tpp := valSet.TotalVotingPower() + assert.True(t, tpp <= l || tpp >= -l, + "expected total priority in (-%d, %d). Got %d", l, l, tpp) + + // verify that priorities are scaled + dist := computeMaxMinPriorityDiff(valSet) + assert.True(t, dist <= PriorityWindowSizeFactor*tvp, + "expected priority distance < %d. Got %d", PriorityWindowSizeFactor*tvp, dist) +} + +func BenchmarkUpdates(b *testing.B) { + const ( + n = 100 + m = 2000 + ) + // Init with n validators + vs := make([]*Validator, n) + for j := 0; j < n; j++ { + vs[j] = newValidator([]byte(fmt.Sprintf("v%d", j)), 100) + } + valSet := NewValidatorSet(vs) + l := len(valSet.Validators) + + // Make m new validators + newValList := make([]*Validator, m) + for j := 0; j < m; j++ { + newValList[j] = newValidator([]byte(fmt.Sprintf("v%d", j+l)), 1000) + } + b.ResetTimer() + + for i := 0; i < b.N; i++ { + // Add m validators to valSetCopy + valSetCopy := valSet.Copy() + assert.NoError(b, valSetCopy.UpdateWithChangeSet(newValList)) + } +} diff --git a/types/vote.go b/types/vote.go index 5a31f0e2..ad05d688 100644 --- a/types/vote.go +++ b/types/vote.go @@ -12,7 +12,7 @@ import ( const ( // MaxVoteBytes is a maximum vote size (including amino overhead). - MaxVoteBytes int64 = 200 + MaxVoteBytes int64 = 223 ) var ( @@ -43,41 +43,34 @@ func NewConflictingVoteError(val *Validator, voteA, voteB *Vote) *ErrVoteConflic } } -// Types of votes -// TODO Make a new type "VoteType" -const ( - VoteTypePrevote = byte(0x01) - VoteTypePrecommit = byte(0x02) -) - -func IsVoteTypeValid(type_ byte) bool { - switch type_ { - case VoteTypePrevote: - return true - case VoteTypePrecommit: - return true - default: - return false - } -} - // Address is hex bytes. type Address = crypto.Address -// Represents a prevote, precommit, or commit vote from validators for consensus. +// Vote represents a prevote, precommit, or commit vote from validators for +// consensus. type Vote struct { - ValidatorAddress Address `json:"validator_address"` - ValidatorIndex int `json:"validator_index"` - Height int64 `json:"height"` - Round int `json:"round"` - Timestamp time.Time `json:"timestamp"` - Type byte `json:"type"` - BlockID BlockID `json:"block_id"` // zero if vote is nil. - Signature []byte `json:"signature"` + Type SignedMsgType `json:"type"` + Height int64 `json:"height"` + Round int `json:"round"` + BlockID BlockID `json:"block_id"` // zero if vote is nil. + Timestamp time.Time `json:"timestamp"` + ValidatorAddress Address `json:"validator_address"` + ValidatorIndex int `json:"validator_index"` + Signature []byte `json:"signature"` +} + +// CommitSig converts the Vote to a CommitSig. +// If the Vote is nil, the CommitSig will be nil. +func (vote *Vote) CommitSig() *CommitSig { + if vote == nil { + return nil + } + cs := CommitSig(*vote) + return &cs } func (vote *Vote) SignBytes(chainID string) []byte { - bz, err := cdc.MarshalBinary(CanonicalizeVote(chainID, vote)) + bz, err := cdc.MarshalBinaryLengthPrefixed(CanonicalizeVote(chainID, vote)) if err != nil { panic(err) } @@ -95,9 +88,9 @@ func (vote *Vote) String() string { } var typeString string switch vote.Type { - case VoteTypePrevote: + case PrevoteType: typeString = "Prevote" - case VoteTypePrecommit: + case PrecommitType: typeString = "Precommit" default: cmn.PanicSanity("Unknown vote type") @@ -112,7 +105,8 @@ func (vote *Vote) String() string { typeString, cmn.Fingerprint(vote.BlockID.Hash), cmn.Fingerprint(vote.Signature), - CanonicalTime(vote.Timestamp)) + CanonicalTime(vote.Timestamp), + ) } func (vote *Vote) Verify(chainID string, pubKey crypto.PubKey) error { @@ -125,3 +119,43 @@ func (vote *Vote) Verify(chainID string, pubKey crypto.PubKey) error { } return nil } + +// ValidateBasic performs basic validation. +func (vote *Vote) ValidateBasic() error { + if !IsVoteTypeValid(vote.Type) { + return errors.New("Invalid Type") + } + if vote.Height < 0 { + return errors.New("Negative Height") + } + if vote.Round < 0 { + return errors.New("Negative Round") + } + + // NOTE: Timestamp validation is subtle and handled elsewhere. + + if err := vote.BlockID.ValidateBasic(); err != nil { + return fmt.Errorf("Wrong BlockID: %v", err) + } + // BlockID.ValidateBasic would not err if we for instance have an empty hash but a + // non-empty PartsSetHeader: + if !vote.BlockID.IsZero() && !vote.BlockID.IsComplete() { + return fmt.Errorf("BlockID must be either empty or complete, got: %v", vote.BlockID) + } + if len(vote.ValidatorAddress) != crypto.AddressSize { + return fmt.Errorf("Expected ValidatorAddress size to be %d bytes, got %d bytes", + crypto.AddressSize, + len(vote.ValidatorAddress), + ) + } + if vote.ValidatorIndex < 0 { + return errors.New("Negative ValidatorIndex") + } + if len(vote.Signature) == 0 { + return errors.New("Signature is missing") + } + if len(vote.Signature) > MaxSignatureSize { + return fmt.Errorf("Signature is too big (max: %d)", MaxSignatureSize) + } + return nil +} diff --git a/types/vote_set.go b/types/vote_set.go index dbcacbbd..14930da4 100644 --- a/types/vote_set.go +++ b/types/vote_set.go @@ -55,7 +55,7 @@ type VoteSet struct { chainID string height int64 round int - type_ byte + type_ SignedMsgType valSet *ValidatorSet mtx sync.Mutex @@ -68,7 +68,7 @@ type VoteSet struct { } // Constructs a new VoteSet struct used to accumulate votes for given height/round. -func NewVoteSet(chainID string, height int64, round int, type_ byte, valSet *ValidatorSet) *VoteSet { +func NewVoteSet(chainID string, height int64, round int, type_ SignedMsgType, valSet *ValidatorSet) *VoteSet { if height == 0 { cmn.PanicSanity("Cannot make VoteSet for height == 0, doesn't make sense.") } @@ -109,7 +109,7 @@ func (voteSet *VoteSet) Type() byte { if voteSet == nil { return 0x00 } - return voteSet.type_ + return byte(voteSet.type_) } func (voteSet *VoteSet) Size() int { @@ -158,7 +158,7 @@ func (voteSet *VoteSet) addVote(vote *Vote) (added bool, err error) { if (vote.Height != voteSet.height) || (vote.Round != voteSet.round) || (vote.Type != voteSet.type_) { - return false, errors.Wrapf(ErrVoteUnexpectedStep, "Got %d/%d/%d, expected %d/%d/%d", + return false, errors.Wrapf(ErrVoteUnexpectedStep, "Expected %d/%d/%d, but got %d/%d/%d", voteSet.height, voteSet.round, voteSet.type_, vote.Height, vote.Round, vote.Type) } @@ -381,7 +381,7 @@ func (voteSet *VoteSet) IsCommit() bool { if voteSet == nil { return false } - if voteSet.type_ != VoteTypePrecommit { + if voteSet.type_ != PrecommitType { return false } voteSet.mtx.Lock() @@ -529,8 +529,8 @@ func (voteSet *VoteSet) sumTotalFrac() (int64, int64, float64) { // Commit func (voteSet *VoteSet) MakeCommit() *Commit { - if voteSet.type_ != VoteTypePrecommit { - cmn.PanicSanity("Cannot MakeCommit() unless VoteSet.Type is VoteTypePrecommit") + if voteSet.type_ != PrecommitType { + cmn.PanicSanity("Cannot MakeCommit() unless VoteSet.Type is PrecommitType") } voteSet.mtx.Lock() defer voteSet.mtx.Unlock() @@ -541,11 +541,13 @@ func (voteSet *VoteSet) MakeCommit() *Commit { } // For every validator, get the precommit - votesCopy := make([]*Vote, len(voteSet.votes)) - copy(votesCopy, voteSet.votes) + commitSigs := make([]*CommitSig, len(voteSet.votes)) + for i, v := range voteSet.votes { + commitSigs[i] = v.CommitSig() + } return &Commit{ BlockID: *voteSet.maj23, - Precommits: votesCopy, + Precommits: commitSigs, } } diff --git a/types/vote_set_test.go b/types/vote_set_test.go index 995fb94b..59205efc 100644 --- a/types/vote_set_test.go +++ b/types/vote_set_test.go @@ -11,7 +11,7 @@ import ( ) // NOTE: privValidators are in order -func randVoteSet(height int64, round int, type_ byte, numValidators int, votingPower int64) (*VoteSet, *ValidatorSet, []PrivValidator) { +func randVoteSet(height int64, round int, type_ SignedMsgType, numValidators int, votingPower int64) (*VoteSet, *ValidatorSet, []PrivValidator) { valSet, privValidators := RandValidatorSet(numValidators, votingPower) return NewVoteSet("test_chain_id", height, round, type_, valSet), valSet, privValidators } @@ -41,7 +41,7 @@ func withRound(vote *Vote, round int) *Vote { // Convenience: Return new vote with different type func withType(vote *Vote, type_ byte) *Vote { vote = vote.Copy() - vote.Type = type_ + vote.Type = SignedMsgType(type_) return vote } @@ -61,12 +61,13 @@ func withBlockPartsHeader(vote *Vote, blockPartsHeader PartSetHeader) *Vote { func TestAddVote(t *testing.T) { height, round := int64(1), 0 - voteSet, _, privValidators := randVoteSet(height, round, VoteTypePrevote, 10, 1) + voteSet, _, privValidators := randVoteSet(height, round, PrevoteType, 10, 1) val0 := privValidators[0] // t.Logf(">> %v", voteSet) - if voteSet.GetByAddress(val0.GetAddress()) != nil { + val0Addr := val0.GetPubKey().Address() + if voteSet.GetByAddress(val0Addr) != nil { t.Errorf("Expected GetByAddress(val0.Address) to be nil") } if voteSet.BitArray().GetIndex(0) { @@ -78,11 +79,11 @@ func TestAddVote(t *testing.T) { } vote := &Vote{ - ValidatorAddress: val0.GetAddress(), + ValidatorAddress: val0Addr, ValidatorIndex: 0, // since privValidators are in order Height: height, Round: round, - Type: VoteTypePrevote, + Type: PrevoteType, Timestamp: tmtime.Now(), BlockID: BlockID{nil, PartSetHeader{}}, } @@ -91,7 +92,7 @@ func TestAddVote(t *testing.T) { t.Error(err) } - if voteSet.GetByAddress(val0.GetAddress()) == nil { + if voteSet.GetByAddress(val0Addr) == nil { t.Errorf("Expected GetByAddress(val0.Address) to be present") } if !voteSet.BitArray().GetIndex(0) { @@ -105,20 +106,21 @@ func TestAddVote(t *testing.T) { func Test2_3Majority(t *testing.T) { height, round := int64(1), 0 - voteSet, _, privValidators := randVoteSet(height, round, VoteTypePrevote, 10, 1) + voteSet, _, privValidators := randVoteSet(height, round, PrevoteType, 10, 1) voteProto := &Vote{ ValidatorAddress: nil, // NOTE: must fill in ValidatorIndex: -1, // NOTE: must fill in Height: height, Round: round, - Type: VoteTypePrevote, + Type: PrevoteType, Timestamp: tmtime.Now(), BlockID: BlockID{nil, PartSetHeader{}}, } // 6 out of 10 voted for nil. for i := 0; i < 6; i++ { - vote := withValidator(voteProto, privValidators[i].GetAddress(), i) + addr := privValidators[i].GetPubKey().Address() + vote := withValidator(voteProto, addr, i) _, err := signAddVote(privValidators[i], vote, voteSet) if err != nil { t.Error(err) @@ -131,7 +133,8 @@ func Test2_3Majority(t *testing.T) { // 7th validator voted for some blockhash { - vote := withValidator(voteProto, privValidators[6].GetAddress(), 6) + addr := privValidators[6].GetPubKey().Address() + vote := withValidator(voteProto, addr, 6) _, err := signAddVote(privValidators[6], withBlockHash(vote, cmn.RandBytes(32)), voteSet) if err != nil { t.Error(err) @@ -144,7 +147,8 @@ func Test2_3Majority(t *testing.T) { // 8th validator voted for nil. { - vote := withValidator(voteProto, privValidators[7].GetAddress(), 7) + addr := privValidators[7].GetPubKey().Address() + vote := withValidator(voteProto, addr, 7) _, err := signAddVote(privValidators[7], vote, voteSet) if err != nil { t.Error(err) @@ -158,7 +162,7 @@ func Test2_3Majority(t *testing.T) { func Test2_3MajorityRedux(t *testing.T) { height, round := int64(1), 0 - voteSet, _, privValidators := randVoteSet(height, round, VoteTypePrevote, 100, 1) + voteSet, _, privValidators := randVoteSet(height, round, PrevoteType, 100, 1) blockHash := crypto.CRandBytes(32) blockPartsTotal := 123 @@ -170,13 +174,14 @@ func Test2_3MajorityRedux(t *testing.T) { Height: height, Round: round, Timestamp: tmtime.Now(), - Type: VoteTypePrevote, + Type: PrevoteType, BlockID: BlockID{blockHash, blockPartsHeader}, } // 66 out of 100 voted for nil. for i := 0; i < 66; i++ { - vote := withValidator(voteProto, privValidators[i].GetAddress(), i) + addr := privValidators[i].GetPubKey().Address() + vote := withValidator(voteProto, addr, i) _, err := signAddVote(privValidators[i], vote, voteSet) if err != nil { t.Error(err) @@ -189,7 +194,8 @@ func Test2_3MajorityRedux(t *testing.T) { // 67th validator voted for nil { - vote := withValidator(voteProto, privValidators[66].GetAddress(), 66) + adrr := privValidators[66].GetPubKey().Address() + vote := withValidator(voteProto, adrr, 66) _, err := signAddVote(privValidators[66], withBlockHash(vote, nil), voteSet) if err != nil { t.Error(err) @@ -202,7 +208,8 @@ func Test2_3MajorityRedux(t *testing.T) { // 68th validator voted for a different BlockParts PartSetHeader { - vote := withValidator(voteProto, privValidators[67].GetAddress(), 67) + addr := privValidators[67].GetPubKey().Address() + vote := withValidator(voteProto, addr, 67) blockPartsHeader := PartSetHeader{blockPartsTotal, crypto.CRandBytes(32)} _, err := signAddVote(privValidators[67], withBlockPartsHeader(vote, blockPartsHeader), voteSet) if err != nil { @@ -216,7 +223,8 @@ func Test2_3MajorityRedux(t *testing.T) { // 69th validator voted for different BlockParts Total { - vote := withValidator(voteProto, privValidators[68].GetAddress(), 68) + addr := privValidators[68].GetPubKey().Address() + vote := withValidator(voteProto, addr, 68) blockPartsHeader := PartSetHeader{blockPartsTotal + 1, blockPartsHeader.Hash} _, err := signAddVote(privValidators[68], withBlockPartsHeader(vote, blockPartsHeader), voteSet) if err != nil { @@ -230,7 +238,8 @@ func Test2_3MajorityRedux(t *testing.T) { // 70th validator voted for different BlockHash { - vote := withValidator(voteProto, privValidators[69].GetAddress(), 69) + addr := privValidators[69].GetPubKey().Address() + vote := withValidator(voteProto, addr, 69) _, err := signAddVote(privValidators[69], withBlockHash(vote, cmn.RandBytes(32)), voteSet) if err != nil { t.Error(err) @@ -243,7 +252,8 @@ func Test2_3MajorityRedux(t *testing.T) { // 71st validator voted for the right BlockHash & BlockPartsHeader { - vote := withValidator(voteProto, privValidators[70].GetAddress(), 70) + addr := privValidators[70].GetPubKey().Address() + vote := withValidator(voteProto, addr, 70) _, err := signAddVote(privValidators[70], vote, voteSet) if err != nil { t.Error(err) @@ -257,7 +267,7 @@ func Test2_3MajorityRedux(t *testing.T) { func TestBadVotes(t *testing.T) { height, round := int64(1), 0 - voteSet, _, privValidators := randVoteSet(height, round, VoteTypePrevote, 10, 1) + voteSet, _, privValidators := randVoteSet(height, round, PrevoteType, 10, 1) voteProto := &Vote{ ValidatorAddress: nil, @@ -265,13 +275,14 @@ func TestBadVotes(t *testing.T) { Height: height, Round: round, Timestamp: tmtime.Now(), - Type: VoteTypePrevote, + Type: PrevoteType, BlockID: BlockID{nil, PartSetHeader{}}, } // val0 votes for nil. { - vote := withValidator(voteProto, privValidators[0].GetAddress(), 0) + addr := privValidators[0].GetPubKey().Address() + vote := withValidator(voteProto, addr, 0) added, err := signAddVote(privValidators[0], vote, voteSet) if !added || err != nil { t.Errorf("Expected VoteSet.Add to succeed") @@ -280,7 +291,8 @@ func TestBadVotes(t *testing.T) { // val0 votes again for some block. { - vote := withValidator(voteProto, privValidators[0].GetAddress(), 0) + addr := privValidators[0].GetPubKey().Address() + vote := withValidator(voteProto, addr, 0) added, err := signAddVote(privValidators[0], withBlockHash(vote, cmn.RandBytes(32)), voteSet) if added || err == nil { t.Errorf("Expected VoteSet.Add to fail, conflicting vote.") @@ -289,7 +301,8 @@ func TestBadVotes(t *testing.T) { // val1 votes on another height { - vote := withValidator(voteProto, privValidators[1].GetAddress(), 1) + addr := privValidators[1].GetPubKey().Address() + vote := withValidator(voteProto, addr, 1) added, err := signAddVote(privValidators[1], withHeight(vote, height+1), voteSet) if added || err == nil { t.Errorf("Expected VoteSet.Add to fail, wrong height") @@ -298,7 +311,8 @@ func TestBadVotes(t *testing.T) { // val2 votes on another round { - vote := withValidator(voteProto, privValidators[2].GetAddress(), 2) + addr := privValidators[2].GetPubKey().Address() + vote := withValidator(voteProto, addr, 2) added, err := signAddVote(privValidators[2], withRound(vote, round+1), voteSet) if added || err == nil { t.Errorf("Expected VoteSet.Add to fail, wrong round") @@ -307,8 +321,9 @@ func TestBadVotes(t *testing.T) { // val3 votes of another type. { - vote := withValidator(voteProto, privValidators[3].GetAddress(), 3) - added, err := signAddVote(privValidators[3], withType(vote, VoteTypePrecommit), voteSet) + addr := privValidators[3].GetPubKey().Address() + vote := withValidator(voteProto, addr, 3) + added, err := signAddVote(privValidators[3], withType(vote, byte(PrecommitType)), voteSet) if added || err == nil { t.Errorf("Expected VoteSet.Add to fail, wrong type") } @@ -317,7 +332,7 @@ func TestBadVotes(t *testing.T) { func TestConflicts(t *testing.T) { height, round := int64(1), 0 - voteSet, _, privValidators := randVoteSet(height, round, VoteTypePrevote, 4, 1) + voteSet, _, privValidators := randVoteSet(height, round, PrevoteType, 4, 1) blockHash1 := cmn.RandBytes(32) blockHash2 := cmn.RandBytes(32) @@ -327,13 +342,14 @@ func TestConflicts(t *testing.T) { Height: height, Round: round, Timestamp: tmtime.Now(), - Type: VoteTypePrevote, + Type: PrevoteType, BlockID: BlockID{nil, PartSetHeader{}}, } + val0Addr := privValidators[0].GetPubKey().Address() // val0 votes for nil. { - vote := withValidator(voteProto, privValidators[0].GetAddress(), 0) + vote := withValidator(voteProto, val0Addr, 0) added, err := signAddVote(privValidators[0], vote, voteSet) if !added || err != nil { t.Errorf("Expected VoteSet.Add to succeed") @@ -342,7 +358,7 @@ func TestConflicts(t *testing.T) { // val0 votes again for blockHash1. { - vote := withValidator(voteProto, privValidators[0].GetAddress(), 0) + vote := withValidator(voteProto, val0Addr, 0) added, err := signAddVote(privValidators[0], withBlockHash(vote, blockHash1), voteSet) if added { t.Errorf("Expected VoteSet.Add to fail, conflicting vote.") @@ -357,7 +373,7 @@ func TestConflicts(t *testing.T) { // val0 votes again for blockHash1. { - vote := withValidator(voteProto, privValidators[0].GetAddress(), 0) + vote := withValidator(voteProto, val0Addr, 0) added, err := signAddVote(privValidators[0], withBlockHash(vote, blockHash1), voteSet) if !added { t.Errorf("Expected VoteSet.Add to succeed, called SetPeerMaj23().") @@ -372,7 +388,7 @@ func TestConflicts(t *testing.T) { // val0 votes again for blockHash1. { - vote := withValidator(voteProto, privValidators[0].GetAddress(), 0) + vote := withValidator(voteProto, val0Addr, 0) added, err := signAddVote(privValidators[0], withBlockHash(vote, blockHash2), voteSet) if added { t.Errorf("Expected VoteSet.Add to fail, duplicate SetPeerMaj23() from peerA") @@ -384,7 +400,8 @@ func TestConflicts(t *testing.T) { // val1 votes for blockHash1. { - vote := withValidator(voteProto, privValidators[1].GetAddress(), 1) + addr := privValidators[1].GetPubKey().Address() + vote := withValidator(voteProto, addr, 1) added, err := signAddVote(privValidators[1], withBlockHash(vote, blockHash1), voteSet) if !added || err != nil { t.Errorf("Expected VoteSet.Add to succeed") @@ -401,7 +418,8 @@ func TestConflicts(t *testing.T) { // val2 votes for blockHash2. { - vote := withValidator(voteProto, privValidators[2].GetAddress(), 2) + addr := privValidators[2].GetPubKey().Address() + vote := withValidator(voteProto, addr, 2) added, err := signAddVote(privValidators[2], withBlockHash(vote, blockHash2), voteSet) if !added || err != nil { t.Errorf("Expected VoteSet.Add to succeed") @@ -421,7 +439,8 @@ func TestConflicts(t *testing.T) { // val2 votes for blockHash1. { - vote := withValidator(voteProto, privValidators[2].GetAddress(), 2) + addr := privValidators[2].GetPubKey().Address() + vote := withValidator(voteProto, addr, 2) added, err := signAddVote(privValidators[2], withBlockHash(vote, blockHash1), voteSet) if !added { t.Errorf("Expected VoteSet.Add to succeed") @@ -447,7 +466,7 @@ func TestConflicts(t *testing.T) { func TestMakeCommit(t *testing.T) { height, round := int64(1), 0 - voteSet, _, privValidators := randVoteSet(height, round, VoteTypePrecommit, 10, 1) + voteSet, _, privValidators := randVoteSet(height, round, PrecommitType, 10, 1) blockHash, blockPartsHeader := crypto.CRandBytes(32), PartSetHeader{123, crypto.CRandBytes(32)} voteProto := &Vote{ @@ -456,13 +475,14 @@ func TestMakeCommit(t *testing.T) { Height: height, Round: round, Timestamp: tmtime.Now(), - Type: VoteTypePrecommit, + Type: PrecommitType, BlockID: BlockID{blockHash, blockPartsHeader}, } // 6 out of 10 voted for some block. for i := 0; i < 6; i++ { - vote := withValidator(voteProto, privValidators[i].GetAddress(), i) + addr := privValidators[i].GetPubKey().Address() + vote := withValidator(voteProto, addr, i) _, err := signAddVote(privValidators[i], vote, voteSet) if err != nil { t.Error(err) @@ -474,7 +494,8 @@ func TestMakeCommit(t *testing.T) { // 7th voted for some other block. { - vote := withValidator(voteProto, privValidators[6].GetAddress(), 6) + addr := privValidators[6].GetPubKey().Address() + vote := withValidator(voteProto, addr, 6) vote = withBlockHash(vote, cmn.RandBytes(32)) vote = withBlockPartsHeader(vote, PartSetHeader{123, cmn.RandBytes(32)}) @@ -486,7 +507,8 @@ func TestMakeCommit(t *testing.T) { // The 8th voted like everyone else. { - vote := withValidator(voteProto, privValidators[7].GetAddress(), 7) + addr := privValidators[7].GetPubKey().Address() + vote := withValidator(voteProto, addr, 7) _, err := signAddVote(privValidators[7], vote, voteSet) if err != nil { t.Error(err) diff --git a/types/vote_test.go b/types/vote_test.go index d0c41a06..e4bf658b 100644 --- a/types/vote_test.go +++ b/types/vote_test.go @@ -7,17 +7,18 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + amino "github.com/tendermint/go-amino" + "github.com/tendermint/tendermint/crypto" "github.com/tendermint/tendermint/crypto/ed25519" "github.com/tendermint/tendermint/crypto/tmhash" - tmtime "github.com/tendermint/tendermint/types/time" ) func examplePrevote() *Vote { - return exampleVote(VoteTypePrevote) + return exampleVote(byte(PrevoteType)) } func examplePrecommit() *Vote { - return exampleVote(VoteTypePrecommit) + return exampleVote(byte(PrecommitType)) } func exampleVote(t byte) *Vote { @@ -27,12 +28,10 @@ func exampleVote(t byte) *Vote { } return &Vote{ - ValidatorAddress: tmhash.Sum([]byte("validator_address")), - ValidatorIndex: 56789, - Height: 12345, - Round: 2, - Timestamp: stamp, - Type: t, + Type: SignedMsgType(t), + Height: 12345, + Round: 2, + Timestamp: stamp, BlockID: BlockID{ Hash: tmhash.Sum([]byte("blockID_hash")), PartsHeader: PartSetHeader{ @@ -40,19 +39,119 @@ func exampleVote(t byte) *Vote { Hash: tmhash.Sum([]byte("blockID_part_set_header_hash")), }, }, + ValidatorAddress: crypto.AddressHash([]byte("validator_address")), + ValidatorIndex: 56789, } } +// Ensure that Vote and CommitSig have the same encoding. +// This ensures using CommitSig isn't a breaking change. +// This test will fail and can be removed once CommitSig contains only sigs and +// timestamps. +func TestVoteEncoding(t *testing.T) { + vote := examplePrecommit() + commitSig := vote.CommitSig() + cdc := amino.NewCodec() + bz1 := cdc.MustMarshalBinaryBare(vote) + bz2 := cdc.MustMarshalBinaryBare(commitSig) + assert.Equal(t, bz1, bz2) +} + func TestVoteSignable(t *testing.T) { vote := examplePrecommit() signBytes := vote.SignBytes("test_chain_id") - expected, err := cdc.MarshalBinary(CanonicalizeVote("test_chain_id", vote)) + expected, err := cdc.MarshalBinaryLengthPrefixed(CanonicalizeVote("test_chain_id", vote)) require.NoError(t, err) require.Equal(t, expected, signBytes, "Got unexpected sign bytes for Vote.") } +func TestVoteSignableTestVectors(t *testing.T) { + vote := CanonicalizeVote("", &Vote{Height: 1, Round: 1}) + + tests := []struct { + canonicalVote CanonicalVote + want []byte + }{ + { + CanonicalizeVote("", &Vote{}), + // NOTE: Height and Round are skipped here. This case needs to be considered while parsing. + // []byte{0x2a, 0x9, 0x9, 0x0, 0x9, 0x6e, 0x88, 0xf1, 0xff, 0xff, 0xff}, + []byte{0x2a, 0xb, 0x8, 0x80, 0x92, 0xb8, 0xc3, 0x98, 0xfe, 0xff, 0xff, 0xff, 0x1}, + }, + // with proper (fixed size) height and round (PreCommit): + { + CanonicalizeVote("", &Vote{Height: 1, Round: 1, Type: PrecommitType}), + []byte{ + 0x8, // (field_number << 3) | wire_type + 0x2, // PrecommitType + 0x11, // (field_number << 3) | wire_type + 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, // height + 0x19, // (field_number << 3) | wire_type + 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, // round + 0x2a, // (field_number << 3) | wire_type + // remaining fields (timestamp): + 0xb, 0x8, 0x80, 0x92, 0xb8, 0xc3, 0x98, 0xfe, 0xff, 0xff, 0xff, 0x1}, + }, + // with proper (fixed size) height and round (PreVote): + { + CanonicalizeVote("", &Vote{Height: 1, Round: 1, Type: PrevoteType}), + []byte{ + 0x8, // (field_number << 3) | wire_type + 0x1, // PrevoteType + 0x11, // (field_number << 3) | wire_type + 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, // height + 0x19, // (field_number << 3) | wire_type + 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, // round + 0x2a, // (field_number << 3) | wire_type + // remaining fields (timestamp): + 0xb, 0x8, 0x80, 0x92, 0xb8, 0xc3, 0x98, 0xfe, 0xff, 0xff, 0xff, 0x1}, + }, + { + vote, + []byte{ + 0x11, // (field_number << 3) | wire_type + 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, // height + 0x19, // (field_number << 3) | wire_type + 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, // round + // remaining fields (timestamp): + 0x2a, + 0xb, 0x8, 0x80, 0x92, 0xb8, 0xc3, 0x98, 0xfe, 0xff, 0xff, 0xff, 0x1}, + }, + // containing non-empty chain_id: + { + CanonicalizeVote("test_chain_id", &Vote{Height: 1, Round: 1}), + []byte{ + 0x11, // (field_number << 3) | wire_type + 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, // height + 0x19, // (field_number << 3) | wire_type + 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, // round + // remaining fields: + 0x2a, // (field_number << 3) | wire_type + 0xb, 0x8, 0x80, 0x92, 0xb8, 0xc3, 0x98, 0xfe, 0xff, 0xff, 0xff, 0x1, // timestamp + 0x32, // (field_number << 3) | wire_type + 0xd, 0x74, 0x65, 0x73, 0x74, 0x5f, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x5f, 0x69, 0x64}, // chainID + }, + } + for i, tc := range tests { + got, err := cdc.MarshalBinaryBare(tc.canonicalVote) + require.NoError(t, err) + + require.Equal(t, tc.want, got, "test case #%v: got unexpected sign bytes for Vote.", i) + } +} + +func TestVoteProposalNotEq(t *testing.T) { + cv := CanonicalizeVote("", &Vote{Height: 1, Round: 1}) + p := CanonicalizeProposal("", &Proposal{Height: 1, Round: 1}) + vb, err := cdc.MarshalBinaryLengthPrefixed(cv) + require.NoError(t, err) + pb, err := cdc.MarshalBinaryLengthPrefixed(p) + require.NoError(t, err) + require.NotEqual(t, vb, pb) +} + func TestVoteVerifySignature(t *testing.T) { privVal := NewMockPV() pubkey := privVal.GetPubKey() @@ -70,9 +169,9 @@ func TestVoteVerifySignature(t *testing.T) { // serialize, deserialize and verify again.... precommit := new(Vote) - bs, err := cdc.MarshalBinary(vote) + bs, err := cdc.MarshalBinaryLengthPrefixed(vote) require.NoError(t, err) - err = cdc.UnmarshalBinary(bs, &precommit) + err = cdc.UnmarshalBinaryLengthPrefixed(bs, &precommit) require.NoError(t, err) // verify the transmitted vote @@ -85,12 +184,12 @@ func TestVoteVerifySignature(t *testing.T) { func TestIsVoteTypeValid(t *testing.T) { tc := []struct { name string - in byte + in SignedMsgType out bool }{ - {"Prevote", VoteTypePrevote, true}, - {"Precommit", VoteTypePrecommit, true}, - {"InvalidType", byte(3), false}, + {"Prevote", PrevoteType, true}, + {"Precommit", PrecommitType, true}, + {"InvalidType", SignedMsgType(0x3), false}, } for _, tt := range tc { @@ -122,13 +221,17 @@ func TestVoteVerify(t *testing.T) { } func TestMaxVoteBytes(t *testing.T) { + // time is varint encoded so need to pick the max. + // year int, month Month, day, hour, min, sec, nsec int, loc *Location + timestamp := time.Date(math.MaxInt64, 0, 0, 0, 0, 0, math.MaxInt64, time.UTC) + vote := &Vote{ - ValidatorAddress: tmhash.Sum([]byte("validator_address")), + ValidatorAddress: crypto.AddressHash([]byte("validator_address")), ValidatorIndex: math.MaxInt64, Height: math.MaxInt64, Round: math.MaxInt64, - Timestamp: tmtime.Now(), - Type: VoteTypePrevote, + Timestamp: timestamp, + Type: PrevoteType, BlockID: BlockID{ Hash: tmhash.Sum([]byte("blockID_hash")), PartsHeader: PartSetHeader{ @@ -142,8 +245,50 @@ func TestMaxVoteBytes(t *testing.T) { err := privVal.SignVote("test_chain_id", vote) require.NoError(t, err) - bz, err := cdc.MarshalBinary(vote) + bz, err := cdc.MarshalBinaryLengthPrefixed(vote) require.NoError(t, err) assert.EqualValues(t, MaxVoteBytes, len(bz)) } + +func TestVoteString(t *testing.T) { + str := examplePrecommit().String() + expected := `Vote{56789:6AF1F4111082 12345/02/2(Precommit) 8B01023386C3 000000000000 @ 2017-12-25T03:00:01.234Z}` + if str != expected { + t.Errorf("Got unexpected string for Vote. Expected:\n%v\nGot:\n%v", expected, str) + } + + str2 := examplePrevote().String() + expected = `Vote{56789:6AF1F4111082 12345/02/1(Prevote) 8B01023386C3 000000000000 @ 2017-12-25T03:00:01.234Z}` + if str2 != expected { + t.Errorf("Got unexpected string for Vote. Expected:\n%v\nGot:\n%v", expected, str2) + } +} + +func TestVoteValidateBasic(t *testing.T) { + privVal := NewMockPV() + + testCases := []struct { + testName string + malleateVote func(*Vote) + expectErr bool + }{ + {"Good Vote", func(v *Vote) {}, false}, + {"Negative Height", func(v *Vote) { v.Height = -1 }, true}, + {"Negative Round", func(v *Vote) { v.Round = -1 }, true}, + {"Invalid BlockID", func(v *Vote) { v.BlockID = BlockID{[]byte{1, 2, 3}, PartSetHeader{111, []byte("blockparts")}} }, true}, + {"Invalid Address", func(v *Vote) { v.ValidatorAddress = make([]byte, 1) }, true}, + {"Invalid ValidatorIndex", func(v *Vote) { v.ValidatorIndex = -1 }, true}, + {"Invalid Signature", func(v *Vote) { v.Signature = nil }, true}, + {"Too big Signature", func(v *Vote) { v.Signature = make([]byte, MaxSignatureSize+1) }, true}, + } + for _, tc := range testCases { + t.Run(tc.testName, func(t *testing.T) { + vote := examplePrecommit() + err := privVal.SignVote("test_chain_id", vote) + require.NoError(t, err) + tc.malleateVote(vote) + assert.Equal(t, tc.expectErr, vote.ValidateBasic() != nil, "Validate Basic had an unexpected result") + }) + } +} diff --git a/types/wire.go b/types/wire.go index f3c314fa..81a7bf76 100644 --- a/types/wire.go +++ b/types/wire.go @@ -2,7 +2,7 @@ package types import ( amino "github.com/tendermint/go-amino" - "github.com/tendermint/tendermint/crypto/encoding/amino" + cryptoAmino "github.com/tendermint/tendermint/crypto/encoding/amino" ) var cdc = amino.NewCodec() @@ -20,3 +20,8 @@ func RegisterBlockAmino(cdc *amino.Codec) { func GetCodec() *amino.Codec { return cdc } + +// For testing purposes only +func RegisterMockEvidencesGlobal() { + RegisterMockEvidences(cdc) +} diff --git a/version/version.go b/version/version.go index 5a089141..b20223c2 100644 --- a/version/version.go +++ b/version/version.go @@ -18,22 +18,30 @@ const ( // TMCoreSemVer is the current version of Tendermint Core. // It's the Semantic Version of the software. // Must be a string because scripts like dist.sh read this file. - TMCoreSemVer = "0.25.0" + // XXX: Don't change the name of this variable or you will break + // automation :) + TMCoreSemVer = "0.29.2" // ABCISemVer is the semantic version of the ABCI library - ABCISemVer = "0.14.0" + ABCISemVer = "0.15.0" ABCIVersion = ABCISemVer ) // Protocol is used for implementation agnostic versioning. type Protocol uint64 +// Uint64 returns the Protocol version as a uint64, +// eg. for compatibility with ABCI types. +func (p Protocol) Uint64() uint64 { + return uint64(p) +} + var ( // P2PProtocol versions all p2p behaviour and msgs. - P2PProtocol Protocol = 4 + P2PProtocol Protocol = 6 // BlockProtocol versions all block data structures and processing. - BlockProtocol Protocol = 7 + BlockProtocol Protocol = 9 ) //------------------------------------------------------------------------