Example of more functional proposer API

consensus/reactor.go

@@ -438,9 +438,9 @@ func (conR *ConsensusReactor) broadcastHasVoteMessage(vote *types.Vote) {
 
 func makeRoundStepMessage(rs *cstypes.RoundState) (nrsMsg *NewRoundStepMessage) {
 	nrsMsg = &NewRoundStepMessage{
-		Height: rs.Height,
-		Round:  rs.Round,
-		Step:   rs.Step,
+		Height:                rs.Height,
+		Round:                 rs.Round,
+		Step:                  rs.Step,
 		SecondsSinceStartTime: int(time.Since(rs.StartTime).Seconds()),
 		LastCommitRound:       rs.LastCommit.Round(),
 	}

consensus/replay.go

@@ -309,7 +309,7 @@ func (h *Handshaker) ReplayBlocks(
 				return nil, fmt.Errorf("Validator set is nil in genesis and still empty after InitChain")
 			}
 		}
-		
+
 		if res.ConsensusParams != nil {
 			state.ConsensusParams = types.PB2TM.ConsensusParams(res.ConsensusParams)
 		}

consensus/state.go

@@ -743,9 +743,13 @@ func (cs *ConsensusState) enterNewRound(height int64, round int) {
 
 	// Increment validators if necessary
 	validators := cs.Validators
-	if cs.Round < round {
-		validators = validators.Copy()
-		validators.IncrementProposerPriority(round - cs.Round)
+	//if cs.Round < round {
+	//	validators = validators.Copy()
+	//	validators.IncrementProposerPriority(round - cs.Round)
+	//}
+
+	for i := cs.Round; i < round; i++ {
+		validators = validators.UpdateProposerPriority()
 	}
 
 	// Setup new round

libs/db/test_4d486965727875417563654d.db/CURRENT

@@ -0,0 +1 @@
+MANIFEST-000000

libs/db/test_4d486965727875417563654d.db/LOG

@@ -0,0 +1,8 @@
+=============== Aug 30, 2018 (CEST) ===============
+14:26:35.934461 log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed
+14:26:35.935099 db@open opening
+14:26:35.937646 version@stat F·[] S·0B[] Sc·[]
+14:26:35.937857 db@janitor F·2 G·0
+14:26:35.937877 db@open done T·2.764376ms
+14:26:35.937934 db@close closing
+14:26:35.937988 db@close done T·51.405µs

libs/db/test_4f37624341786461414b6967.db/CURRENT

@@ -0,0 +1 @@
+MANIFEST-000000

libs/db/test_4f37624341786461414b6967.db/LOG

@@ -0,0 +1,8 @@
+=============== Sep 14, 2018 (CEST) ===============
+14:21:29.692786 log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed
+14:21:29.693349 db@open opening
+14:21:29.693582 version@stat F·[] S·0B[] Sc·[]
+14:21:29.693784 db@janitor F·2 G·0
+14:21:29.693809 db@open done T·448.919µs
+14:21:29.693863 db@close closing
+14:21:29.693911 db@close done T·46.842µs

libs/db/test_4f6453636f6f426f39495765.db/CURRENT

@@ -0,0 +1 @@
+MANIFEST-000000

libs/db/test_4f6453636f6f426f39495765.db/LOG

@@ -0,0 +1,8 @@
+=============== Sep 19, 2018 (CEST) ===============
+14:37:09.911532 log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed
+14:37:09.912117 db@open opening
+14:37:09.914566 version@stat F·[] S·0B[] Sc·[]
+14:37:09.914829 db@janitor F·2 G·0
+14:37:09.914859 db@open done T·2.726834ms
+14:37:09.914925 db@close closing
+14:37:09.914975 db@close done T·49.799µs

libs/db/test_5072554f4d39643159514836.db/CURRENT

@@ -0,0 +1 @@
+MANIFEST-000000

libs/db/test_5072554f4d39643159514836.db/LOG

@@ -0,0 +1,8 @@
+=============== Sep 13, 2018 (CEST) ===============
+14:44:55.955262 log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed
+14:44:55.955874 db@open opening
+14:44:55.958189 version@stat F·[] S·0B[] Sc·[]
+14:44:55.958409 db@janitor F·2 G·0
+14:44:55.958432 db@open done T·2.545213ms
+14:44:55.958493 db@close closing
+14:44:55.958591 db@close done T·94.523µs

libs/db/test_634d416e304c6e6e6c577641.db/CURRENT

@@ -0,0 +1 @@
+MANIFEST-000000

libs/db/test_634d416e304c6e6e6c577641.db/LOG

@@ -0,0 +1,8 @@
+=============== Sep 13, 2018 (CEST) ===============
+15:28:02.333289 log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed
+15:28:02.333838 db@open opening
+15:28:02.336552 version@stat F·[] S·0B[] Sc·[]
+15:28:02.336795 db@janitor F·2 G·0
+15:28:02.336819 db@open done T·2.967026ms
+15:28:02.336874 db@close closing
+15:28:02.336919 db@close done T·44.064µs

libs/db/test_6e4e6e653530765a50726c44.db/CURRENT

@@ -0,0 +1 @@
+MANIFEST-000000

libs/db/test_6e4e6e653530765a50726c44.db/LOG

@@ -0,0 +1,8 @@
+=============== Sep 19, 2018 (CEST) ===============
+13:11:03.442075 log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed
+13:11:03.442667 db@open opening
+13:11:03.445054 version@stat F·[] S·0B[] Sc·[]
+13:11:03.445349 db@janitor F·2 G·0
+13:11:03.445379 db@open done T·2.700203ms
+13:11:03.445456 db@close closing
+13:11:03.445536 db@close done T·77.839µs

libs/db/test_6e546b586568583737436b6b.db/CURRENT

@@ -0,0 +1 @@
+MANIFEST-000000

libs/db/test_6e546b586568583737436b6b.db/LOG

@@ -0,0 +1,8 @@
+=============== Sep 14, 2018 (CEST) ===============
+13:49:47.358574 log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed
+13:49:47.359172 db@open opening
+13:49:47.359435 version@stat F·[] S·0B[] Sc·[]
+13:49:47.359678 db@janitor F·2 G·0
+13:49:47.359699 db@open done T·512.135µs
+13:49:47.359751 db@close closing
+13:49:47.359816 db@close done T·60.197µs

libs/db/test_7250714f47516e5454566873.db/CURRENT

@@ -0,0 +1 @@
+MANIFEST-000000

libs/db/test_7250714f47516e5454566873.db/LOG

@@ -0,0 +1,8 @@
+=============== Sep 13, 2018 (CEST) ===============
+15:21:32.553457 log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed
+15:21:32.554059 db@open opening
+15:21:32.556630 version@stat F·[] S·0B[] Sc·[]
+15:21:32.556897 db@janitor F·2 G·0
+15:21:32.556920 db@open done T·2.846122ms
+15:21:32.556970 db@close closing
+15:21:32.557032 db@close done T·60.719µs

libs/db/test_77377739514d334f67563239.db/CURRENT

@@ -0,0 +1 @@
+MANIFEST-000000

libs/db/test_77377739514d334f67563239.db/LOG

@@ -0,0 +1,8 @@
+=============== Sep 19, 2018 (CEST) ===============
+13:35:56.985348 log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed
+13:35:56.985952 db@open opening
+13:35:56.988432 version@stat F·[] S·0B[] Sc·[]
+13:35:56.988667 db@janitor F·2 G·0
+13:35:56.988689 db@open done T·2.72436ms
+13:35:56.988750 db@close closing
+13:35:56.988822 db@close done T·70.216µs

libs/db/test_786e30656d4f53665354304b.db/CURRENT

@@ -0,0 +1 @@
+MANIFEST-000000

libs/db/test_786e30656d4f53665354304b.db/LOG

@@ -0,0 +1,8 @@
+=============== Sep 19, 2018 (CEST) ===============
+14:35:04.677460 log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed
+14:35:04.678109 db@open opening
+14:35:04.678354 version@stat F·[] S·0B[] Sc·[]
+14:35:04.678550 db@janitor F·2 G·0
+14:35:04.678573 db@open done T·449.111µs
+14:35:04.678626 db@close closing
+14:35:04.678695 db@close done T·65.663µs

p2p/metrics.go

@@ -62,7 +62,7 @@ func PrometheusMetrics(namespace string) *Metrics {
 // NopMetrics returns no-op Metrics.
 func NopMetrics() *Metrics {
 	return &Metrics{
-		Peers: discard.NewGauge(),
+		Peers:                 discard.NewGauge(),
 		PeerReceiveBytesTotal: discard.NewCounter(),
 		PeerSendBytesTotal:    discard.NewCounter(),
 		PeerPendingSendBytes:  discard.NewGauge(),

state/execution.go

@@ -343,6 +343,72 @@ func validateValidatorUpdates(abciUpdates []abci.ValidatorUpdate,
 	return nil
 }
 
+func NextValidators(currentSet *types.ValidatorSet, updates []*types.Validator) (*types.ValidatorSet, error) {
+	nValSet := currentSet.UpdateProposerPriority()
+
+	// apply updates
+	for _, valUpdate := range updates {
+		// should already have been checked
+		if valUpdate.VotingPower < 0 {
+			return nil, fmt.Errorf("Voting power can't be negative %v", valUpdate)
+		}
+
+		address := valUpdate.Address
+		_, val := nValSet.GetByAddress(address)
+		// valUpdate.VotingPower is ensured to be non-negative in validation method
+		if valUpdate.VotingPower == 0 { // remove val
+			_, removed := nValSet.Remove(address)
+			if !removed {
+				return nil, fmt.Errorf("Failed to remove validator %X", address)
+			}
+		} else if val == nil { // add val
+			// make sure we do not exceed MaxTotalVotingPower by adding this validator:
+			totalVotingPower := currentSet.TotalVotingPower()
+			updatedVotingPower := valUpdate.VotingPower + totalVotingPower
+			overflow := updatedVotingPower > types.MaxTotalVotingPower || updatedVotingPower < 0
+			if overflow {
+				return nil, fmt.Errorf(
+					"Failed to add new validator %v. Adding it would exceed max allowed total voting power %v",
+					valUpdate,
+					types.MaxTotalVotingPower)
+			}
+
+			// TODO: issue #1558 update spec according to the following:
+			// Set ProposerPriority to -C*totalVotingPower (with C ~= 1.125) to make sure validators can't
+			// unbond/rebond to reset their (potentially previously negative) ProposerPriority to zero.
+			//
+			// Contract: totalVotingPower < MaxTotalVotingPower to ensure ProposerPriority does
+			// not exceed the bounds of int64.
+			//
+			// Compute ProposerPriority = -1.125*totalVotingPower == -(totalVotingPower + (totalVotingPower >> 3)).
+			valUpdate.ProposerPriority = -(totalVotingPower + (totalVotingPower >> 3))
+			added := nValSet.Add(valUpdate)
+			if !added {
+				return nil, fmt.Errorf("Failed to add new validator %v", valUpdate)
+			}
+		} else { // update val
+			// make sure we do not exceed MaxTotalVotingPower by updating this validator:
+			totalVotingPower := nValSet.TotalVotingPower()
+			curVotingPower := val.VotingPower
+			updatedVotingPower := totalVotingPower - curVotingPower + valUpdate.VotingPower
+			overflow := updatedVotingPower > types.MaxTotalVotingPower || updatedVotingPower < 0
+			if overflow {
+				return nil, fmt.Errorf(
+					"Failed to update existing validator %v. Updating it would exceed max allowed total voting power %v",
+					valUpdate,
+					types.MaxTotalVotingPower)
+			}
+
+			updated := nValSet.Update(valUpdate)
+			if !updated {
+				return nil, fmt.Errorf("Failed to update validator %X to %v", address, valUpdate)
+			}
+		}
+	}
+
+	return nValSet, nil
+}
+
 // If more or equal than 1/3 of total voting power changed in one block, then
 // a light client could never prove the transition externally. See
 // ./lite/doc.go for details on how a light client tracks validators.
@@ -418,12 +484,13 @@ func updateState(
 
 	// Copy the valset so we can apply changes from EndBlock
 	// and update s.LastValidators and s.Validators.
-	nValSet := state.NextValidators.Copy()
+	nValSet, err := state.NextValidators, error(nil)
 
 	// Update the validator set with the latest abciResponses.
 	lastHeightValsChanged := state.LastHeightValidatorsChanged
 	if len(validatorUpdates) > 0 {
-		err := updateValidators(nValSet, validatorUpdates)
+		//err := updateValidators(nValSet, validatorUpdates)
+		nValSet, err = NextValidators(nValSet, validatorUpdates)
 		if err != nil {
 			return state, fmt.Errorf("Error changing validator set: %v", err)
 		}
@@ -431,8 +498,11 @@ func updateState(
 		lastHeightValsChanged = header.Height + 1 + 1
 	}
 
+	//TODO: Given validator set at height h, when we move to height h+1, should we increment
+	//proposer priority with the voting power they have at height h, or at height h+1?
+
 	// Update validator proposer priority and set state variables.
-	nValSet.IncrementProposerPriority(1)
+	//nValSet.IncrementProposerPriority(1)
 
 	// Update the params with the latest abciResponses.
 	nextParams := state.ConsensusParams

state/execution_test.go

@@ -354,6 +354,45 @@ func TestEndBlockValidatorUpdates(t *testing.T) {
 	}
 }
 
+func TestNextValidatorsWithEmptyUpdates(t *testing.T) {
+	pubkey1 := ed25519.GenPrivKey().PubKey()
+	val1 := types.NewValidator(pubkey1, 10)
+	pubkey2 := ed25519.GenPrivKey().PubKey()
+	val2 := types.NewValidator(pubkey2, 20)
+
+	currentSet := types.NewValidatorSet([]*types.Validator{val1, val2})
+	t.Log(currentSet)
+
+	newSet, error := NextValidators(currentSet, nil)
+	assert.Nil(t, error)
+	t.Log(newSet)
+
+	newSet2, error := NextValidators(currentSet, []*types.Validator{})
+	assert.Nil(t, error)
+	t.Log(newSet2)
+}
+
+func TestNextValidatorsWithUpdates(t *testing.T) {
+	pubkey1 := ed25519.GenPrivKey().PubKey()
+	val1 := types.NewValidator(pubkey1, 10)
+	pubkey2 := ed25519.GenPrivKey().PubKey()
+	val2 := types.NewValidator(pubkey2, 20)
+	pubkey3 := ed25519.GenPrivKey().PubKey()
+	val3 := types.NewValidator(pubkey3, 30)
+
+	currentSet := types.NewValidatorSet([]*types.Validator{val1, val2, val3})
+	t.Log(currentSet)
+
+	val1.VotingPower = 0
+	newSet, error := NextValidators(currentSet, []*types.Validator{val1})
+	assert.Nil(t, error)
+	t.Log("Remove v1", newSet)
+
+	newSet2, error := NextValidators(newSet, []*types.Validator{val2})
+	assert.Nil(t, error)
+	t.Log("add v2", newSet2)
+}
+
 //----------------------------------------------------------------------------
 
 // make some bogus txs

state/state.go

@@ -226,7 +226,8 @@ func MakeGenesisState(genDoc *types.GenesisDoc) (State, error) {
 			validators[i] = types.NewValidator(val.PubKey, val.Power)
 		}
 		validatorSet = types.NewValidatorSet(validators)
-		nextValidatorSet = types.NewValidatorSet(validators).CopyIncrementProposerPriority(1)
+		//nextValidatorSet = types.NewValidatorSet(validators).CopyIncrementProposerPriority(1)
+		nextValidatorSet = validatorSet.UpdateProposerPriority()
 	}
 
 	return State{

tools/tm-monitor/monitor/node.go

@@ -55,13 +55,13 @@ func NewNode(rpcAddr string, options ...func(*Node)) *Node {
 
 func NewNodeWithEventMeterAndRpcClient(rpcAddr string, em eventMeter, rpcClient rpc_client.HTTPClient, options ...func(*Node)) *Node {
 	n := &Node{
-		rpcAddr:   rpcAddr,
-		em:        em,
-		rpcClient: rpcClient,
-		Name:      rpcAddr,
-		quit:      make(chan struct{}),
+		rpcAddr:                  rpcAddr,
+		em:                       em,
+		rpcClient:                rpcClient,
+		Name:                     rpcAddr,
+		quit:                     make(chan struct{}),
 		checkIsValidatorInterval: 5 * time.Second,
-		logger: log.NewNopLogger(),
+		logger:                   log.NewNopLogger(),
 	}
 
 	for _, option := range options {

types/block_test.go

@@ -341,3 +341,10 @@ func TestBlockMaxDataBytesUnknownEvidence(t *testing.T) {
 		}
 	}
 }
+
+func TestNilBlock(t *testing.T) {
+	nilBlock := BlockID{nil, PartSetHeader{}}
+	t.Log(nilBlock)
+	t.Log(len(nilBlock.Hash))
+	t.Log(nilBlock.IsZero())
+}

types/evidence_test.go

@@ -61,7 +61,7 @@ func TestEvidence(t *testing.T) {
 		{vote1, makeVote(val, chainID, 0, 10, 3, 1, blockID2), false},    // wrong round
 		{vote1, makeVote(val, chainID, 0, 10, 2, 2, blockID2), false},    // wrong step
 		{vote1, makeVote(val2, chainID, 0, 10, 2, 1, blockID), false},    // wrong validator
-		{vote1, badVote, false},                                          // signed by wrong key
+		{vote1, badVote, false}, // signed by wrong key
 	}
 
 	pubKey := val.GetPubKey()

types/validator_set.go

@@ -36,7 +36,7 @@ type ValidatorSet struct {
 	Proposer   *Validator   `json:"proposer"`
 
 	// cached (unexported)
-	totalVotingPower int64
+	totalVotingPower       int64
 }
 
 // NewValidatorSet initializes a ValidatorSet by copying over the
@@ -44,17 +44,24 @@ type ValidatorSet struct {
 // the new ValidatorSet will have an empty list of Validators.
 func NewValidatorSet(valz []*Validator) *ValidatorSet {
 	validators := make([]*Validator, len(valz))
+	propPriorities := make([]int64, len(valz))
 	for i, val := range valz {
 		validators[i] = val.Copy()
 	}
 	sort.Sort(ValidatorsByAddress(validators))
+
+	for i, val := range validators {
+		propPriorities[i] = val.ProposerPriority
+	}
+
 	vals := &ValidatorSet{
-		Validators: validators,
-	}
-	if len(valz) > 0 {
-		vals.IncrementProposerPriority(1)
+		Validators:             validators,
 	}
 
+	//if len(valz) > 0 {
+	//	vals.IncrementProposerPriority(1)
+	//}
+
 	return vals
 }
 
@@ -93,6 +100,26 @@ func (vals *ValidatorSet) IncrementProposerPriority(times int) {
 	vals.Proposer = proposer
 }
 
+// UpdateProposerPriority update proposer priority between rounds. Proposer priority of
+// every proposer is augmented with its voting power; proposer priority of the coordinator is
+// decreased by the total voting power.
+func (vals *ValidatorSet) UpdateProposerPriority() *ValidatorSet {
+	nValSet := vals.Copy()
+
+	// just pick the proposer with the highest proposer priority
+	// without any modification to it
+	proposer := vals.GetProposer()
+	// update proposer priority
+	for _, val := range nValSet.Validators {
+		// Check for overflow for sum.
+		val.ProposerPriority = safeAddClip(val.ProposerPriority, val.VotingPower)
+		if bytes.Equal(proposer.Address, val.Address) {
+			val.ProposerPriority = safeSubClip(val.ProposerPriority, nValSet.TotalVotingPower())
+		}
+	}
+	return nValSet
+}
+
 func (vals *ValidatorSet) incrementProposerPriority(subAvg bool) *Validator {
 	for _, val := range vals.Validators {
 		// Check for overflow for sum.
@@ -255,7 +282,8 @@ func (vals *ValidatorSet) Add(val *Validator) (added bool) {
 		vals.Validators = append(vals.Validators, val)
 		// Invalidate cache
 		vals.Proposer = nil
-		vals.totalVotingPower = 0
+		//vals.totalVotingPower = 0
+		vals.totalVotingPower = vals.totalVotingPower + val.VotingPower
 		return true
 	} else if bytes.Equal(vals.Validators[idx].Address, val.Address) {
 		return false
@@ -267,7 +295,8 @@ func (vals *ValidatorSet) Add(val *Validator) (added bool) {
 		vals.Validators = newValidators
 		// Invalidate cache
 		vals.Proposer = nil
-		vals.totalVotingPower = 0
+		//vals.totalVotingPower = 0
+		vals.totalVotingPower = vals.totalVotingPower + val.VotingPower
 		return true
 	}
 }
@@ -291,7 +320,7 @@ func (vals *ValidatorSet) Update(val *Validator) (updated bool) {
 	vals.Validators[index] = val.Copy()
 	// Invalidate cache
 	vals.Proposer = nil
-	vals.totalVotingPower = 0
+	vals.totalVotingPower = vals.totalVotingPower - sameVal.VotingPower + val.VotingPower
 	return true
 }