wordings and clarifications, unnecessary code uncommenting

consensus/height_vote_set.go

@@ -18,7 +18,8 @@ type RoundVoteSet struct {
 Keeps track of all VoteSets from round 0 to round 'round'.
 
 Also keeps track of up to one RoundVoteSet greater than
-'round' from each peer, to facilitate fast-forward syncing.
+'round' from each peer, to facilitate catchup syncing of commits.
+
 A commit is +2/3 precommits for a block at a round,
 but which round is not known in advance, so when a peer
 provides a precommit for a round greater than mtx.round,
@@ -29,18 +30,18 @@ type HeightVoteSet struct {
 	height uint
 	valSet *sm.ValidatorSet
 
-	mtx             sync.Mutex
+	mtx               sync.Mutex
-	round           uint                  // max tracked round
+	round             uint                  // max tracked round
-	roundVoteSets   map[uint]RoundVoteSet // keys: [0...round]
+	roundVoteSets     map[uint]RoundVoteSet // keys: [0...round]
-	peerFastForward map[string]uint       // keys: peer.Key; values: round
+	peerCatchupRounds map[string]uint       // keys: peer.Key; values: round
 }
 
 func NewHeightVoteSet(height uint, valSet *sm.ValidatorSet) *HeightVoteSet {
 	hvs := &HeightVoteSet{
-		height:          height,
+		height:            height,
-		valSet:          valSet,
+		valSet:            valSet,
-		roundVoteSets:   make(map[uint]RoundVoteSet),
+		roundVoteSets:     make(map[uint]RoundVoteSet),
-		peerFastForward: make(map[string]uint),
+		peerCatchupRounds: make(map[string]uint),
 	}
 	hvs.addRound(0)
 	hvs.round = 0
@@ -66,7 +67,7 @@ func (hvs *HeightVoteSet) SetRound(round uint) {
 	}
 	for r := hvs.round + 1; r <= round; r++ {
 		if _, ok := hvs.roundVoteSets[r]; ok {
-			continue // Already exists because peerFastForward.
+			continue // Already exists because peerCatchupRounds.
 		}
 		hvs.addRound(round)
 	}
@@ -92,9 +93,9 @@ func (hvs *HeightVoteSet) AddByAddress(address []byte, vote *types.Vote, peerKey
 	defer hvs.mtx.Unlock()
 	voteSet := hvs.getVoteSet(vote.Round, vote.Type)
 	if voteSet == nil {
-		if _, ok := hvs.peerFastForward[peerKey]; !ok {
+		if _, ok := hvs.peerCatchupRounds[peerKey]; !ok {
 			hvs.addRound(vote.Round)
-			hvs.peerFastForward[peerKey] = vote.Round
+			hvs.peerCatchupRounds[peerKey] = vote.Round
 		} else {
 			// Peer has sent a vote that does not match our round,
 			// for more than one round.  Bad peer!
@@ -160,7 +161,7 @@ func (hvs *HeightVoteSet) StringIndented(indent string) string {
 		voteSetString = hvs.roundVoteSets[round].Precommits.StringShort()
 		vsStrings = append(vsStrings, voteSetString)
 	}
-	// all other peer fast-forward rounds
+	// all other peer catchup rounds
 	for round, roundVoteSet := range hvs.roundVoteSets {
 		if round <= hvs.round {
 			continue

consensus/state.go

@@ -1,19 +1,31 @@
 /*
 
-Consensus State Machine Overview:
+	Consensus State Machine Overview:
 
-* NewHeight, NewRound, Propose, Prevote, Precommit represent state machine steps. (aka RoundStep).
+	NewHeight, NewRound, Propose, Prevote, Precommit represent state machine steps. (aka RoundStep).
-* To "prevote/precommit" something means to broadcast a prevote/precommit vote for something.
+
-* During NewHeight/NewRound/Propose/Prevote/Precommit:
+	To "prevote/precommit" something means to broadcast a prevote/precommit vote for something.
-  * Nodes gossip the proposal block proposed by the designated proposer for that round.
+
-  * Nodes gossip prevotes/precommits for rounds [0...currentRound+1] (currentRound+1 for catch-up)
+	During NewHeight/NewRound/Propose/Prevote/Precommit:
-  * Nodes also gossip prevotes for the proposal's POL (proof-of-lock) round if proposed.
+	* Nodes gossip the proposal block proposed by the designated proposer for that round.
-* Upon each state transition, the height/round/step is broadcast to neighboring peers.
+	* Nodes gossip prevotes/precommits for rounds [0...currentRound+1] (currentRound+1 to allow round-skipping)
-* The set of +2/3 of precommits at the same round for the same block is called a Commit, or Validation.
+	* Nodes gossip prevotes for the proposal's POL (proof-of-lock) round if proposed.
-* A block contains the last block's Validation, which includes the Commit precommits.
+	* Nodes gossip to late nodes (lagging in height) with precommits of the commit round (aka catchup)
-  While all the precommits in the Validation are from the same height & round (ordered by validator index),
+
-  some precommits may be nil (if the validator's precommit vote didn't reach the proposer in time),
+	Upon each state transition, the height/round/step is broadcast to neighboring peers.
-  or some precommits may be for different blockhashes for the last block hash (which is fine).
+
+	The set of +2/3 of precommits at the same round for the same block is called a Commit, or Validation.
+
+	A block contains the last block's Validation, which includes the Commit precommits.
+	While all the precommits in the Validation are from the same height & round (ordered by validator index),
+	some precommits may be <nil> (if the validator's precommit vote didn't reach the proposer in time),
+	or some precommits may be for different blockhashes for the last block hash (which is fine).
+
+	Each unlock/change-of-lock should be justifiable by an POL where +2/3 prevoted for
+	some block or <nil> at some round.
+
+		POL = Proof-of-Lock = +2/3 prevotes for block B or <nil> for (H,R)
+		lockRound < POLRound <= unlockOrChangeLockRound
 
 * NewRound(height:H,round:R):
   * Set up new round.                                                --> goto Propose(H,R)
@@ -273,15 +285,7 @@ func (cs *ConsensusState) reconstructLastCommit(state *sm.State) {
 	lastPrecommits := NewVoteSet(state.LastBlockHeight, 0, types.VoteTypePrecommit, state.LastBondedValidators)
 	seenValidation := cs.blockStore.LoadSeenValidation(state.LastBlockHeight)
 	for idx, precommit := range seenValidation.Precommits {
-		precommitVote := &types.Vote{
+		added, _, err := lastPrecommits.AddByIndex(uint(idx), precommit)
-			Height:     state.LastBlockHeight,
-			Round:      seenValidation.Round(),
-			Type:       types.VoteTypePrecommit,
-			BlockHash:  state.LastBlockHash,
-			BlockParts: state.LastBlockParts,
-			Signature:  precommit.Signature,
-		}
-		added, _, err := lastPrecommits.AddByIndex(uint(idx), precommitVote)
 		if !added || err != nil {
 			panic(Fmt("Failed to reconstruct LastCommit: %v", err))
 		}
@@ -320,11 +324,12 @@ func (cs *ConsensusState) Start() {
 	}
 }
 
+// EnterNewRound(height, 0) at cs.StartTime.
 func (cs *ConsensusState) scheduleRound0(height uint) {
-	log.Debug("scheduleRound0", "now", time.Now(), "startTime", cs.StartTime)
+	//log.Debug("scheduleRound0", "now", time.Now(), "startTime", cs.StartTime)
 	sleepDuration := cs.StartTime.Sub(time.Now())
 	go func() {
-		if sleepDuration > 0 {
+		if 0 < sleepDuration {
 			time.Sleep(sleepDuration)
 		}
 		cs.EnterNewRound(height, 0)
@@ -346,7 +351,7 @@ func (cs *ConsensusState) IsStopped() bool {
 // The round becomes 0 and cs.Step becomes RoundStepNewHeight.
 func (cs *ConsensusState) updateToState(state *sm.State, contiguous bool) {
 	// SANITY CHECK
-	if contiguous && cs.Height > 0 && cs.Height != state.LastBlockHeight {
+	if contiguous && 0 < cs.Height && cs.Height != state.LastBlockHeight {
 		panic(Fmt("updateToState() expected state height of %v but found %v",
 			cs.Height, state.LastBlockHeight))
 	}
@@ -357,6 +362,9 @@ func (cs *ConsensusState) updateToState(state *sm.State, contiguous bool) {
 	height := state.LastBlockHeight + 1 // next desired block height
 	lastPrecommits := (*VoteSet)(nil)
 	if contiguous && cs.Votes != nil {
+		if !cs.Votes.Precommits(cs.Round).HasTwoThirdsMajority() {
+			panic("updateToState(state, true) called but last Precommit round didn't have +2/3")
+		}
 		lastPrecommits = cs.Votes.Precommits(cs.Round)
 	}
 
@@ -424,9 +432,9 @@ func (cs *ConsensusState) SetPrivValidator(priv *sm.PrivValidator) {
 
 //-----------------------------------------------------------------------------
 
-// Enter: +2/3 precommits for nil from previous round
+// Enter: +2/3 precommits for nil from (height,round-1)
-// Enter: `timeoutPrecommits` after any +2/3 precommits
+// Enter: `timeoutPrecommits` after any +2/3 precommits from (height,round-1)
-// Enter: `commitTime+timeoutCommit` from NewHeight
+// Enter: `startTime = commitTime+timeoutCommit` from NewHeight(height)
 // NOTE: cs.StartTime was already set for height.
 func (cs *ConsensusState) EnterNewRound(height uint, round uint) {
 	cs.mtx.Lock()
@@ -453,17 +461,17 @@ func (cs *ConsensusState) EnterNewRound(height uint, round uint) {
 	cs.Proposal = nil
 	cs.ProposalBlock = nil
 	cs.ProposalBlockParts = nil
-	cs.Votes.SetRound(round + 1) // track next round.
+	cs.Votes.SetRound(round + 1) // also track next round (round+1) to allow round-skipping
 
 	// Immediately go to EnterPropose.
 	go cs.EnterPropose(height, round)
 }
 
-// Enter: from NewRound.
+// Enter: from NewRound(height,round).
 func (cs *ConsensusState) EnterPropose(height uint, round uint) {
 	cs.mtx.Lock()
 	defer cs.mtx.Unlock()
-	if cs.Height != height || cs.Round > round || (cs.Round == round && cs.Step >= RoundStepPropose) {
+	if cs.Height != height || round < cs.Round || (cs.Round == round && RoundStepPropose <= cs.Step) {
 		log.Debug(Fmt("EnterPropose(%v/%v): Invalid args. Current step: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
 		return
 	}
@@ -486,21 +494,21 @@ func (cs *ConsensusState) EnterPropose(height uint, round uint) {
 		cs.EnterPrevote(height, round)
 	}()
 
-	// Nothing to do if it's not our turn.
+	// Nothing more to do if we're not a validator
 	if cs.privValidator == nil {
 		return
 	}
 
-	// See if it is our turn to propose
 	if !bytes.Equal(cs.Validators.Proposer().Address, cs.privValidator.Address) {
 		log.Debug("EnterPropose: Not our turn to propose", "proposer", cs.Validators.Proposer().Address, "privValidator", cs.privValidator)
-		return
 	} else {
 		log.Debug("EnterPropose: Our turn to propose", "proposer", cs.Validators.Proposer().Address, "privValidator", cs.privValidator)
+		cs.decideProposal(height, round)
 	}
+}
 
-	// We are going to propose a block.
+// Decides on the next proposal and sets them onto cs.Proposal*
-
+func (cs *ConsensusState) decideProposal(height uint, round uint) {
 	var block *types.Block
 	var blockParts *types.PartSet
 
@@ -514,21 +522,23 @@ func (cs *ConsensusState) EnterPropose(height uint, round uint) {
 	}
 
 	// Make proposal
-	proposal := NewProposal(cs.Height, cs.Round, blockParts.Header(), cs.Votes.POLRound())
+	proposal := NewProposal(height, round, blockParts.Header(), cs.Votes.POLRound())
 	err := cs.privValidator.SignProposal(cs.state.ChainID, proposal)
 	if err == nil {
-		log.Info("Signed and set proposal", "height", cs.Height, "round", cs.Round, "proposal", proposal)
+		log.Info("Signed and set proposal", "height", height, "round", round, "proposal", proposal)
 		log.Debug(Fmt("Signed and set proposal block: %v", block))
 		// Set fields
 		cs.Proposal = proposal
 		cs.ProposalBlock = block
 		cs.ProposalBlockParts = blockParts
 	} else {
-		log.Warn("EnterPropose: Error signing proposal", "height", cs.Height, "round", cs.Round, "error", err)
+		log.Warn("EnterPropose: Error signing proposal", "height", height, "round", round, "error", err)
 	}
 
 }
 
+// Returns true if the proposal block is complete &&
+// (if POLRound was proposed, we have +2/3 prevotes from there).
 func (cs *ConsensusState) isProposalComplete() bool {
 	if cs.Proposal == nil || cs.ProposalBlock == nil {
 		return false
@@ -585,32 +595,30 @@ func (cs *ConsensusState) createProposalBlock() (block *types.Block, blockParts
 	return block, blockParts
 }
 
-// Enter: `timeoutPropose` after start of Propose.
+// Enter: `timeoutPropose` after entering Propose.
 // Enter: proposal block and POL is ready.
-// Enter: any +2/3 prevotes for next round.
+// Enter: any +2/3 prevotes for future round.
-// Prevote for LockedBlock if we're locked, or ProposealBlock if valid.
+// Prevote for LockedBlock if we're locked, or ProposalBlock if valid.
 // Otherwise vote nil.
 func (cs *ConsensusState) EnterPrevote(height uint, round uint) {
 	cs.mtx.Lock()
 	defer cs.mtx.Unlock()
-	if cs.Height != height || cs.Round > round || (cs.Round == round && cs.Step >= RoundStepPrevote) {
+	if cs.Height != height || round < cs.Round || (cs.Round == round && RoundStepPrevote <= cs.Step) {
 		log.Debug(Fmt("EnterPrevote(%v/%v): Invalid args. Current step: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
 		return
 	}
 
-	defer func() {
-		// Done EnterPrevote:
-		cs.Round = round
-		cs.Step = RoundStepPrevote
-		cs.newStepCh <- cs.getRoundState()
-		// Maybe immediately go to EnterPrevoteWait.
-		if cs.Votes.Prevotes(round).HasTwoThirdsAny() {
-			go cs.EnterPrevoteWait(height, round)
-		}
-	}()
-
 	// Sign and broadcast vote as necessary
 	cs.doPrevote(height, round)
+
+	// Done EnterPrevote:
+	cs.Round = round
+	cs.Step = RoundStepPrevote
+	cs.newStepCh <- cs.getRoundState()
+	/* This isn't necessary because addVote() does it for us.
+	if cs.Votes.Prevotes(round).HasTwoThirdsAny() {
+		go cs.EnterPrevoteWait(height, round)
+	}*/
 }
 
 func (cs *ConsensusState) doPrevote(height uint, round uint) {
@@ -646,7 +654,7 @@ func (cs *ConsensusState) doPrevote(height uint, round uint) {
 func (cs *ConsensusState) EnterPrevoteWait(height uint, round uint) {
 	cs.mtx.Lock()
 	defer cs.mtx.Unlock()
-	if cs.Height != height || cs.Round > round || (cs.Round == round && cs.Step >= RoundStepPrevoteWait) {
+	if cs.Height != height || round < cs.Round || (cs.Round == round && RoundStepPrevoteWait <= cs.Step) {
 		log.Debug(Fmt("EnterPrevoteWait(%v/%v): Invalid args. Current step: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
 		return
 	}
@@ -675,7 +683,7 @@ func (cs *ConsensusState) EnterPrevoteWait(height uint, round uint) {
 func (cs *ConsensusState) EnterPrecommit(height uint, round uint) {
 	cs.mtx.Lock()
 	defer cs.mtx.Unlock()
-	if cs.Height != height || cs.Round > round || (cs.Round == round && cs.Step >= RoundStepPrecommit) {
+	if cs.Height != height || round < cs.Round || (cs.Round == round && RoundStepPrecommit <= cs.Step) {
 		log.Debug(Fmt("EnterPrecommit(%v/%v): Invalid args. Current step: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
 		return
 	}
@@ -685,10 +693,10 @@ func (cs *ConsensusState) EnterPrecommit(height uint, round uint) {
 		cs.Round = round
 		cs.Step = RoundStepPrecommit
 		cs.newStepCh <- cs.getRoundState()
-		// Maybe immediately go to EnterPrecommitWait.
+		/* This isn't necessary because addVote() does it for us.
 		if cs.Votes.Precommits(round).HasTwoThirdsAny() {
 			go cs.EnterPrecommitWait(height, round)
-		}
+		}*/
 	}()
 
 	hash, partsHeader, ok := cs.Votes.Prevotes(round).TwoThirdsMajority()
@@ -741,7 +749,11 @@ func (cs *ConsensusState) EnterPrecommit(height uint, round uint) {
 	}
 
 	// Otherwise, we need to fetch the +2/3 prevoted block.
-	// We don't have the block yet so we can't lock/precommit it.
+	// Unlock and precommit nil.
+	// The +2/3 prevotes for this round is the POL for our unlock.
+	if cs.Votes.POLRound() < round {
+		panic(Fmt("This POLRound shold be %v but got %", round, cs.Votes.POLRound()))
+	}
 	cs.LockedBlock = nil
 	cs.LockedBlockParts = nil
 	if !cs.ProposalBlockParts.HasHeader(partsHeader) {
@@ -756,7 +768,7 @@ func (cs *ConsensusState) EnterPrecommit(height uint, round uint) {
 func (cs *ConsensusState) EnterPrecommitWait(height uint, round uint) {
 	cs.mtx.Lock()
 	defer cs.mtx.Unlock()
-	if cs.Height != height || cs.Round > round || (cs.Round == round && cs.Step >= RoundStepPrecommitWait) {
+	if cs.Height != height || round < cs.Round || (cs.Round == round && RoundStepPrecommitWait <= cs.Step) {
 		log.Debug(Fmt("EnterPrecommitWait(%v/%v): Invalid args. Current step: %v/%v/%v", height, round, cs.Height, cs.Round, cs.Step))
 		return
 	}
@@ -784,7 +796,7 @@ func (cs *ConsensusState) EnterPrecommitWait(height uint, round uint) {
 func (cs *ConsensusState) EnterCommit(height uint) {
 	cs.mtx.Lock()
 	defer cs.mtx.Unlock()
-	if cs.Height != height || cs.Step >= RoundStepCommit {
+	if cs.Height != height || RoundStepCommit <= cs.Step {
 		log.Debug(Fmt("EnterCommit(%v): Invalid args. Current step: %v/%v/%v", height, cs.Height, cs.Round, cs.Step))
 		return
 	}
@@ -912,7 +924,7 @@ func (cs *ConsensusState) SetProposal(proposal *Proposal) error {
 	}
 
 	// We don't care about the proposal if we're already in RoundStepCommit.
-	if cs.Step >= RoundStepCommit {
+	if RoundStepCommit <= cs.Step {
 		return nil
 	}
 
@@ -959,6 +971,7 @@ func (cs *ConsensusState) AddProposalBlockPart(height uint, round uint, part *ty
 		if cs.Step == RoundStepPropose && cs.isProposalComplete() {
 			go cs.EnterPrevote(height, round)
 		} else if cs.Step == RoundStepCommit {
+			/// XXX How about, EnterCommit()?
 			cs.tryFinalizeCommit(height)
 		}
 		return true, err
@@ -1011,7 +1024,7 @@ func (cs *ConsensusState) addVote(address []byte, vote *types.Vote, peerKey stri
 							cs.EnterPrevoteWait(height, cs.Round)
 						}()
 					}
-				} else if cs.Proposal != nil && cs.Proposal.POLRound >= 0 && uint(cs.Proposal.POLRound) == vote.Round {
+				} else if cs.Proposal != nil && 0 <= cs.Proposal.POLRound && uint(cs.Proposal.POLRound) == vote.Round {
 					if cs.isProposalComplete() {
 						go cs.EnterPrevote(height, cs.Round)
 					}