p2p: peer state init too late and pex message too soon (#3634)

* fix peer state init to late

Peer does not have a state yet. We set it in AddPeer.
We need an new interface before mconnection is started

* pex message to soon

fix reconnection pex send too fast,
error is caused lastReceivedRequests is still
not deleted when a peer reconnected

* add test case for initpeer

* add prove case

* remove potentially infinite loop

* Update consensus/reactor.go

Co-Authored-By: guagualvcha <baifudong@lancai.cn>

* Update consensus/reactor_test.go

Co-Authored-By: guagualvcha <baifudong@lancai.cn>

* document Reactor interface better

* refactor TestReactorReceiveDoesNotPanicIfAddPeerHasntBeenCalledYet

* fix merge conflicts

* blockchain: remove peer's ID from the pool in InitPeer

Refs #3338

* pex: resetPeersRequestsInfo both upon InitPeer and RemovePeer

* ensure RemovePeer is always called before InitPeer

by removing the peer from the switch last (after we've stopped it and
removed from all reactors)

* add some comments for ConsensusReactor#InitPeer

* fix pex reactor

* format code

* fix spelling

* update changelog

* remove unused methods

* do not clear lastReceivedRequests upon error

only in RemovePeer

* call InitPeer before we start the peer!

* add a comment to InitPeer

* write a test

* use waitUntilSwitchHasAtLeastNPeers func

* bring back timeouts

* Test to ensure Receive panics if InitPeer has not been called

p2p/switch_test.go

@@ -12,6 +12,7 @@ import (
 	"regexp"
 	"strconv"
 	"sync"
+	"sync/atomic"
 	"testing"
 	"time"
 
@@ -603,6 +604,71 @@ func TestSwitchAcceptRoutineErrorCases(t *testing.T) {
 	})
 }
 
+// mockReactor checks that InitPeer never called before RemovePeer. If that's
+// not true, InitCalledBeforeRemoveFinished will return true.
+type mockReactor struct {
+	*BaseReactor
+
+	// atomic
+	removePeerInProgress           uint32
+	initCalledBeforeRemoveFinished uint32
+}
+
+func (r *mockReactor) RemovePeer(peer Peer, reason interface{}) {
+	atomic.StoreUint32(&r.removePeerInProgress, 1)
+	defer atomic.StoreUint32(&r.removePeerInProgress, 0)
+	time.Sleep(100 * time.Millisecond)
+}
+
+func (r *mockReactor) InitPeer(peer Peer) Peer {
+	if atomic.LoadUint32(&r.removePeerInProgress) == 1 {
+		atomic.StoreUint32(&r.initCalledBeforeRemoveFinished, 1)
+	}
+
+	return peer
+}
+
+func (r *mockReactor) InitCalledBeforeRemoveFinished() bool {
+	return atomic.LoadUint32(&r.initCalledBeforeRemoveFinished) == 1
+}
+
+// see stopAndRemovePeer
+func TestSwitchInitPeerIsNotCalledBeforeRemovePeer(t *testing.T) {
+	// make reactor
+	reactor := &mockReactor{}
+	reactor.BaseReactor = NewBaseReactor("mockReactor", reactor)
+
+	// make switch
+	sw := MakeSwitch(cfg, 1, "testing", "123.123.123", func(i int, sw *Switch) *Switch {
+		sw.AddReactor("mock", reactor)
+		return sw
+	})
+	err := sw.Start()
+	require.NoError(t, err)
+	defer sw.Stop()
+
+	// add peer
+	rp := &remotePeer{PrivKey: ed25519.GenPrivKey(), Config: cfg}
+	rp.Start()
+	defer rp.Stop()
+	_, err = rp.Dial(sw.NetAddress())
+	require.NoError(t, err)
+	// wait till the switch adds rp to the peer set
+	time.Sleep(50 * time.Millisecond)
+
+	// stop peer asynchronously
+	go sw.StopPeerForError(sw.Peers().Get(rp.ID()), "test")
+
+	// simulate peer reconnecting to us
+	_, err = rp.Dial(sw.NetAddress())
+	require.NoError(t, err)
+	// wait till the switch adds rp to the peer set
+	time.Sleep(50 * time.Millisecond)
+
+	// make sure reactor.RemovePeer is finished before InitPeer is called
+	assert.False(t, reactor.InitCalledBeforeRemoveFinished())
+}
+
 func BenchmarkSwitchBroadcast(b *testing.B) {
 	s1, s2 := MakeSwitchPair(b, func(i int, sw *Switch) *Switch {
 		// Make bar reactors of bar channels each