[p2p/pex] connect to more than 10 peers (#2169)

* [p2p/pex] connect to more than 10 peers

also, remove DefaultMinNumOutboundPeers because a) I am not sure it's
needed b) it's super confusing

look closely

```
maxPeers := sw.config.MaxNumPeers - DefaultMinNumOutboundPeers
if maxPeers <= sw.peers.Size() {
  sw.Logger.Info("Ignoring inbound connection: already have enough peers", "address", inConn.RemoteAddr().String(), "numPeers", sw.peers.Size(), "max", maxPeers)
```

we print maxPeers = config.MaxPeers - DefaultMinNumOutboundPeers. So we
may not have enough peers even though we say we have enough.

Refs #2130

* update spec

* replace MaxNumPeers with MaxNumInboundPeers/MaxNumOutboundPeers

Refs #2130

* update changelog

* make max rpc conns formula visible to users

* update spec

* docs: note max outbound peers excludes persistent

CHANGELOG_PENDING.md

@@ -10,6 +10,7 @@ BREAKING CHANGES:
 - [abci] Added address of the original proposer of the block to Header.
 - [abci] Change ABCI Header to match Tendermint exactly
 - [libs] Remove cmn.Fmt, in favor of fmt.Sprintf
+- [config] Replace MaxNumPeers with MaxNumInboundPeers and MaxNumOutboundPeers
 
 FEATURES:
 

config/config.go

@@ -239,6 +239,8 @@ type RPCConfig struct {
 	// If you want to accept more significant number than the default, make sure
 	// you increase your OS limits.
 	// 0 - unlimited.
+	// Should be < {ulimit -Sn} - {MaxNumInboundPeers} - {MaxNumOutboundPeers} - {N of wal, db and other open files}
+	// 1024 - 40 - 10 - 50 = 924 = ~900
 	MaxOpenConnections int `mapstructure:"max_open_connections"`
 }
 
@@ -248,11 +250,9 @@ func DefaultRPCConfig() *RPCConfig {
 		ListenAddress: "tcp://0.0.0.0:26657",
 
 		GRPCListenAddress:      "",
-		GRPCMaxOpenConnections: 900, // no ipv4
+		GRPCMaxOpenConnections: 900,
 
 		Unsafe:             false,
-		// should be < {ulimit -Sn} - {MaxNumPeers} - {N of wal, db and other open files}
-		// 1024 - 50 - 50 = 924 = ~900
 		MaxOpenConnections: 900,
 	}
 }
@@ -295,8 +295,11 @@ type P2PConfig struct {
 	// Set true for strict address routability rules
 	AddrBookStrict bool `mapstructure:"addr_book_strict"`
 
-	// Maximum number of peers to connect to
+	// Maximum number of inbound peers
-	MaxNumPeers int `mapstructure:"max_num_peers"`
+	MaxNumInboundPeers int `mapstructure:"max_num_inbound_peers"`
+
+	// Maximum number of outbound peers to connect to, excluding persistent peers
+	MaxNumOutboundPeers int `mapstructure:"max_num_outbound_peers"`
 
 	// Time to wait before flushing messages out on the connection, in ms
 	FlushThrottleTimeout int `mapstructure:"flush_throttle_timeout"`
@@ -346,7 +349,8 @@ func DefaultP2PConfig() *P2PConfig {
 		UPNP:                    false,
 		AddrBook:                defaultAddrBookPath,
 		AddrBookStrict:          true,
-		MaxNumPeers:             50,
+		MaxNumInboundPeers:      40,
+		MaxNumOutboundPeers:     10,
 		FlushThrottleTimeout:    100,
 		MaxPacketMsgPayloadSize: 1024,    // 1 kB
 		SendRate:                5120000, // 5 mB/s

config/toml.go

@@ -124,6 +124,8 @@ grpc_laddr = "{{ .RPC.GRPCListenAddress }}"
 # If you want to accept more significant number than the default, make sure
 # you increase your OS limits.
 # 0 - unlimited.
+# Should be < {ulimit -Sn} - {MaxNumInboundPeers} - {MaxNumOutboundPeers} - {N of wal, db and other open files}
+# 1024 - 40 - 10 - 50 = 924 = ~900
 grpc_max_open_connections = {{ .RPC.GRPCMaxOpenConnections }}
 
 # Activate unsafe RPC commands like /dial_seeds and /unsafe_flush_mempool
@@ -134,6 +136,8 @@ unsafe = {{ .RPC.Unsafe }}
 # If you want to accept more significant number than the default, make sure
 # you increase your OS limits.
 # 0 - unlimited.
+# Should be < {ulimit -Sn} - {MaxNumInboundPeers} - {MaxNumOutboundPeers} - {N of wal, db and other open files}
+# 1024 - 40 - 10 - 50 = 924 = ~900
 max_open_connections = {{ .RPC.MaxOpenConnections }}
 
 ##### peer to peer configuration options #####
@@ -166,8 +170,11 @@ addr_book_strict = {{ .P2P.AddrBookStrict }}
 # Time to wait before flushing messages out on the connection, in ms
 flush_throttle_timeout = {{ .P2P.FlushThrottleTimeout }}
 
-# Maximum number of peers to connect to
+# Maximum number of inbound peers
-max_num_peers = {{ .P2P.MaxNumPeers }}
+max_num_inbound_peers = {{ .P2P.MaxNumInboundPeers }}
+
+# Maximum number of outbound peers to connect to, excluding persistent peers
+max_num_outbound_peers = {{ .P2P.MaxNumOutboundPeers }}
 
 # Maximum size of a message packet payload, in bytes
 max_packet_msg_payload_size = {{ .P2P.MaxPacketMsgPayloadSize }}

docs/spec/reactors/pex/pex.md

@@ -15,6 +15,9 @@ we will not put them in the address book or gossip them to others.
 All peers except private peers and peers coming from them are tracked using the
 address book.
 
+The rest of our peers are only distinguished by being either
+inbound (they dialed our public address) or outbound (we dialed them).
+
 ## Discovery
 
 Peer discovery begins with a list of seeds.
@@ -26,15 +29,15 @@ and will attempt to maintain persistent connections with them. If the connection
 we will redial every 5s for a few minutes, then switch to an exponential backoff schedule,
 and after about a day of trying, stop dialing the peer.
 
-So long as we have less than `MinNumOutboundPeers`, we periodically request additional peers
+So long as we have less than `MaxNumOutboundPeers`, we periodically request additional peers
 from each of our own. If sufficient time goes by and we still can't find enough peers,
 we try the seeds again.
 
 ## Listening
 
 Peers listen on a configurable ListenAddr that they self-report in their
-NodeInfo during handshakes with other peers. Peers accept up to (MaxNumPeers -
+NodeInfo during handshakes with other peers. Peers accept up to
-MinNumOutboundPeers) incoming peers.
+`MaxNumInboundPeers` incoming peers.
 
 ## Address Book
 
@@ -73,10 +76,11 @@ a trust metric (see below), but it's best to start with something simple.
 
 ## Select Peers to Dial
 
-When we need more peers, we pick them randomly from the addrbook with some
+When we need more peers, we pick addresses randomly from the addrbook with some
-configurable bias for unvetted peers. The bias should be lower when we have fewer peers
+configurable bias for unvetted peers. The bias should be lower when we have
-and can increase as we obtain more, ensuring that our first peers are more trustworthy,
+fewer peers and can increase as we obtain more, ensuring that our first peers
-but always giving us the chance to discover new good peers.
+are more trustworthy, but always giving us the chance to discover new good
+peers.
 
 We track the last time we dialed a peer and the number of unsuccessful attempts
 we've made. If too many attempts are made, we mark the peer as bad.
@@ -85,7 +89,8 @@ Connection attempts are made with exponential backoff (plus jitter). Because
 the selection process happens every `ensurePeersPeriod`, we might not end up
 dialing a peer for much longer than the backoff duration.
 
-If we fail to connect to the peer after 16 tries (with exponential backoff), we remove from address book completely.
+If we fail to connect to the peer after 16 tries (with exponential backoff), we
+remove from address book completely.
 
 ## Select Peers to Exchange
 

docs/tendermint-core/configuration.md

@@ -77,6 +77,8 @@ grpc_laddr = ""
 # If you want to accept more significant number than the default, make sure
 # you increase your OS limits.
 # 0 - unlimited.
+# Should be < {ulimit -Sn} - {MaxNumInboundPeers} - {MaxNumOutboundPeers} - {N of wal, db and other open files}
+# 1024 - 40 - 10 - 50 = 924 = ~900
 grpc_max_open_connections = 900
 
 # Activate unsafe RPC commands like /dial_seeds and /unsafe_flush_mempool
@@ -87,7 +89,9 @@ unsafe = false
 # If you want to accept more significant number than the default, make sure
 # you increase your OS limits.
 # 0 - unlimited.
-max_open_connections = 450
+# Should be < {ulimit -Sn} - {MaxNumInboundPeers} - {MaxNumOutboundPeers} - {N of wal, db and other open files}
+# 1024 - 40 - 10 - 50 = 924 = ~900
+max_open_connections = 900
 
 ##### peer to peer configuration options #####
 [p2p]
@@ -113,8 +117,11 @@ addr_book_strict = true
 # Time to wait before flushing messages out on the connection, in ms
 flush_throttle_timeout = 100
 
-# Maximum number of peers to connect to
+# Maximum number of inbound peers
-max_num_peers = 50
+max_num_inbound_peers = 40
+
+# Maximum number of outbound peers to connect to
+max_num_outbound_peers = 10
 
 # Maximum size of a message packet payload, in bytes
 max_packet_msg_payload_size = 1024

p2p/pex/pex_reactor.go

@@ -32,7 +32,6 @@ const (
 
 	// ensure we have enough peers
 	defaultEnsurePeersPeriod = 30 * time.Second
-	defaultMinNumOutboundPeers = p2p.DefaultMinNumOutboundPeers
 
 	// Seed/Crawler constants
 
@@ -362,7 +361,7 @@ func (r *PEXReactor) ensurePeersRoutine() {
 func (r *PEXReactor) ensurePeers() {
 	var (
 		out, in, dial = r.Switch.NumPeers()
-		numToDial     = defaultMinNumOutboundPeers - (out + dial)
+		numToDial     = r.Switch.MaxNumOutboundPeers() - (out + dial)
 	)
 	r.Logger.Info(
 		"Ensure peers",

p2p/switch.go

@@ -26,10 +26,6 @@ const (
 	// ie. 3**10 = 16hrs
 	reconnectBackOffAttempts    = 10
 	reconnectBackOffBaseSeconds = 3
-
-	// keep at least this many outbound peers
-	// TODO: move to config
-	DefaultMinNumOutboundPeers = 10
 )
 
 //-----------------------------------------------------------------------------
@@ -268,6 +264,11 @@ func (sw *Switch) NumPeers() (outbound, inbound, dialing int) {
 	return
 }
 
+// MaxNumOutboundPeers returns a maximum number of outbound peers.
+func (sw *Switch) MaxNumOutboundPeers() int {
+	return sw.config.MaxNumOutboundPeers
+}
+
 // Peers returns the set of peers that are connected to the switch.
 func (sw *Switch) Peers() IPeerSet {
 	return sw.peers
@@ -491,11 +492,15 @@ func (sw *Switch) listenerRoutine(l Listener) {
 			break
 		}
 
-		// ignore connection if we already have enough
+		// Ignore connection if we already have enough peers.
-		// leave room for MinNumOutboundPeers
+		_, in, _ := sw.NumPeers()
-		maxPeers := sw.config.MaxNumPeers - DefaultMinNumOutboundPeers
+		if in >= sw.config.MaxNumInboundPeers {
-		if maxPeers <= sw.peers.Size() {
+			sw.Logger.Info(
-			sw.Logger.Info("Ignoring inbound connection: already have enough peers", "address", inConn.RemoteAddr().String(), "numPeers", sw.peers.Size(), "max", maxPeers)
+				"Ignoring inbound connection: already have enough inbound peers",
+				"address", inConn.RemoteAddr().String(),
+				"have", in,
+				"max", sw.config.MaxNumInboundPeers,
+			)
 			inConn.Close()
 			continue
 		}