Close and retry a RemoteSigner on err (#2923)

* Close and recreate a RemoteSigner on err

* Update changelog

* Address Anton's comments / suggestions:

 - update changelog
 - restart TCPVal
 - shut down on `ErrUnexpectedResponse`

* re-init remote signer client with fresh connection if Ping fails

- add/update TODOs in secret connection
- rename tcp.go -> tcp_client.go, same with ipc to clarify their purpose

* account for `conn returned by waitConnection can be `nil`

- also add TODO about RemoteSigner conn field

* Tests for retrying: IPC / TCP

 - shorter info log on success
 - set conn and use it in tests to close conn

* Tests for retrying: IPC / TCP

 - shorter info log on success
 - set conn and use it in tests to close conn
 - add rwmutex for conn field in IPC

* comments and doc.go

* fix ipc tests. fixes #2677

* use constants for tests

* cleanup some error statements

* fixes #2784, race in tests

* remove print statement

* minor fixes from review

* update comment on sts spec

* cosmetics

* p2p/conn: add failing tests

* p2p/conn: make SecretConnection thread safe

* changelog

* IPCVal signer refactor

- use a .reset() method
- don't use embedded RemoteSignerClient
- guard RemoteSignerClient with mutex
- drop the .conn
- expose Close() on RemoteSignerClient

* apply IPCVal refactor to TCPVal

* remove mtx from RemoteSignerClient

* consolidate IPCVal and TCPVal, fixes #3104

- done in tcp_client.go
- now called SocketVal
- takes a listener in the constructor
- make tcpListener and unixListener contain all the differences

* delete ipc files

* introduce unix and tcp dialer for RemoteSigner

* rename files

- drop tcp_ prefix
- rename priv_validator.go to file.go

* bring back listener options

* fix node

* fix priv_val_server

* fix node test

* minor cleanup and comments

privval/socket_test.go

@@ -0,0 +1,84 @@
+package privval
+
+import (
+	"net"
+	"testing"
+	"time"
+
+	"github.com/tendermint/tendermint/crypto/ed25519"
+)
+
+//-------------------------------------------
+// helper funcs
+
+func newPrivKey() ed25519.PrivKeyEd25519 {
+	return ed25519.GenPrivKey()
+}
+
+//-------------------------------------------
+// tests
+
+func TestTCPListenerAcceptDeadline(t *testing.T) {
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tcpLn := NewTCPListener(ln, newPrivKey())
+	TCPListenerAcceptDeadline(time.Millisecond)(tcpLn)
+	TCPListenerConnDeadline(time.Second)(tcpLn)
+
+	_, err = tcpLn.Accept()
+	opErr, ok := err.(*net.OpError)
+	if !ok {
+		t.Fatalf("have %v, want *net.OpError", err)
+	}
+
+	if have, want := opErr.Op, "accept"; have != want {
+		t.Errorf("have %v, want %v", have, want)
+	}
+}
+
+func TestTCPListenerConnDeadline(t *testing.T) {
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tcpLn := NewTCPListener(ln, newPrivKey())
+	TCPListenerAcceptDeadline(time.Second)(tcpLn)
+	TCPListenerConnDeadline(time.Millisecond)(tcpLn)
+
+	readyc := make(chan struct{})
+	donec := make(chan struct{})
+	go func(ln net.Listener) {
+		defer close(donec)
+
+		c, err := ln.Accept()
+		if err != nil {
+			t.Fatal(err)
+		}
+		<-readyc
+
+		time.Sleep(2 * time.Millisecond)
+
+		msg := make([]byte, 200)
+		_, err = c.Read(msg)
+		opErr, ok := err.(*net.OpError)
+		if !ok {
+			t.Fatalf("have %v, want *net.OpError", err)
+		}
+
+		if have, want := opErr.Op, "read"; have != want {
+			t.Errorf("have %v, want %v", have, want)
+		}
+	}(tcpLn)
+
+	dialer := DialTCPFn(ln.Addr().String(), testConnDeadline, newPrivKey())
+	_, err = dialer()
+	if err != nil {
+		t.Fatal(err)
+	}
+	close(readyc)
+	<-donec
+}