rpc: add support for batched requests/responses (#3534)

Continues from #3280 in building support for batched requests/responses in the JSON RPC (as per issue #3213).

* Add JSON RPC batching for client and server

As per #3213, this adds support for [JSON RPC batch requests and
responses](https://www.jsonrpc.org/specification#batch).

* Add additional checks to ensure client responses are the same as results

* Fix case where a notification is sent and no response is expected

* Add test to check that JSON RPC notifications in a batch are left out in responses

* Update CHANGELOG_PENDING.md

* Update PR number now that PR has been created

* Make errors start with lowercase letter

* Refactor batch functionality to be standalone

This refactors the batching functionality to rather act in a standalone
way. In light of supporting concurrent goroutines making use of the same
client, it would make sense to have batching functionality where one
could create a batch of requests per goroutine and send that batch
without interfering with a batch from another goroutine.

* Add examples for simple and batch HTTP client usage

* Check errors from writer and remove nolinter directives

* Make error strings start with lowercase letter

* Refactor examples to make them testable

* Use safer deferred shutdown for example Tendermint test node

* Recompose rpcClient interface from pre-existing interface components

* Rename WaitGroup for brevity

* Replace empty ID string with request ID

* Remove extraneous test case

* Convert first letter of errors.Wrap() messages to lowercase

* Remove extraneous function parameter

* Make variable declaration terse

* Reorder WaitGroup.Done call to help prevent race conditions in the face of failure

* Swap mutex to value representation and remove initialization

* Restore empty JSONRPC string ID in response to prevent nil

* Make JSONRPCBufferedRequest private

* Revert PR hard link in CHANGELOG_PENDING

* Add client ID for JSONRPCClient

This adds code to automatically generate a randomized client ID for the
JSONRPCClient, and adds a check of the IDs in the responses (if one was
set in the requests).

* Extract response ID validation into separate function

* Remove extraneous comments

* Reorder fields to indicate clearly which are protected by the mutex

* Refactor for loop to remove indexing

* Restructure and combine loop

* Flatten conditional block for better readability

* Make multi-variable declaration slightly more readable

* Change for loop style

* Compress error check statements

* Make function description more generic to show that we support different protocols

* Preallocate memory for request and result objects
This commit is contained in:
Thane Thomson
2019-04-17 11:10:12 -04:00
committed by Anton Kaliaev
parent 621c0e629d
commit 90465f727f
11 changed files with 747 additions and 133 deletions

View File

@ -10,9 +10,11 @@ import (
"net/url"
"reflect"
"strings"
"sync"
"github.com/pkg/errors"
amino "github.com/tendermint/go-amino"
cmn "github.com/tendermint/tendermint/libs/common"
types "github.com/tendermint/tendermint/rpc/lib/types"
)
@ -83,25 +85,56 @@ func makeHTTPClient(remoteAddr string) (string, *http.Client) {
//------------------------------------------------------------------------------------
// jsonRPCBufferedRequest encapsulates a single buffered request, as well as its
// anticipated response structure.
type jsonRPCBufferedRequest struct {
request types.RPCRequest
result interface{} // The result will be deserialized into this object.
}
// JSONRPCRequestBatch allows us to buffer multiple request/response structures
// into a single batch request. Note that this batch acts like a FIFO queue, and
// is thread-safe.
type JSONRPCRequestBatch struct {
client *JSONRPCClient
mtx sync.Mutex
requests []*jsonRPCBufferedRequest
}
// JSONRPCClient takes params as a slice
type JSONRPCClient struct {
address string
client *http.Client
id types.JSONRPCStringID
cdc *amino.Codec
}
// JSONRPCCaller implementers can facilitate calling the JSON RPC endpoint.
type JSONRPCCaller interface {
Call(method string, params map[string]interface{}, result interface{}) (interface{}, error)
}
// Both JSONRPCClient and JSONRPCRequestBatch can facilitate calls to the JSON
// RPC endpoint.
var _ JSONRPCCaller = (*JSONRPCClient)(nil)
var _ JSONRPCCaller = (*JSONRPCRequestBatch)(nil)
// NewJSONRPCClient returns a JSONRPCClient pointed at the given address.
func NewJSONRPCClient(remote string) *JSONRPCClient {
address, client := makeHTTPClient(remote)
return &JSONRPCClient{
address: address,
client: client,
id: types.JSONRPCStringID("jsonrpc-client-" + cmn.RandStr(8)),
cdc: amino.NewCodec(),
}
}
// Call will send the request for the given method through to the RPC endpoint
// immediately, without buffering of requests.
func (c *JSONRPCClient) Call(method string, params map[string]interface{}, result interface{}) (interface{}, error) {
request, err := types.MapToRequest(c.cdc, types.JSONRPCStringID("jsonrpc-client"), method, params)
request, err := types.MapToRequest(c.cdc, c.id, method, params)
if err != nil {
return nil, err
}
@ -109,9 +142,7 @@ func (c *JSONRPCClient) Call(method string, params map[string]interface{}, resul
if err != nil {
return nil, err
}
// log.Info(string(requestBytes))
requestBuf := bytes.NewBuffer(requestBytes)
// log.Info(Fmt("RPC request to %v (%v): %v", c.remote, method, string(requestBytes)))
httpResponse, err := c.client.Post(c.address, "text/json", requestBuf)
if err != nil {
return nil, err
@ -122,8 +153,40 @@ func (c *JSONRPCClient) Call(method string, params map[string]interface{}, resul
if err != nil {
return nil, err
}
// log.Info(Fmt("RPC response: %v", string(responseBytes)))
return unmarshalResponseBytes(c.cdc, responseBytes, result)
return unmarshalResponseBytes(c.cdc, responseBytes, c.id, result)
}
// NewRequestBatch starts a batch of requests for this client.
func (c *JSONRPCClient) NewRequestBatch() *JSONRPCRequestBatch {
return &JSONRPCRequestBatch{
requests: make([]*jsonRPCBufferedRequest, 0),
client: c,
}
}
func (c *JSONRPCClient) sendBatch(requests []*jsonRPCBufferedRequest) ([]interface{}, error) {
reqs := make([]types.RPCRequest, 0, len(requests))
results := make([]interface{}, 0, len(requests))
for _, req := range requests {
reqs = append(reqs, req.request)
results = append(results, req.result)
}
// serialize the array of requests into a single JSON object
requestBytes, err := json.Marshal(reqs)
if err != nil {
return nil, err
}
httpResponse, err := c.client.Post(c.address, "text/json", bytes.NewBuffer(requestBytes))
if err != nil {
return nil, err
}
defer httpResponse.Body.Close() // nolint: errcheck
responseBytes, err := ioutil.ReadAll(httpResponse.Body)
if err != nil {
return nil, err
}
return unmarshalResponseBytesArray(c.cdc, responseBytes, c.id, results)
}
func (c *JSONRPCClient) Codec() *amino.Codec {
@ -136,6 +199,57 @@ func (c *JSONRPCClient) SetCodec(cdc *amino.Codec) {
//-------------------------------------------------------------
// Count returns the number of enqueued requests waiting to be sent.
func (b *JSONRPCRequestBatch) Count() int {
b.mtx.Lock()
defer b.mtx.Unlock()
return len(b.requests)
}
func (b *JSONRPCRequestBatch) enqueue(req *jsonRPCBufferedRequest) {
b.mtx.Lock()
defer b.mtx.Unlock()
b.requests = append(b.requests, req)
}
// Clear empties out the request batch.
func (b *JSONRPCRequestBatch) Clear() int {
b.mtx.Lock()
defer b.mtx.Unlock()
return b.clear()
}
func (b *JSONRPCRequestBatch) clear() int {
count := len(b.requests)
b.requests = make([]*jsonRPCBufferedRequest, 0)
return count
}
// Send will attempt to send the current batch of enqueued requests, and then
// will clear out the requests once done. On success, this returns the
// deserialized list of results from each of the enqueued requests.
func (b *JSONRPCRequestBatch) Send() ([]interface{}, error) {
b.mtx.Lock()
defer func() {
b.clear()
b.mtx.Unlock()
}()
return b.client.sendBatch(b.requests)
}
// Call enqueues a request to call the given RPC method with the specified
// parameters, in the same way that the `JSONRPCClient.Call` function would.
func (b *JSONRPCRequestBatch) Call(method string, params map[string]interface{}, result interface{}) (interface{}, error) {
request, err := types.MapToRequest(b.client.cdc, b.client.id, method, params)
if err != nil {
return nil, err
}
b.enqueue(&jsonRPCBufferedRequest{request: request, result: result})
return result, nil
}
//-------------------------------------------------------------
// URI takes params as a map
type URIClient struct {
address string
@ -168,7 +282,7 @@ func (c *URIClient) Call(method string, params map[string]interface{}, result in
if err != nil {
return nil, err
}
return unmarshalResponseBytes(c.cdc, responseBytes, result)
return unmarshalResponseBytes(c.cdc, responseBytes, "", result)
}
func (c *URIClient) Codec() *amino.Codec {
@ -181,7 +295,7 @@ func (c *URIClient) SetCodec(cdc *amino.Codec) {
//------------------------------------------------
func unmarshalResponseBytes(cdc *amino.Codec, responseBytes []byte, result interface{}) (interface{}, error) {
func unmarshalResponseBytes(cdc *amino.Codec, responseBytes []byte, expectedID types.JSONRPCStringID, result interface{}) (interface{}, error) {
// Read response. If rpc/core/types is imported, the result will unmarshal
// into the correct type.
// log.Notice("response", "response", string(responseBytes))
@ -189,19 +303,71 @@ func unmarshalResponseBytes(cdc *amino.Codec, responseBytes []byte, result inter
response := &types.RPCResponse{}
err = json.Unmarshal(responseBytes, response)
if err != nil {
return nil, errors.Errorf("Error unmarshalling rpc response: %v", err)
return nil, errors.Errorf("error unmarshalling rpc response: %v", err)
}
if response.Error != nil {
return nil, errors.Errorf("Response error: %v", response.Error)
return nil, errors.Errorf("response error: %v", response.Error)
}
// From the JSON-RPC 2.0 spec:
// id: It MUST be the same as the value of the id member in the Request Object.
if err := validateResponseID(response, expectedID); err != nil {
return nil, err
}
// Unmarshal the RawMessage into the result.
err = cdc.UnmarshalJSON(response.Result, result)
if err != nil {
return nil, errors.Errorf("Error unmarshalling rpc response result: %v", err)
return nil, errors.Errorf("error unmarshalling rpc response result: %v", err)
}
return result, nil
}
func unmarshalResponseBytesArray(cdc *amino.Codec, responseBytes []byte, expectedID types.JSONRPCStringID, results []interface{}) ([]interface{}, error) {
var (
err error
responses []types.RPCResponse
)
err = json.Unmarshal(responseBytes, &responses)
if err != nil {
return nil, errors.Errorf("error unmarshalling rpc response: %v", err)
}
// No response error checking here as there may be a mixture of successful
// and unsuccessful responses.
if len(results) != len(responses) {
return nil, errors.Errorf("expected %d result objects into which to inject responses, but got %d", len(responses), len(results))
}
for i, response := range responses {
// From the JSON-RPC 2.0 spec:
// id: It MUST be the same as the value of the id member in the Request Object.
if err := validateResponseID(&response, expectedID); err != nil {
return nil, errors.Errorf("failed to validate response ID in response %d: %v", i, err)
}
if err := cdc.UnmarshalJSON(responses[i].Result, results[i]); err != nil {
return nil, errors.Errorf("error unmarshalling rpc response result: %v", err)
}
}
return results, nil
}
func validateResponseID(res *types.RPCResponse, expectedID types.JSONRPCStringID) error {
// we only validate a response ID if the expected ID is non-empty
if len(expectedID) == 0 {
return nil
}
if res.ID == nil {
return errors.Errorf("missing ID in response")
}
id, ok := res.ID.(types.JSONRPCStringID)
if !ok {
return errors.Errorf("expected ID string in response but got: %v", id)
}
if expectedID != id {
return errors.Errorf("response ID (%s) does not match request ID (%s)", id, expectedID)
}
return nil
}
func argsToURLValues(cdc *amino.Codec, args map[string]interface{}) (url.Values, error) {
values := make(url.Values)
if len(args) == 0 {

View File

@ -103,7 +103,7 @@ func makeJSONRPCHandler(funcMap map[string]*RPCFunc, cdc *amino.Codec, logger lo
return func(w http.ResponseWriter, r *http.Request) {
b, err := ioutil.ReadAll(r.Body)
if err != nil {
WriteRPCResponseHTTP(w, types.RPCInvalidRequestError(types.JSONRPCStringID(""), errors.Wrap(err, "Error reading request body")))
WriteRPCResponseHTTP(w, types.RPCInvalidRequestError(types.JSONRPCStringID(""), errors.Wrap(err, "error reading request body")))
return
}
// if its an empty request (like from a browser),
@ -113,49 +113,59 @@ func makeJSONRPCHandler(funcMap map[string]*RPCFunc, cdc *amino.Codec, logger lo
return
}
var request types.RPCRequest
err = json.Unmarshal(b, &request)
if err != nil {
WriteRPCResponseHTTP(w, types.RPCParseError(types.JSONRPCStringID(""), errors.Wrap(err, "Error unmarshalling request")))
return
}
// A Notification is a Request object without an "id" member.
// The Server MUST NOT reply to a Notification, including those that are within a batch request.
if request.ID == types.JSONRPCStringID("") {
logger.Debug("HTTPJSONRPC received a notification, skipping... (please send a non-empty ID if you want to call a method)")
return
}
if len(r.URL.Path) > 1 {
WriteRPCResponseHTTP(w, types.RPCInvalidRequestError(request.ID, errors.Errorf("Path %s is invalid", r.URL.Path)))
return
}
rpcFunc := funcMap[request.Method]
if rpcFunc == nil || rpcFunc.ws {
WriteRPCResponseHTTP(w, types.RPCMethodNotFoundError(request.ID))
return
}
ctx := &types.Context{JSONReq: &request, HTTPReq: r}
args := []reflect.Value{reflect.ValueOf(ctx)}
if len(request.Params) > 0 {
fnArgs, err := jsonParamsToArgs(rpcFunc, cdc, request.Params)
if err != nil {
WriteRPCResponseHTTP(w, types.RPCInvalidParamsError(request.ID, errors.Wrap(err, "Error converting json params to arguments")))
// first try to unmarshal the incoming request as an array of RPC requests
var (
requests []types.RPCRequest
responses []types.RPCResponse
)
if err := json.Unmarshal(b, &requests); err != nil {
// next, try to unmarshal as a single request
var request types.RPCRequest
if err := json.Unmarshal(b, &request); err != nil {
WriteRPCResponseHTTP(w, types.RPCParseError(types.JSONRPCStringID(""), errors.Wrap(err, "error unmarshalling request")))
return
}
args = append(args, fnArgs...)
requests = []types.RPCRequest{request}
}
returns := rpcFunc.f.Call(args)
logger.Info("HTTPJSONRPC", "method", request.Method, "args", args, "returns", returns)
result, err := unreflectResult(returns)
if err != nil {
WriteRPCResponseHTTP(w, types.RPCInternalError(request.ID, err))
return
for _, request := range requests {
// A Notification is a Request object without an "id" member.
// The Server MUST NOT reply to a Notification, including those that are within a batch request.
if request.ID == types.JSONRPCStringID("") {
logger.Debug("HTTPJSONRPC received a notification, skipping... (please send a non-empty ID if you want to call a method)")
continue
}
if len(r.URL.Path) > 1 {
responses = append(responses, types.RPCInvalidRequestError(request.ID, errors.Errorf("path %s is invalid", r.URL.Path)))
continue
}
rpcFunc, ok := funcMap[request.Method]
if !ok || rpcFunc.ws {
responses = append(responses, types.RPCMethodNotFoundError(request.ID))
continue
}
ctx := &types.Context{JSONReq: &request, HTTPReq: r}
args := []reflect.Value{reflect.ValueOf(ctx)}
if len(request.Params) > 0 {
fnArgs, err := jsonParamsToArgs(rpcFunc, cdc, request.Params)
if err != nil {
responses = append(responses, types.RPCInvalidParamsError(request.ID, errors.Wrap(err, "error converting json params to arguments")))
continue
}
args = append(args, fnArgs...)
}
returns := rpcFunc.f.Call(args)
logger.Info("HTTPJSONRPC", "method", request.Method, "args", args, "returns", returns)
result, err := unreflectResult(returns)
if err != nil {
responses = append(responses, types.RPCInternalError(request.ID, err))
continue
}
responses = append(responses, types.NewRPCSuccessResponse(cdc, request.ID, result))
}
if len(responses) > 0 {
WriteRPCResponseArrayHTTP(w, responses)
}
WriteRPCResponseHTTP(w, types.NewRPCSuccessResponse(cdc, request.ID, result))
}
}
@ -194,7 +204,7 @@ func mapParamsToArgs(rpcFunc *RPCFunc, cdc *amino.Codec, params map[string]json.
func arrayParamsToArgs(rpcFunc *RPCFunc, cdc *amino.Codec, params []json.RawMessage, argsOffset int) ([]reflect.Value, error) {
if len(rpcFunc.argNames) != len(params) {
return nil, errors.Errorf("Expected %v parameters (%v), got %v (%v)",
return nil, errors.Errorf("expected %v parameters (%v), got %v (%v)",
len(rpcFunc.argNames), rpcFunc.argNames, len(params), params)
}
@ -236,7 +246,7 @@ func jsonParamsToArgs(rpcFunc *RPCFunc, cdc *amino.Codec, raw []byte) ([]reflect
}
// Otherwise, bad format, we cannot parse
return nil, errors.Errorf("Unknown type for JSON params: %v. Expected map or array", err)
return nil, errors.Errorf("unknown type for JSON params: %v. Expected map or array", err)
}
// rpc.json
@ -261,7 +271,7 @@ func makeHTTPHandler(rpcFunc *RPCFunc, cdc *amino.Codec, logger log.Logger) func
fnArgs, err := httpParamsToArgs(rpcFunc, cdc, r)
if err != nil {
WriteRPCResponseHTTP(w, types.RPCInvalidParamsError(types.JSONRPCStringID(""), errors.Wrap(err, "Error converting http params to arguments")))
WriteRPCResponseHTTP(w, types.RPCInvalidParamsError(types.JSONRPCStringID(""), errors.Wrap(err, "error converting http params to arguments")))
return
}
args = append(args, fnArgs...)
@ -372,7 +382,7 @@ func _nonJSONStringToArg(cdc *amino.Codec, rt reflect.Type, arg string) (reflect
if isHexString {
if !expectingString && !expectingByteSlice {
err := errors.Errorf("Got a hex string arg, but expected '%s'",
err := errors.Errorf("got a hex string arg, but expected '%s'",
rt.Kind().String())
return reflect.ValueOf(nil), err, false
}
@ -631,7 +641,7 @@ func (wsc *wsConnection) readRoutine() {
var request types.RPCRequest
err = json.Unmarshal(in, &request)
if err != nil {
wsc.WriteRPCResponse(types.RPCParseError(types.JSONRPCStringID(""), errors.Wrap(err, "Error unmarshaling request")))
wsc.WriteRPCResponse(types.RPCParseError(types.JSONRPCStringID(""), errors.Wrap(err, "error unmarshaling request")))
continue
}
@ -654,7 +664,7 @@ func (wsc *wsConnection) readRoutine() {
if len(request.Params) > 0 {
fnArgs, err := jsonParamsToArgs(rpcFunc, wsc.cdc, request.Params)
if err != nil {
wsc.WriteRPCResponse(types.RPCInternalError(request.ID, errors.Wrap(err, "Error converting json params to arguments")))
wsc.WriteRPCResponse(types.RPCInternalError(request.ID, errors.Wrap(err, "error converting json params to arguments")))
continue
}
args = append(args, fnArgs...)

View File

@ -154,6 +154,72 @@ func TestRPCNotification(t *testing.T) {
require.Equal(t, len(blob), 0, "a notification SHOULD NOT be responded to by the server")
}
func TestRPCNotificationInBatch(t *testing.T) {
mux := testMux()
tests := []struct {
payload string
expectCount int
}{
{
`[
{"jsonrpc": "2.0","id": ""},
{"jsonrpc": "2.0","method":"c","id":"abc","params":["a","10"]}
]`,
1,
},
{
`[
{"jsonrpc": "2.0","id": ""},
{"jsonrpc": "2.0","method":"c","id":"abc","params":["a","10"]},
{"jsonrpc": "2.0","id": ""},
{"jsonrpc": "2.0","method":"c","id":"abc","params":["a","10"]}
]`,
2,
},
}
for i, tt := range tests {
req, _ := http.NewRequest("POST", "http://localhost/", strings.NewReader(tt.payload))
rec := httptest.NewRecorder()
mux.ServeHTTP(rec, req)
res := rec.Result()
// Always expecting back a JSONRPCResponse
assert.True(t, statusOK(res.StatusCode), "#%d: should always return 2XX", i)
blob, err := ioutil.ReadAll(res.Body)
if err != nil {
t.Errorf("#%d: err reading body: %v", i, err)
continue
}
var responses []types.RPCResponse
// try to unmarshal an array first
err = json.Unmarshal(blob, &responses)
if err != nil {
// if we were actually expecting an array, but got an error
if tt.expectCount > 1 {
t.Errorf("#%d: expected an array, couldn't unmarshal it\nblob: %s", i, blob)
continue
} else {
// we were expecting an error here, so let's unmarshal a single response
var response types.RPCResponse
err = json.Unmarshal(blob, &response)
if err != nil {
t.Errorf("#%d: expected successful parsing of an RPCResponse\nblob: %s", i, blob)
continue
}
// have a single-element result
responses = []types.RPCResponse{response}
}
}
if tt.expectCount != len(responses) {
t.Errorf("#%d: expected %d response(s), but got %d\nblob: %s", i, tt.expectCount, len(responses), blob)
continue
}
for _, response := range responses {
assert.NotEqual(t, response, new(types.RPCResponse), "#%d: not expecting a blank RPCResponse", i)
}
}
}
func TestUnknownRPCPath(t *testing.T) {
mux := testMux()
req, _ := http.NewRequest("GET", "http://localhost/unknownrpcpath", nil)

View File

@ -76,7 +76,7 @@ func GetParamUint(r *http.Request, param string) (uint, error) {
func GetParamRegexp(r *http.Request, param string, re *regexp.Regexp) (string, error) {
s := GetParam(r, param)
if !re.MatchString(s) {
return "", errors.Errorf(param, "Did not match regular expression %v", re.String())
return "", errors.Errorf(param, "did not match regular expression %v", re.String())
}
return s, nil
}

View File

@ -98,7 +98,9 @@ func WriteRPCResponseHTTPError(
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(httpCode)
w.Write(jsonBytes) // nolint: errcheck, gas
if _, err := w.Write(jsonBytes); err != nil {
panic(err)
}
}
func WriteRPCResponseHTTP(w http.ResponseWriter, res types.RPCResponse) {
@ -108,12 +110,33 @@ func WriteRPCResponseHTTP(w http.ResponseWriter, res types.RPCResponse) {
}
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(200)
w.Write(jsonBytes) // nolint: errcheck, gas
if _, err := w.Write(jsonBytes); err != nil {
panic(err)
}
}
// WriteRPCResponseArrayHTTP will do the same as WriteRPCResponseHTTP, except it
// can write arrays of responses for batched request/response interactions via
// the JSON RPC.
func WriteRPCResponseArrayHTTP(w http.ResponseWriter, res []types.RPCResponse) {
if len(res) == 1 {
WriteRPCResponseHTTP(w, res[0])
} else {
jsonBytes, err := json.MarshalIndent(res, "", " ")
if err != nil {
panic(err)
}
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(200)
if _, err := w.Write(jsonBytes); err != nil {
panic(err)
}
}
}
//-----------------------------------------------------------------------------
// Wraps an HTTP handler, adding error logging.
// RecoverAndLogHandler wraps an HTTP handler, adding error logging.
// If the inner function panics, the outer function recovers, logs, sends an
// HTTP 500 error response.
func RecoverAndLogHandler(handler http.Handler, logger log.Logger) http.Handler {
@ -191,14 +214,14 @@ func Listen(addr string, config *Config) (listener net.Listener, err error) {
parts := strings.SplitN(addr, "://", 2)
if len(parts) != 2 {
return nil, errors.Errorf(
"Invalid listening address %s (use fully formed addresses, including the tcp:// or unix:// prefix)",
"invalid listening address %s (use fully formed addresses, including the tcp:// or unix:// prefix)",
addr,
)
}
proto, addr := parts[0], parts[1]
listener, err = net.Listen(proto, addr)
if err != nil {
return nil, errors.Errorf("Failed to listen on %v: %v", addr, err)
return nil, errors.Errorf("failed to listen on %v: %v", addr, err)
}
if config.MaxOpenConnections > 0 {
listener = netutil.LimitListener(listener, config.MaxOpenConnections)