diff --git a/config/config.go b/config/config.go index c6cb6549..1a0dfaf3 100644 --- a/config/config.go +++ b/config/config.go @@ -45,34 +45,37 @@ type Config struct { BaseConfig `mapstructure:",squash"` // Options for services - RPC *RPCConfig `mapstructure:"rpc"` - P2P *P2PConfig `mapstructure:"p2p"` - Mempool *MempoolConfig `mapstructure:"mempool"` - Consensus *ConsensusConfig `mapstructure:"consensus"` - TxIndex *TxIndexConfig `mapstructure:"tx_index"` + RPC *RPCConfig `mapstructure:"rpc"` + P2P *P2PConfig `mapstructure:"p2p"` + Mempool *MempoolConfig `mapstructure:"mempool"` + Consensus *ConsensusConfig `mapstructure:"consensus"` + TxIndex *TxIndexConfig `mapstructure:"tx_index"` + Instrumentation *InstrumentationConfig `mapstructure:"instrumentation"` } // DefaultConfig returns a default configuration for a Tendermint node func DefaultConfig() *Config { return &Config{ - BaseConfig: DefaultBaseConfig(), - RPC: DefaultRPCConfig(), - P2P: DefaultP2PConfig(), - Mempool: DefaultMempoolConfig(), - Consensus: DefaultConsensusConfig(), - TxIndex: DefaultTxIndexConfig(), + BaseConfig: DefaultBaseConfig(), + RPC: DefaultRPCConfig(), + P2P: DefaultP2PConfig(), + Mempool: DefaultMempoolConfig(), + Consensus: DefaultConsensusConfig(), + TxIndex: DefaultTxIndexConfig(), + Instrumentation: DefaultInstrumentationConfig(), } } // TestConfig returns a configuration that can be used for testing func TestConfig() *Config { return &Config{ - BaseConfig: TestBaseConfig(), - RPC: TestRPCConfig(), - P2P: TestP2PConfig(), - Mempool: TestMempoolConfig(), - Consensus: TestConsensusConfig(), - TxIndex: TestTxIndexConfig(), + BaseConfig: TestBaseConfig(), + RPC: TestRPCConfig(), + P2P: TestP2PConfig(), + Mempool: TestMempoolConfig(), + Consensus: TestConsensusConfig(), + TxIndex: TestTxIndexConfig(), + Instrumentation: TestInstrumentationConfig(), } } @@ -142,10 +145,6 @@ type BaseConfig struct { // Database directory DBPath string `mapstructure:"db_dir"` - - // When true, metrics are served under `/metrics` using a Prometheus client - // Check out the documentation for the list of available metrics. - Monitoring bool `mapstructure:"monitoring"` } // DefaultBaseConfig returns a default base configuration for a Tendermint node @@ -163,7 +162,6 @@ func DefaultBaseConfig() BaseConfig { FilterPeers: false, DBBackend: "leveldb", DBPath: "data", - Monitoring: false, } } @@ -578,6 +576,35 @@ func TestTxIndexConfig() *TxIndexConfig { return DefaultTxIndexConfig() } +//----------------------------------------------------------------------------- +// InstrumentationConfig + +// InstrumentationConfig defines the configuration for metrics reporting. +type InstrumentationConfig struct { + // When true, Prometheus metrics are served under /metrics on + // PrometheusListenAddr. + // Check out the documentation for the list of available metrics. + Prometheus bool `mapstructure:"prometheus"` + + // Address to listen for Prometheus collector(s) connections. + PrometheusListenAddr string `mapstructure:"prometheus_listen_addr"` +} + +// DefaultInstrumentationConfig returns a default configuration for metrics +// reporting. +func DefaultInstrumentationConfig() *InstrumentationConfig { + return &InstrumentationConfig{ + Prometheus: false, + PrometheusListenAddr: ":26660", + } +} + +// TestInstrumentationConfig returns a default configuration for metrics +// reporting. +func TestInstrumentationConfig() *InstrumentationConfig { + return DefaultInstrumentationConfig() +} + //----------------------------------------------------------------------------- // Utils diff --git a/config/toml.go b/config/toml.go index 8927f15d..c3d41a9b 100644 --- a/config/toml.go +++ b/config/toml.go @@ -107,10 +107,6 @@ prof_laddr = "{{ .BaseConfig.ProfListenAddress }}" # so the app can decide if we should keep the connection or not filter_peers = {{ .BaseConfig.FilterPeers }} -# When true, metrics are served under /metrics using a Prometheus client -# Check out the documentation for the list of available metrics. -monitoring = {{ .BaseConfig.Monitoring }} - ##### advanced configuration options ##### ##### rpc server configuration options ##### @@ -236,6 +232,17 @@ index_tags = "{{ .TxIndex.IndexTags }}" # desirable (see the comment above). IndexTags has a precedence over # IndexAllTags (i.e. when given both, IndexTags will be indexed). index_all_tags = {{ .TxIndex.IndexAllTags }} + +##### instrumentation configuration options ##### +[instrumentation] + +# When true, Prometheus metrics are served under /metrics on +# PrometheusListenAddr. +# Check out the documentation for the list of available metrics. +prometheus = {{ .Instrumentation.Prometheus }} + +# Address to listen for Prometheus collector(s) connections +prometheus_listen_addr = "{{ .Instrumentation.PrometheusListenAddr }}" ` /****** these are for test settings ***********/ diff --git a/node/node.go b/node/node.go index 0d3ecdc1..df1d10cb 100644 --- a/node/node.go +++ b/node/node.go @@ -2,6 +2,7 @@ package node import ( "bytes" + "context" "errors" "fmt" "net" @@ -210,6 +211,7 @@ type Node struct { rpcListeners []net.Listener // rpc servers txIndexer txindex.TxIndexer indexerService *txindex.IndexerService + prometheusSrv *http.Server } // NewNode returns a new, ready to go, Tendermint Node. @@ -311,7 +313,7 @@ func NewNode(config *cfg.Config, p2pMetrics *p2p.Metrics memplMetrics *mempl.Metrics ) - if config.BaseConfig.Monitoring { + if config.Instrumentation.Prometheus { csMetrics, p2pMetrics, memplMetrics = metricsProvider() } else { csMetrics, p2pMetrics, memplMetrics = NopMetricsProvider() @@ -520,6 +522,10 @@ func (n *Node) OnStart() error { n.rpcListeners = listeners } + if n.config.Instrumentation.Prometheus { + n.prometheusSrv = n.StartPrometheusServer(n.config.Instrumentation.PrometheusListenAddr) + } + // Start the switch (the P2P server). err = n.sw.Start() if err != nil { @@ -561,6 +567,13 @@ func (n *Node) OnStop() { n.Logger.Error("Error stopping priv validator socket client", "err", err) } } + + if n.prometheusSrv != nil { + if err := n.prometheusSrv.Shutdown(context.Background()); err != nil { + // Error from closing listeners, or context timeout: + n.Logger.Error("Prometheus HTTP server Shutdown", "err", err) + } + } } // RunForever waits for an interrupt signal and stops the node. @@ -615,9 +628,6 @@ func (n *Node) startRPC() ([]net.Listener, error) { wm := rpcserver.NewWebsocketManager(rpccore.Routes, coreCodec, rpcserver.EventSubscriber(n.eventBus)) wm.SetLogger(rpcLogger.With("protocol", "websocket")) mux.HandleFunc("/websocket", wm.WebsocketHandler) - if n.config.BaseConfig.Monitoring { - mux.Handle("/metrics", promhttp.Handler()) - } rpcserver.RegisterRPCFuncs(mux, rpccore.Routes, coreCodec, rpcLogger) listener, err := rpcserver.StartHTTPServer(listenAddr, mux, rpcLogger) if err != nil { @@ -639,6 +649,22 @@ func (n *Node) startRPC() ([]net.Listener, error) { return listeners, nil } +// StartPrometheusServer starts a Prometheus HTTP server, listening for metrics +// collectors on addr. +func (n *Node) StartPrometheusServer(addr string) *http.Server { + srv := &http.Server{ + Addr: addr, + Handler: promhttp.Handler(), + } + go func(s *http.Server, logger log.Logger) { + if err := s.ListenAndServe(); err != http.ErrServerClosed { + // Error starting or closing listener: + logger.Error("Prometheus HTTP server ListenAndServe", "err", err) + } + }(srv, n.Logger) + return srv +} + // Switch returns the Node's Switch. func (n *Node) Switch() *p2p.Switch { return n.sw