different way to get stats (#90)

Refs #62

tm-bench/Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.6
+FROM alpine:3.7
 
 WORKDIR /app
 COPY tm-bench /app/tm-bench

tm-bench/Gopkg.lock

@@ -5,7 +5,13 @@
   branch = "master"
   name = "github.com/btcsuite/btcd"
   packages = ["btcec"]
-  revision = "2be2f12b358dc57d70b8f501b00be450192efbc3"
+  revision = "675abc5df3c5531bc741b56a765e35623459da6d"
+
+[[projects]]
+  name = "github.com/davecgh/go-spew"
+  packages = ["spew"]
+  revision = "346938d642f2ec3594ed81d874461961cd0faa76"
+  version = "v1.1.0"
 
 [[projects]]
   branch = "master"
@@ -94,7 +100,7 @@
   branch = "master"
   name = "github.com/rcrowley/go-metrics"
   packages = ["."]
-  revision = "8732c616f52954686704c8645fe1a9d59e9df7c1"
+  revision = "e2704e165165ec55d062f5919b4b29494e9fa790"
 
 [[projects]]
   branch = "master"
@@ -113,7 +119,7 @@
     "leveldb/table",
     "leveldb/util"
   ]
-  revision = "714f901b98fdb3aa954b4193d8cbd64a28d80cad"
+  revision = "ae970a0732be3a1f5311da86118d37b9f4bd2a5a"
 
 [[projects]]
   name = "github.com/tendermint/abci"
@@ -123,8 +129,8 @@
     "example/kvstore",
     "types"
   ]
-  revision = "46686763ba8ea595ede16530ed4a40fb38f49f94"
+  revision = "78a8905690ef54f9d57e3b2b0ee7ad3a04ef3f1f"
-  version = "v0.10.2"
+  version = "v0.10.3"
 
 [[projects]]
   branch = "master"
@@ -137,63 +143,66 @@
   revision = "d8387025d2b9d158cf4efb07e7ebf814bcce2057"
 
 [[projects]]
-  name = "github.com/tendermint/go-crypto"
+  name = "github.com/tendermint/go-amino"
   packages = ["."]
-  revision = "c3e19f3ea26f5c3357e0bcbb799b0761ef923755"
+  revision = "ed62928576cfcaf887209dc96142cd79cdfff389"
-  version = "v0.5.0"
+  version = "0.9.9"
 
 [[projects]]
-  name = "github.com/tendermint/go-wire"
+  name = "github.com/tendermint/go-crypto"
-  packages = [
+  packages = ["."]
-    ".",
+  revision = "915416979bf70efa4bcbf1c6cd5d64c5fff9fc19"
-    "data"
+  version = "v0.6.2"
-  ]
-  revision = "fa721242b042ecd4c6ed1a934ee740db4f74e45c"
-  source = "github.com/tendermint/go-amino"
-  version = "v0.7.3"
 
 [[projects]]
   name = "github.com/tendermint/tendermint"
   packages = [
+    "blockchain",
     "config",
+    "consensus",
     "consensus/types",
+    "evidence",
+    "mempool",
+    "node",
     "p2p",
     "p2p/conn",
+    "p2p/pex",
     "p2p/upnp",
     "proxy",
+    "rpc/client",
+    "rpc/core",
     "rpc/core/types",
+    "rpc/grpc",
+    "rpc/lib",
     "rpc/lib/client",
+    "rpc/lib/server",
     "rpc/lib/types",
     "state",
+    "state/txindex",
+    "state/txindex/kv",
+    "state/txindex/null",
     "types",
-    "wire"
+    "types/priv_validator",
+    "version"
   ]
-  revision = "6f9956990c444d53f62f2a3905ed410cfe9afe77"
+  revision = "64408a40419a5fa6f156dd96a2ab2fffa3c8ab96"
-  version = "v0.17.1"
+  version = "v0.19.2"
 
 [[projects]]
   name = "github.com/tendermint/tmlibs"
   packages = [
+    "autofile",
+    "clist",
     "common",
     "db",
-    "events",
     "flowrate",
     "log",
     "merkle",
     "pubsub",
     "pubsub/query"
   ]
-  revision = "24da7009c3d8c019b40ba4287495749e3160caca"
+  revision = "d94e312673e16a11ea55d742cefb3e331228f898"
-  version = "v0.7.1"
+  version = "v0.8.2"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/tendermint/tools"
-  packages = [
-    "tm-monitor/eventmeter",
-    "tm-monitor/monitor"
-  ]
-  revision = "a66b20aff45868e53b64b54a9f6feccd6e3ca3ba"
 
 [[projects]]
   branch = "master"
@@ -208,13 +217,14 @@
     "ripemd160",
     "salsa20/salsa"
   ]
-  revision = "12892e8c234f4fe6f6803f052061de9057903bb2"
+  revision = "1f94bef427e370e425e55b172f3b5e300ef38304"
 
 [[projects]]
   branch = "master"
   name = "golang.org/x/net"
   packages = [
     "context",
+    "http/httpguts",
     "http2",
     "http2/hpack",
     "idna",
@@ -222,7 +232,7 @@
     "lex/httplex",
     "trace"
   ]
-  revision = "b68f30494add4df6bd8ef5e82803f308e7f7c59c"
+  revision = "640f4622ab692b87c2f3a94265e6f579fe38263d"
 
 [[projects]]
   name = "golang.org/x/text"
@@ -249,7 +259,7 @@
   branch = "master"
   name = "google.golang.org/genproto"
   packages = ["googleapis/rpc/status"]
-  revision = "ab0870e398d5dd054b868c0db1481ab029b9a9f2"
+  revision = "86e600f69ee4704c6efbf6a2a40a5c10700e76c2"
 
 [[projects]]
   name = "google.golang.org/grpc"
@@ -278,6 +288,6 @@
 [solve-meta]
   analyzer-name = "dep"
   analyzer-version = 1
-  inputs-digest = "9e81caf774c988dc43578eac01f7b783d5179226a14e60b31db5d50a29cf633f"
+  inputs-digest = "0da857a5c1642e5e47aaa00e5b82b15dcdd1b08e44d0b027676adc085501e6a6"
   solver-name = "gps-cdcl"
   solver-version = 1

tm-bench/Gopkg.toml

@@ -31,11 +31,11 @@
 
 [[constraint]]
   name = "github.com/gorilla/websocket"
-  version = "1.2.0"
+  version = "^1.2.0"
 
 [[constraint]]
   name = "github.com/pkg/errors"
-  version = "0.8.0"
+  version = "^0.8.0"
 
 [[constraint]]
   branch = "master"
@@ -43,15 +43,11 @@
 
 [[constraint]]
   name = "github.com/tendermint/tendermint"
-  version = "0.17.1"
+  version = "~0.19.2"
 
 [[constraint]]
   name = "github.com/tendermint/tmlibs"
-  version = "0.7.1"
+  version = "~0.8.2"
-
-[[constraint]]
-  name = "github.com/tendermint/tools"
-  branch = "master"
 
 [prune]
   go-tests = true

tm-bench/main.go

@@ -13,9 +13,8 @@ import (
 
 	"text/tabwriter"
 
-	tmtypes "github.com/tendermint/tendermint/types"
+	tmrpc "github.com/tendermint/tendermint/rpc/client"
 	"github.com/tendermint/tmlibs/log"
-	"github.com/tendermint/tools/tm-monitor/monitor"
 )
 
 var version = "0.3.0"
@@ -23,9 +22,8 @@ var version = "0.3.0"
 var logger = log.NewNopLogger()
 
 type statistics struct {
-	BlockTimeSample    metrics.Histogram
+	TxsThroughput    metrics.Histogram `json:"txs_per_sec"`
-	TxThroughputSample metrics.Histogram
+	BlocksThroughput metrics.Histogram `json:"blocks_per_sec"`
-	BlockLatency       metrics.Histogram
 }
 
 func main() {
@@ -60,7 +58,7 @@ Examples:
 
 	if verbose {
 		if outputFormat == "json" {
-			fmt.Println("Verbose mode not supported with json output.")
+			fmt.Fprintln(os.Stderr, "Verbose mode not supported with json output.")
 			os.Exit(1)
 		}
 		// Color errors red
@@ -79,71 +77,91 @@ Examples:
 
 	endpoints := strings.Split(flag.Arg(0), ",")
 
-	blockCh := make(chan tmtypes.Header, 100)
+	client := tmrpc.NewHTTP(endpoints[0], "/websocket")
-	blockLatencyCh := make(chan float64, 100)
 
-	nodes := startNodes(endpoints, blockCh, blockLatencyCh)
+	minHeight := latestBlockHeight(client)
+	logger.Info("Latest block height", "h", minHeight)
+
+	// record time start
+	timeStart := time.Now()
+	logger.Info("Time started", "t", timeStart)
 
 	transacters := startTransacters(endpoints, connections, txsRate)
 
-	stats := &statistics{
-		BlockTimeSample:    metrics.NewHistogram(metrics.NewUniformSample(1000)),
-		TxThroughputSample: metrics.NewHistogram(metrics.NewUniformSample(1000)),
-		BlockLatency:       metrics.NewHistogram(metrics.NewUniformSample(1000)),
-	}
-
-	lastBlockHeight := int64(-1)
-
-	durationTimer := time.After(time.Duration(duration) * time.Second)
-	ticker := time.NewTicker(1 * time.Second)
-	var blocks int
-	var txs int64
-	for {
 	select {
-		case b := <-blockCh:
+	case <-time.After(time.Duration(duration) * time.Second):
-			if lastBlockHeight < b.Height {
-				blocks++
-				txs += b.NumTxs
-				lastBlockHeight = b.Height
-			}
-		case l := <-blockLatencyCh:
-			stats.BlockLatency.Update(int64(l))
-		case <-ticker.C:
-			stats.BlockTimeSample.Update(int64(blocks))
-			stats.TxThroughputSample.Update(txs)
-			blocks = 0
-			txs = 0
-		case <-durationTimer:
 		for _, t := range transacters {
 			t.Stop()
 		}
+		timeStop := time.Now()
+		logger.Info("Time stopped", "t", timeStop)
+
+		stats := calculateStatistics(client, minHeight, timeStart, timeStop)
 
 		printStatistics(stats, outputFormat)
 
-			for _, n := range nodes {
-				n.Stop()
-			}
 		return
 	}
 }
-}
 
-func startNodes(endpoints []string, blockCh chan<- tmtypes.Header, blockLatencyCh chan<- float64) []*monitor.Node {
+func latestBlockHeight(client tmrpc.Client) int64 {
-	nodes := make([]*monitor.Node, len(endpoints))
+	status, err := client.Status()
-
+	if err != nil {
-	for i, e := range endpoints {
+		fmt.Fprintln(os.Stderr, err)
-		n := monitor.NewNode(e)
-		n.SetLogger(logger.With("node", e))
-		n.SendBlocksTo(blockCh)
-		n.SendBlockLatenciesTo(blockLatencyCh)
-		if err := n.Start(); err != nil {
-			fmt.Println(err)
 		os.Exit(1)
 	}
-		nodes[i] = n
+	return status.SyncInfo.LatestBlockHeight
 }
 
-	return nodes
+func calculateStatistics(client tmrpc.Client, minHeight int64, timeStart, timeStop time.Time) *statistics {
+	stats := &statistics{
+		BlocksThroughput: metrics.NewHistogram(metrics.NewUniformSample(1000)),
+		TxsThroughput:    metrics.NewHistogram(metrics.NewUniformSample(1000)),
+	}
+
+	// get blocks between minHeight and last height
+	info, err := client.BlockchainInfo(minHeight, 0)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+
+	numBlocksPerSec := make(map[int64]int64)
+	numTxsPerSec := make(map[int64]int64)
+	for _, blockMeta := range info.BlockMetas {
+		// check if block was created before timeStop
+		if blockMeta.Header.Time.After(timeStop) {
+			break
+		}
+
+		sec := secondsSinceTimeStart(timeStart, blockMeta.Header.Time)
+
+		// increase number of blocks for that second
+		if _, ok := numBlocksPerSec[sec]; !ok {
+			numBlocksPerSec[sec] = 0
+		}
+		numBlocksPerSec[sec]++
+
+		// increase number of txs for that second
+		if _, ok := numTxsPerSec[sec]; !ok {
+			numTxsPerSec[sec] = 0
+		}
+		numTxsPerSec[sec] += blockMeta.Header.NumTxs
+	}
+
+	for _, n := range numBlocksPerSec {
+		stats.BlocksThroughput.Update(n)
+	}
+
+	for _, n := range numTxsPerSec {
+		stats.TxsThroughput.Update(n)
+	}
+
+	return stats
+}
+
+func secondsSinceTimeStart(timeStart, timePassed time.Time) int64 {
+	return int64(timePassed.Sub(timeStart).Seconds())
 }
 
 func startTransacters(endpoints []string, connections int, txsRate int) []*transacter {
@@ -153,7 +171,7 @@ func startTransacters(endpoints []string, connections int, txsRate int) []*trans
 		t := newTransacter(e, connections, txsRate)
 		t.SetLogger(logger)
 		if err := t.Start(); err != nil {
-			fmt.Println(err)
+			fmt.Fprintln(os.Stderr, err)
 			os.Exit(1)
 		}
 		transacters[i] = t
@@ -162,47 +180,28 @@ func startTransacters(endpoints []string, connections int, txsRate int) []*trans
 	return transacters
 }
 
-type Results struct {
-	BlockLatencyMean   float64
-	BlockLatencyMax    int64
-	BlockLatencyStdDev float64
-	BlockTimeMean      float64
-	BlockTimeMax       int64
-	BlockTimeStdDev    float64
-	TxThroughputMean   float64
-	TxThroughputMax    int64
-	TxThroughputStdDev float64
-}
-
 func printStatistics(stats *statistics, outputFormat string) {
 	if outputFormat == "json" {
-		result, _ := json.Marshal(Results{
+		result, err := json.Marshal(struct {
-			stats.BlockLatency.Mean() / 1000000.0,
+			TxsThroughput    float64 `json:"txs_per_sec_avg"`
-			stats.BlockLatency.Max() / 1000000.0,
+			BlocksThroughput float64 `json:"blocks_per_sec_avg"`
-			stats.BlockLatency.StdDev() / 1000000.0,
+		}{stats.TxsThroughput.Mean(), stats.BlocksThroughput.Mean()})
-			stats.BlockTimeSample.Mean(),
+		if err != nil {
-			stats.BlockTimeSample.Max(),
+			fmt.Fprintln(os.Stderr, err)
-			stats.BlockTimeSample.StdDev(),
+			os.Exit(1)
-			stats.TxThroughputSample.Mean(),
+		}
-			stats.TxThroughputSample.Max(),
-			stats.TxThroughputSample.StdDev(),
-		})
 		fmt.Println(string(result))
 	} else {
 		w := tabwriter.NewWriter(os.Stdout, 0, 0, 5, ' ', 0)
-		fmt.Fprintln(w, "Stats\tAvg\tStdev\tMax\t")
+		fmt.Fprintln(w, "Stats\tAvg\tStdDev\tMax\t")
-		fmt.Fprintln(w, fmt.Sprintf("Block latency\t%.2fms\t%.2fms\t%dms\t",
-			stats.BlockLatency.Mean()/1000000.0,
-			stats.BlockLatency.StdDev()/1000000.0,
-			stats.BlockLatency.Max()/1000000))
-		fmt.Fprintln(w, fmt.Sprintf("Blocks/sec\t%.3f\t%.3f\t%d\t",
-			stats.BlockTimeSample.Mean(),
-			stats.BlockTimeSample.StdDev(),
-			stats.BlockTimeSample.Max()))
 		fmt.Fprintln(w, fmt.Sprintf("Txs/sec\t%.0f\t%.0f\t%d\t",
-			stats.TxThroughputSample.Mean(),
+			stats.TxsThroughput.Mean(),
-			stats.TxThroughputSample.StdDev(),
+			stats.TxsThroughput.StdDev(),
-			stats.TxThroughputSample.Max()))
+			stats.TxsThroughput.Max()))
+		fmt.Fprintln(w, fmt.Sprintf("Blocks/sec\t%.3f\t%.3f\t%d\t",
+			stats.BlocksThroughput.Mean(),
+			stats.BlocksThroughput.StdDev(),
+			stats.BlocksThroughput.Max()))
 		w.Flush()
 	}
 }