bench by nblocks or ntxs

handlers/handlers.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"sort"
 	"sync"
+	"time"
 
 	"github.com/tendermint/go-event-meter"
 	"github.com/tendermint/go-wire"
@@ -128,7 +129,15 @@ func (tn *TendermintNetwork) RegisterChain(chainConfig *types.BlockchainConfig)
 	for _, v := range chainConfig.Validators {
 		v.Status = &types.ValidatorStatus{}
 
-		if err := v.Start(); err != nil {
+		var err error
+	RETRYLOOP:
+		for i := 0; i < 10; i++ {
+			if err = v.Start(); err == nil {
+				break RETRYLOOP
+			}
+			time.Sleep(time.Second)
+		}
+		if err != nil {
 			return nil, fmt.Errorf("Error starting validator %s: %v", v.Config.Validator.ID, err)
 		}
 

main.go

@@ -83,6 +83,18 @@ func main() {
 			Action: func(c *cli.Context) {
 				cmdBench(c)
 			},
+			Flags: []cli.Flag{
+				cli.IntFlag{
+					Name:  "n_txs",
+					Value: 0,
+					Usage: "run benchmark until this many txs have been committed",
+				},
+				cli.IntFlag{
+					Name:  "n_blocks",
+					Value: 0,
+					Usage: "run benchmark until this many blocks have been committed",
+				},
+			},
 		},
 	}
 	app.Run(os.Args)
@@ -185,17 +197,16 @@ func cmdMonitor(c *cli.Context) {
 
 func cmdBench(c *cli.Context) {
 	args := c.Args()
-	if len(args) < 4 {
+	if len(args) < 2 {
-		Exit("bench expectes at least 4 args")
+		Exit("bench expects at least 2 args")
 	}
 	chainsAndValsFile := args[0]
 	resultsDir := args[1]
-	nTxS := args[2]
+	// extra args are a program to run locally
-	args = args[3:]
+	if len(args) > 2 {
-
+		args = args[2:]
-	nTxs, err := strconv.Atoi(nTxS)
+	} else {
-	if err != nil {
+		args = args[:0]
-		Exit(err.Error())
 	}
 
 	chainsAndVals, err := LoadChainsAndValsFromFile(chainsAndValsFile)
@@ -212,15 +223,29 @@ func cmdBench(c *cli.Context) {
 	// we should only have one chain for a benchmark run
 	chAndValIDs, _ := network.Status()
 	chain, _ := network.GetChain(chAndValIDs.ChainIDs[0])
-	chain.Status.Benchmark(done, nTxs, args)
 
+	// setup benchresults struct and fire txs
+	if nTxs := c.Int("n_txs"); nTxs != 0 {
+		chain.Status.BenchmarkTxs(done, nTxs, args)
+	} else if nBlocks := c.Int("n_blocks"); nBlocks != 0 {
+		chain.Status.BenchmarkBlocks(done, nBlocks, args)
+	} else {
+		Exit("Must specify one of n_txs or n_blocks")
+	}
 	results := <-done
 
-	b, _ := json.Marshal(results)
+	b, err := json.Marshal(results)
-	if err := ioutil.WriteFile(path.Join(resultsDir, "results.dat"), b, 0600); err != nil {
+	if err != nil {
+		Exit(err.Error())
+	}
+	fmt.Println(string(b))
+	if err := ioutil.WriteFile(path.Join(resultsDir, "netmon.log"), b, 0600); err != nil {
+		Exit(err.Error())
+	}
+	finalResults := fmt.Sprintf("%f,%f\n", results.MeanLatency, results.MeanThroughput)
+	if err := ioutil.WriteFile(path.Join(resultsDir, "final_results"), []byte(finalResults), 0600); err != nil {
 		Exit(err.Error())
 	}
-	fmt.Println(results)
 }
 
 func registerNetwork(chainsAndVals *ChainsAndValidators) *handlers.TendermintNetwork {

types/chain.go

@@ -2,6 +2,7 @@ package types
 
 import (
 	"fmt"
+	"os"
 	"os/exec"
 	"sync"
 	"time"
@@ -120,16 +121,38 @@ type BlockchainStatus struct {
 	benchResults *BenchmarkResults
 }
 
-func (bc *BlockchainStatus) Benchmark(done chan *BenchmarkResults, nTxs int, args []string) {
+func (bc *BlockchainStatus) BenchmarkTxs(results chan *BenchmarkResults, nTxs int, args []string) {
+	log.Notice("Running benchmark", "ntxs", nTxs)
 	bc.benchResults = &BenchmarkResults{
 		StartTime: time.Now(),
-		NumTxs:    nTxs,
+		nTxs:      nTxs,
-		done:      done,
+		results:   results,
 	}
 
-	// TODO: capture output to file
+	if len(args) > 0 {
-	cmd := exec.Command(args[0], args[1:]...)
+		// TODO: capture output to file
-	go cmd.Run()
+		cmd := exec.Command(args[0], args[1:]...)
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stderr
+		go cmd.Run()
+	}
+}
+
+func (bc *BlockchainStatus) BenchmarkBlocks(results chan *BenchmarkResults, nBlocks int, args []string) {
+	log.Notice("Running benchmark", "nblocks", nBlocks)
+	bc.benchResults = &BenchmarkResults{
+		StartTime: time.Now(),
+		nBlocks:   nBlocks,
+		results:   results,
+	}
+
+	if len(args) > 0 {
+		// TODO: capture output to file
+		cmd := exec.Command(args[0], args[1:]...)
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stderr
+		go cmd.Run()
+	}
 }
 
 type Block struct {
@@ -140,19 +163,25 @@ type Block struct {
 
 type BenchmarkResults struct {
 	StartTime      time.Time `json:"start_time"`
+	StartBlock     int       `json:"start_block"`
 	TotalTime      float64   `json:"total_time"` // seconds
+	Blocks         []*Block  `json:"blocks"`
 	NumBlocks      int       `json:"num_blocks"`
 	NumTxs         int       `json:"num_txs`
-	Blocks         []*Block  `json:"blocks"`
 	MeanLatency    float64   `json:"latency"`    // seconds per block
 	MeanThroughput float64   `json:"throughput"` // txs per second
 
-	done chan *BenchmarkResults
+	// either we wait for n blocks or n txs
+	nBlocks int
+	nTxs    int
+
+	done    bool
+	results chan *BenchmarkResults
 }
 
 // Return the total time to commit all txs, in seconds
 func (br *BenchmarkResults) ElapsedTime() float64 {
-	return float64(br.Blocks[br.NumBlocks].Time.Sub(br.StartTime)) / float64(1000000000)
+	return float64(br.Blocks[br.NumBlocks-1].Time.Sub(br.StartTime)) / float64(1000000000)
 }
 
 // Return the avg seconds/block
@@ -166,10 +195,12 @@ func (br *BenchmarkResults) Throughput() float64 {
 }
 
 func (br *BenchmarkResults) Done() {
+	log.Info("Done benchmark", "num blocks", br.NumBlocks, "block len", len(br.Blocks))
+	br.done = true
 	br.TotalTime = br.ElapsedTime()
 	br.MeanThroughput = br.Throughput()
 	br.MeanLatency = br.Latency()
-	br.done <- br
+	br.results <- br
 }
 
 type UptimeData struct {
@@ -198,20 +229,28 @@ func (s *BlockchainStatus) NewBlock(block *tmtypes.Block) {
 	s.mtx.Lock()
 	defer s.mtx.Unlock()
 	if block.Header.Height > s.Height {
+		numTxs := block.Header.NumTxs
 		s.Height = block.Header.Height
 		s.blockTimeMeter.Mark(1)
-		s.txThroughputMeter.Mark(int64(block.Header.NumTxs))
+		s.txThroughputMeter.Mark(int64(numTxs))
-		s.MeanBlockTime = (1 / s.blockTimeMeter.Rate1()) * 1000 // 1/s to ms
+		s.MeanBlockTime = (1.0 / s.blockTimeMeter.Rate1()) * 1000 // 1/s to ms
 		s.TxThroughput = s.txThroughputMeter.Rate1()
 
-		if s.benchResults != nil {
+		log.Debug("New Block", "height", s.Height, "ntxs", numTxs)
+		if s.benchResults != nil && !s.benchResults.done {
+			if s.benchResults.StartBlock == 0 && numTxs > 0 {
+				s.benchResults.StartBlock = s.Height
+			}
 			s.benchResults.Blocks = append(s.benchResults.Blocks, &Block{
 				Time:   time.Now(),
 				Height: s.Height,
-				NumTxs: block.Header.NumTxs,
+				NumTxs: numTxs,
 			})
-			if s.txThroughputMeter.Count() >= int64(s.benchResults.NumTxs) {
+			s.benchResults.NumTxs += numTxs
-				// XXX: do we need to be more careful than just counting?!
+			s.benchResults.NumBlocks += 1
+			if s.benchResults.nTxs > 0 && s.benchResults.NumTxs >= s.benchResults.nTxs {
+				s.benchResults.Done()
+			} else if s.benchResults.nBlocks > 0 && s.benchResults.NumBlocks >= s.benchResults.nBlocks {
 				s.benchResults.Done()
 			}
 		}