Unclean shutdown on SIGINT / SIGTERM (#3308)

* libs/common: TrapSignal accepts logger as a first parameter

 and does not block anymore
* previously it was dumping "captured ..." msg to os.Stdout
* TrapSignal should not be responsible for blocking thread of execution

Refs #3238

* exit with zero (0) code upon receiving SIGTERM/SIGINT

Refs #3238

* fix formatting in docs/app-dev/abci-cli.md

Co-Authored-By: melekes <anton.kalyaev@gmail.com>

* fix formatting in docs/app-dev/abci-cli.md

Co-Authored-By: melekes <anton.kalyaev@gmail.com>

CHANGELOG_PENDING.md

@@ -12,6 +12,9 @@ Special thanks to external contributors on this release:
 * Apps
 
 * Go API
+- [libs/common] TrapSignal accepts logger as a first parameter and does not block anymore
+  * previously it was dumping "captured ..." msg to os.Stdout
+  * TrapSignal should not be responsible for blocking thread of execution
 
 * Blockchain Protocol
 
@@ -23,6 +26,7 @@ Special thanks to external contributors on this release:
   `/num_unconfirmed_txs` and `/unconfirmed_txs` RPC endpoints.
 
 ### IMPROVEMENTS:
+- [libs/common] \#3238 exit with zero (0) code upon receiving SIGTERM/SIGINT
 
 ### BUG FIXES:
 

abci/cmd/abci-cli/abci-cli.go

@@ -636,9 +636,7 @@ func cmdQuery(cmd *cobra.Command, args []string) error {
 }
 
 func cmdCounter(cmd *cobra.Command, args []string) error {
-
 	app := counter.NewCounterApplication(flagSerial)
-
 	logger := log.NewTMLogger(log.NewSyncWriter(os.Stdout))
 
 	// Start the listener
@@ -651,12 +649,14 @@ func cmdCounter(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	// Wait forever
+	// Stop upon receiving SIGTERM or CTRL-C.
-	cmn.TrapSignal(func() {
+	cmn.TrapSignal(logger, func() {
 		// Cleanup
 		srv.Stop()
 	})
-	return nil
+
+	// Run forever.
+	select {}
 }
 
 func cmdKVStore(cmd *cobra.Command, args []string) error {
@@ -681,12 +681,14 @@ func cmdKVStore(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	// Wait forever
+	// Stop upon receiving SIGTERM or CTRL-C.
-	cmn.TrapSignal(func() {
+	cmn.TrapSignal(logger, func() {
 		// Cleanup
 		srv.Stop()
 	})
-	return nil
+
+	// Run forever.
+	select {}
 }
 
 //--------------------------------------------------------------------------------

cmd/priv_val_server/main.go

@@ -54,10 +54,14 @@ func main() {
 		panic(err)
 	}
 
-	cmn.TrapSignal(func() {
+	// Stop upon receiving SIGTERM or CTRL-C.
+	cmn.TrapSignal(logger, func() {
 		err := rs.Stop()
 		if err != nil {
 			panic(err)
 		}
 	})
+
+	// Run forever.
+	select {}
 }

cmd/tendermint/commands/lite.go

@@ -59,6 +59,11 @@ func ensureAddrHasSchemeOrDefaultToTCP(addr string) (string, error) {
 }
 
 func runProxy(cmd *cobra.Command, args []string) error {
+	// Stop upon receiving SIGTERM or CTRL-C.
+	cmn.TrapSignal(logger, func() {
+		// TODO: close up shop
+	})
+
 	nodeAddr, err := ensureAddrHasSchemeOrDefaultToTCP(nodeAddr)
 	if err != nil {
 		return err
@@ -86,9 +91,6 @@ func runProxy(cmd *cobra.Command, args []string) error {
 		return cmn.ErrorWrap(err, "starting proxy")
 	}
 
-	cmn.TrapSignal(func() {
+	// Run forever
-		// TODO: close up shop
+	select {}
-	})
-
-	return nil
 }

cmd/tendermint/commands/run_node.go

@@ -2,12 +2,10 @@ package commands
 
 import (
 	"fmt"
-	"os"
-	"os/signal"
-	"syscall"
 
 	"github.com/spf13/cobra"
 
+	cmn "github.com/tendermint/tendermint/libs/common"
 	nm "github.com/tendermint/tendermint/node"
 )
 
@@ -57,25 +55,19 @@ func NewRunNodeCmd(nodeProvider nm.NodeProvider) *cobra.Command {
 				return fmt.Errorf("Failed to create node: %v", err)
 			}
 
-			// Stop upon receiving SIGTERM or CTRL-C
+			// Stop upon receiving SIGTERM or CTRL-C.
-			c := make(chan os.Signal, 1)
+			cmn.TrapSignal(logger, func() {
-			signal.Notify(c, os.Interrupt, syscall.SIGTERM)
+				if n.IsRunning() {
-			go func() {
+					n.Stop()
-				for sig := range c {
-					logger.Error(fmt.Sprintf("captured %v, exiting...", sig))
-					if n.IsRunning() {
-						n.Stop()
-					}
-					os.Exit(1)
 				}
-			}()
+			})
 
 			if err := n.Start(); err != nil {
 				return fmt.Errorf("Failed to start node: %v", err)
 			}
 			logger.Info("Started node", "nodeInfo", n.Switch().NodeInfo())
 
-			// Run forever
+			// Run forever.
 			select {}
 		},
 	}

docs/app-dev/abci-cli.md

@@ -89,12 +89,14 @@ func cmdKVStore(cmd *cobra.Command, args []string) error {
         return err
     }
 
-    // Wait forever
+    // Stop upon receiving SIGTERM or CTRL-C.
-    cmn.TrapSignal(func() {
+    cmn.TrapSignal(logger, func() {
         // Cleanup
         srv.Stop()
     })
-    return nil
+
+    // Run forever.
+    select {}
 }
 ```
 
@@ -238,12 +240,14 @@ func cmdCounter(cmd *cobra.Command, args []string) error {
         return err
     }
 
-    // Wait forever
+    // Stop upon receiving SIGTERM or CTRL-C.
-    cmn.TrapSignal(func() {
+    cmn.TrapSignal(logger, func() {
         // Cleanup
         srv.Stop()
     })
-    return nil
+
+    // Run forever.
+    select {}
 }
 ```
 

libs/autofile/cmd/logjack.go

@@ -29,7 +29,21 @@ func parseFlags() (headPath string, chopSize int64, limitSize int64, version boo
 	return
 }
 
+type fmtLogger struct{}
+
+func (fmtLogger) Info(msg string, keyvals ...interface{}) {
+	strs := make([]string, len(keyvals))
+	for i, kv := range keyvals {
+		strs[i] = fmt.Sprintf("%v", kv)
+	}
+	fmt.Printf("%s %s\n", msg, strings.Join(strs, ","))
+}
+
 func main() {
+	// Stop upon receiving SIGTERM or CTRL-C.
+	cmn.TrapSignal(fmtLogger{}, func() {
+		fmt.Println("logjack shutting down")
+	})
 
 	// Read options
 	headPath, chopSize, limitSize, version := parseFlags()
@@ -51,29 +65,22 @@ func main() {
 		os.Exit(1)
 	}
 
-	go func() {
+	// Forever read from stdin and write to AutoFile.
-		// Forever, read from stdin and write to AutoFile.
+	buf := make([]byte, readBufferSize)
-		buf := make([]byte, readBufferSize)
+	for {
-		for {
+		n, err := os.Stdin.Read(buf)
-			n, err := os.Stdin.Read(buf)
+		group.Write(buf[:n])
-			group.Write(buf[:n])
+		group.Flush()
-			group.Flush()
+		if err != nil {
-			if err != nil {
+			group.Stop()
-				group.Stop()
+			if err == io.EOF {
-				if err == io.EOF {
+				os.Exit(0)
-					os.Exit(0)
+			} else {
-				} else {
+				fmt.Println("logjack errored")
-					fmt.Println("logjack errored")
+				os.Exit(1)
-					os.Exit(1)
-				}
 			}
 		}
-	}()
+	}
-
-	// Trap signal
-	cmn.TrapSignal(func() {
-		fmt.Println("logjack shutting down")
-	})
 }
 
 func parseBytesize(chopSize string) int64 {

libs/common/os.go

@@ -34,21 +34,24 @@ func GoPath() string {
 	return path
 }
 
-// TrapSignal catches the SIGTERM and executes cb function. After that it exits
+type logger interface {
-// with code 1.
+	Info(msg string, keyvals ...interface{})
-func TrapSignal(cb func()) {
+}
+
+// TrapSignal catches the SIGTERM/SIGINT and executes cb function. After that it exits
+// with code 0.
+func TrapSignal(logger logger, cb func()) {
 	c := make(chan os.Signal, 1)
 	signal.Notify(c, os.Interrupt, syscall.SIGTERM)
 	go func() {
 		for sig := range c {
-			fmt.Printf("captured %v, exiting...\n", sig)
+			logger.Info(fmt.Sprintf("captured %v, exiting...", sig))
 			if cb != nil {
 				cb()
 			}
-			os.Exit(1)
+			os.Exit(0)
 		}
 	}()
-	select {}
 }
 
 // Kill the running process by sending itself SIGTERM.

libs/common/os_test.go

@@ -5,7 +5,7 @@ import (
 	"testing"
 )
 
-func TestGoPath(t *testing.T) {
+func TestOSGoPath(t *testing.T) {
 	// restore original gopath upon exit
 	path := os.Getenv("GOPATH")
 	defer func() {
@@ -28,7 +28,7 @@ func TestGoPath(t *testing.T) {
 	}
 }
 
-func TestGoPathWithoutEnvVar(t *testing.T) {
+func TestOSGoPathWithoutEnvVar(t *testing.T) {
 	// restore original gopath upon exit
 	path := os.Getenv("GOPATH")
 	defer func() {

rpc/lib/test/main.go

@@ -24,17 +24,19 @@ type Result struct {
 }
 
 func main() {
-	mux := http.NewServeMux()
+	var (
-	cdc := amino.NewCodec()
+		mux    = http.NewServeMux()
-	logger := log.NewTMLogger(log.NewSyncWriter(os.Stdout))
+		cdc    = amino.NewCodec()
+		logger = log.NewTMLogger(log.NewSyncWriter(os.Stdout))
+	)
+
+	// Stop upon receiving SIGTERM or CTRL-C.
+	cmn.TrapSignal(logger, func() {})
+
 	rpcserver.RegisterRPCFuncs(mux, routes, cdc, logger)
 	listener, err := rpcserver.Listen("0.0.0.0:8008", rpcserver.Config{})
 	if err != nil {
 		cmn.Exit(err.Error())
 	}
-	go rpcserver.StartHTTPServer(listener, mux, logger)
+	rpcserver.StartHTTPServer(listener, mux, logger)
-	// Wait forever
-	cmn.TrapSignal(func() {
-	})
-
 }

tools/tm-bench/main.go

@@ -4,14 +4,13 @@ import (
 	"flag"
 	"fmt"
 	"os"
-	"os/signal"
 	"strings"
 	"sync"
-	"syscall"
 	"time"
 
 	"github.com/go-kit/kit/log/term"
 
+	cmn "github.com/tendermint/tendermint/libs/common"
 	"github.com/tendermint/tendermint/libs/log"
 	tmrpc "github.com/tendermint/tendermint/rpc/client"
 )
@@ -94,18 +93,12 @@ Examples:
 		"broadcast_tx_"+broadcastTxMethod,
 	)
 
-	// Quit when interrupted or received SIGTERM.
+	// Stop upon receiving SIGTERM or CTRL-C.
-	c := make(chan os.Signal, 1)
+	cmn.TrapSignal(logger, func() {
-	signal.Notify(c, os.Interrupt, syscall.SIGTERM)
+		for _, t := range transacters {
-	go func() {
+			t.Stop()
-		for sig := range c {
-			fmt.Printf("captured %v, exiting...\n", sig)
-			for _, t := range transacters {
-				t.Stop()
-			}
-			os.Exit(1)
 		}
-	}()
+	})
 
 	// Wait until transacters have begun until we get the start time.
 	timeStart := time.Now()

tools/tm-monitor/main.go

@@ -58,13 +58,17 @@ Examples:
 		ton.Start()
 	}
 
-	cmn.TrapSignal(func() {
+	// Stop upon receiving SIGTERM or CTRL-C.
+	cmn.TrapSignal(logger, func() {
 		if !noton {
 			ton.Stop()
 		}
 		monitor.Stop()
 		listener.Close()
 	})
+
+	// Run forever.
+	select {}
 }
 
 func startMonitor(endpoints string) *monitor.Monitor {