tendermint/libs/fail/fail.go
Ethan Buchman 39eba4e154
WAL: better errors and new fail point (#3246)
* privval: more info in errors

* wal: change Debug logs to Info

* wal: log and return error on corrupted wal instead of panicing

* fail: Exit right away instead of sending interupt

* consensus: FAIL before handling our own vote

allows to replicate #3089:
- run using `FAIL_TEST_INDEX=0`
- delete some bytes from the end of the WAL
- start normally

Results in logs like:

```
I[2019-02-03|18:12:58.225] Searching for height module=consensus wal=/Users/ethanbuchman/.tendermint/data/cs.wal/wal height=1 min=0 max=0
E[2019-02-03|18:12:58.225] Error on catchup replay. Proceeding to start ConsensusState anyway module=consensus err="failed to read data: EOF"
I[2019-02-03|18:12:58.225] Started node module=main nodeInfo="{ProtocolVersion:{P2P:6 Block:9 App:1} ID_:35e87e93f2e31f305b65a5517fd2102331b56002 ListenAddr:tcp://0.0.0.0:26656 Network:test-chain-J8JvJH Version:0.29.1 Channels:4020212223303800 Moniker:Ethans-MacBook-Pro.local Other:{TxIndex:on RPCAddress:tcp://0.0.0.0:26657}}"
E[2019-02-03|18:12:58.226] Couldn't connect to any seeds module=p2p
I[2019-02-03|18:12:59.229] Timed out module=consensus dur=998.568ms height=1 round=0 step=RoundStepNewHeight
I[2019-02-03|18:12:59.230] enterNewRound(1/0). Current: 1/0/RoundStepNewHeight module=consensus height=1 round=0
I[2019-02-03|18:12:59.230] enterPropose(1/0). Current: 1/0/RoundStepNewRound module=consensus height=1 round=0
I[2019-02-03|18:12:59.230] enterPropose: Our turn to propose module=consensus height=1 round=0 proposer=AD278B7767B05D7FBEB76207024C650988FA77D5 privValidator="PrivValidator{AD278B7767B05D7FBEB76207024C650988FA77D5 LH:1, LR:0, LS:2}"
E[2019-02-03|18:12:59.230] enterPropose: Error signing proposal module=consensus height=1 round=0 err="Error signing proposal: Step regression at height 1 round 0. Got 1, last step 2"
I[2019-02-03|18:13:02.233] Timed out module=consensus dur=3s height=1 round=0 step=RoundStepPropose
I[2019-02-03|18:13:02.233] enterPrevote(1/0). Current: 1/0/RoundStepPropose module=consensus
I[2019-02-03|18:13:02.233] enterPrevote: ProposalBlock is nil module=consensus height=1 round=0
E[2019-02-03|18:13:02.234] Error signing vote module=consensus height=1 round=0 vote="Vote{0:AD278B7767B0 1/00/1(Prevote) 000000000000 000000000000 @ 2019-02-04T02:13:02.233897Z}" err="Error signing vote: Conflicting data"
```

Notice the EOF, the step regression, and the conflicting data.

* wal: change errors to be DataCorruptionError

* exit on corrupt WAL

* fix log

* fix new line
2019-02-04 13:00:06 -05:00

80 lines
1.4 KiB
Go

package fail
import (
"fmt"
"math/rand"
"os"
"strconv"
)
var callIndexToFail int
func init() {
callIndexToFailS := os.Getenv("FAIL_TEST_INDEX")
if callIndexToFailS == "" {
callIndexToFail = -1
} else {
var err error
callIndexToFail, err = strconv.Atoi(callIndexToFailS)
if err != nil {
callIndexToFail = -1
}
}
}
// Fail when FAIL_TEST_INDEX == callIndex
var (
callIndex int //indexes Fail calls
callRandIndex int // indexes a run of FailRand calls
callRandIndexToFail = -1 // the callRandIndex to fail on in FailRand
)
func Fail() {
if callIndexToFail < 0 {
return
}
if callIndex == callIndexToFail {
Exit()
}
callIndex += 1
}
// FailRand should be called n successive times.
// It will fail on a random one of those calls
// n must be greater than 0
func FailRand(n int) {
if callIndexToFail < 0 {
return
}
if callRandIndexToFail < 0 {
// first call in the loop, pick a random index to fail at
callRandIndexToFail = rand.Intn(n)
callRandIndex = 0
}
if callIndex == callIndexToFail {
if callRandIndex == callRandIndexToFail {
Exit()
}
}
callRandIndex += 1
if callRandIndex == n {
callIndex += 1
}
}
func Exit() {
fmt.Printf("*** fail-test %d ***\n", callIndex)
os.Exit(1)
// proc, _ := os.FindProcess(os.Getpid())
// proc.Signal(os.Interrupt)
// panic(fmt.Sprintf("*** fail-test %d ***", callIndex))
}