test: refactor bash; test fastsync (failing)

scripts/install_tmsp_apps.sh

@@ -2,6 +2,7 @@
 
 go get github.com/tendermint/tmsp/...
 
+# get the tmsp commit used by tendermint
 COMMIT=`bash scripts/glide/parse.sh $(pwd)/glide.lock tmsp`
 
 cd $GOPATH/src/github.com/tendermint/tmsp

test/docker/Dockerfile

@@ -14,10 +14,9 @@ WORKDIR $TENDERMINT_ORG/tendermint
 
 RUN make get_vendor_deps
 
+
 RUN go install ./cmd/tendermint
-
 RUN bash scripts/install_tmsp_apps.sh
 
-
 EXPOSE 46656
 EXPOSE 46657

test/docker/DockerfileDev

@@ -0,0 +1,3 @@
+FROM tester
+
+VOLUME /go/bin

test/docker/build.sh

@@ -0,0 +1,3 @@
+#! /bin/bash
+
+docker build -t tester -f ./test/docker/Dockerfile .

test/docker/update.sh

@@ -0,0 +1,9 @@
+#! /bin/bash
+
+# update the `tester` image by copying in the latest tendermint binary
+
+docker run --name builder tester true
+docker cp $GOPATH/bin/tendermint builder:/go/bin/tendermint
+docker commit builder tester
+docker rm -vf builder
+

test/net/test.sh

@@ -1,6 +1,8 @@
 #! /bin/bash
 set -eu
 
+# start a testnet and benchmark throughput using mintnet+netmon via the network_testing repo
+
 DATACENTER=single
 VALSETSIZE=4
 BLOCKSIZE=8092

test/p2p/a.sh

@@ -0,0 +1,8 @@
+#! /bin/bash
+
+h1=10
+h2=0
+while [ $h2 -lt $(($h1+3)) ]; do
+	echo "$h2"
+	h2=$(($h2+1))
+done

test/p2p/atomic_broadcast/test.sh

@@ -1,9 +1,20 @@
 #! /bin/bash
 
+###################################################################
+# wait for all peers to come online
+# for each peer:
+# 	wait to have 3 peers
+#	wait to be at height > 1
+#	send a tx, wait for commit
+#	assert app hash on every peer reflects the post tx state
+###################################################################
+
+N=4
+
 # wait for everyone to come online
 echo "Waiting for nodes to come online"
-for i in `seq 1 4`; do
+for i in `seq 1 $N`; do
-	addr="172.57.0.$((100+$i)):46657"
+	addr=$(test/p2p/ip.sh $i):46657
 	curl -s $addr/status > /dev/null
 	ERR=$?
 	while [ "$ERR" != 0 ]; do
@@ -16,8 +27,8 @@ done
 
 echo ""
 # run the test on each of them
-for i in `seq 1 4`; do
+for i in `seq 1 $N`; do
-	addr="172.57.0.$((100+$i)):46657"
+	addr=$(test/p2p/ip.sh $i):46657
 
 	# - assert everyone has 3 other peers
 	N_PEERS=`curl -s $addr/net_info | jq '.result[1].peers | length'`
@@ -61,9 +72,10 @@ for i in `seq 1 4`; do
 	fi
 
 	# check we get the same new hash on all other nodes
-	for j in `seq 1 4`; do
+	for j in `seq 1 $N`; do
 		if [[ "$i" != "$j" ]]; then
-			HASH3=`curl -s 172.57.0.$((100+$j)):46657/status | jq .result[1].latest_app_hash`
+			addrJ=$(test/p2p/ip.sh $j):46657
+			HASH3=`curl -s $addrJ/status | jq .result[1].latest_app_hash`
 			
 			if [[ "$HASH2" != "$HASH3" ]]; then
 				echo "App hash for node $j doesn't match. Got $HASH3, expected $HASH2"
@@ -77,3 +89,4 @@ done
 
 echo ""
 echo "PASS"
+echo ""

test/p2p/clean.sh

@@ -1,4 +1,5 @@
 #! /bin/bash
 
+# clean everything
 docker rm -vf $(docker ps -aq)
 docker network rm local_testnet

test/p2p/client.sh

@@ -5,12 +5,14 @@ DOCKER_IMAGE=$1
 NETWORK_NAME=$2
 CMD=$3
 
+echo "starting test client container with CMD=$CMD"
 # run the test container on the local network
 docker run -t \
-	--rm \
 	-v $GOPATH/src/github.com/tendermint/tendermint/test/p2p/:/go/src/github.com/tendermint/tendermint/test/p2p \
 	--net=$NETWORK_NAME \
-	--ip=172.57.0.99 \
+	--ip=$(test/p2p/ip.sh "-1") \
 	--name test_container \
 	--entrypoint bash \
 	$DOCKER_IMAGE $CMD
+
+docker rm -vf test_container

test/p2p/fast_sync/test.sh

@@ -0,0 +1,44 @@
+#! /bin/bash
+set -eu
+set -o pipefail
+
+###############################################################
+# for each peer:
+# 	kill peer
+# 	bring it back online via fast sync
+# 	check app hash
+###############################################################
+
+ID=$1
+
+addr=$(test/p2p/ip.sh $ID):46657
+peerID=$(( $(($ID % 4)) + 1  )) # 1->2 ... 3->4 ... 4->1
+peer_addr=$(test/p2p/ip.sh $peerID):46657
+
+# get another peer's height
+h1=`curl -s $peer_addr/status | jq .result[1].latest_block_height`
+
+# get another peer's state
+root1=`curl -s $peer_addr/status | jq .result[1].latest_app_hash`
+
+echo "Other peer is on height $h1 with state $root1"
+echo "Waiting for peer $ID to catch up"
+
+# wait for it to sync to past its previous height
+set +e
+set +o pipefail
+h2="0"
+while [[ "$h2" -lt "$(($h1+3))" ]]; do
+	sleep 1
+	h2=`curl -s $addr/status | jq .result[1].latest_block_height`
+	echo "... $h2"
+done
+
+# check the app hash
+root2=`curl -s $addr/status | jq .result[1].latest_app_hash`
+
+if [[ "$root1" != "$root2" ]]; then
+	echo "App hash after fast sync does not match. Got $root2; expected $root1"
+	exit 1
+fi
+echo "... fast sync successful"

test/p2p/ip.sh

@@ -0,0 +1,7 @@
+#! /bin/bash
+set -eu
+
+ID=$1
+echo "172.57.0.$((100+$ID))"
+
+

test/p2p/local_testnet.sh

@@ -10,19 +10,12 @@ cd $GOPATH/src/github.com/tendermint/tendermint
 docker network create --driver bridge --subnet 172.57.0.0/16 $NETWORK_NAME
 
 N=4
-seeds="172.57.0.101:46656"
+seeds="$(test/p2p/ip.sh 1):46656"
 for i in `seq 2 $N`; do
-	seeds="$seeds,172.57.0.$((100+$i)):46656"
+	seeds="$seeds,$(test/p2p/ip.sh $i):46656"
 done
 echo "Seeds: $seeds"
 
 for i in `seq 1 $N`; do
-	# start tendermint container
+	bash test/p2p/peer.sh $DOCKER_IMAGE $NETWORK_NAME $i $seeds
-	docker run -d \
-		--net=$NETWORK_NAME \
-		--ip=172.57.0.$((100+$i)) \
-		--name local_testnet_$i \
-		--entrypoint tendermint \
-		-e TMROOT=/go/src/github.com/tendermint/tendermint/test/p2p/data/mach$i/core \
-		$DOCKER_IMAGE node --seeds $seeds --proxy_app=dummy
 done

test/p2p/peer.sh

@@ -0,0 +1,23 @@
+#! /bin/bash
+set -eu
+
+DOCKER_IMAGE=$1
+NETWORK_NAME=$2
+ID=$3
+
+set +u
+SEEDS=$4
+set -u
+if [[ "$SEEDS" != "" ]]; then
+	SEEDS=" --seeds $SEEDS "
+fi
+
+echo "starting tendermint peer ID=$ID"
+# start tendermint container on the network
+docker run -d \
+	--net=$NETWORK_NAME \
+	--ip=$(test/p2p/ip.sh $ID) \
+	--name local_testnet_$ID \
+	--entrypoint tendermint \
+	-e TMROOT=/go/src/github.com/tendermint/tendermint/test/p2p/data/mach$ID/core \
+	$DOCKER_IMAGE node $SEEDS --proxy_app=dummy

test/p2p/test.sh

@@ -1,4 +1,5 @@
 #! /bin/bash
+set -eu
 
 DOCKER_IMAGE=$1
 NETWORK_NAME=local_testnet
@@ -6,5 +7,28 @@ NETWORK_NAME=local_testnet
 # start the testnet on a local network
 bash test/p2p/local_testnet.sh $DOCKER_IMAGE $NETWORK_NAME
 
-# run the test
+# test atomic broadcast
-bash test/p2p/test_client.sh $DOCKER_IMAGE $NETWORK_NAME test/p2p/run_test.sh
+bash test/p2p/client.sh $DOCKER_IMAGE $NETWORK_NAME test/p2p/atomic_broadcast/test.sh
+
+# test fast sync (from current state of network)
+# run it on each of them
+N=4
+for i in `seq 1 $N`; do
+	echo "Testing fasysync on node $i"
+
+	# kill peer 
+	docker rm -vf local_testnet_$i
+
+	# restart peer - should have an empty blockchain
+	SEEDS="$(test/p2p/ip.sh 1):46656"
+	for j in `seq 2 $N`; do
+		SEEDS="$SEEDS,$(test/p2p/ip.sh $j):46656"
+	done
+	bash test/p2p/peer.sh $DOCKER_IMAGE $NETWORK_NAME $i $SEEDS
+
+	bash test/p2p/client.sh $DOCKER_IMAGE $NETWORK_NAME "test/p2p/fast_sync/test.sh $i"
+done
+echo ""
+echo "PASS"
+echo ""
+

test/test.sh

@@ -5,17 +5,17 @@ set -eu
 # See the github.com/tendermint/tendermint/test/README.md
 
 echo ""
-echo "* building docker file"
+echo "* building docker image"
-docker build -t tester -f ./test/Dockerfile .
+bash ./test/docker/build.sh
 
 echo ""
-echo "* running go tests and app tests"
+echo "* running go tests and app tests in docker container"
 docker run -t tester bash test/run_test.sh
 
 # test basic network connectivity
 # by starting a local testnet and checking peers connect and make blocks
 echo ""
-echo "* running basic peer tests"
+echo "* running p2p tests on a local docker network"
 bash test/p2p/test.sh tester
 
 # only run the cloud benchmark for releases