mirror of
https://github.com/fluencelabs/redis
synced 2025-06-12 08:41:21 +00:00
diskless replication on slave side (don't store rdb to file), plus some other related fixes
The implementation of the diskless replication was currently diskless only on the master side. The slave side was still storing the received rdb file to the disk before loading it back in and parsing it. This commit adds two modes to load rdb directly from socket: 1) when-empty 2) using "swapdb" the third mode of using diskless slave by flushdb is risky and currently not included. other changes: -------------- distinguish between aof configuration and state so that we can re-enable aof only when sync eventually succeeds (and not when exiting from readSyncBulkPayload after a failed attempt) also a CONFIG GET and INFO during rdb loading would have lied When loading rdb from the network, don't kill the server on short read (that can be a network error) Fix rdb check when performed on preamble AOF tests: run replication tests for diskless slave too make replication test a bit more aggressive Add test for diskless load swapdb
This commit is contained in:
Oran Agra
@ -1,12 +1,3 @@
|
||||
proc start_bg_complex_data {host port db ops} {
|
||||
set tclsh [info nameofexecutable]
|
||||
exec $tclsh tests/helpers/bg_complex_data.tcl $host $port $db $ops &
|
||||
}
|
||||
|
||||
proc stop_bg_complex_data {handle} {
|
||||
catch {exec /bin/kill -9 $handle}
|
||||
}
|
||||
|
||||
start_server {tags {"repl"}} {
|
||||
start_server {} {
|
||||
|
||||
|
||||
@ -1,12 +1,3 @@
|
||||
proc start_bg_complex_data {host port db ops} {
|
||||
set tclsh [info nameofexecutable]
|
||||
exec $tclsh tests/helpers/bg_complex_data.tcl $host $port $db $ops &
|
||||
}
|
||||
|
||||
proc stop_bg_complex_data {handle} {
|
||||
catch {exec /bin/kill -9 $handle}
|
||||
}
|
||||
|
||||
# Creates a master-slave pair and breaks the link continuously to force
|
||||
# partial resyncs attempts, all this while flooding the master with
|
||||
# write queries.
|
||||
@ -17,7 +8,7 @@ proc stop_bg_complex_data {handle} {
|
||||
# If reconnect is > 0, the test actually try to break the connection and
|
||||
# reconnect with the master, otherwise just the initial synchronization is
|
||||
# checked for consistency.
|
||||
proc test_psync {descr duration backlog_size backlog_ttl delay cond diskless reconnect} {
|
||||
proc test_psync {descr duration backlog_size backlog_ttl delay cond mdl sdl reconnect} {
|
||||
start_server {tags {"repl"}} {
|
||||
start_server {} {
|
||||
|
||||
@ -28,8 +19,9 @@ proc test_psync {descr duration backlog_size backlog_ttl delay cond diskless rec
|
||||
|
||||
$master config set repl-backlog-size $backlog_size
|
||||
$master config set repl-backlog-ttl $backlog_ttl
|
||||
$master config set repl-diskless-sync $diskless
|
||||
$master config set repl-diskless-sync $mdl
|
||||
$master config set repl-diskless-sync-delay 1
|
||||
$slave config set repl-diskless-load $sdl
|
||||
|
||||
set load_handle0 [start_bg_complex_data $master_host $master_port 9 100000]
|
||||
set load_handle1 [start_bg_complex_data $master_host $master_port 11 100000]
|
||||
@ -54,7 +46,7 @@ proc test_psync {descr duration backlog_size backlog_ttl delay cond diskless rec
|
||||
}
|
||||
}
|
||||
|
||||
test "Test replication partial resync: $descr (diskless: $diskless, reconnect: $reconnect)" {
|
||||
test "Test replication partial resync: $descr (diskless: $mdl, $sdl, reconnect: $reconnect)" {
|
||||
# Now while the clients are writing data, break the maste-slave
|
||||
# link multiple times.
|
||||
if ($reconnect) {
|
||||
@ -132,23 +124,25 @@ proc test_psync {descr duration backlog_size backlog_ttl delay cond diskless rec
|
||||
}
|
||||
}
|
||||
|
||||
foreach diskless {no yes} {
|
||||
test_psync {no reconnection, just sync} 6 1000000 3600 0 {
|
||||
} $diskless 0
|
||||
foreach mdl {no yes} {
|
||||
foreach sdl {disabled swapdb} {
|
||||
test_psync {no reconnection, just sync} 6 1000000 3600 0 {
|
||||
} $mdl $sdl 0
|
||||
|
||||
test_psync {ok psync} 6 100000000 3600 0 {
|
||||
test_psync {ok psync} 6 100000000 3600 0 {
|
||||
assert {[s -1 sync_partial_ok] > 0}
|
||||
} $diskless 1
|
||||
} $mdl $sdl 1
|
||||
|
||||
test_psync {no backlog} 6 100 3600 0.5 {
|
||||
test_psync {no backlog} 6 100 3600 0.5 {
|
||||
assert {[s -1 sync_partial_err] > 0}
|
||||
} $diskless 1
|
||||
} $mdl $sdl 1
|
||||
|
||||
test_psync {ok after delay} 3 100000000 3600 3 {
|
||||
test_psync {ok after delay} 3 100000000 3600 3 {
|
||||
assert {[s -1 sync_partial_ok] > 0}
|
||||
} $diskless 1
|
||||
} $mdl $sdl 1
|
||||
|
||||
test_psync {backlog expired} 3 100000000 1 3 {
|
||||
test_psync {backlog expired} 3 100000000 1 3 {
|
||||
assert {[s -1 sync_partial_err] > 0}
|
||||
} $diskless 1
|
||||
} $mdl $sdl 1
|
||||
}
|
||||
}
|
||||
|
||||
@ -183,85 +183,92 @@ start_server {tags {"repl"}} {
|
||||
}
|
||||
}
|
||||
|
||||
foreach dl {no yes} {
|
||||
start_server {tags {"repl"}} {
|
||||
set master [srv 0 client]
|
||||
$master config set repl-diskless-sync $dl
|
||||
set master_host [srv 0 host]
|
||||
set master_port [srv 0 port]
|
||||
set slaves {}
|
||||
set load_handle0 [start_write_load $master_host $master_port 3]
|
||||
set load_handle1 [start_write_load $master_host $master_port 5]
|
||||
set load_handle2 [start_write_load $master_host $master_port 20]
|
||||
set load_handle3 [start_write_load $master_host $master_port 8]
|
||||
set load_handle4 [start_write_load $master_host $master_port 4]
|
||||
start_server {} {
|
||||
lappend slaves [srv 0 client]
|
||||
foreach mdl {no yes} {
|
||||
foreach sdl {disabled swapdb} {
|
||||
start_server {tags {"repl"}} {
|
||||
set master [srv 0 client]
|
||||
$master config set repl-diskless-sync $mdl
|
||||
$master config set repl-diskless-sync-delay 1
|
||||
set master_host [srv 0 host]
|
||||
set master_port [srv 0 port]
|
||||
set slaves {}
|
||||
set load_handle0 [start_bg_complex_data $master_host $master_port 9 100000000]
|
||||
set load_handle1 [start_bg_complex_data $master_host $master_port 11 100000000]
|
||||
set load_handle2 [start_bg_complex_data $master_host $master_port 12 100000000]
|
||||
set load_handle3 [start_write_load $master_host $master_port 8]
|
||||
set load_handle4 [start_write_load $master_host $master_port 4]
|
||||
after 5000 ;# wait for some data to accumulate so that we have RDB part for the fork
|
||||
start_server {} {
|
||||
lappend slaves [srv 0 client]
|
||||
start_server {} {
|
||||
lappend slaves [srv 0 client]
|
||||
test "Connect multiple replicas at the same time (issue #141), diskless=$dl" {
|
||||
# Send SLAVEOF commands to slaves
|
||||
[lindex $slaves 0] slaveof $master_host $master_port
|
||||
[lindex $slaves 1] slaveof $master_host $master_port
|
||||
[lindex $slaves 2] slaveof $master_host $master_port
|
||||
start_server {} {
|
||||
lappend slaves [srv 0 client]
|
||||
test "Connect multiple replicas at the same time (issue #141), master diskless=$mdl, replica diskless=$sdl" {
|
||||
# Send SLAVEOF commands to slaves
|
||||
[lindex $slaves 0] config set repl-diskless-load $sdl
|
||||
[lindex $slaves 1] config set repl-diskless-load $sdl
|
||||
[lindex $slaves 2] config set repl-diskless-load $sdl
|
||||
[lindex $slaves 0] slaveof $master_host $master_port
|
||||
[lindex $slaves 1] slaveof $master_host $master_port
|
||||
[lindex $slaves 2] slaveof $master_host $master_port
|
||||
|
||||
# Wait for all the three slaves to reach the "online"
|
||||
# state from the POV of the master.
|
||||
set retry 500
|
||||
while {$retry} {
|
||||
set info [r -3 info]
|
||||
if {[string match {*slave0:*state=online*slave1:*state=online*slave2:*state=online*} $info]} {
|
||||
break
|
||||
} else {
|
||||
incr retry -1
|
||||
after 100
|
||||
# Wait for all the three slaves to reach the "online"
|
||||
# state from the POV of the master.
|
||||
set retry 500
|
||||
while {$retry} {
|
||||
set info [r -3 info]
|
||||
if {[string match {*slave0:*state=online*slave1:*state=online*slave2:*state=online*} $info]} {
|
||||
break
|
||||
} else {
|
||||
incr retry -1
|
||||
after 100
|
||||
}
|
||||
}
|
||||
if {$retry == 0} {
|
||||
error "assertion:Slaves not correctly synchronized"
|
||||
}
|
||||
}
|
||||
if {$retry == 0} {
|
||||
error "assertion:Replicas not correctly synchronized"
|
||||
}
|
||||
|
||||
# Wait that slaves acknowledge they are online so
|
||||
# we are sure that DBSIZE and DEBUG DIGEST will not
|
||||
# fail because of timing issues.
|
||||
wait_for_condition 500 100 {
|
||||
[lindex [[lindex $slaves 0] role] 3] eq {connected} &&
|
||||
[lindex [[lindex $slaves 1] role] 3] eq {connected} &&
|
||||
[lindex [[lindex $slaves 2] role] 3] eq {connected}
|
||||
} else {
|
||||
fail "Replicas still not connected after some time"
|
||||
# Wait that slaves acknowledge they are online so
|
||||
# we are sure that DBSIZE and DEBUG DIGEST will not
|
||||
# fail because of timing issues.
|
||||
wait_for_condition 500 100 {
|
||||
[lindex [[lindex $slaves 0] role] 3] eq {connected} &&
|
||||
[lindex [[lindex $slaves 1] role] 3] eq {connected} &&
|
||||
[lindex [[lindex $slaves 2] role] 3] eq {connected}
|
||||
} else {
|
||||
fail "Slaves still not connected after some time"
|
||||
}
|
||||
|
||||
# Stop the write load
|
||||
stop_bg_complex_data $load_handle0
|
||||
stop_bg_complex_data $load_handle1
|
||||
stop_bg_complex_data $load_handle2
|
||||
stop_write_load $load_handle3
|
||||
stop_write_load $load_handle4
|
||||
|
||||
# Make sure that slaves and master have same
|
||||
# number of keys
|
||||
wait_for_condition 500 100 {
|
||||
[$master dbsize] == [[lindex $slaves 0] dbsize] &&
|
||||
[$master dbsize] == [[lindex $slaves 1] dbsize] &&
|
||||
[$master dbsize] == [[lindex $slaves 2] dbsize]
|
||||
} else {
|
||||
fail "Different number of keys between master and replica after too long time."
|
||||
}
|
||||
|
||||
# Check digests
|
||||
set digest [$master debug digest]
|
||||
set digest0 [[lindex $slaves 0] debug digest]
|
||||
set digest1 [[lindex $slaves 1] debug digest]
|
||||
set digest2 [[lindex $slaves 2] debug digest]
|
||||
assert {$digest ne 0000000000000000000000000000000000000000}
|
||||
assert {$digest eq $digest0}
|
||||
assert {$digest eq $digest1}
|
||||
assert {$digest eq $digest2}
|
||||
}
|
||||
|
||||
# Stop the write load
|
||||
stop_write_load $load_handle0
|
||||
stop_write_load $load_handle1
|
||||
stop_write_load $load_handle2
|
||||
stop_write_load $load_handle3
|
||||
stop_write_load $load_handle4
|
||||
|
||||
# Make sure that slaves and master have same
|
||||
# number of keys
|
||||
wait_for_condition 500 100 {
|
||||
[$master dbsize] == [[lindex $slaves 0] dbsize] &&
|
||||
[$master dbsize] == [[lindex $slaves 1] dbsize] &&
|
||||
[$master dbsize] == [[lindex $slaves 2] dbsize]
|
||||
} else {
|
||||
fail "Different number of keys between masted and replica after too long time."
|
||||
}
|
||||
|
||||
# Check digests
|
||||
set digest [$master debug digest]
|
||||
set digest0 [[lindex $slaves 0] debug digest]
|
||||
set digest1 [[lindex $slaves 1] debug digest]
|
||||
set digest2 [[lindex $slaves 2] debug digest]
|
||||
assert {$digest ne 0000000000000000000000000000000000000000}
|
||||
assert {$digest eq $digest0}
|
||||
assert {$digest eq $digest1}
|
||||
assert {$digest eq $digest2}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -309,3 +316,70 @@ start_server {tags {"repl"}} {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
test {slave fails full sync and diskless load swapdb recoveres it} {
|
||||
start_server {tags {"repl"}} {
|
||||
set slave [srv 0 client]
|
||||
set slave_host [srv 0 host]
|
||||
set slave_port [srv 0 port]
|
||||
set slave_log [srv 0 stdout]
|
||||
start_server {} {
|
||||
set master [srv 0 client]
|
||||
set master_host [srv 0 host]
|
||||
set master_port [srv 0 port]
|
||||
|
||||
# Put different data sets on the master and slave
|
||||
# we need to put large keys on the master since the slave replies to info only once in 2mb
|
||||
$slave debug populate 2000 slave 10
|
||||
$master debug populate 200 master 100000
|
||||
$master config set rdbcompression no
|
||||
|
||||
# Set master and slave to use diskless replication
|
||||
$master config set repl-diskless-sync yes
|
||||
$master config set repl-diskless-sync-delay 0
|
||||
$slave config set repl-diskless-load swapdb
|
||||
|
||||
# Set master with a slow rdb generation, so that we can easily disconnect it mid sync
|
||||
# 10ms per key, with 200 keys is 2 seconds
|
||||
$master config set rdb-key-save-delay 10000
|
||||
|
||||
# Start the replication process...
|
||||
$slave slaveof $master_host $master_port
|
||||
|
||||
# wait for the slave to start reading the rdb
|
||||
wait_for_condition 50 100 {
|
||||
[s -1 loading] eq 1
|
||||
} else {
|
||||
fail "Replica didn't get into loading mode"
|
||||
}
|
||||
|
||||
# make sure that next sync will not start immediately so that we can catch the slave in betweeen syncs
|
||||
$master config set repl-diskless-sync-delay 5
|
||||
# for faster server shutdown, make rdb saving fast again (the fork is already uses the slow one)
|
||||
$master config set rdb-key-save-delay 0
|
||||
|
||||
# waiting slave to do flushdb (key count drop)
|
||||
wait_for_condition 50 100 {
|
||||
2000 != [scan [regexp -inline {keys\=([\d]*)} [$slave info keyspace]] keys=%d]
|
||||
} else {
|
||||
fail "Replica didn't flush"
|
||||
}
|
||||
|
||||
# make sure we're still loading
|
||||
assert_equal [s -1 loading] 1
|
||||
|
||||
# kill the slave connection on the master
|
||||
set killed [$master client kill type slave]
|
||||
|
||||
# wait for loading to stop (fail)
|
||||
wait_for_condition 50 100 {
|
||||
[s -1 loading] eq 0
|
||||
} else {
|
||||
fail "Replica didn't disconnect"
|
||||
}
|
||||
|
||||
# make sure the original keys were restored
|
||||
assert_equal [$slave dbsize] 2000
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -399,3 +399,15 @@ proc lshuffle {list} {
|
||||
}
|
||||
return $slist
|
||||
}
|
||||
|
||||
# Execute a background process writing complex data for the specified number
|
||||
# of ops to the specified Redis instance.
|
||||
proc start_bg_complex_data {host port db ops} {
|
||||
set tclsh [info nameofexecutable]
|
||||
exec $tclsh tests/helpers/bg_complex_data.tcl $host $port $db $ops &
|
||||
}
|
||||
|
||||
# Stop a process generating write load executed with start_bg_complex_data.
|
||||
proc stop_bg_complex_data {handle} {
|
||||
catch {exec /bin/kill -9 $handle}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user