fluencelabs/redis
1
0
Fork 0
mirror of https://github.com/fluencelabs/redis synced 2025-06-12 16:51:22 +00:00
Code Issues Projects Releases Wiki Activity

Abstract file/buffer I/O to support in-memory serialization

Browse Source
This commit is contained in:
Pieter Noordhuis
2011-05-13 17:31:00 +02:00
parent 70bc5f7724
commit 2e4b0e7727
8 changed files with 371 additions and 279 deletions
Download Patch File Download Diff File Expand all files Collapse all files

190
src/cluster.c
View File

@ -1383,11 +1383,10 @@ void clusterCommand(redisClient *c) {
/* RESTORE key ttl serialized-value */
void restoreCommand(redisClient *c) {
FILE *fp;
char buf[64];
robj *o;
unsigned char *data;
long ttl;
rio payload;
unsigned char *data;
/* Make sure this key does not already exist here... */
if (dbExists(c->db,c->argv[1])) {
@ -1403,41 +1402,20 @@ void restoreCommand(redisClient *c) {
return;
}
/* rdbLoadObject() only works against file descriptors so we need to
* dump the serialized object into a file and reload. */
snprintf(buf,sizeof(buf),"redis-restore-%d.tmp",getpid());
fp = fopen(buf,"w+");
if (!fp) {
redisLog(REDIS_WARNING,"Can't open tmp file for RESTORE: %s",
strerror(errno));
addReplyErrorFormat(c,"RESTORE failed, tmp file creation error: %s",
strerror(errno));
return;
}
unlink(buf);
/* Temporary hack to get RDB-aligned payload. */
payload = rioInitWithBuffer(sdsnewlen(c->argv[3]->ptr+1, sdslen(c->argv[3]->ptr)-1));
data = c->argv[3]->ptr;
/* Write the actual data and rewind the file */
data = (unsigned char*) c->argv[3]->ptr;
if (fwrite(data+1,sdslen((sds)data)-1,1,fp) != 1) {
redisLog(REDIS_WARNING,"Can't write against tmp file for RESTORE: %s",
strerror(errno));
addReplyError(c,"RESTORE failed, tmp file I/O error.");
fclose(fp);
return;
}
rewind(fp);
/* Finally create the object from the serialized dump and
* store it at the specified key. */
/* Create the object from the serialized dump. */
if ((data[0] > 4 && data[0] < 9) ||
data[0] > 11 ||
(o = rdbLoadObject(data[0],fp)) == NULL)
(o = rdbLoadObject(data[0],&payload)) == NULL)
{
addReplyError(c,"Bad data format.");
fclose(fp);
sdsfree(payload.io.buffer.ptr);
return;
}
fclose(fp);
sdsfree(payload.io.buffer.ptr);
/* Create the key and set the TTL if any */
dbAdd(c->db,c->argv[1],o);
@ -1450,12 +1428,10 @@ void migrateCommand(redisClient *c) {
int fd;
long timeout;
long dbid;
char buf[64];
FILE *fp;
time_t ttl;
robj *o;
unsigned char type;
off_t payload_len;
rio cmd, payload;
/* Sanity check */
if (getLongFromObjectOrReply(c,c->argv[5],&timeout,NULL) != REDIS_OK)
@ -1485,54 +1461,51 @@ void migrateCommand(redisClient *c) {
return;
}
/* Create temp file */
snprintf(buf,sizeof(buf),"redis-migrate-%d.tmp",getpid());
fp = fopen(buf,"w+");
if (!fp) {
redisLog(REDIS_WARNING,"Can't open tmp file for MIGRATE: %s",
strerror(errno));
addReplyErrorFormat(c,"MIGRATE failed, tmp file creation error: %s.",
strerror(errno));
return;
}
unlink(buf);
/* Build the SELECT + RESTORE query writing it in our temp file. */
if (fwriteBulkCount(fp,'*',2) == 0) goto file_wr_err;
if (fwriteBulkString(fp,"SELECT",6) == 0) goto file_wr_err;
if (fwriteBulkLongLong(fp,dbid) == 0) goto file_wr_err;
cmd = rioInitWithBuffer(sdsempty());
redisAssert(rioWriteBulkCount(&cmd,'*',2));
redisAssert(rioWriteBulkString(&cmd,"SELECT",6));
redisAssert(rioWriteBulkLongLong(&cmd,dbid));
ttl = getExpire(c->db,c->argv[3]);
type = o->type;
if (fwriteBulkCount(fp,'*',4) == 0) goto file_wr_err;
if (fwriteBulkString(fp,"RESTORE",7) == 0) goto file_wr_err;
if (fwriteBulkObject(fp,c->argv[3]) == 0) goto file_wr_err;
if (fwriteBulkLongLong(fp, (ttl == -1) ? 0 : ttl) == 0) goto file_wr_err;
if (type == REDIS_LIST && o->encoding == REDIS_ENCODING_ZIPLIST)
type = REDIS_LIST_ZIPLIST;
else if (type == REDIS_HASH && o->encoding == REDIS_ENCODING_ZIPMAP)
type = REDIS_HASH_ZIPMAP;
else if (type == REDIS_SET && o->encoding == REDIS_ENCODING_INTSET)
type = REDIS_SET_INTSET;
else
type = o->type;
redisAssert(rioWriteBulkCount(&cmd,'*',4));
redisAssert(rioWriteBulkString(&cmd,"RESTORE",7));
redisAssert(c->argv[3]->encoding == REDIS_ENCODING_RAW);
redisAssert(rioWriteBulkString(&cmd,c->argv[3]->ptr,sdslen(c->argv[3]->ptr)));
redisAssert(rioWriteBulkLongLong(&cmd,(ttl == -1) ? 0 : ttl));
/* Finally the last argument that is the serailized object payload
* in the form: <type><rdb-serailized-object>. */
payload_len = rdbSavedObjectLen(o);
if (fwriteBulkCount(fp,'$',payload_len+1) == 0) goto file_wr_err;
if (fwrite(&type,1,1,fp) == 0) goto file_wr_err;
if (rdbSaveObject(fp,o) == -1) goto file_wr_err;
if (fwrite("\r\n",2,1,fp) == 0) goto file_wr_err;
* in the form: <type><rdb-serialized-object>. */
payload = rioInitWithBuffer(sdsempty());
redisAssert(rioWrite(&payload,&type,1));
redisAssert(rdbSaveObject(&payload,o) != -1);
redisAssert(rioWriteBulkString(&cmd,payload.io.buffer.ptr,sdslen(payload.io.buffer.ptr)));
sdsfree(payload.io.buffer.ptr);
/* Tranfer the query to the other node */
rewind(fp);
/* Tranfer the query to the other node in 64K chunks. */
{
char buf[4096];
size_t nread;
sds buf = cmd.io.buffer.ptr;
size_t pos = 0, towrite;
int nwritten = 0;
while ((nread = fread(buf,1,sizeof(buf),fp)) != 0) {
int nwritten;
nwritten = syncWrite(fd,buf,nread,timeout);
if (nwritten != (signed)nread) goto socket_wr_err;
while ((towrite = sdslen(buf)-pos) > 0) {
towrite = (towrite > (64*1024) ? (64*1024) : towrite);
nwritten = syncWrite(fd,buf+nwritten,towrite,timeout);
if (nwritten != (signed)towrite) goto socket_wr_err;
pos += nwritten;
}
if (ferror(fp)) goto file_rd_err;
}
/* Read back the reply */
/* Read back the reply. */
{
char buf1[1024];
char buf2[1024];
@ -1541,7 +1514,7 @@ void migrateCommand(redisClient *c) {
if (syncReadLine(fd, buf1, sizeof(buf1), timeout) <= 0)
goto socket_rd_err;
if (syncReadLine(fd, buf2, sizeof(buf2), timeout) <= 0)
goto socket_rd_err;
goto socket_rd_err;
if (buf1[0] == '-' || buf2[0] == '-') {
addReplyErrorFormat(c,"Target instance replied with error: %s",
(buf1[0] == '-') ? buf1+1 : buf2+1);
@ -1550,25 +1523,8 @@ void migrateCommand(redisClient *c) {
addReply(c,shared.ok);
}
}
fclose(fp);
close(fd);
return;
file_wr_err:
redisLog(REDIS_WARNING,"Can't write on tmp file for MIGRATE: %s",
strerror(errno));
addReplyErrorFormat(c,"MIGRATE failed, tmp file write error: %s.",
strerror(errno));
fclose(fp);
close(fd);
return;
file_rd_err:
redisLog(REDIS_WARNING,"Can't read from tmp file for MIGRATE: %s",
strerror(errno));
addReplyErrorFormat(c,"MIGRATE failed, tmp file read error: %s.",
strerror(errno));
fclose(fp);
sdsfree(cmd.io.buffer.ptr);
close(fd);
return;
@ -1577,7 +1533,7 @@ socket_wr_err:
strerror(errno));
addReplyErrorFormat(c,"MIGRATE failed, writing to target node: %s.",
strerror(errno));
fclose(fp);
sdsfree(cmd.io.buffer.ptr);
close(fd);
return;
@ -1586,7 +1542,7 @@ socket_rd_err:
strerror(errno));
addReplyErrorFormat(c,"MIGRATE failed, reading from target node: %s.",
strerror(errno));
fclose(fp);
sdsfree(cmd.io.buffer.ptr);
close(fd);
return;
}
@ -1595,11 +1551,9 @@ socket_rd_err:
* DUMP is actually not used by Redis Cluster but it is the obvious
* complement of RESTORE and can be useful for different applications. */
void dumpCommand(redisClient *c) {
char buf[64];
FILE *fp;
robj *o, *dumpobj;
sds dump = NULL;
off_t payload_len;
rio payload;
unsigned int type;
/* Check if the key is here. */
@ -1608,27 +1562,15 @@ void dumpCommand(redisClient *c) {
return;
}
/* Create temp file */
snprintf(buf,sizeof(buf),"redis-dump-%d.tmp",getpid());
fp = fopen(buf,"w+");
if (!fp) {
redisLog(REDIS_WARNING,"Can't open tmp file for MIGRATE: %s",
strerror(errno));
addReplyErrorFormat(c,"DUMP failed, tmp file creation error: %s.",
strerror(errno));
return;
}
unlink(buf);
/* Dump the serailized object and read it back in memory. We prefix it
* with a one byte containing the type ID. This is the serialization
* format understood by RESTORE. */
payload = rioInitWithBuffer(sdsempty());
redisAssert(rdbSaveObject(&payload,o)); /* always write >= 1 bytes. */
dump = sdsnewlen(NULL,sdslen(payload.io.buffer.ptr)+1);
memcpy(dump+1,payload.io.buffer.ptr,sdslen(payload.io.buffer.ptr));
sdsfree(payload.io.buffer.ptr);
/* Dump the serailized object and read it back in memory.
* We prefix it with a one byte containing the type ID.
* This is the serialization format understood by RESTORE. */
if (rdbSaveObject(fp,o) == -1) goto file_wr_err;
payload_len = ftello(fp);
if (fseeko(fp,0,SEEK_SET) == -1) goto file_rd_err;
dump = sdsnewlen(NULL,payload_len+1);
if (payload_len && fread(dump+1,payload_len,1,fp) != 1) goto file_rd_err;
fclose(fp);
type = o->type;
if (type == REDIS_LIST && o->encoding == REDIS_ENCODING_ZIPLIST)
type = REDIS_LIST_ZIPLIST;
@ -1645,24 +1587,6 @@ void dumpCommand(redisClient *c) {
addReplyBulk(c,dumpobj);
decrRefCount(dumpobj);
return;
file_wr_err:
redisLog(REDIS_WARNING,"Can't write on tmp file for DUMP: %s",
strerror(errno));
addReplyErrorFormat(c,"DUMP failed, tmp file write error: %s.",
strerror(errno));
sdsfree(dump);
fclose(fp);
return;
file_rd_err:
redisLog(REDIS_WARNING,"Can't read from tmp file for DUMP: %s",
strerror(errno));
addReplyErrorFormat(c,"DUMP failed, tmp file read error: %s.",
strerror(errno));
sdsfree(dump);
fclose(fp);
return;
}
/* -----------------------------------------------------------------------------