diff --git a/src/config.c b/src/config.c index 612d59c2..cd39b8d9 100644 --- a/src/config.c +++ b/src/config.c @@ -226,8 +226,7 @@ void loadServerConfigFromString(char *config) { } } else if (!strcasecmp(argv[0],"maxmemory") && argc == 2) { server.maxmemory = memtoll(argv[1],NULL); - server.maxmemory_adjusted = server.maxmemory; - server.maxmemory_enforced = server.maxmemory; + server.maxmemory_enforced = (double) server.maxmemory / server.maxmemory_frag_guess; } else if (!strcasecmp(argv[0],"rss-aware-maxmemory") && argc==2) { if ((server.rss_aware_maxmemory = yesnotoi(argv[1])) == -1) { err = "argument must be 'yes' or 'no'"; goto loaderr; @@ -637,8 +636,7 @@ void configSetCommand(redisClient *c) { if (getLongLongFromObject(o,&ll) == REDIS_ERR || ll < 0) goto badfmt; server.maxmemory = ll; - server.maxmemory_adjusted = ll; - server.maxmemory_enforced = ll; + server.maxmemory_enforced = (double) server.maxmemory / server.maxmemory_frag_guess; if (server.maxmemory) { if (server.maxmemory < zmalloc_used_memory()) { redisLog(REDIS_WARNING,"WARNING: the new maxmemory value set via CONFIG SET is smaller than the current memory usage. This will result in keys eviction and/or inability to accept new write commands depending on the maxmemory-policy."); diff --git a/src/redis.c b/src/redis.c index fbe8a882..d07bb7c5 100644 --- a/src/redis.c +++ b/src/redis.c @@ -1433,8 +1433,8 @@ void initServerConfig(void) { server.maxclients = REDIS_MAX_CLIENTS; server.bpop_blocked_clients = 0; server.maxmemory = REDIS_DEFAULT_MAXMEMORY; - server.maxmemory_enforced = REDIS_DEFAULT_MAXMEMORY; - server.maxmemory_adjusted = REDIS_DEFAULT_MAXMEMORY; + server.maxmemory_frag_guess = REDIS_DEFAULT_MAXMEMORY_FRAG_GUESS; + server.maxmemory_enforced = (double) REDIS_DEFAULT_MAXMEMORY / server.maxmemory_frag_guess; server.maxmemory_policy = REDIS_DEFAULT_MAXMEMORY_POLICY; server.maxmemory_samples = REDIS_DEFAULT_MAXMEMORY_SAMPLES; server.rss_aware_maxmemory = REDIS_DEFAULT_RSS_AWARE_MAXMEMORY; @@ -3157,7 +3157,7 @@ void evictionPoolPopulate(dict *sampledict, dict *keydict, struct evictionPoolEn } int freeMemoryIfNeeded(void) { - size_t mem_used, mem_tofree, mem_freed; + size_t mem_used, mem_tofree, mem_freed, mem_target = server.maxmemory; int slaves = listLength(server.slaves); mstime_t latency; @@ -3183,14 +3183,72 @@ int freeMemoryIfNeeded(void) { mem_used -= aofRewriteBufferSize(); } + /* If we use RSS aware maxmemory, update the target memory using + * the current fragmentation figure. */ +#ifdef HAVE_RSS_REPORTING + if (server.rss_aware_maxmemory && + server.maxmemory_policy != REDIS_MAXMEMORY_NO_EVICTION) + { + static unsigned long iterations = 0; + static unsigned long sampling_stage = 1; + static float last_observed_frag = 0; + + /* For some time, we analyze what happens during memory pressure, when + * objects are evicted and reallocated. */ + if (mem_used > server.maxmemory_enforced) { + unsigned long sample_cycles = 1000000; + + /* Every sample_cycle cycles we sample the fragmentation, and + * compare it with the previos one. If it is no longer raising, + * we take it as a guess of the fragmentation with this workload. */ + if (sampling_stage && iterations < sample_cycles) { + iterations++; + if (iterations == sample_cycles) { + float current_frag = zmalloc_get_fragmentation_ratio(server.resident_set_size); + if (last_observed_frag == 0) { + /* First sample we get. */ + last_observed_frag = current_frag; + } else { + if (current_frag <= last_observed_frag) { + size_t enforced_new; + + sampling_stage = 0; + server.maxmemory_frag_guess = current_frag; + /* Update the global fragmentation guess and use + * it (also used it for successive + * "CONFIG SET maxmemory" commands). */ + if (server.maxmemory_frag_guess < 1) + server.maxmemory_frag_guess = 1; + else if (server.maxmemory_frag_guess > 2) + server.maxmemory_frag_guess = 2; + + /* Only set the new limit if it is higher than our + * initial guess, otherwise it is futile: RSS will + * not go backward anyway. */ + enforced_new = (double) server.maxmemory / + server.maxmemory_frag_guess; + if (enforced_new > server.maxmemory_enforced) + server.maxmemory_enforced = enforced_new; + redisLog(REDIS_NOTICE,"RSS aware maxmemory, fragmentation looks stable at: %f", server.maxmemory_frag_guess); + } + last_observed_frag = current_frag; + } + iterations = 0; + } + } + } + mem_target = server.maxmemory_enforced; + } +#endif + /* Check if we are over the memory limit. */ - if (mem_used <= server.maxmemory) return REDIS_OK; + if (mem_used <= mem_target) return REDIS_OK; if (server.maxmemory_policy == REDIS_MAXMEMORY_NO_EVICTION) return REDIS_ERR; /* We need to free memory, but policy forbids. */ /* Compute how much memory we need to free. */ - mem_tofree = mem_used - server.maxmemory; + mem_tofree = mem_used - mem_target; mem_freed = 0; latencyStartMonitor(latency); while (mem_freed < mem_tofree) { diff --git a/src/redis.h b/src/redis.h index b047c13a..b396eb70 100644 --- a/src/redis.h +++ b/src/redis.h @@ -121,6 +121,7 @@ typedef long long mstime_t; /* millisecond time type. */ #define REDIS_DEFAULT_REPL_DISABLE_TCP_NODELAY 0 #define REDIS_DEFAULT_MAXMEMORY 0 #define REDIS_DEFAULT_MAXMEMORY_SAMPLES 5 +#define REDIS_DEFAULT_MAXMEMORY_FRAG_GUESS 1.4 #define REDIS_DEFAULT_RSS_AWARE_MAXMEMORY 0 #define REDIS_DEFAULT_AOF_FILENAME "appendonly.aof" #define REDIS_DEFAULT_AOF_NO_FSYNC_ON_REWRITE 0 @@ -840,14 +841,10 @@ struct redisServer { unsigned long long maxmemory; /* Max number of memory bytes to use */ int maxmemory_policy; /* Policy for key eviction */ int maxmemory_samples; /* Pricision of random sampling */ - /* RSS aware maxmemory additional state: - * the adjusted maxmemory is adjusted for fragmentation, however we - * enforce maxmemory_enforced instead, which follows the adjusted value - * in small steps at eviction cycle to avoid too fast changes in the - * enforced maxmemory value. */ + /* RSS aware maxmemory additional state. */ int rss_aware_maxmemory; /* Non zero if enabled. */ - unsigned long long maxmemory_adjusted; /* Maxmemory adjusted for frag. */ unsigned long long maxmemory_enforced; /* Currently enforced maxmemory. */ + float maxmemory_frag_guess; /* Guessed fragmentation. */ /* Blocked clients */ unsigned int bpop_blocked_clients; /* Number of clients blocked by lists */ list *unblocked_clients; /* list of clients to unblock before next loop */