From 0200e8ada63ddff5d5a7413ff481bf8f1d5465ba Mon Sep 17 00:00:00 2001 From: Moti Cohen Date: Tue, 25 Feb 2025 00:38:44 +0200 Subject: [PATCH] Fix multiple issues with "INFO KEYSIZES" (#13825) This commit addresses several issues related to the `INFO KEYSIZES` feature: - HyperLogLog commands: `KEYSIZES` hooks were not properly set or tested. - HFE lazy expiration: `KEYSIZES` hooks were not properly set or tested. - Empty DB & SYNC flow: On `blocking_async=0` flow, global `keysizes` histogram were not reset (can reproduced using `DEBUG RELOAD`). - Empty string handling: Fix histogram for strings of size 0. Not relevant to other data-types. --- src/db.c | 43 +++-- src/hyperloglog.c | 9 ++ src/kvstore.c | 3 + src/kvstore.h | 2 +- src/server.c | 4 +- src/server.h | 19 +-- src/t_hash.c | 14 +- tests/unit/info-keysizes.tcl | 299 +++++++++++++++++++++++++++++++---- 8 files changed, 330 insertions(+), 63 deletions(-) diff --git a/src/db.c b/src/db.c index e3019d2121..d1b3ab8c51 100644 --- a/src/db.c +++ b/src/db.c @@ -57,36 +57,51 @@ void updateLFU(robj *val) { * The histogram is a base-2 logarithmic histogram, with 64 bins. The i'th bin * represents the number of keys with a size in the range 2^i and 2^(i+1) * exclusive. oldLen/newLen must be smaller than 2^48, and if their value - * equals 0, it means that the key is being created/deleted, respectively. Each + * equals -1, it means that the key is being created/deleted, respectively. Each * data type has its own histogram and it is per database (In addition, there is * histogram per slot for future cluster use). - * - * Examples to LEN values and corresponding bins in histogram: - * [1,2)->0 [2,4)->1 [4,8)->2 [8,16)->3 + * + * Example mapping of key lengths to bins: + * [1,2)->1 [2,4)->2 [4,8)->3 [8,16)->4 ... + * + * Since strings can be zero length, the histogram also tracks: + * [0,1)->0 */ -void updateKeysizesHist(redisDb *db, int didx, uint32_t type, uint64_t oldLen, uint64_t newLen) { +void updateKeysizesHist(redisDb *db, int didx, uint32_t type, int64_t oldLen, int64_t newLen) { if(unlikely(type >= OBJ_TYPE_BASIC_MAX)) return; kvstoreDictMetadata *dictMeta = kvstoreGetDictMetadata(db->keys, didx); kvstoreMetadata *kvstoreMeta = kvstoreGetMetadata(db->keys); - if (oldLen != 0) { - int old_bin = log2ceil(oldLen); + if (oldLen > 0) { + int old_bin = log2ceil(oldLen) + 1; debugServerAssertWithInfo(server.current_client, NULL, old_bin < MAX_KEYSIZES_BINS); /* If following a key deletion it is last one in slot's dict, then * slot's dict might get released as well. Verify if metadata is not NULL. */ if(dictMeta) dictMeta->keysizes_hist[type][old_bin]--; kvstoreMeta->keysizes_hist[type][old_bin]--; + } else { + /* here, oldLen can be either 0 or -1 */ + if (oldLen == 0) { + if (dictMeta) dictMeta->keysizes_hist[type][0]--; + kvstoreMeta->keysizes_hist[type][0]--; + } } - if (newLen != 0) { - int new_bin = log2ceil(newLen); + if (newLen > 0) { + int new_bin = log2ceil(newLen) + 1; debugServerAssertWithInfo(server.current_client, NULL, new_bin < MAX_KEYSIZES_BINS); /* If following a key deletion it is last one in slot's dict, then * slot's dict might get released as well. Verify if metadata is not NULL. */ if(dictMeta) dictMeta->keysizes_hist[type][new_bin]++; kvstoreMeta->keysizes_hist[type][new_bin]++; + } else { + /* here, newLen can be either 0 or -1 */ + if (newLen == 0) { + if (dictMeta) dictMeta->keysizes_hist[type][0]++; + kvstoreMeta->keysizes_hist[type][0]++; + } } } @@ -249,7 +264,7 @@ static dictEntry *dbAddInternal(redisDb *db, robj *key, robj *val, int update_if kvstoreDictSetVal(db->keys, slot, de, val); signalKeyAsReady(db, key, val->type); notifyKeyspaceEvent(NOTIFY_NEW,"new",key,db->id); - updateKeysizesHist(db, slot, val->type, 0, getObjectLength(val)); /* add hist */ + updateKeysizesHist(db, slot, val->type, -1, getObjectLength(val)); /* add hist */ return de; } @@ -295,7 +310,7 @@ int dbAddRDBLoad(redisDb *db, sds key, robj *val) { int slot = getKeySlot(key); dictEntry *de = kvstoreDictAddRaw(db->keys, slot, key, NULL); if (de == NULL) return 0; - updateKeysizesHist(db, slot, val->type, 0, getObjectLength(val)); /* add hist */ + updateKeysizesHist(db, slot, val->type, -1, getObjectLength(val)); /* add hist */ initObjectLRUOrLFU(val); kvstoreDictSetVal(db->keys, slot, de, val); return 1; @@ -320,7 +335,7 @@ static void dbSetValue(redisDb *db, robj *key, robj *val, int overwrite, dictEnt robj *old = dictGetVal(de); /* Remove old key from keysizes histogram */ - updateKeysizesHist(db, slot, old->type, getObjectLength(old), 0); /* remove hist */ + updateKeysizesHist(db, slot, old->type, getObjectLength(old), -1); /* remove hist */ val->lru = old->lru; @@ -341,7 +356,7 @@ static void dbSetValue(redisDb *db, robj *key, robj *val, int overwrite, dictEnt kvstoreDictSetVal(db->keys, slot, de, val); /* Add new key to keysizes histogram */ - updateKeysizesHist(db, slot, val->type, 0, getObjectLength(val)); + updateKeysizesHist(db, slot, val->type, -1, getObjectLength(val)); /* if hash with HFEs, take care to remove from global HFE DS */ if (old->type == OBJ_HASH) @@ -457,7 +472,7 @@ int dbGenericDelete(redisDb *db, robj *key, int async, int flags) { robj *val = dictGetVal(de); /* remove key from histogram */ - updateKeysizesHist(db, slot, val->type, getObjectLength(val), 0); + updateKeysizesHist(db, slot, val->type, getObjectLength(val), -1); /* If hash object with expiry on fields, remove it from HFE DS of DB */ if (val->type == OBJ_HASH) diff --git a/src/hyperloglog.c b/src/hyperloglog.c index 0533e0f755..9f6e7c27e3 100644 --- a/src/hyperloglog.c +++ b/src/hyperloglog.c @@ -1413,6 +1413,7 @@ invalid: /* PFADD var ele ele ele ... ele => :0 or :1 */ void pfaddCommand(client *c) { + uint64_t oldLen; robj *o = lookupKeyWrite(c->db,c->argv[1]); struct hllhdr *hdr; int updated = 0, j; @@ -1428,6 +1429,8 @@ void pfaddCommand(client *c) { if (isHLLObjectOrReply(c,o) != C_OK) return; o = dbUnshareStringValue(c->db,c->argv[1],o); } + oldLen = stringObjectLen(o); + /* Perform the low level ADD operation for every element. */ for (j = 2; j < c->argc; j++) { int retval = hllAdd(o, (unsigned char*)c->argv[j]->ptr, @@ -1447,6 +1450,7 @@ void pfaddCommand(client *c) { signalModifiedKey(c,c->db,c->argv[1]); notifyKeyspaceEvent(NOTIFY_STRING,"pfadd",c->argv[1],c->db->id); server.dirty += updated; + updateKeysizesHist(c->db, getKeySlot(c->argv[1]->ptr), OBJ_STRING, oldLen, stringObjectLen(o)); } addReply(c, updated ? shared.cone : shared.czero); } @@ -1592,6 +1596,8 @@ void pfmergeCommand(client *c) { o = dbUnshareStringValue(c->db,c->argv[1],o); } + uint64_t oldLen = stringObjectLen(o); + /* Convert the destination object to dense representation if at least * one of the inputs was dense. */ if (use_dense && hllSparseToDense(o) == C_ERR) { @@ -1622,6 +1628,9 @@ void pfmergeCommand(client *c) { /* We generate a PFADD event for PFMERGE for semantical simplicity * since in theory this is a mass-add of elements. */ notifyKeyspaceEvent(NOTIFY_STRING,"pfadd",c->argv[1],c->db->id); + + updateKeysizesHist(c->db, getKeySlot(c->argv[1]->ptr), + OBJ_STRING, oldLen, stringObjectLen(o)); server.dirty++; addReply(c,shared.ok); } diff --git a/src/kvstore.c b/src/kvstore.c index fdb9b61a68..d7948c9ab1 100644 --- a/src/kvstore.c +++ b/src/kvstore.c @@ -310,6 +310,9 @@ void kvstoreEmpty(kvstore *kvs, void(callback)(dict*)) { freeDictIfNeeded(kvs, didx); } + if (kvs->flags & KVSTORE_ALLOC_META_KEYS_HIST) + memset(kvstoreGetMetadata(kvs), 0, sizeof(kvstoreMetadata)); + listEmpty(kvs->rehashing); kvs->key_count = 0; diff --git a/src/kvstore.h b/src/kvstore.h index 8b9fd7348f..c077dd625a 100644 --- a/src/kvstore.h +++ b/src/kvstore.h @@ -18,7 +18,7 @@ #include "adlist.h" /* maximum number of bins of keysizes histogram */ -#define MAX_KEYSIZES_BINS 48 +#define MAX_KEYSIZES_BINS 60 #define MAX_KEYSIZES_TYPES 5 /* static_assert at db.c verifies == OBJ_TYPE_BASIC_MAX */ /* When creating kvstore with flag `KVSTORE_ALLOC_META_KEYS_HIST`, then kvstore diff --git a/src/server.c b/src/server.c index e4030be47e..2f464605d4 100644 --- a/src/server.c +++ b/src/server.c @@ -6317,13 +6317,13 @@ sds genRedisInfoString(dict *section_dict, int all_sections, int everything) { for (int dbnum = 0; dbnum < server.dbnum; dbnum++) { char *expSizeLabels[] = { - "1", "2", "4", "8", "16", "32", "64", "128", "256", "512", /* Byte */ + "0", "1", "2", "4", "8", "16", "32", "64", "128", "256", "512", /* Byte */ "1K", "2K", "4K", "8K", "16K", "32K", "64K", "128K", "256K", "512K", /* Kilo */ "1M", "2M", "4M", "8M", "16M", "32M", "64M", "128M", "256M", "512M", /* Mega */ "1G", "2G", "4G", "8G", "16G", "32G", "64G", "128G", "256G", "512G", /* Giga */ "1T", "2T", "4T", "8T", "16T", "32T", "64T", "128T", "256T", "512T", /* Tera */ "1P", "2P", "4P", "8P", "16P", "32P", "64P", "128P", "256P", "512P", /* Peta */ - "1E", "2E", "4E", "8E" /* Exa */ + "1E", "2E", "4E" /* Exa */ }; if (kvstoreSize(server.db[dbnum].keys) == 0) diff --git a/src/server.h b/src/server.h index 585a654a83..91b859e4d5 100644 --- a/src/server.h +++ b/src/server.h @@ -3366,14 +3366,15 @@ typedef struct dictExpireMetadata { #define HASH_SET_COPY 0 /* Hash field lazy expiration flags. Used by core hashTypeGetValue() and its callers */ -#define HFE_LAZY_EXPIRE (0) /* Delete expired field, and if last field also the hash */ -#define HFE_LAZY_AVOID_FIELD_DEL (1<<0) /* Avoid deleting expired field */ -#define HFE_LAZY_AVOID_HASH_DEL (1<<1) /* Avoid deleting hash if the field is the last one */ -#define HFE_LAZY_NO_NOTIFICATION (1<<2) /* Do not send notification, used when multiple fields - * may expire and only one notification is desired. */ -#define HFE_LAZY_NO_SIGNAL (1<<3) /* Do not send signal, used when multiple fields - * may expire and only one signal is desired. */ -#define HFE_LAZY_ACCESS_EXPIRED (1<<4) /* Avoid lazy expire and allow access to expired fields */ +#define HFE_LAZY_EXPIRE (0) /* Delete expired field, and if last field also the hash */ +#define HFE_LAZY_AVOID_FIELD_DEL (1<<0) /* Avoid deleting expired field */ +#define HFE_LAZY_AVOID_HASH_DEL (1<<1) /* Avoid deleting hash if the field is the last one */ +#define HFE_LAZY_NO_NOTIFICATION (1<<2) /* Do not send notification, used when multiple fields + * may expire and only one notification is desired. */ +#define HFE_LAZY_NO_SIGNAL (1<<3) /* Do not send signal, used when multiple fields + * may expire and only one signal is desired. */ +#define HFE_LAZY_ACCESS_EXPIRED (1<<4) /* Avoid lazy expire and allow access to expired fields */ +#define HFE_LAZY_NO_UPDATE_KEYSIZES (1<<5) /* If field lazy deleted, avoid updating keysizes histogram */ void hashTypeConvert(robj *o, int enc, ebuckets *hexpires); void hashTypeTryConversion(redisDb *db, robj *subject, robj **argv, int start, int end); @@ -3516,7 +3517,7 @@ long long getModuleNumericConfig(ModuleConfig *module_config); int setModuleNumericConfig(ModuleConfig *config, long long val, const char **err); /* db.c -- Keyspace access API */ -void updateKeysizesHist(redisDb *db, int didx, uint32_t type, uint64_t oldLen, uint64_t newLen); +void updateKeysizesHist(redisDb *db, int didx, uint32_t type, int64_t oldLen, int64_t newLen); int removeExpire(redisDb *db, robj *key); void deleteExpiredKeyAndPropagate(redisDb *db, robj *keyobj); void deleteEvictedKeyAndPropagate(redisDb *db, robj *keyobj, long long *key_mem_freed); diff --git a/src/t_hash.c b/src/t_hash.c index b4d31bd0e6..80bd26a973 100644 --- a/src/t_hash.c +++ b/src/t_hash.c @@ -777,6 +777,11 @@ GetFieldRes hashTypeGetValue(redisDb *db, robj *o, sds field, unsigned char **vs propagateHashFieldDeletion(db, key, field, sdslen(field)); server.stat_expired_subkeys++; + if (!(hfeFlags & HFE_LAZY_NO_UPDATE_KEYSIZES)) { + uint64_t l = hashTypeLength(o, 0); + updateKeysizesHist(db, getKeySlot(key), OBJ_HASH, l+1, l); + } + /* If the field is the last one in the hash, then the hash will be deleted */ res = GETF_EXPIRED; robj *keyObj = createStringObject(key, sdslen(key)); @@ -2410,7 +2415,8 @@ void hsetexCommand(client *c) { sds field = c->argv[first_field_pos + (i * 2)]->ptr; const int opt = HFE_LAZY_NO_NOTIFICATION | HFE_LAZY_NO_SIGNAL | - HFE_LAZY_AVOID_HASH_DEL; + HFE_LAZY_AVOID_HASH_DEL | + HFE_LAZY_NO_UPDATE_KEYSIZES; int exists = hashTypeExists(c->db, o, field, opt, NULL); found += (exists != 0); @@ -2715,7 +2721,8 @@ void hgetdelCommand(client *c) { for (int i = 4; i < c->argc; i++) { const int flags = HFE_LAZY_NO_NOTIFICATION | HFE_LAZY_NO_SIGNAL | - HFE_LAZY_AVOID_HASH_DEL; + HFE_LAZY_AVOID_HASH_DEL | + HFE_LAZY_NO_UPDATE_KEYSIZES; res = addHashFieldToReply(c, o, c->argv[i]->ptr, flags); expired += (res == GETF_EXPIRED); /* Try to delete only if it's found and not expired lazily. */ @@ -2837,7 +2844,8 @@ void hgetexCommand(client *c) { for (int i = num_fields_pos + 1; i < c->argc; i++) { const int flags = HFE_LAZY_NO_NOTIFICATION | HFE_LAZY_NO_SIGNAL | - HFE_LAZY_AVOID_HASH_DEL; + HFE_LAZY_AVOID_HASH_DEL | + HFE_LAZY_NO_UPDATE_KEYSIZES; sds field = c->argv[i]->ptr; int res = addHashFieldToReply(c, o, field, flags); expired += (res == GETF_EXPIRED); diff --git a/tests/unit/info-keysizes.tcl b/tests/unit/info-keysizes.tcl index a866efcfd6..e5fd81551b 100644 --- a/tests/unit/info-keysizes.tcl +++ b/tests/unit/info-keysizes.tcl @@ -20,6 +20,11 @@ proc get_stripped_info {server} { # Instead of the output of "strings_len_exp_distrib" write "STR". # Similarly for LIST, SET, ZSET and HASH. Spaces and newlines are # ignored. +# +# Alternatively, you can set "__EVAL_DB_HIST__". The function +# will read all the keys from the server for selected db index, +# ask for their length and compute the expected output. + # waitCond - If set to 1, the function wait_for_condition 50x50msec for the # expOutput to match the actual output. # @@ -28,50 +33,122 @@ proc get_stripped_info {server} { # for the replica to catch up and verify the output on the replica # as well. Otherwise, just run the command on the leader and verify # the output. -proc run_cmd_verify_hist {cmd expOutput {waitCond 0} } { +proc run_cmd_verify_hist {cmd expOutput {waitCond 0}} { uplevel 1 $cmd global replicaMode - # ref the leader with `server` variable - if {$replicaMode eq 1} { set server [srv -1 client] } else { set server [srv 0 client] } - + # ref the leader with `server` variable + if {$replicaMode eq 1} { + set server [srv -1 client] + set replica [srv 0 client] + } else { + set server [srv 0 client] + } + + # If need eval expected histogram by reading all the keys & lengths from the + # server. + if {[regexp {^__EVAL_DB_HIST__\s+(\d+)$} $expOutput -> dbid]} { + set expOutput [eval_db_histogram $server $dbid] + } + # Replace all placeholders with the actual values. Remove spaces & newlines. set expStripped [string map { - "STR" "distrib_strings_sizes" - "LIST" "distrib_lists_items" - "SET" "distrib_sets_items" - "ZSET" "distrib_zsets_items" + "STR" "distrib_strings_sizes" + "LIST" "distrib_lists_items" + "SET" "distrib_sets_items" + "ZSET" "distrib_zsets_items" "HASH" "distrib_hashes_items" " " "" "\n" "" "\r" "" } $expOutput] - + + # Compare the stripped expected output with the actual output from the server if {$waitCond} { - wait_for_condition 50 50 { + wait_for_condition 50 50 { $expStripped eq [get_stripped_info $server] - } else { + } else { fail "Unexpected KEYSIZES. Expected: `$expStripped` \ but got: `[get_stripped_info $server]`. Failed after command: $cmd" - - } + } } else { - set infoStripped [get_stripped_info $server] + set infoStripped [get_stripped_info $server] if {$expStripped ne $infoStripped} { fail "Unexpected KEYSIZES. Expected: `$expStripped` \ but got: `$infoStripped`. Failed after command: $cmd" } } - - # If we are testing `replicaMode` then need to wait for the replica to catch up - if {$replicaMode eq 1} { - wait_for_condition 50 50 { - $expStripped eq [get_stripped_info $server] - } else { + + # If we are testing `replicaMode` then need to wait for the replica to catch up + if {$replicaMode eq 1} { + wait_for_condition 50 50 { + $expStripped eq [get_stripped_info $replica] + } else { fail "Unexpected replica KEYSIZES. Expected: `$expStripped` \ - but got: `[get_stripped_info $server]`. Failed after command: $cmd" - } + but got: `[get_stripped_info $replica]`. Failed after command: $cmd" + } } } +## eval_db_histogram - eval The expected histogram for current db, by +## reading all the keys from the server, query for their length, and computing +## the expected output. +proc eval_db_histogram {server dbid} { + array set type_counts {} + + set keys [$server keys *] + foreach key $keys { + set key_type [$server type $key] + switch -exact $key_type { + "string" { + set value [$server strlen $key] + set type "STR" + } + "list" { + set value [$server llen $key] + set type "LIST" + } + "set" { + set value [$server scard $key] + set type "SET" + } + "zset" { + set value [$server zcard $key] + set type "ZSET" + } + "hash" { + set value [$server hlen $key] + set type "HASH" + } + default { + continue ; # Skip unknown types + } + } + + set power 1 + while { ($power * 2) <= $value } { set power [expr {$power * 2}] } + # Store counts by type and size bucket + incr type_counts($type,$power) + } + + set result {} + foreach type {STR LIST SET ZSET HASH} { + if {[array exists type_counts] && [array names type_counts $type,*] ne ""} { + set sorted_powers [lsort -integer [lmap item [array names type_counts $type,*] { + lindex [split $item ,] 1 ; # Extracts only the numeric part + }]] + + set type_result {} + foreach power $sorted_powers { + set display_power $power + if { $power >= 1024 } { set display_power "[expr {$power / 1024}]K" } + lappend type_result "$display_power=$type_counts($type,$power)" + } + lappend result "db${dbid}_$type: [join $type_result ", "]" + } + } + + return [join $result " | "] +} + proc test_all_keysizes { {replMode 0} } { # If in replica mode then update global var `replicaMode` so function # `run_cmd_verify_hist` knows to run the command on the leader and then @@ -80,7 +157,8 @@ proc test_all_keysizes { {replMode 0} } { set replicaMode $replMode # ref the leader with `server` variable if {$replicaMode eq 1} { - set server [srv -1 client] + set server [srv -1 client] + set replica [srv 0 client] set suffixRepl "(replica)" } else { set server [srv 0 client] @@ -94,7 +172,7 @@ proc test_all_keysizes { {replMode 0} } { append base_string "x" set log_value [expr {1 << int(log($i) / log(2))}] #puts "Iteration $i: $base_string (Log base 2 pattern: $log_value)" - run_cmd_verify_hist {$server set mykey $base_string} "db0_STR:$log_value=1" + run_cmd_verify_hist {$server set mykey $base_string} "db0_STR:$log_value=1" } } @@ -119,13 +197,47 @@ proc test_all_keysizes { {replMode 0} } { run_cmd_verify_hist {$server APPEND x [$server get x]} {db0_STR:1M=1} run_cmd_verify_hist {$server APPEND x [$server get x]} {db0_STR:2M=1} } + + # It is difficult to predict the actual string length of hyperloglog. To address + # this, we will create expected output by indicating __EVAL_DB_HIST__ to read + # all keys & lengths from server. Based on it, generate the expected output. + test "KEYSIZES - Test hyperloglog $suffixRepl" { + run_cmd_verify_hist {$server FLUSHALL} {} + # PFADD (sparse & dense) + for {set i 1} {$i <= 3000} {incr i} { + run_cmd_verify_hist {$server PFADD hll1 a$i b$i c$i} {__EVAL_DB_HIST__ 0} + run_cmd_verify_hist {$server PFADD hll2 x$i y$i z$i} {__EVAL_DB_HIST__ 0} + } + # PFMERGE, PFCOUNT (sparse & dense) + for {set i 1} {$i <= 3000} {incr i} { + run_cmd_verify_hist {$server PFADD hll3 x$i y$i z$i} {__EVAL_DB_HIST__ 0} + run_cmd_verify_hist {$server PFMERGE hll4 hll1 hll2 hll3} {__EVAL_DB_HIST__ 0} + run_cmd_verify_hist {$server PFCOUNT hll1 hll2 hll3 hll4} {__EVAL_DB_HIST__ 0} + } + # DEL + run_cmd_verify_hist {$server DEL hll4} {__EVAL_DB_HIST__ 0} + run_cmd_verify_hist {$server DEL hll3} {__EVAL_DB_HIST__ 0} + run_cmd_verify_hist {$server DEL hll1} {__EVAL_DB_HIST__ 0} + run_cmd_verify_hist {$server DEL hll2} {} + # SET overwrites + run_cmd_verify_hist {$server FLUSHALL} {} + run_cmd_verify_hist {$server PFADD hll1 a b c d e f g h i j k l m} {db0_STR:32=1} + run_cmd_verify_hist {$server SET hll1 1234567} {db0_STR:4=1} + catch {run_cmd_verify_hist {$server PFADD hll1 a b c d e f g h i j k l m} {db0_STR:4=1}} + run_cmd_verify_hist {} {db0_STR:4=1} + # EXPIRE + run_cmd_verify_hist {$server FLUSHALL} {} + run_cmd_verify_hist {$server PFADD hll1 a b c d e f g h i j k l m} {db0_STR:32=1} + run_cmd_verify_hist {$server PEXPIRE hll1 50} {db0_STR:32=1} + run_cmd_verify_hist {} {} 1 + } {} {cluster:skip} test "KEYSIZES - Test List $suffixRepl" { # FLUSHALL run_cmd_verify_hist {$server FLUSHALL} {} # RPUSH run_cmd_verify_hist {$server RPUSH l1 1 2 3 4 5} {db0_LIST:4=1} - run_cmd_verify_hist {$server RPUSH l1 6 7 8 9} {db0_LIST:8=1} + run_cmd_verify_hist {$server RPUSH l1 6 7 8 9} {db0_LIST:8=1} # Test also LPUSH, RPUSH, LPUSHX, RPUSHX run_cmd_verify_hist {$server LPUSH l2 1} {db0_LIST:1=1,8=1} run_cmd_verify_hist {$server LPUSH l2 2} {db0_LIST:2=1,8=1} @@ -313,6 +425,20 @@ proc test_all_keysizes { {replMode 0} } { run_cmd_verify_hist {$server SET s1 842} {db0_STR:2=1} run_cmd_verify_hist {$server SET s1 2} {db0_STR:1=1} run_cmd_verify_hist {$server SET s1 1234567} {db0_STR:4=1} + + # SET (string of length 0) + run_cmd_verify_hist {$server FLUSHALL} {} + run_cmd_verify_hist {$server SET s1 ""} {db0_STR:0=1} + run_cmd_verify_hist {$server SET s1 ""} {db0_STR:0=1} + run_cmd_verify_hist {$server SET s2 ""} {db0_STR:0=2} + run_cmd_verify_hist {$server SET s2 "bla"} {db0_STR:0=1,2=1} + run_cmd_verify_hist {$server SET s2 ""} {db0_STR:0=2} + run_cmd_verify_hist {$server HSET h f v} {db0_STR:0=2 db0_HASH:1=1} + run_cmd_verify_hist {$server SET h ""} {db0_STR:0=3} + run_cmd_verify_hist {$server DEL h} {db0_STR:0=2} + run_cmd_verify_hist {$server DEL s2} {db0_STR:0=1} + run_cmd_verify_hist {$server DEL s1} {} + } {} {cluster:skip} foreach type {listpackex hashtable} { @@ -364,7 +490,7 @@ proc test_all_keysizes { {replMode 0} } { run_cmd_verify_hist {$server HMSET h1 1 1 2 2 3 3} {db0_HASH:2=1} run_cmd_verify_hist {$server HMSET h1 1 1 2 2 3 3} {db0_HASH:2=1} run_cmd_verify_hist {$server HMSET h1 1 1 2 2 3 3 4 4} {db0_HASH:4=1} - + # HINCRBY run_cmd_verify_hist {$server FLUSHALL} {} run_cmd_verify_hist {$server hincrby h1 f1 10} {db0_HASH:1=1} @@ -386,7 +512,103 @@ proc test_all_keysizes { {replMode 0} } { run_cmd_verify_hist {$server HPEXPIRE h1 50 FIELDS 1 f3} {db0_HASH:1=1} run_cmd_verify_hist {} {} 1 } - } + + test "KEYSIZES - Test Hash field lazy expiration ($type) $suffixRepl" { + $server debug set-active-expire 0 + + # HGET + run_cmd_verify_hist {$server FLUSHALL} {} + run_cmd_verify_hist {$server HSETEX h1 PX 1 FIELDS 2 f1 v1 f2 v2} {db0_HASH:2=1} + run_cmd_verify_hist {after 5} {db0_HASH:2=1} + run_cmd_verify_hist {$server HGET h1 f1} {db0_HASH:1=1} + run_cmd_verify_hist {$server HGET h1 f2} {} + + # HMGET + run_cmd_verify_hist {$server HSETEX h1 PX 1 FIELDS 2 f1 v1 f2 v2} {db0_HASH:2=1} + run_cmd_verify_hist {after 5} {db0_HASH:2=1} + run_cmd_verify_hist {$server HMGET h1 f1} {db0_HASH:1=1} + run_cmd_verify_hist {$server HMGET h1 f2} {} + + # HGETDEL + run_cmd_verify_hist {$server FLUSHALL} {} + run_cmd_verify_hist {$server HSETEX h1 PX 1 FIELDS 2 f1 v1 f2 v2} {db0_HASH:2=1} + run_cmd_verify_hist {after 5} {db0_HASH:2=1} + run_cmd_verify_hist {$server HGETDEL h1 FIELDS 1 f1} {db0_HASH:1=1} + run_cmd_verify_hist {$server HGETDEL h1 FIELDS 1 f2} {} + + # HGETEX + run_cmd_verify_hist {$server FLUSHALL} {} + run_cmd_verify_hist {$server HSETEX h1 PX 1 FIELDS 2 f1 v1 f2 v2} {db0_HASH:2=1} + run_cmd_verify_hist {after 5} {db0_HASH:2=1} + run_cmd_verify_hist {$server HGETEX h1 PX 1 FIELDS 1 f1} {db0_HASH:1=1} + run_cmd_verify_hist {$server HGETEX h1 PX 1 FIELDS 1 f2} {} + + # HSETNX + run_cmd_verify_hist {$server FLUSHALL} {} + run_cmd_verify_hist {$server HSETEX h1 PX 1 FIELDS 1 f1 v1} {db0_HASH:1=1} + run_cmd_verify_hist {after 5} {db0_HASH:1=1} + run_cmd_verify_hist {$server HSETNX h1 f1 v1} {db0_HASH:1=1} + run_cmd_verify_hist {$server DEL h1} {} + run_cmd_verify_hist {$server HSETEX h2 PX 1 FIELDS 2 f1 v1 f2 v2} {db0_HASH:2=1} + run_cmd_verify_hist {after 5} {db0_HASH:2=1} + run_cmd_verify_hist {$server HSETNX h2 f1 v1} {db0_HASH:2=1} + run_cmd_verify_hist {$server HSETNX h2 f2 v2} {db0_HASH:2=1} + + # HSETEX + run_cmd_verify_hist {$server FLUSHALL} {} + run_cmd_verify_hist {$server HSETEX h1 PX 1 FIELDS 1 f1 v1} {db0_HASH:1=1} + run_cmd_verify_hist {after 5} {db0_HASH:1=1} + run_cmd_verify_hist {$server HSETEX h1 PX 1 FIELDS 1 f1 v1} {db0_HASH:1=1} + run_cmd_verify_hist {$server HSETEX h1 PX 1 FIELDS 1 f2 v2} {db0_HASH:2=1} + run_cmd_verify_hist {after 5} {db0_HASH:2=1} + run_cmd_verify_hist {$server HGET h1 f1} {db0_HASH:1=1} + run_cmd_verify_hist {$server HGET h1 f2} {} + + # HEXISTS + run_cmd_verify_hist {$server FLUSHALL} {} + run_cmd_verify_hist {$server HSETEX h1 PX 1 FIELDS 2 f1 v1 f2 v2} {db0_HASH:2=1} + run_cmd_verify_hist {after 5} {db0_HASH:2=1} + run_cmd_verify_hist {$server HEXISTS h1 f1} {db0_HASH:1=1} + run_cmd_verify_hist {$server HEXISTS h1 f2} {} + + # HSTRLEN + run_cmd_verify_hist {$server FLUSHALL} {} + run_cmd_verify_hist {$server HSETEX h1 PX 1 FIELDS 2 f1 v1 f2 v2} {db0_HASH:2=1} + run_cmd_verify_hist {after 5} {db0_HASH:2=1} + run_cmd_verify_hist {$server HSTRLEN h1 f1} {db0_HASH:1=1} + run_cmd_verify_hist {$server HSTRLEN h1 f2} {} + + # HINCRBYFLOAT + run_cmd_verify_hist {$server FLUSHALL} {} + run_cmd_verify_hist {$server HSETEX h1 PX 1 FIELDS 1 f1 1} {db0_HASH:1=1} + run_cmd_verify_hist {after 5} {db0_HASH:1=1} + run_cmd_verify_hist {$server HINCRBYFLOAT h1 f1 1.5} {db0_HASH:1=1} + + # HINCRBY + run_cmd_verify_hist {$server FLUSHALL} {} + run_cmd_verify_hist {$server HSETEX h1 PX 1 FIELDS 1 f1 1} {db0_HASH:1=1} + run_cmd_verify_hist {after 5} {db0_HASH:1=1} + run_cmd_verify_hist {$server HINCRBY h1 f1 1} {db0_HASH:1=1} + run_cmd_verify_hist {$server HSETEX h1 PX 1 FIELDS 1 f2 1} {db0_HASH:2=1} + run_cmd_verify_hist {after 5} {db0_HASH:2=1} + run_cmd_verify_hist {$server HINCRBY h1 f2 1} {db0_HASH:2=1} + + # SORT + run_cmd_verify_hist {$server FLUSHALL} {} + run_cmd_verify_hist {$server RPUSH user_ids 1 2} {db0_LIST:2=1} + run_cmd_verify_hist {$server HSET user:1 name "Alice" score 50} {db0_LIST:2=1 db0_HASH:2=1} + run_cmd_verify_hist {$server HSETEX user:2 PX 1 FIELDS 2 name "Bob" score 70} {db0_LIST:2=1 db0_HASH:2=2} + run_cmd_verify_hist {after 5} {db0_LIST:2=1 db0_HASH:2=2} + run_cmd_verify_hist {$server SORT user_ids BY user:*->score GET user:*->name GET user:*->score} {db0_LIST:2=1 db0_HASH:2=1} + run_cmd_verify_hist {$server DEL user_ids} {db0_HASH:2=1} + run_cmd_verify_hist {$server RPUSH user_ids 1} {db0_LIST:1=1 db0_HASH:2=1} + run_cmd_verify_hist {$server HSETEX user:1 PX 1 FIELDS 2 name "Alice" score 50} {db0_LIST:1=1 db0_HASH:2=1} + run_cmd_verify_hist {after 5} {db0_LIST:1=1 db0_HASH:2=1} + run_cmd_verify_hist {$server SORT user_ids BY user:*->score GET user:*->name GET user:*->score} {db0_LIST:1=1} + + $server debug set-active-expire 1 + } {OK} {cluster:skip needs:debug} + } test "KEYSIZES - Test STRING BITS $suffixRepl" { # BITOPS @@ -415,8 +637,8 @@ proc test_all_keysizes { {replMode 0} } { set encoded [$server dump l10] run_cmd_verify_hist {$server del l10} {} run_cmd_verify_hist {$server restore l11 0 $encoded} {db0_LIST:4=1} - } - + } + test "KEYSIZES - Test RENAME $suffixRepl" { run_cmd_verify_hist {$server FLUSHALL} {} run_cmd_verify_hist {$server RPUSH l12 1 2 3 4} {db0_LIST:4=1} @@ -439,15 +661,25 @@ proc test_all_keysizes { {replMode 0} } { $server select 0 } {OK} {singledb:skip} + test "KEYSIZES - DEBUG RELOAD reset keysizes $suffixRepl" { + run_cmd_verify_hist {$server FLUSHALL} {} + run_cmd_verify_hist {$server RPUSH l10 1 2 3 4} {db0_LIST:4=1} + run_cmd_verify_hist {$server SET s2 1234567890} {db0_STR:8=1 db0_LIST:4=1} + run_cmd_verify_hist {$server DEBUG RELOAD} {db0_STR:8=1 db0_LIST:4=1} + run_cmd_verify_hist {$server DEL l10} {db0_STR:8=1} + run_cmd_verify_hist {$server DEBUG RELOAD} {db0_STR:8=1} + } {} {cluster:skip needs:debug} + test "KEYSIZES - Test RDB $suffixRepl" { run_cmd_verify_hist {$server FLUSHALL} {} - # Write list, set and zset to db0 + # Write list, set and zset to db0 run_cmd_verify_hist {$server RPUSH l1 1 2 3 4} {db0_LIST:4=1} run_cmd_verify_hist {$server SADD s1 1 2 3 4 5} {db0_LIST:4=1 db0_SET:4=1} run_cmd_verify_hist {$server ZADD z1 1 a 2 b 3 c} {db0_LIST:4=1 db0_SET:4=1 db0_ZSET:2=1} run_cmd_verify_hist {$server SAVE} {db0_LIST:4=1 db0_SET:4=1 db0_ZSET:2=1} - if {$replicaMode eq 1} { - run_cmd_verify_hist {restart_server -1 true false} {db0_LIST:4=1 db0_SET:4=1 db0_ZSET:2=1} + if {$replicaMode eq 1} { + run_cmd_verify_hist {$replica SAVE} {db0_LIST:4=1 db0_SET:4=1 db0_ZSET:2=1} + run_cmd_verify_hist {restart_server 0 true false} {db0_LIST:4=1 db0_SET:4=1 db0_ZSET:2=1} 1 } else { run_cmd_verify_hist {restart_server 0 true false} {db0_LIST:4=1 db0_SET:4=1 db0_ZSET:2=1} } @@ -458,7 +690,6 @@ start_server {} { # Test KEYSIZES on a single server r select 0 test_all_keysizes 0 - # Start another server to test replication of KEYSIZES start_server {tags {needs:repl external:skip}} { # Set the outer layer server as primary @@ -471,7 +702,7 @@ start_server {} { # Server should have role replica $replica replicaof $primary_host $primary_port wait_for_condition 50 100 { [s 0 role] eq {slave} } else { fail "Replication not started." } - + # Test KEYSIZES on leader and replica $primary select 0 test_all_keysizes 1