Redis数据类型使用场景及有序集合SortedSet底层实现详解

redisObject源码(server.h)

typedef struct redisObject {
unsigned type:4;//对象类型,用于分辨字符串、列表、
//集合、有序集合、字典,有序集合为REDIS_ZSET
unsigned encoding:4;//编码,标识底层数据结构,
//有序集合有REDIS_ENCODING_ZIPLIST(压缩列表)、REDIS_ENCODING_SKIPLIST(跳表)
//记录键最近一次被访问的时间,长时间不被访问的对象可被内存回收
unsigned lru:LRU_BITS; 

/* LRU time (relative to global lru_clock) or
* LFU data (least significant 8 bits frequency
* and most significant 16 bits access time). */
int refcount;//引用计数,用于对象内存回收,
//当为0时回收内存,引用计数可实现不同键相同值的共享,
//事实上,redis会初始化创建0到9999个整数对象用于共享,从而节约内存
void *ptr;//指向底层数据结构实例的指针
} robj;

zaddGenericCommand函数

if (server.zset_max_ziplist_entries == 0 ||
server.zset_max_ziplist_value < sdslen(c->argv[scoreidx+1]->ptr))
{
zobj = createZsetObject(); //创建zset
} else {
zobj = createZsetZiplistObject();//创建ziplist
}

Field

含义

zlbytes(uint32_t)

ziplist占用的内存字节数,包括zlbytes本身

zltail(uint32_t)

最后一个entry的offset偏移值

zllen(uint16_t)

数据项entry的个数

entry(变长)

数据项

zlend(uint8_t)

标识ziplist的结束,值为255

Field

含义

prevlen

上一个数据项entry的长度。当长度小于254字节,则prevlen占1字节,当长度大于或等于254字节,则prevlen占5字节,首字节值为254,剩下4字节表示上一entry长度。

encoding

encoding的值依赖于数据entry-data。首字节的前两个bit为00、01、10,标识entry-data为字符串,同时表示encoding的长度分别为1、2、5字节,除前两个bit，剩下的bit表示字符串长度;前两个bit为11,表示entry-data为整型,接下来的2 bit表示整数类型。entry-data不同类型及encoding如下:

1)       |00pppppp| - 1 byte,字符串且长度小于等于63字节(6bit)

2)       |01pppppp|qqqqqqqq| - 2 bytes,字符串且长度小于等于16383字节(14bit)

3)       |10000000|qqqqqqqq|rrrrrrrr|ssssssss|tttttttt| - 5 bytes,字符串且长度大于等于16384(后面四个字节表示长度,首字节的低位6bit设为0)

4)       |11000000| - 1 bytes,len字段为1字节,后面的entry-data为整型且类型为int16_t (2 bytes)

5)       |11010000| - 1 bytes, entry-data为整型且类型为int32_t (4 bytes)

6)       |11100000| - 1 bytes, entry-data为整型且类型为int64_t (8 bytes)

7)       |11110000| - 1 bytes, entry-data为整型且占3 bytes

8)       |11111110| - 1 bytes, entry-data为整型且占1 bytes

9)       |1111xxxx| - (with xxxx between 0000 and 1101),xxxx的值从1到13,可用于表示entry-data(1到12),encoding包含entry-data的值,从而不需要entry-data域

10)    |11111111| - 用于标识ziplist的结束

entry-data

具体的数据

ziplist插入逻辑源码(ziplist.c)

/* Insert item at "p". */

unsigned char *__ziplistInsert(unsigned char *zl, unsigned char *p, unsigned char *s, unsigned int slen) {

size_t curlen = intrev32ifbe(ZIPLIST_BYTES(zl)), reqlen;

unsigned int prevlensize, prevlen = 0;

size_t offset;

int nextdiff = 0;

unsigned char encoding = 0;

long long value = 123456789; /* initialized to avoid warning. Using a value

that is easy to see if for some reason

we use it uninitialized. */

zlentry tail;

/* Find out prevlen for the entry that is inserted. */

//插入位置前面一个entry节点占用的字节数prevlen

if (p[0] != ZIP_END) {//插入节点不在末尾节点,直接从p的前面字节读

ZIP_DECODE_PREVLEN(p, prevlensize, prevlen);

} else {//插入节点在末尾位置,找到末尾节点

unsigned char *ptail = ZIPLIST_ENTRY_TAIL(zl);

if (ptail[0] != ZIP_END) {

prevlen = zipRawEntryLength(ptail);

}

}

/* See if the entry can be encoded */

if (zipTryEncoding(s,slen,&value,&encoding)) {//判断s是否可以转化为整数,并将整数值和enconding分别存在value和encoding指针

/* 'encoding' is set to the appropriate integer encoding */

reqlen = zipIntSize(encoding);//整数值长度

} else {

/* 'encoding' is untouched, however zipStoreEntryEncoding will use the

* string length to figure out how to encode it. */

reqlen = slen;//字符串长度

}

/* We need space for both the length of the previous entry and

* the length of the payload. */

//得出新插入节点占用的总字节数reqlen

reqlen += zipStorePrevEntryLength(NULL,prevlen);

reqlen += zipStoreEntryEncoding(NULL,encoding,slen);

/* When the insert position is not equal to the tail, we need to

* make sure that the next entry can hold this entry's length in

* its prevlen field. */

//插入新节点不在末尾位置,则插入位置p所指向的entry节点的prevlen,

//值会变成新插入节点的总长度,且prevlen所占用的字节数可能会变化,

//nextdiff表示新插入节点下一节点的prevlen需要空间的变化,负值表示变小,

//正值表示扩大

int forcelarge = 0;

nextdiff = (p[0] != ZIP_END) ? zipPrevLenByteDiff(p,reqlen) : 0;

if (nextdiff == -4 && reqlen < 4) {

nextdiff = 0;

forcelarge = 1;

}

/* Store offset because a realloc may change the address of zl. */

offset = p-zl;

zl = ziplistResize(zl,curlen+reqlen+nextdiff);//重新分配空间,并将zl的每字节都填充到新分配的内存中

p = zl+offset;

//将p后面的数据项进行移动

/* Apply memory move when necessary and update tail offset. */

if (p[0] != ZIP_END) {

/* Subtract one because of the ZIP_END bytes */

memmove(p+reqlen,p-nextdiff,curlen-offset-1+nextdiff);

/* Encode this entry's raw length in the next entry. */

if (forcelarge)//设置下一个节点的prevlen

zipStorePrevEntryLengthLarge(p+reqlen,reqlen);

else

zipStorePrevEntryLength(p+reqlen,reqlen);

/* Update offset for tail */

ZIPLIST_TAIL_OFFSET(zl) =

intrev32ifbe(intrev32ifbe(ZIPLIST_TAIL_OFFSET(zl))+reqlen);

/* When the tail contains more than one entry, we need to take

* "nextdiff" in account as well. Otherwise, a change in the

* size of prevlen doesn't have an effect on the *tail* offset. */

zipEntry(p+reqlen, &tail);

if (p[reqlen+tail.headersize+tail.len] != ZIP_END) {

ZIPLIST_TAIL_OFFSET(zl) =

intrev32ifbe(intrev32ifbe(ZIPLIST_TAIL_OFFSET(zl))+nextdiff);

}

} else {

/* This element will be the new tail. */

ZIPLIST_TAIL_OFFSET(zl) = intrev32ifbe(p-zl);

}

/* When nextdiff != 0, the raw length of the next entry has changed, so

* we need to cascade the update throughout the ziplist */

if (nextdiff != 0) {

offset = p-zl;

zl = __ziplistCascadeUpdate(zl,p+reqlen);

p = zl+offset;

}

/* Write the entry */

//将新数据项放入插入位置

p += zipStorePrevEntryLength(p,prevlen);

p += zipStoreEntryEncoding(p,encoding,slen);

if (ZIP_IS_STR(encoding)) {

memcpy(p,s,slen);

} else {

zipSaveInteger(p,value,encoding);

}

ZIPLIST_INCR_LENGTH(zl,1);

return zl;

}

zset定义(server.h)

typedef struct zset {

dict *dict;//字典

zskiplist *zsl;//跳表

} zset;

zslRandomLevel函数(t_zset.c)

int zslRandomLevel(void) {

//随机生成节点层数,当第i层节点存在时，第i+1节点存在的概率为ZSKIPLIST_P = 1/4

//ZSKIPLIST_MAXLEVEL 64,表示节点的最大层数

int level = 1;

while ((random()&0xFFFF) < (ZSKIPLIST_P * 0xFFFF))

level += 1;

return (level<ZSKIPLIST_MAXLEVEL) ? level : ZSKIPLIST_MAXLEVEL;

}

zskiplist及zskiplistNode(server.h)

/* ZSETs use a specialized version of Skiplists */

typedef struct zskiplistNode {

sds ele;//实际存储的数据

double score;//分值

struct zskiplistNode *backward;//后退指针,指向前一个节点

struct zskiplistLevel {

struct zskiplistNode *forward;//前进指针,指向下一个节点

unsigned long span;//跨度,表示该层链表的这一节点到下一节点跨越的节点数,用于计算rank

} level[];//层级数组,每个层级都有到下一个节点的指针和跨度

} zskiplistNode;//跳表节点

typedef struct zskiplist {

struct zskiplistNode *header, *tail;//跳表头节点和尾节点

unsigned long length;//跳表元素个数

int level;//跳表的最高层数(不包括头节点,头节点实际上并不存储数据)

} zskiplist;

zslInsert(t_zset.c)

zskiplistNode *zslInsert(zskiplist *zsl, double score, sds ele) {

zskiplistNode *update[ZSKIPLIST_MAXLEVEL], *x;

unsigned int rank[ZSKIPLIST_MAXLEVEL];

int i, level;

serverAssert(!isnan(score));

x = zsl->header;

for (i = zsl->level-1; i >= 0; i--) {

/* store rank that is crossed to reach the insert position */

//rank[i]初始化为rank[i+1],所以rank[i]-rank[i+1]表示在i层走过的节点数

rank[i] = i == (zsl->level-1) ? 0 : rank[i+1];

while (x->level[i].forward &&

(x->level[i].forward->score < score ||

(x->level[i].forward->score == score &&

sdscmp(x->level[i].forward->ele,ele) < 0)))

{

rank[i] += x->level[i].span;

x = x->level[i].forward;

}

// 记录将要和新节点相连接的节点,x表示新节点在i层连接的上一节点

update[i] = x;

}

/* we assume the element is not already inside, since we allow duplicated

* scores, reinserting the same element should never happen since the

* caller of zslInsert() should test in the hash table if the element is

* already inside or not. */

level = zslRandomLevel();//随机生成此节点的层数

if (level > zsl->level) {

for (i = zsl->level; i < level; i++) {

rank[i] = 0;

update[i] = zsl->header;

update[i]->level[i].span = zsl->length;

}

zsl->level = level;

}

x = zslCreateNode(level,score,ele);

for (i = 0; i < level; i++) {

x->level[i].forward = update[i]->level[i].forward;

update[i]->level[i].forward = x;

/* update span covered by update[i] as x is inserted here */

//rank[0]表示0层链表，插入节点x左边的节点数

//rank[i]表示i层链表,插入节点x左边的节点数

//rank[0] - rank[i]+1表示i层链表,x前一节点到x的跨度

x->level[i].span = update[i]->level[i].span - (rank[0] - rank[i]);

update[i]->level[i].span = (rank[0] - rank[i]) + 1;

}

/* increment span for untouched levels */

//在level及之上的每层,update[i]到下一节点的距离由于插入了x节点而加1

for (i = level; i < zsl->level; i++) {

update[i]->level[i].span++;

}

//更新后退指针

x->backward = (update[0] == zsl->header) ? NULL : update[0];

if (x->level[0].forward)

x->level[0].forward->backward = x;

else

zsl->tail = x;

zsl->length++;

return x;

}