结合redis设计与实现的redis源码学习-9-zipmap（压缩图)

在学习了所有redis设计与实现的数据结构后，我发现在redis中还有一些特殊的数据结构，今天我们来学习zipmap。

zipmap和ziplist比较相似，使用特殊的内存存储方法来提升个数较少的键值对。具有以下特点：

1、使用1个字节的len来存储元素个数，当长度大于最大值那么只能遍历获得size了；

2、len后面是key的长度，一般使用1个字节或者5个字节表示；

3、free用来表示value长度的空闲值，当value的数据被改变后可能会发生变化，一般情况下free的值较小，如果空闲太多，zipmap会进行调整使整体变得尽可能小；

4、数据布局格式：

<zmlen><len>"foo"<len><free>"bar"<len>"hello"<len><free>"world"

<总长度>

#ifndef _ZIPMAP_H
#define _ZIPMAP_H

unsigned char *zipmapNew(void);//创建一个新压缩图
unsigned char *zipmapSet(unsigned char *zm, unsigned char *key, unsigned int klen, unsigned char *
4000
val, unsigned int vlen, int *update);//设置一个键值对
unsigned char *zipmapDel(unsigned char *zm, unsigned char *key, unsigned int klen, int *deleted);//删除一个键值对
unsigned char *zipmapRewind(unsigned char *zm);//
unsigned char *zipmapNext(unsigned char *zm, unsigned char **key, unsigned int *klen, unsigned char **value, unsigned int *vlen);//取得下一个键值对
int zipmapGet(unsigned char *zm, unsigned char *key, unsigned int klen, unsigned char **value, unsigned int *vlen);//获取某个键值对
int zipmapExists(unsigned char *zm, unsigned char *key, unsigned int klen);//判断某个键值对是否存在
unsigned int zipmapLen(unsigned char *zm);//获取键值对个数
size_t zipmapBlobLen(unsigned char *zm);//获取序列化到文件中所需的大小
void zipmapRepr(unsigned char *p);//输出压缩图的具体信息

#ifdef REDIS_TEST
int zipmapTest(int argc, char *argv[]);
#endif

#endif

zipmap.c

#include <stdio.h>
#include <string.h>
#include "zmalloc.h"
#include "endianconv.h"

#define ZIPMAP_BIGLEN 254
#define ZIPMAP_END 255

/* The following defines the max value for the <free> field described in the
* comments above, that is, the max number of trailing bytes in a value. */
#define ZIPMAP_VALUE_MAX_FREE 4

/* The following macro returns the number of bytes needed to encode the length
* for the integer value _l, that is, 1 byte for lengths < ZIPMAP_BIGLEN and 返回应该开辟的大小
* 5 bytes for all the other lengths. */
#define ZIPMAP_LEN_BYTES(_l) (((_l) < ZIPMAP_BIGLEN) ? 1 : sizeof(unsigned int)+1)

/* Create a new empty zipmap. */
unsigned char *zipmapNew(void) {
unsigned char *zm = zmalloc(2);
//一个空的压缩图只有长度标志位和结束符
zm[0] = 0; /* Length */
zm[1] = ZIPMAP_END;
return zm;
}
/* Decode the encoded length pointed by 'p' 返回zipmap的长度*/
static unsigned int zipmapDecodeLength(unsigned char *p) {
unsigned int len = *p;

if (len < ZIPMAP_BIGLEN) return len;
memcpy(&len,p+1,sizeof(unsigned int));
memrev32ifbe(&len);
return len;
}

/* Encode the length 'l' writing it in 'p'. If p is NULL it just returns
* the amount of bytes required to encode such a length. 返回zipmap的编码长度*/
static unsigned int zipmapEncodeLength(unsigned char *p, unsigned int len) {
if (p == NULL) {
return ZIPMAP_LEN_BYTES(len);
} else {
if (len < ZIPMAP_BIGLEN) {//小于最大长度,返回1，p[0]可以表示该长度范围
p[0] = len;
return 1;
} else {
p[0] = ZIPMAP_BIGLEN;//等于最大长度,返回长度+1
memcpy(p+1,&len,sizeof(len));
memrev32ifbe(p+1);
return 1+sizeof(len);
}
}
}
/* Search for a matching key, returning a pointer to the entry inside the
* zipmap. Returns NULL if the key is not found.
*
* If NULL is returned, and totlen is not NULL, it is set to the entire
* size of the zimap, so that the calling function will be able to
* reallocate the original zipmap to make room for more entries. */
static unsigned char *zipmapLookupRaw(unsigned char *zm, unsigned char *key, unsigned int klen, unsigned int *totlen) {
unsigned char *p = zm+1, *k = NULL;//p是存储数据的开始位置
unsigned int l,llen;

while(*p != ZIPMAP_END) {//没有到结束位
unsigned char free;

/* Match or skip the key 对比或者跳过key*/
l = zipmapDecodeLength(p);//
llen = zipmapEncodeLength(NULL,l);
if (key != NULL && k == NULL && l == klen && !memcmp(p+llen,key,l)) {
/* Only return when the user doesn't care
* for the total length of the zipmap. 只有在用户不在意zipmap的总长度的时候返回*/
if (totlen != NULL) {
k = p;
} else {
return p;
}
}
p += llen+l;
/* Skip the value as well 总是跳过该值*/
l = zipmapDecodeLength(p);
p += zipmapEncodeLength(NULL,l);
free = p[0];
p += l+1+free; /* +1 to skip the free byte */
}
if (totlen != NULL) *totlen = (unsigned int)(p-zm)+1;
return k;
}
//返回zipmap要求的长度，根据zipmap的最大长度判断，key或value大于就+4
static unsigned long zipmapRequiredLength(unsigned int klen, unsigned int vlen) {
unsigned int l;

l = klen+vlen+3;
if (klen >= ZIPMAP_BIGLEN) l += 4;
if (vlen >= ZIPMAP_BIGLEN) l += 4;
return l;
}

/* Return the total amount used by a key (encoded length + payload) 返回key的长度*/
static unsigned int zipmapRawKeyLength(unsigned char *p) {
unsigned int l = zipmapDecodeLength(p);
return zipmapEncodeLength(NULL,l) + l;
}
/* Return the total amount used by a value
* (encoded length + single byte free count + payload) 获取值占用的长度*/
static unsigned int zipmapRawValueLength(unsigned char *p) {
unsigned int l = zipmapDecodeLength(p);
unsigned int used;

used = zipmapEncodeLength(NULL,l);
used += p[used] + 1 + l;
return used;
}

/* If 'p' points to a key, this function returns the total amount of
* bytes used to store this entry (entry = key + associated value + trailing
* free space if any). 如果p是key的话，会返回条目总共使用的长度*/
static unsigned int zipmapRawEntryLength(unsigned char *p) {
unsigned int l = zipmapRawKeyLength(p);
return l + zipmapRawValueLength(p+l);
}
//重置zipmap的大小
static inline unsigned char *zipmapResize(unsigned char *zm, unsigned int len) {
zm = zrealloc(zm, len);
zm[len-1] = ZIPMAP_END;
return zm;
}
/* Set key to value, creating the key if it does not already exist.
* If 'update' is not NULL, *update is set to 1 if the key was
* already preset, otherwise to 0. 设置一个还不存在的key和他的value，如果update不是空的话且key已经存在的话,会把他的值设为一*/
unsigned char *zipmapSet(unsigned char *zm, unsigned char *key, unsigned int klen, unsigned char *val, unsigned int vlen, int *update) {
unsigned int zmlen, offset;
unsigned int freelen, reqlen = zipmapRequiredLength(klen,vlen);
unsigned int empty, vempty;
unsigned char *p;

freelen = reqlen;
if (update) *update = 0;
p = zipmapLookupRaw(zm,key,klen,&zmlen);
if (p == NULL) {
/* Key not found: enlarge 找不到key，重设大小，插入key*/
zm = zipmapResize(zm, zmlen+reqlen);
p = zm+zmlen-1;
zmlen = zmlen+reqlen;

/* Increase zipmap length (this is an insert) */
if (zm[0] < ZIPMAP_BIGLEN) zm[0]++;
} else {
/* Key found. Is there enough space for the new value? */
/* Compute the total length: 找到key了，是否有足够的空间给新值，计算总长度*/
if (update) *update = 1;
freel
c42c
en = zipmapRawEntryLength(p);
if (freelen < reqlen) {
/* Store the offset of this key within the current zipmap, so
* it can be resized. Then, move the tail backwards so this
* pair fits at the current position. */
offset = p-zm;
zm = zipmapResize(zm, zmlen-freelen+reqlen);
p = zm+offset;

/* The +1 in the number of bytes to be moved is caused by the
* end-of-zipmap byte. Note: the *original* zmlen is used. */
memmove(p+reqlen, p+freelen, zmlen-(offset+freelen+1));
zmlen = zmlen-freelen+reqlen;
freelen = reqlen;
}
}

/* We now have a suitable block where the key/value entry can
* be written. If there is too much free space, move the tail
* of the zipmap a few bytes to the front and shrink the zipmap,
* as we want zipmaps to be very space efficient. */
empty = freelen-reqlen;
if (empty >= ZIPMAP_VALUE_MAX_FREE) {
/* First, move the tail <empty> bytes to the front, then resize
* the zipmap to be <empty> bytes smaller. 移动结尾的空字节到开始位置，然后重置zipmap的大小*/
offset = p-zm;
memmove(p+reqlen, p+freelen, zmlen-(offset+freelen+1));
zmlen -= empty;
zm = zipmapResize(zm, zmlen);
p = zm+offset;
vempty = 0;
} else {
vempty = empty;
}

/* Just write the key + value and we are done. 现在将key和value存入*/
/* Key: */
p += zipmapEncodeLength(p,klen);
memcpy(p,key,klen);
p += klen;
/* Value: */
p += zipmapEncodeLength(p,vlen);
*p++ = vempty;
memcpy(p,val,vlen);
return zm;
}
/* Remove the specified key. If 'deleted' is not NULL the pointed integer is
* set to 0 if the key was not found, to 1 if it was found and deleted. 删除指定的key，返回0是没有找到，1的话就是正常删除*/
unsigned char *zipmapDel(unsigned char *zm, unsigned char *key, unsigned int klen, int *deleted) {
unsigned int zmlen, freelen;
unsigned char *p = zipmapLookupRaw(zm,key,klen,&zmlen);//找啊找
if (p) {
freelen = zipmapRawEntryLength(p);
memmove(p, p+freelen, zmlen-((p-zm)+freelen+1));
zm = zipmapResize(zm, zmlen-freelen);

/* Decrease zipmap length */
if (zm[0] < ZIPMAP_BIGLEN) zm[0]--;

if (deleted) *deleted = 1;
} else {
if (deleted) *deleted = 0;
}
return zm;
}
/* Call before iterating through elements via zipmapNext() 返回第一个元素的地址，配合zipmapNext来遍历zipmap*/
unsigned char *zipmapRewind(unsigned char *zm) {
return zm+1;
}
* In the next calls what zipmapNext returns is used as first argument.
* Example:
* unsigned char *i = zipmapRewind(my_zipmap);
* while((i = zipmapNext(i,&key,&klen,&value,&vlen)) != NULL) {
*     printf("%d bytes key at $p\n", klen, key);
*     printf("%d bytes value at $p\n", vlen, value);
* }
可以用来遍历zipmap*/
unsigned char *zipmapNext(unsigned char *zm, unsigned char **key, unsigned int *klen, unsigned char **value, unsigned int *vlen) {
if (zm[0] == ZIPMAP_END) return NULL;
if (key) {
*key = zm;
*klen = zipmapDecodeLength(zm);
*key += ZIPMAP_LEN_BYTES(*klen);
}
zm += zipmapRawKeyLength(zm);
if (value) {
*value = zm+1;
*vlen = zipmapDecodeLength(zm);
*value += ZIPMAP_LEN_BYTES(*vlen);
}
zm += zipmapRawValueLength(zm);
return zm;
}
/* Search a key and retrieve the pointer and len of the associated value.
* If the key is found the function returns 1, otherwise 0. 查找一个key*/
int zipmapGet(unsigned char *zm, unsigned char *key, unsigned int klen, unsigned char **value, unsigned int *vlen) {
unsigned char *p;
if ((p = zipmapLookupRaw(zm,key,klen,NULL)) == NULL) return 0;
p += zipmapRawKeyLength(p);
*vlen = zipmapDecodeLength(p);
*value = p + ZIPMAP_LEN_BYTES(*vlen) + 1;
return 1;
}
/* Return 1 if the key exists, otherwise 0 is returned. 判断key是否存在*/
int zipmapExists(unsigned char *zm, unsigned char *key, unsigned int klen) {
return zipmapLookupRaw(zm,key,klen,NULL) != NULL;
}
/* Return the number of entries inside a zipmap 返回zipmap的元素个数*/
unsigned int zipmapLen(unsigned char *zm) {
unsigned int len = 0;
if (zm[0] < ZIPMAP_BIGLEN) {
len = zm[0];
} else {
unsigned char *p = zipmapRewind(zm);//如果个数太多就要遍历计数
while((p = zipmapNext(p,NULL,NULL,NULL,NULL)) != NULL) len++;

/* Re-store length if small enough 防止判断失误*/
if (len < ZIPMAP_BIGLEN) zm[0] = len;
}
return len;
}
/* 返回zipmap的总长度 Return the raw size in bytes of a zipmap, so that we can serialize
* the zipmap on disk (or everywhere is needed) just writing the returned
* amount of bytes of the C array starting at the zipmap pointer. */
size_t zipmapBlobLen(unsigned char *zm) {
unsigned int totlen;
zipmapLookupRaw(zm,NULL,0,&totlen);
return totlen;
}