redis cluster安装部署(测试环境)
2015-06-19 21:44
661 查看
redis 应用于web前端，做缓存和数据存取的速度是挺可观的，最近看了一些资料，手痒了，就弄了一个测试环境，两台方案，试用一下。
##Redis 集群部署##
一，方案调研:
参考博客: http://jolestar.com/redis-ha/ http://www.luocs.com/archives/tag/redis https://github.com/wandoulabs/codis/blob/master/doc/tutorial_zh.md https://github.com/twitter/twemproxy
二，部署架构
三,方案简介：
需求: 缓存:定时任务临时写入和读取的数据量大概在10G左右 存储:日常存储量大概在40G左右 需要7＊24小时不间断提供数据服务，所以要求redis集群要稳定，在有故障时可自动切换和数据即时恢复，并在业务增长或下降时，集群可伸缩或扩展（手动），并不影响在线业务，故调研测试这两种方案，以codis 为代理的codis server redis的集群，以twemproxy为代理的redis集群 codis 集群技术原理参考 https://github.com/wandoulabs/codis/blob/master/doc/tutorial_zh.md twemproxy 集群技术原理参考: https://github.com/twitter/twemproxy http://blog.jpush.cn/redis-twemproxy-benchmark/
四,部署架构规划
服务器操作系统： centos 6.x 注意:centos 5版本的内核支持不了codis 安装编译 服务器部署规划: 192.168.5.14 keepalived ,codis相关服务 192.168.5.15 keepalived,codis相关服务 192.168.5.16 twemporxy,redis,haproxy 192.168.5.44 twemporxy,redis,haproxy
五,部署安装
1,codis 方案安装
1.1安装go 环境: 首先按照golang，下载地址：https://golang.org/dl/，最新的1.4.2版本。 如果被墙使用golang中国下载http://golangtc.com/download。 cd /usr/loca/ wget https://storage.googleapis.com/golang/go1.4.2.linux-amd64.tar.gz tar -zxvf go1.4.2.linux-amd64.tar.gz vim /etc/profile 最后追加 #set go & codis environment GOROOT=/usr/local/go PATH=$PATH:$GOROOT/bin GOPATH=/usr/local/codis export GOROOT PATH GOPATH source /etc/profile 1.2 安装配置codis 相关配置 cd /usr/local git clone https://github.com/wandoulabs/codis.git #将codis的pkg包拷贝到$GOPATH目录。 mkdir -p /usr/local/codis/src/github.com/wandoulabs/codis cp /usr/local/codis/pkg/ /usr/local/codis/cmd/ /usr/local/codis/src/github.com/wandoulabs/codis -R cd /usr/local/codis/ ; sh bootstrap.sh #安装编译需要些时间，过程略过...... 大概3分钟左右，我的是这样，你的环境，哈哈，你懂的， 1.3配置脚本: github上原码中有示例，可以参考试用 https://github.com/wandoulabs/codis 在$path/sample 目录下: cat startall.sh ./start_dashboard.sh sleep 3 ./start_redis.sh ./add_group.sh ./initslot.sh ./start_proxy.sh ./set_proxy_online.sh cat start_dashboard.sh #!/bin/sh nohup ../bin/codis-config -c config.ini -L ./log/dashboard.log dashboard --addr=:18087 --http-log=./log/requests.log &>/dev/null & 启动前注意一下redis的配置文件./redis_conf/*.conf cat start_redis.sh #!/bin/sh nohup ../bin/codis-server ./redis_conf/6381.conf &> ./log/redis_6381.log & nohup ../bin/codis-server ./redis_conf/6382.conf &> ./log/redis_6382.log & echo "sleep 3s" sleep 3 tail -n 30 ./log/redis_6381.log tail -n 30 ./log/redis_6382.log cat add_group.sh #!/bin/sh echo "add group 1 with a master(localhost:6381), Notice: do not use localhost when in produciton" ../bin/codis-config -c config.ini -L ./log/cconfig.log server add 5 192.168.5.44:6381 master echo "add group 2 with a master(localhost:6382), Notice: do not use localhost when in produciton" ../bin/codis-config -c config.ini -L ./log/cconfig.log server add 6 192.168.5.44:6382 master cat ./initslot.sh #!/bin/sh echo "slots initializing..." #../bin/codis-config -c config.ini slot init -f echo "done" echo "set slot ranges to server groups..." ../bin/codis-config -c config.ini slot range-set 0 511 5 online ../bin/codis-config -c config.ini slot range-set 512 1023 6 online echo "done" cat ./start_proxy.sh #!/bin/sh echo "shut down proxy_g5_g6.." ../bin/codis-config -c config.ini proxy offline proxy_g5_g6 echo "done" echo "start new proxy..." nohup ../bin/codis-proxy --log-level info -c config.ini -L ./log/proxy.log --cpu=8 --addr=0.0.0.0:19000 --http-addr=0.0.0.0:11000 & echo "done" echo "sleep 3s" sleep 3 tail -n 30 ./log/proxy.log cat ./set_proxy_online.sh #!/bin/sh echo "set proxy_1 online" ../bin/codis-config -c config.ini proxy online proxy_g5_g6 echo "done"
1.4有自已编写的一个类上面所有脚本的python脚本：在测试中，
#!/usr/bin/env python
# coding:utf8
#author:shantuwqk@163.com
import os,sys,commands,time
from subprocess import Popen,PIPE
from mako.template import Template
codis_config = "config.ini"
codis_root = "/data/setup/codis/data"
def codis_dashboard(opt):
if opt == "start":
#os.chdir(codis_root)
exec_cmd = "cd %s; nohup ../bin/codis-config -c %s -L ./log/dashboard.log dashboard --addr=:18087 --http-log=./log/http.log &>/dev/null &" %(codis_root,codis_config)
s = os.system(exec_cmd)
if s == 0:
print "\033[32;1m codis dashboard start .... OK\033[0m"
else:
print "\033[31;1m codis dashboard start .... Error\033[0m"
elif opt == "stop":
dashboard_id = "ps aux |grep \"codis-config\"| grep \"dashboard\"|awk '{print $2}'"
s,v = commands.getstatusoutput("kill -9 `%s`" %dashboard_id)
if s == 0:
print "\033[32;1m KILL codis dashboard id:[%s] OK\033[0m"%dashboard_id
else:
print "\033[31;1m KILL codis dashboard id:[%s] Error\033[0m"%dashboard_id
else:
pass
def codis_redis(opt,port):
if opt == "start":
exec_cmd = "cd %s; nohup ../bin/codis-server ./conf/%s.conf &> ./log/%s.log &" %(codis_root,port,port)
print exec_cmd
print os.system(exec_cmd)
s = os.system(exec_cmd)
if s == 0:
print "\033[32;1m start redis %s server is OK \033[0m"%(port)
else:
print "\033[31;1m start redis %s server is Error\033[0m"%(port)
elif opt == "stop":
redis_id = "ps aux | grep codis-server| grep %s|awk '{print $2}'"%port
s,v = commands.getstatusoutput("kill -9 `%s`" %redis_id)
if s == 0:
print "\033[32;1m KILL redis port:[%s] pid:[%s] is OK\033[0m"%(port,redis_id)
print v,
else:
print "\033[31;1m KILL redis port:[%s] pid:[%s] is Error\033[0m"%(port,redis_id)
print v,
else:
pass
def codis_group(gid,addr,gtag):
exec_cmd = "cd %s; ../bin/codis-config -c %s -L ./log/%s_addgroup.log server add %s %s %s" %(codis_root,codis_config,addr.split(':')[1],gid,addr,gtag)
print "start add codis cluster group info.....","\n",exec_cmd
s,v = commands.getstatusoutput(exec_cmd)
print s,v,
if s == 0:
print "\033[32;1m add group:[%s] with a gtag:[%s] addr:(%s) OK\033[0m" %(gid,gtag,addr)
else:
print "\033[31;1m add group:[%s] with a gtag:[%s] addr:(%s) Error\033[0m" %(gid,gtag,addr)
def remove_fenc():
remove_fenc = "cd %s;../bin/codis-config -c %s action remove-fence"%(codis_root,codis_config)
s,v = commands.getstatusoutput(remove_fenc)
print "remove fenc proxy info",v,
def slot_init():
init_cmd = "cd %s; ../bin/codis-config -c %s slot init -f" %(codis_root,codis_config)
print "INIT SLOT ........","\n",commands.getstatusoutput(init_cmd)[1]
def codis_initslot(gid,slot_range):
exec_cmd = "cd %s; ../bin/codis-config -c %s slot range-set %s %s %s online"%(codis_root,codis_config,slot_range[0],slot_range[1],gid)
print exec_cmd
s,v = commands.getstatusoutput(exec_cmd)
print v,
if s == 0:
print "\033[32;1m slot init:[%s],gid:[%s] OK\033[0m"%(slot_range,gid)
else:
print "\033[31;1m slot init:[%s],gid:[%s] Error\033[0m"%(slot_range,gid)
def offline_proxy():
proxy_tag = commands.getstatusoutput("cat %s|grep proxy_id"%codis_config)[1].split('=')[1]
exec_down_cmd = "cd %s; ../bin/codis-config -c %s proxy offline %s" %(codis_root,codis_config,proxy_tag)
print "Shutdown %s offline....." %proxy_tag
print exec_down_cmd
commands.getstatusoutput(exec_down_cmd)
def codis_proxy(opt):
proxy_tag = commands.getstatusoutput("cat %s|grep proxy_id"%codis_config)[1].split('=')[1]
if opt == "start":
exec_new_cmd = "cd %s; nohup ../bin/codis-proxy --log-level info -c %s -L ./log/%s.log --cpu=8 --addr=0.0.0.0:19000 --http-addr=0.0.0.0:11000 &" %(codis_root,codis_config,proxy_tag)
print exec_new_cmd
s = os.system(exec_new_cmd)
if s == 0:
print "\033[32;1m codis proxy tag:[%s]start OK\033[0m"%(proxy_tag)
else:
print "\033[31;1m codis proxy tag:[%s]start Error\033[0m"%(proxy_tag)
time.sleep(5)
exec_online_cmd = "cd %s;../bin/codis-config -c %s proxy online %s" %(codis_root,codis_config,proxy_tag)
print "Set %s online .....!!" %(proxy_tag)
print exec_online_cmd
print commands.getstatusoutput(exec_online_cmd)[1]
elif opt == "stop":
print "Shutdown %s offline....." %proxy_tag
exec_down_cmd = "cd %s; ../bin/codis-config -c %s proxy offline %s" %(codis_root,codis_config,proxy_tag)
print commands.getstatusoutput(exec_down_cmd)[1]
exec_proxy_id = "ps aux |grep codis-proxy| grep -v grep|awk '{print $2}'"
print commands.getstatusoutput("kill -9 `%s`"%exec_proxy_id)
def get_client_ip():
eth_inter= commands.getstatusoutput("ifconfig -a| awk '/^em/ {;a=$1;FS=\":\"; nextline=NR+1; next}{ if (NR==nextline) { split($2,b,\" \")}{ if ($2 ~ /[0-9]\./) {print a,b[1]}; FS=\" \"}}'|uniq -c|awk '{print $2,$3}'")
if eth_inter[0] == 0:
return eth_inter[1].split()[1]
else:
print "get client ip error"
def slot_range(n,group):
slot_dict = {}
num=0
l = range(n)
per = len(l) / group
#改变i的索引值
for i in l[::per]:
avg=l[i:i+per]
tupv01 = avg[0],avg[-1]
#print type(tupv01),tupv01
#指定当前添加的组数
num+=1
slot_dict[num] = []
slot_dict[num].append(tupv01)
#如果只剩下一组，则将剩余元素全部追加至列表
if num==group-1:
tupv02 = l[i+per:][0],l[i+per:][-1]
#print tupv02
v02key = num + 1
slot_dict[v02key] = []
slot_dict[v02key].append(tupv02)
break
for k,port in redis_instance_port.items():
if k in slot_dict.keys():
slot_dict[k].append(port)
return slot_dict
def help_prompt():
print """
This program prints files to the standard output.
Options include:
--version : Prints the version number
--help : Helpful tips
--task : To operate on missions:
[init_codis_cluster]
[start_dashborad|stop_dashboard|start_redis|stop_redis|start_proxy|stop_proxy|addgroup|initslot]
sample : python es_manage.py --task init_codis_cluster
: python es_manage.py --task [start_dashborad|stop_dashboard|start_redis|stop_redis|start_proxy|stop_proxy|addgroup|initslot]
"""
if __name__ == "__main__":
if len(sys.argv) < 2:
print "no argument"
sys.exit()
if sys.argv[1].startswith('--'):
option = sys.argv[1][2:]
if option == 'version':
print 'Version 0.1'
elif option == 'help':
help_prompt()
if option == "task" and sys.argv[2] == "init_codis_cluster" and sys.argv[2] is not None:
codis_dashboard('start')
#redis_port = ['6381','6382'] #定义所要启动 redis实例的端口号
redis_instance_port = {1:'6381',2:"6382"}
for k,p in redis_instance_port.items():
#生成redis启动配置文件
template_file = Template(filename="./conf/redis.master.conf.template",module_directory='tmp/test').render(port="%s"%p,memsize=4)
conf = open('./conf/%s.conf'%p,'w')
conf.write(template_file)
conf.close()
#启动redis master 实例
codis_redis('start',p)
print slot_range(1024,len(redis_instance_port.keys()))
time.sleep(5)
remove_fenc()
slot_init()
for k,v in slot_range(1024,len(redis_instance_port.keys())).items():
codis_group(k,'%s:%s'%(get_client_ip(),v[1]),'master')
for k,v in slot_range(1024,len(redis_instance_port.keys())).items():
codis_initslot(k,v[0])
offline_proxy()
codis_proxy('start')
elif option == "task" and sys.argv[2] == "start_dashborad":
codis_dashboard('start')
elif option == "task" and sys.argv[2] == "stop_dashboard":
codis_dashboard("stop")
elif option == "task" and sys.argv[2] == "start_redis" and sys.argv[3] is not None:
#redis_tmp_port = "6381"
codis_redis('start',sys.argv[3])
elif option == "task" and sys.argv[2] == "stop_redis" and sys.argv[3] is not None:
codis_redis('stop',sys.argv[3])
elif option == "task" and sys.argv[2] == "start_proxy":
codis_proxy("start")
elif option == "task" and sys.argv[2] == "stop_proxy":
codis_proxy("stop")
elif option == "task" and sys.argv[2] == "stopall":
codis_proxy("stop")
commands.getstatusoutput("killall codis-server")
codis_dashboard("stop")
else:
help_prompt()
#一般情况下codis集群在安装部署初始化的时候，需要先期规划，已经做好addgrop和initslot工作，后续看需求再补充2,twemproxy方案安装:
安装过程参考: https://github.com/twitter/twemproxy 2.1注意，configure 编译之前需要系统安装autoreconf 当前用的 autoconf-2.64.tar.gz 这个包， tar zxvf autoconf-2.64.tar.gz ; cd autoconf-2.64 ; make && make install 即可 2.2 下载安装 twemproxy git clone git@github.com:twitter/twemproxy.git cd twemproxy autoreconf -fvi mkdir /usr/local/twemporxy ./configure –prefix=/usr/local/twemproxy --enable-debug=full make make install src/nutcracker -h
2.3 配置 twemproxy 代码 cat /usr/local/twemproxy/etc/544cluster.yml
alpha: listen: 0.0.0.0:12000 hash: fnv1a_64 distribution: ketama auto_eject_hosts: true redis: true server_retry_timeout: 2000 server_failure_limit: 1 servers: - 192.168.5.44:6479:1 - 192.168.5.16:6479:1
配置详解参考如下:
http://cpjsjxy.iteye.com/blog/2090333
3.haproxy 安装部署：
3.1下载 haproxy-1.5.10.tar.gz tar zxvf haproxy-1.5.10.tar.gz ; cd haproxy-1.5.10; make TARGET=linux26 ARCH=x86_64&& make install 3.2 配置文件：
global
daemon
nbproc 6
pidfile /var/run/haproxy.pid
ulimit-n 65535
defaults
mode tcp #mode { tcp|http|health }，tcp 表示4层，http表示7层，health仅作为健康检查使用
retries 2 #尝试2次失败则从集群摘除
option redispatch #如果失效则强制转换其他服务器
option abortonclose #连接数过大自动关闭
maxconn 1024 #最大连接数
timeout connect 1d #连接超时时间，重要，hive查询数据能返回结果的保证
timeout client 1d #同上
timeout server 1d #同上
timeout check 2000 #健康检查时间
log 127.0.0.1 local0 err #[err warning info debug]
listen admin_stats #定义管理界面
bind 0.0.0.0:8888 #管理界面访问IP和端口
mode http #管理界面所使用的协议
maxconn 10 #最大连接数
stats refresh 30s #30秒自动刷新
stats uri / #访问url
stats realm Hive\ Haproxy #验证窗口提示
stats auth admin:123456 #401验证用户名密码
listen codis-proxy-ha-20000 #codis-proxy
bind 0.0.0.0:20000 #ha作为proxy所绑定的IP和端口
mode tcp #以4层方式代理，重要
balance leastconn #调度算法 'leastconn' 最少连接数分配，或者 'roundrobin'，轮询分配
maxconn 1024 #最大连接数
server codis-proxy2-master 192.168.5.15:19000 check inter 5000 rise 1 fall 2
server codis-proxy1-master 192.168.5.14:19000 check inter 5000 rise 1 fall 2
#server codis-server-5156381-master 192.168.5.15:6381 check inter 5000 rise 1 fall 2
#server codis-server-5156382-master 192.168.5.15:6382 check inter 5000 rise 1 fall 2
listen twemproxy-ha-21000
bind 0.0.0.0:21000
mode tcp
balance leastconn
maxconn 1024
server redis-544-master 192.168.5.44:6479 check inter 5000 rise 1 fall 2
server redis-516-master 192.168.5.16:6479 check inter 5000 rise 1 fall 2
3.3 haproxy 启动关闭：
/usr/local/sbin/haproxy -f /data/setup/haproxy-1.5.10/conf/haproxy.cfg4.keepalived安装部署: yum install keepalived
主从配置是一样的，特别注意的配置项：
state BACKUP
为了避免网络不稳定的情况下，主从抢占，配为BACKUP
interface em2
VIP 邦定的网卡接口指定
cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
notification_email {
wangqiankun@lashou-inc.com
}
notification_email_from nagios@lashou.com
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id ha_01
}
vrrp_instance HA_01 {
state BACKUP
nopreempt
interface em2
virtual_router_id 15
priority 99
advert_int 1
authentication {
auth_type PASS
auth_pass 2222
}
virtual_ipaddress {
192.168.5.90
192.168.5.91
}
}
virtual_server 192.168.5.90 20000 {
delay_loop 6
lb_algo lc
lb_kind DR
nat_mask 255.255.255.0
# persistence_timeout 10
protocol TCP
real_server 192.168.5.44 20000 {
weight 1
TCP_CHECK {
connect_timeout 5
nb_get_retry 3
delay_before_retry 3
connect_port 20000
}
}
real_server 192.168.5.16 20000 {
weight 1
TCP_CHECK {
connect_timeout 5
nb_get_retry 3
delay_before_retry 3
connect_port 20000
}
}
}
virtual_server 192.168.5.91 21000 {
delay_loop 6
lb_algo lc
lb_kind DR
nat_mask 255.255.255.0
# persistence_timeout 10
protocol TCP
real_server 192.168.5.44 21000 {
weight 1
TCP_CHECK {
connect_timeout 5
nb_get_retry 3
delay_before_retry 3
connect_port 21000
}
}
real_server 192.168.5.16 21000 {
weight 1
TCP_CHECK {
connect_timeout 5
nb_get_retry 3
delay_before_retry 3
connect_port 21000
}
}
}proxy 做为keepalived + lvs 的后端真实机 需要安装lvs 并加如下脚本
cat lvs_re.sh
#!/bin/bash
WEB_VIP1=192.168.5.90
WEB_VIP2=192.168.5.91
. /etc/rc.d/init.d/functions
case "$1" in
start)
ifconfig lo:0 $WEB_VIP1 netmask 255.255.255.255 broadcast $WEB_VIP1
ifconfig lo:1 $WEB_VIP2 netmask 255.255.255.255 broadcast $WEB_VIP2
/sbin/route add -host $WEB_VIP1 dev lo:0
/sbin/route add -host $WEB_VIP2 dev lo:1
echo "1" >/proc/sys/net/ipv4/conf/lo/arp_ignore
echo "2" >/proc/sys/net/ipv4/conf/lo/arp_announce
echo "1" >/proc/sys/net/ipv4/conf/all/arp_ignore
echo "2" >/proc/sys/net/ipv4/conf/all/arp_announce
sysctl -p >/dev/null 2>&1
echo "RealServer Start OK"
;;
stop)
ifconfig lo:0 down
ifconfig lo:1 down
route del $WEB_VIP1 >/dev/null 2>&1
route del $WEB_VIP2 >/dev/null 2>&1
echo "0" >/proc/sys/net/ipv4/conf/lo/arp_ignore
echo "0" >/proc/sys/net/ipv4/conf/lo/arp_announce
echo "0" >/proc/sys/net/ipv4/conf/all/arp_ignore
echo "0" >/proc/sys/net/ipv4/conf/all/arp_announce
echo "RealServer Stoped"
;;
status)
# Status of LVS-DR real server.
islothere=`/sbin/ifconfig lo:0 | grep "$WEB_VIP1"`
islothere=`/sbin/ifconfig lo:1 | grep "$WEB_VIP2"`
isrothere=`netstat -rn | grep "lo:0" | grep "$WEB_VIP"`
if [ ! "$islothere" -o ! "isrothere" ];then
# Either the route or the lo:0 device
# not found.
echo "LVS-DR real server Stopped."
else
echo "LVS-DR Running."
fi
;;
*)
# Invalid entry.
echo "$0: Usage: $0 {start|status|stop}"
exit 1
;;
esac
exit 04.2.启动keepalived 和 lvs 服务加载配置
/etc/init.d/keepalived start ./lvs_re.sh start
六： 日常维护记录： 后续维护过程中待记录
内容来自用户分享和网络整理，不保证内容的准确性，如有侵权内容，可联系管理员处理 
相关文章推荐
- Redis数据类型之LIST类型
- Redis服务器学习笔记(一)
- Redis在.net中的应用学习
- redis3.0.2安装
- ubuntu安装redis的方法以及PHP安装redis扩展、CI框架sess使用redis的方法
- redis 学习
- redis自启动过程中出现的系统警告处理
- centos redis自启动相关
- redis 通过 shell 将一个键值对从一个DB移动到另外一个DB
- redis-3.0.2集群部署
- nginx+tomcat7+redis集群配置
- Redis 之 Jedis（1）
- Redis数据类型之HASH类型
- Redis 2.8 配置文件说明
- Redis命令学习-Set（集合）
- redis安装配置
- redis集群部署之codis 维护脚本
- redis 数据库维护之 key 大小获取
- Redis主从配置
- Redis学习第八课：Redis高级实用特性（二）