一、需求背景
發(fā)現(xiàn)公司好多項目都用到memched。并且都是單點,memched服務(wù)器 掛了就會影響業(yè)務(wù),于是百度之,發(fā)現(xiàn)memched不可以集群,也沒有看到有什么高可用方案,但是發(fā)現(xiàn)了repcached(memched主從復(fù)制),結(jié)合LVS的NAT模式不同端口轉(zhuǎn)發(fā)實現(xiàn)memched雙機HA,本文章記錄本人的部署過程,從屬實戰(zhàn),沒有文字性的說明,歡迎各位指點。
二、環(huán)境:
1、系統(tǒng)環(huán)境:
-
CentOS release 6.4 (Final)
2、網(wǎng)絡(luò)環(huán)境
-
調(diào)度機(master):
-
vip:172.28.26.100
-
vip1:172.28.16.100(lvs轉(zhuǎn)發(fā)網(wǎng)關(guān) )
-
eth1:172.28.26.101 (內(nèi)網(wǎng))
-
eth2:172.28.16.101(lvs轉(zhuǎn)發(fā)網(wǎng)段)
-
調(diào)度機(backup):
-
eth1:172.28.26.99 (內(nèi)網(wǎng))
-
eth2:172.28.16.99(lvs轉(zhuǎn)發(fā)網(wǎng)段)
-
memched備節(jié)點:
-
eth1:172.28.26.102 (內(nèi)網(wǎng))
-
eth2:172.28.16.102(lvs轉(zhuǎn)發(fā)網(wǎng)段)
-
memched主節(jié)點:
-
eth1:172.28.26.103 (內(nèi)網(wǎng))
-
eth2:172.28.16.103(lvs轉(zhuǎn)發(fā)網(wǎng)段)
3、路由策略
-
調(diào)度機(master):
-
echo '201 eth1' >> /etc/iproute2/rt_tables
-
echo 'default table eth1 via 172.28.26.1 dev eth1' > /etc/sysconfig/network-scripts/route-eth1
-
echo 'from 172.28.26.101/255.255.255.255 table eth1' > /etc/sysconfig/network-scripts/rule-eth1
-
echo 'from 172.28.26.100/255.255.255.255 table eth1' >> /etc/sysconfig/network-scripts/rule-eth1
-
echo 'from 172.28.16.101/32 table ZW_LVS_LAN' > /etc/sysconfig/network-scripts/rule-eth2
-
memched節(jié)點1:
-
echo 'default table ZW_LVS_LAN via 172.28.16.100' > /etc/sysconfig/network-scripts/route-eth2
-
echo 'from 172.28.16.0/24 table ZW_LVS_LAN' > /etc/sysconfig/network-scripts/rule-eth2
-
echo '202 ZW_LVS_LAN' >> /etc/iproute2/rt_tables
-
memched節(jié)點2:
-
echo 'default table ZW_LVS_LAN via 172.28.16.100' > /etc/sysconfig/network-scripts/route-eth2
-
echo 'from 172.28.16.0/24 table ZW_LVS_LAN' > /etc/sysconfig/network-scripts/rule-eth2
-
echo '202 ZW_LVS_LAN' >> /etc/iproute2/rt_tables
4、內(nèi)核參數(shù)
-
fs.file-max = 1000000
-
kernel.core_uses_pid = 1
-
kernel.msgmax = 1048560
-
kernel.msgmnb = 1073741824
-
kernel.shmall = 4294967296
-
kernel.shmmax = 68719476736
-
kernel.sysrq = 0
-
net.core.netdev_max_backlog = 1048576
-
net.core.rmem_default = 2097152
-
net.core.rmem_max = 16777216
-
net.core.somaxconn = 1048576
-
net.core.wmem_default = 2097152
-
net.core.wmem_max = 16777216
-
net.ipv4.conf.default.accept_source_route = 0
-
net.ipv4.conf.default.rp_filter = 1
-
net.ipv4.ip_forward = 1
-
net.ipv4.ip_local_port_range = 1024 65000
-
net.ipv4.neigh.default.gc_thresh1 = 10240
-
net.ipv4.neigh.default.gc_thresh2 = 40960
-
net.ipv4.neigh.default.gc_thresh3 = 81920
-
net.ipv4.tcp_fin_timeout = 1
-
net.ipv4.tcp_keepalive_intvl = 15
-
net.ipv4.tcp_keepalive_probes = 5
-
net.ipv4.tcp_keepalive_time = 30
-
net.ipv4.tcp_max_orphans = 3276800
-
net.ipv4.tcp_max_syn_backlog = 1048576
-
net.ipv4.tcp_max_tw_buckets = 50000
-
net.ipv4.tcp_mem = 94500000 915000000 927000000
-
net.ipv4.tcp_orphan_retries = 3
-
net.ipv4.tcp_reordering = 5
-
net.ipv4.tcp_retrans_collapse = 0
-
net.ipv4.tcp_retries2 = 5
-
net.ipv4.tcp_rmem = 4096 87380 4194304
-
net.ipv4.tcp_sack = 1
-
net.ipv4.tcp_synack_retries = 1
-
net.ipv4.tcp_syncookies = 0
-
net.ipv4.tcp_syn_retries = 1
-
net.ipv4.tcp_timestamps = 1
-
net.ipv4.tcp_tw_recycle = 1
-
net.ipv4.tcp_tw_reuse = 1
-
net.ipv4.tcp_window_scaling = 1
-
net.ipv4.tcp_wmem = 4096 16384 4194304
-
net.ipv6.conf.all.disable_ipv6 = 1
-
net.ipv6.conf.default.disable_ipv6 = 1
-
net.ipv4.conf.eth0.rp_filter = 1 #memched節(jié)點不需要
-
net.ipv4.conf.eth1.rp_filter = 1 #memched節(jié)點不需要
-
net.ipv4.conf.eth2.rp_filter = 1 #memched節(jié)點不需要
-
net.ipv4.conf.all.rp_filter = 0 #memched節(jié)點不需要
三、服務(wù)安裝
1、調(diào)度機(master、backup)
-
yum -y install ipvsadm keepalived sendmail
2、memched節(jié)點1、memched節(jié)點2
-
wget http://memcached./files/memcached-1.4.15.tar.gz
-
wget http://www./~provos/libevent-1.4.13-stable.tar.gz
-
wget http://downloads./repcached/memcached-1.2.8-repcached-2.2.tar.gz
-
tar -zxvf libevent-1.4.13-stable.tar.gz
-
cd libevent-1.4.13-stable
-
./configure --prefix=/usr/local
-
make
-
make install
-
-
tar -zxvf memcached-1.4.15.tar.gz
-
cd memcached-1.4.15
-
./configure --with-libevent=/usr/local
-
make
-
make install
-
-
tar -zxvf memcached-1.2.8-repcached-2.2.tar.gz
-
cd memcached-1.2.8-repcached-2.2
-
./configure --enable-replication
-
make
-
make install
四、keepalived配置(master、backup)
[root@LVS1 keepalived]# cat /etc/keepalived/gobal_module
-
! global configure file
-
-
global_defs {
-
notification_email {
-
navyaijm@qq.com
-
}
-
notification_email_from navyaijm@qq.com
-
smtp_server 127.0.0.1
-
smtp_connect_timeout 30
-
router_id ZH_DG_LVS1
-
}
-
-
[root@LVS1 keepalived]# cat /etc/keepalived/keepalived.conf
-
! Keepalived main configure file
-
include /etc/keepalived/gobal_module
-
include /etc/keepalived/vrrpd_module
-
include /etc/keepalived/lvs_module
-
[root@LVS1 keepalived]# cat /etc/keepalived/vrrpd_module
-
vrrp_sync_group ZH_DG_Memcached1 {
-
group {
-
ZH_DG_WAN1
-
ZH_DG_LAN1
-
}
-
smtp_alter
-
}
-
-
vrrp_instance ZH_DG_WAN1 {
-
state MASTER (從上為:backup)
-
interface eth1
-
virtual_router_id 10 (從為:9)
-
priority 100
-
advert_int 1
-
authentication {
-
auth_type PASS
-
auth_pass 08756CD0
-
}
-
virtual_ipaddress {
-
172.28.26.100 # vip
-
}
-
}
-
-
-
vrrp_instance ZH_DG_LAN2 {
-
state MASTER (從上為:backup)
-
interface eth2
-
virtual_router_id 10 (從為:9)
-
priority 100
-
advert_int 1
-
authentication {
-
auth_type PASS
-
auth_pass 6F8DBC2E
-
}
-
virtual_ipaddress {
-
172.28.16.100 # Memcached 虛擬網(wǎng)關(guān)
-
}
-
}
[root@LVS1 keepalived]# cat /etc/keepalived/lvs_module
-
virtual_server 172.28.26.100 11211 {
-
delay_loop 6
-
lb_algo wrr
-
lb_kind NAT
-
persistence_timeout 60
-
protocol TCP
-
-
include /etc/keepalived/realserver/172.28.26.102_11234.conf
-
include /etc/keepalived/realserver/172.28.26.103_11233.conf
-
}
-
-
-
[root@LVS1 keepalived]# cat /etc/keepalived/realserver/172.28.26.102_11234.conf
-
real_server 172.28.16.102 11234 {
-
weight 1 #權(quán)重
-
inhibit_on_failure #故障會修改權(quán)重為零
-
TCP_CHECK {
-
connect_timeout 10 #10秒無響應(yīng)超時
-
nb_get_retry 3 #重連次數(shù)
-
delay_before_retry 3 #重連間隔,單位為秒
-
connect_port 11234 #檢測端口
-
}
-
}
-
-
-
[root@LVS1 keepalived]# cat /etc/keepalived/realserver/172.28.26.103_11233.conf
-
real_server 172.28.16.103 11233 {
-
weight 1 #權(quán)重
-
inhibit_on_failure #故障會修改權(quán)重為零
-
TCP_CHECK {
-
connect_timeout 10 #10秒無響應(yīng)超時
-
nb_get_retry 3 #重連次數(shù)
-
delay_before_retry 3 #重連間隔,單位為秒
-
connect_port 11233 #檢測端口
-
}
-
}
五、啟動服務(wù)
1、keepalived啟動(master和backup)
-
/etc/init.d/keepalived start
memcached的啟動
主節(jié)點:
-
/usr/local/bin/memcached -d -v -l 0.0.0.0 -p 11233 -X 11244 -uroot (11244是數(shù)據(jù)同步端口)
備節(jié)點:
-
/usr/local/bin/memcached -d -v -l 0.0.0.0 -p 11234 -uroot -x 172.28.16.103 -X 11244(11234是服務(wù)監(jiān)聽端口,11244是監(jiān)聽數(shù)據(jù)同步端口;-x 指定masterIP,-X指定master數(shù)據(jù)同步監(jiān)聽端口)
-
PS:
-
1、如果啟動的時候報錯如下:
-
[root@yw_memcached_slave ~]#/usr/local/bin/memcached -d -v -l 0.0.0.0 -p 11233 -X 11244 -uroot
-
/usr/local/bin/memcached: error while loading shared libraries: libevent-1.4.so.2: cannot open shared object file: No such file or directory
-
請yum安裝libeven
-
[root@yw_memcached_slave ~]# yum -y install libeven
-
2、repcached是日本人開發(fā)的實現(xiàn)memcached復(fù)制功能,它是一個單 master單 slave的方案,但它的 master/slave都是可讀寫的,而且可以相互同步,如果 master壞掉, slave偵測到連接斷了,它會自動 listen而成為 master;而如果 slave壞掉, master也會偵測到連接斷,它就會重新 listen等待新的 slave加入 ;master沒有搶占功能,如果master掛掉再起來只能是從了,并且永遠只能是從。
六、測試
1、寫一個php測試文件
vi session.php
-
<?php
-
ini_set('session.save_handler', 'memcached');
-
ini_set('session.save_path', "172.28.26.100:11211");
-
session_start();
-
$_SESSION['aa'] = "bb";
-
-
echo session_id();
-
?>
2、用php執(zhí)行這個文件會得到一個字符串,telnet 172.28.26.100 11211 或者telnet 172.28.26.103 11233再或者telnet 172.28.26.103 11234,如果能得到值說明session已經(jīng)寫入memched,down到任何一臺memched數(shù)據(jù)不會丟。
本文出自 “為了夢想奮斗” 博客,轉(zhuǎn)載請與作者聯(lián)系!
原文地址:http://navyaijm.blog.51cto.com/4647068/1177508
|