CentOS6.4搭建mysql高可用架构MHA

参考/content/4993962.html
参考http://www.cnblogs.com/gomysql/p/3675429.html
一、环境

系统 CentOS 6.4x64最小化安装
manager 192.168.3.51
master 192.168.3.52
slave1 192.168.3.53 (备用master)
slave2 192.168.3.54
二、配置hosts本地解析
4台机都配置相同的hosts解析，内容如下

[root@manager ~]# cat /etc/hosts
127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
192.168.3.51	manager
192.168.3.52	master
192.168.3.53	slave1
192.168.3.54	slave2

三、配置四台主机之间ssh免秘钥登陆

manager:

[root@manager ~]# ssh-keygen
[root@manager ~]# ssh-copy-id -i ~/.ssh/id_rsa.pub root@master
[root@manager ~]# ssh-copy-id -i ~/.ssh/id_rsa.pub root@slave1
[root@manager ~]# ssh-copy-id -i ~/.ssh/id_rsa.pub root@slave2

master:

[root@master ~]# ssh-keygen
[root@master ~]# ssh-copy-id -i ~/.ssh/id_rsa.pub root@manager
[root@master ~]# ssh-copy-id -i ~/.ssh/id_rsa.pub root@slave1
[root@master ~]# ssh-copy-id -i ~/.ssh/id_rsa.pub root@slave2

slave1:

[root@slave1 ~]# ssh-keygen
[root@slave1 ~]# ssh-copy-id -i ~/.ssh/id_rsa.pub root@manager
[root@slave1 ~]# ssh-copy-id -i ~/.ssh/id_rsa.pub root@master
[root@slave1 ~]# ssh-copy-id -i ~/.ssh/id_rsa.pub root@slave2

slave2:

[root@slave2 ~]# ssh-keygen
[root@slave2 ~]# ssh-copy-id -i ~/.ssh/id_rsa.pub root@manager
[root@slave2 ~]# ssh-copy-id -i ~/.ssh/id_rsa.pub root@master
[root@slave2 ~]# ssh-copy-id -i ~/.ssh/id_rsa.pub root@slave1

四、安装mysql
在master,slave1,slave2上安装mysql服务。这里安装的是mysql-5.5.37.tar.gz,使用脚本进行安装

脚本内容如下

[root@master ~]# cat mysql_install.sh
#!/bin/bash

DATADIR='/data/mysql/data'
VERSION='mysql-5.5.37'
export LANG=zh_CN.UTF-8

#Source function library.
. /etc/init.d/functions

#camke install mysql5.5.X
install_mysql(){
read -p "please input a password for root: " PASSWD
if [ ! -d $DATADIR ];then
mkdir -p $DATADIR
fi
yum install cmake make gcc-c++ bison-devel ncurses-devel -y
id mysql &>/dev/null
if [ $? -ne 0 ];then
useradd mysql -s /sbin/nologin -M
fi
#useradd mysql -s /sbin/nologin -M
#change datadir owner to mysql
chown -R mysql.mysql $DATADIR
cd
#wget http://mirrors.sohu.com/mysql/MySQL-5.5/mysql-5.5.38.tar.gz tar xf $VERSION.tar.gz
cd $VERSION
cmake . -DCMAKE_INSTALL_PREFIX=/usr/local/$VERSION \
-DMYSQL_DATADIR=$DATADIR \
-DMYSQL_UNIX_ADDR=$DATADIR/mysql.sock \
-DDEFAULT_CHARSET=utf8 \
-DDEFAULT_COLLATION=utf8_general_ci \
-DENABLED_LOCAL_INFILE=ON \
-DWITH_INNOBASE_STORAGE_ENGINE=1 \
-DWITH_FEDERATED_STORAGE_ENGINE=1 \
-DWITH_BLACKHOLE_STORAGE_ENGINE=1 \
-DWITHOUT_EXAMPLE_STORAGE_ENGINE=1 \
-DWITHOUT_PARTITION_STORAGE_ENGINE=1
make && make install
if [ $? -ne 0 ];then
action "install mysql is failed"  /bin/false
exit $?
fi
sleep 2
#link
ln -s /usr/local/$VERSION/ /usr/local/mysql
ln -s /usr/local/mysql/bin/* /usr/bin/
#copy config and start file
/bin/cp /usr/local/mysql/support-files/my-small.cnf /etc/my.cnf
cp /usr/local/mysql/support-files/mysql.server /etc/init.d/mysqld
chmod 700 /etc/init.d/mysqld
#init mysql
/usr/local/mysql/scripts/mysql_install_db  --basedir=/usr/local/mysql --datadir=$DATADIR --user=mysql
if [ $? -ne 0 ];then
action "install mysql is failed"  /bin/false
exit $?
fi
#check mysql
/etc/init.d/mysqld start
if [ $? -ne 0 ];then
action "mysql start is failed"  /bin/false
exit $?
fi
chkconfig --add mysqld
chkconfig mysqld on
/usr/local/mysql/bin/mysql -e "update mysql.user set password=password('$PASSWD') where host='localhost' and user='root';"
/usr/local/mysql/bin/mysql -e "update mysql.user set password=password('$PASSWD') where host='127.0.0.1' and user='root';"
/usr/local/mysql/bin/mysql -e "delete from mysql.user where password='';"
/usr/local/mysql/bin/mysql -e "flush privileges;"
#/usr/local/mysql/bin/mysql -e "select version();" >/dev/null 2>&1
if [ $? -eq 0 ];then
echo "+---------------------------+"
echo "+------mysql安装完成--------+"
echo "+---------------------------+"
fi
#/etc/init.d/mysqld stop
}

install_mysql

建立master,slave1,slave2之间的主从复制
修改3台机的server-id，确保是唯一的。打开log-bin选项

#master的server-id
[root@master ~]# egrep "log-bin|server-id" /etc/my.cnf
server-id	= 1
log-bin=mysql-bin

#slave1的server-id
[root@slave1 ~]# egrep "log-bin|server-id" /etc/my.cnf
server-id	= 53
log-bin=mysql-bin

#slave2的server-id
[root@slave2 ~]# egrep "log-bin|server-id" /etc/my.cnf
server-id	= 54
log-bin=mysql-bin

在master,slave1上配置主从同步用的账号。slave1是备用的master，这个也需要进行授权。

#创建主从同步用的账号
mysql> grant all privileges on *.* to 'rep'@'192.168.3.%' identified by 'rep123';
Query OK, 0 rows affected (0.00 sec)

mysql> flush privileges;
Query OK, 0 rows affected (0.01 sec)

查看master的状态信息，在master上执行

mysql> show master status;
+------------------+----------+--------------+------------------+
| File             | Position | Binlog_Do_DB | Binlog_Ignore_DB |
+------------------+----------+--------------+------------------+
| mysql-bin.000002 |      330 |              |                  |
+------------------+----------+--------------+------------------+
1 row in set (0.00 sec)

在slave1,slave2上执行主从同步

mysql> change master to master_host='192.168.3.52', master_port=3306,master_user='rep',master_password='rep123',master_log_file='mysql-bin.000002',master_log_pos=330;
Query OK, 0 rows affected (0.02 sec)

mysql> start slave;
Query OK, 0 rows affected (0.01 sec)

#查看主从同步结果
mysql> show slave status\G
*************************** 1. row ***************************
Slave_IO_State: Waiting for master to send event
Master_Host: 192.168.3.52
Master_User: rep
Master_Port: 3306
Connect_Retry: 60
Master_Log_File: mysql-bin.000002
Read_Master_Log_Pos: 330
Relay_Log_File: slave1-relay-bin.000002
Relay_Log_Pos: 253
Relay_Master_Log_File: mysql-bin.000002
Slave_IO_Running: Yes            #显示YES为正常
Slave_SQL_Running: Yes            #显示YES为正常
Replicate_Do_DB:
Replicate_Ignore_DB:
Replicate_Do_Table:
Replicate_Ignore_Table:
Replicate_Wild_Do_Table:
Replicate_Wild_Ignore_Table:
Last_Errno: 0
Last_Error:
Skip_Counter: 0
Exec_Master_Log_Pos: 330
Relay_Log_Space: 410
Until_Condition: None
Until_Log_File:
Until_Log_Pos: 0
Master_SSL_Allowed: No
Master_SSL_CA_File:
Master_SSL_CA_Path:
Master_SSL_Cert:
Master_SSL_Cipher:
Master_SSL_Key:
Seconds_Behind_Master: 0
Master_SSL_Verify_Server_Cert: No
Last_IO_Errno: 0
Last_IO_Error:
Last_SQL_Errno: 0
Last_SQL_Error:
Replicate_Ignore_Server_Ids:
Master_Server_Id: 1
1 row in set (0.00 sec)

到这里主从同步已配置成功
五、创建MHA管理用的复制账号

#每台数据库上都要创建4个账号
mysql> grant all privileges on *.* to 'mha_rep'@'192.168.3.52' identified by '123456';
Query OK, 0 rows affected (0.00 sec)

mysql> grant all privileges on *.* to 'mha_rep'@'192.168.3.51' identified by '123456';
Query OK, 0 rows affected (0.00 sec)

mysql> grant all privileges on *.* to 'mha_rep'@'192.168.3.53' identified by '123456';
Query OK, 0 rows affected (0.00 sec)

mysql> grant all privileges on *.* to 'mha_rep'@'192.168.3.54' identified by '123456';
Query OK, 0 rows affected (0.00 sec)

mysql> flush privileges;
Query OK, 0 rows affected (0.00 sec)

六、在所有节点安装mha4mysql-node

#安装基础软件包
[root@manager ~]# yum install gcc perl-DBD-MySQL perl-CPAN perl-ExtUtils-CBuilder perl-ExtUtils-MakeMaker -y
[root@manager ~]# tar xf mha4mysql-node-0.56.tar.gz
[root@manager ~]# cd mha4mysql-node-0.56
[root@manager mha4mysql-node-0.56]# perl Makefile.PL
*** Module::AutoInstall version 1.03
*** Checking for Perl dependencies...
[Core Features]
- DBI        ...loaded. (1.609)
- DBD::mysql ...loaded. (4.013)
*** Module::AutoInstall configuration finished.
Checking if your kit is complete...
Looks good
Writing Makefile for mha4mysql::node
[root@manager mha4mysql-node-0.56]# make && make install

七、在manager安装mha4mysql-manager

#安装基础软件包
[root@manager ~]# yum install perl-DBD-MySQL perl-Config-Tiny perl-Log-Dispatch perl-Parallel-ForkManager perl-Time-HiRes -y
[root@manager ~]# tar xf mha4mysql-manager-0.56.tar.gz
[root@manager ~]# cd mha4mysql-manager-0.56
[root@manager mha4mysql-manager-0.56]# perl Makefile.PL
*** Module::AutoInstall version 1.03
*** Checking for Perl dependencies...
[Core Features]
- DBI                   ...loaded. (1.609)
- DBD::mysql            ...loaded. (4.013)
- Time::HiRes           ...loaded. (1.9721)
- Config::Tiny          ...loaded. (2.12)
- Log::Dispatch         ...loaded. (2.26)
- Parallel::ForkManager ...loaded. (0.7.9)
- MHA::NodeConst        ...loaded. (0.56)
*** Module::AutoInstall configuration finished.
Checking if your kit is complete...
Looks good
Writing Makefile for mha4mysql::manager
[root@manager mha4mysql-manager-0.56]# make && make install

#复制相关文件
[root@manager mha4mysql-manager-0.56]# mkdir /etc/masterha
[root@manager mha4mysql-manager-0.56]# mkdir -p /master/app1
[root@manager mha4mysql-manager-0.56]# mkdir -p /scripts
[root@manager mha4mysql-manager-0.56]# cp samples/conf/* /etc/masterha/
[root@manager mha4mysql-manager-0.56]# cp samples/scripts/* /scripts/

八、修改manager相关配置文件
A.修改配置文件

[root@manager ~]# cat /etc/masterha/app1.cnf
[server default]
user=mha_rep                                    #MHA管理mysql的用户名
password=123456                                 #MHA管理mysql的密码
manager_workdir=/masterha/app1                  #MHA的工作目录
manager_log=/masterha/app1/manager.log          #MHA的日志路径
ssh_user=root                                   #免秘钥登陆的用户名
repl_user=rep                                   #主从复制账号，用来在主从之间同步数据
repl_password=rep123
ping_interval=1                                 #ping间隔时间，用来检查master是否正常

[server1]
hostname=192.168.3.52
master_binlog_dir=/data/mysql/data/
candidate_master=1                              #master宕机后，优先启用这台作为master

[server2]
hostname=192.168.3.53
master_binlog_dir=/data/mysql/data/
candidate_master=1

[server3]
hostname=192.168.3.54
master_binlog_dir=/data/mysql/data/
no_master=1                                     #设置na_master=1，使服务器不能成为master

B.检查ssh互信是否成功

[root@manager ~]# masterha_check_ssh --conf=/etc/masterha/app1.cnf
Wed Jul  8 10:37:52 2015 - [warning] Global configuration file /etc/masterha_default.cnf not found. Skipping.
Wed Jul  8 10:37:52 2015 - [info] Reading application default configuration from /etc/masterha/app1.cnf..
Wed Jul  8 10:37:52 2015 - [info] Reading server configuration from /etc/masterha/app1.cnf..
Wed Jul  8 10:37:52 2015 - [info] Starting SSH connection tests..
Wed Jul  8 10:37:53 2015 - [debug]
Wed Jul  8 10:37:52 2015 - [debug]  Connecting via SSH from root@192.168.3.52(192.168.3.52:22) to root@192.168.3.53(192.168.3.53:22)..
Wed Jul  8 10:37:53 2015 - [debug]   ok.
Wed Jul  8 10:37:53 2015 - [debug]  Connecting via SSH from root@192.168.3.52(192.168.3.52:22) to root@192.168.3.54(192.168.3.54:22)..
Wed Jul  8 10:37:53 2015 - [debug]   ok.
Wed Jul  8 10:37:54 2015 - [debug]
Wed Jul  8 10:37:53 2015 - [debug]  Connecting via SSH from root@192.168.3.53(192.168.3.53:22) to root@192.168.3.52(192.168.3.52:22)..
Wed Jul  8 10:37:54 2015 - [debug]   ok.
Wed Jul  8 10:37:54 2015 - [debug]  Connecting via SSH from root@192.168.3.53(192.168.3.53:22) to root@192.168.3.54(192.168.3.54:22)..
Wed Jul  8 10:37:54 2015 - [debug]   ok.
Wed Jul  8 10:37:54 2015 - [debug]
Wed Jul  8 10:37:53 2015 - [debug]  Connecting via SSH from root@192.168.3.54(192.168.3.54:22) to root@192.168.3.52(192.168.3.52:22)..
Wed Jul  8 10:37:54 2015 - [debug]   ok.
Wed Jul  8 10:37:54 2015 - [debug]  Connecting via SSH from root@192.168.3.54(192.168.3.54:22) to root@192.168.3.53(192.168.3.53:22)..
Wed Jul  8 10:37:54 2015 - [debug]   ok.
Wed Jul  8 10:37:54 2015 - [info] All SSH connection tests passed successfully.

C.masterha_check_repl工具检查mysql主从复制是否成功

[root@manager ~]# masterha_check_repl --conf=/etc/masterha/app1.cnf
Wed Jul  8 10:42:05 2015 - [warning] Global configuration file /etc/masterha_default.cnf not found. Skipping.
Wed Jul  8 10:42:05 2015 - [info] Reading application default configuration from /etc/masterha/app1.cnf..
Wed Jul  8 10:42:05 2015 - [info] Reading server configuration from /etc/masterha/app1.cnf..
Wed Jul  8 10:42:05 2015 - [info] MHA::MasterMonitor version 0.56.
Wed Jul  8 10:42:05 2015 - [info] GTID failover mode = 0
Wed Jul  8 10:42:05 2015 - [info] Dead Servers:
Wed Jul  8 10:42:05 2015 - [info] Alive Servers:
Wed Jul  8 10:42:05 2015 - [info]   192.168.3.52(192.168.3.52:3306)
Wed Jul  8 10:42:05 2015 - [info]   192.168.3.53(192.168.3.53:3306)
Wed Jul  8 10:42:05 2015 - [info]   192.168.3.54(192.168.3.54:3306)
Wed Jul  8 10:42:05 2015 - [info] Alive Slaves:
Wed Jul  8 10:42:05 2015 - [info]   192.168.3.53(192.168.3.53:3306)  Version=5.5.37-log (oldest major version between slaves) log-bin:enabled
Wed Jul  8 10:42:05 2015 - [info]     Replicating from 192.168.3.52(192.168.3.52:3306)
Wed Jul  8 10:42:05 2015 - [info]     Primary candidate for the new Master (candidate_master is set)
Wed Jul  8 10:42:05 2015 - [info]   192.168.3.54(192.168.3.54:3306)  Version=5.5.37-log (oldest major version between slaves) log-bin:enabled
Wed Jul  8 10:42:05 2015 - [info]     Replicating from 192.168.3.52(192.168.3.52:3306)
Wed Jul  8 10:42:05 2015 - [info]     Not candidate for the new Master (no_master is set)
Wed Jul  8 10:42:05 2015 - [info] Current Alive Master: 192.168.3.52(192.168.3.52:3306)
Wed Jul  8 10:42:05 2015 - [info] Checking slave configurations..
Wed Jul  8 10:42:05 2015 - [info]  read_only=1 is not set on slave 192.168.3.53(192.168.3.53:3306).
Wed Jul  8 10:42:05 2015 - [warning]  relay_log_purge=0 is not set on slave 192.168.3.53(192.168.3.53:3306).
Wed Jul  8 10:42:05 2015 - [info]  read_only=1 is not set on slave 192.168.3.54(192.168.3.54:3306).
Wed Jul  8 10:42:05 2015 - [warning]  relay_log_purge=0 is not set on slave 192.168.3.54(192.168.3.54:3306).
Wed Jul  8 10:42:05 2015 - [info] Checking replication filtering settings..
Wed Jul  8 10:42:05 2015 - [info]  binlog_do_db= , binlog_ignore_db=
Wed Jul  8 10:42:05 2015 - [info]  Replication filtering check ok.
Wed Jul  8 10:42:05 2015 - [info] GTID (with auto-pos) is not supported
Wed Jul  8 10:42:05 2015 - [info] Starting SSH connection tests..
Wed Jul  8 10:42:06 2015 - [info] All SSH connection tests passed successfully.
Wed Jul  8 10:42:06 2015 - [info] Checking MHA Node version..
Wed Jul  8 10:42:07 2015 - [info]  Version check ok.
Wed Jul  8 10:42:07 2015 - [info] Checking SSH publickey authentication settings on the current master..
Wed Jul  8 10:42:07 2015 - [info] HealthCheck: SSH to 192.168.3.52 is reachable.
Wed Jul  8 10:42:07 2015 - [info] Master MHA Node version is 0.56.
Wed Jul  8 10:42:07 2015 - [info] Checking recovery script configurations on 192.168.3.52(192.168.3.52:3306)..
Wed Jul  8 10:42:07 2015 - [info]   Executing command: save_binary_logs --command=test --start_pos=4 --binlog_dir=/data/mysql/data/ --output_file=/var/tmp/save_binary_logs_test --manager_version=0.56 --start_file=mysql-bin.000002
Wed Jul  8 10:42:07 2015 - [info]   Connecting to root@192.168.3.52(192.168.3.52:22)..
Creating /var/tmp if not exists..    ok.
Checking output directory is accessible or not..
ok.
Binlog found at /data/mysql/data/, up to mysql-bin.000002
Wed Jul  8 10:42:07 2015 - [info] Binlog setting check done.
Wed Jul  8 10:42:07 2015 - [info] Checking SSH publickey authentication and checking recovery script configurations on all alive slave servers..
Wed Jul  8 10:42:07 2015 - [info]   Executing command : apply_diff_relay_logs --command=test --slave_user='mha_rep' --slave_host=192.168.3.53 --slave_ip=192.168.3.53 --slave_port=3306 --workdir=/var/tmp --target_version=5.5.37-log --manager_version=0.56 --relay_log_info=/data/mysql/data/relay-log.info  --relay_dir=/data/mysql/data/  --slave_pass=xxx
Wed Jul  8 10:42:07 2015 - [info]   Connecting to root@192.168.3.53(192.168.3.53:22)..
Checking slave recovery environment settings..
Opening /data/mysql/data/relay-log.info ... ok.
Relay log found at /data/mysql/data, up to slave1-relay-bin.000002
Temporary relay log file is /data/mysql/data/slave1-relay-bin.000002
Testing mysql connection and privileges.. done.
Testing mysqlbinlog output.. done.
Cleaning up test file(s).. done.
Wed Jul  8 10:42:08 2015 - [info]   Executing command : apply_diff_relay_logs --command=test --slave_user='mha_rep' --slave_host=192.168.3.54 --slave_ip=192.168.3.54 --slave_port=3306 --workdir=/var/tmp --target_version=5.5.37-log --manager_version=0.56 --relay_log_info=/data/mysql/data/relay-log.info  --relay_dir=/data/mysql/data/  --slave_pass=xxx
Wed Jul  8 10:42:08 2015 - [info]   Connecting to root@192.168.3.54(192.168.3.54:22)..
Checking slave recovery environment settings..
Opening /data/mysql/data/relay-log.info ... ok.
Relay log found at /data/mysql/data, up to slave2-relay-bin.000002
Temporary relay log file is /data/mysql/data/slave2-relay-bin.000002
Testing mysql connection and privileges.. done.
Testing mysqlbinlog output.. done.
Cleaning up test file(s).. done.
Wed Jul  8 10:42:08 2015 - [info] Slaves settings check done.
Wed Jul  8 10:42:08 2015 - [info]
192.168.3.52(192.168.3.52:3306) (current master)
+--192.168.3.53(192.168.3.53:3306)
+--192.168.3.54(192.168.3.54:3306)

Wed Jul  8 10:42:08 2015 - [info] Checking replication health on 192.168.3.53..
Wed Jul  8 10:42:08 2015 - [info]  ok.
Wed Jul  8 10:42:08 2015 - [info] Checking replication health on 192.168.3.54..
Wed Jul  8 10:42:08 2015 - [info]  ok.
Wed Jul  8 10:42:08 2015 - [warning] master_ip_failover_script is not defined.
Wed Jul  8 10:42:08 2015 - [warning] shutdown_script is not defined.
Wed Jul  8 10:42:08 2015 - [info] Got exit code 0 (Not master dead).

MySQL Replication Health is OK.

D.启动MHA manager,并监控日志文件

[root@manager ~]# nohup masterha_manager --conf=/etc/masterha/app1.cnf > /tmp/mha_manager.log 2>&1 &

E.测试master宕机后，时候会自动切换

#测试前查看slave1,slave2的主从同步情况
#slave1
[root@slave1 ~]# mysql -u root -plyao36843 -h 127.0.0.1 -e 'show slave status\G' |grep Yes
Slave_IO_Running: Yes
Slave_SQL_Running: Yes

#slave2
[root@slave2 ~]# mysql -u root -plyao36843 -h 127.0.0.1 -e 'show slave status\G' |grep Yes
Slave_IO_Running: Yes
Slave_SQL_Running: Yes

#停止master的mysql服务
[root@master ~]# service mysqld stop
Shutting down MySQL... SUCCESS!

#查看manager的日志文件
[root@manager ~]# cat /masterha/app1/manager.log
----- Failover Report -----

app1: MySQL Master failover 192.168.3.52(192.168.3.52:3306) to 192.168.3.53(192.168.3.53:3306) succeeded        #表示master已从192.168.3.52切换到192.168.3.53

Master 192.168.3.52(192.168.3.52:3306) is down!        #显示master已down机

Check MHA Manager logs at manager:/masterha/app1/manager.log for details.

Started automated(non-interactive) failover.
The latest slave 192.168.3.53(192.168.3.53:3306) has all relay logs for recovery.
Selected 192.168.3.53(192.168.3.53:3306) as a new master.
192.168.3.53(192.168.3.53:3306): OK: Applying all logs succeeded.
192.168.3.54(192.168.3.54:3306): This host has the latest relay log events.
Generating relay diff files from the latest slave succeeded.
192.168.3.54(192.168.3.54:3306): OK: Applying all logs succeeded. Slave started, replicating from 192.168.3.53(192.168.3.53:3306)
192.168.3.53(192.168.3.53:3306): Resetting slave info succeeded.
Master failover to 192.168.3.53(192.168.3.53:3306) completed successfully.

#我们查看slave2的主从同步信息
[root@slave2 ~]# mysql -u root -plyao36843 -h 127.0.0.1 -e 'show slave status\G' |egrep 'Yes|Master_Host'
Master_Host: 192.168.3.53        #同步IP地址切换到slave1上了
Slave_IO_Running: Yes
Slave_SQL_Running: Yes                 #主从同步正常

F.恢复master

#删除故障转移文件
[root@manager ~]# rm -rf  /masterha/app1/app1.failover.complete

#启动master的mysql服务
[root@master ~]# service mysqld start
Starting MySQL.. SUCCESS!
[root@master ~]# netstat -tunlp |grep mysql
tcp        0      0 0.0.0.0:3306                0.0.0.0:*                   LISTEN      9088/mysqld

#在manager的日子文件中找到主从同步的sql语句
[root@manager ~]# grep MASTER_HOST /masterha/app1/manager.log
Wed Jul  8 11:02:13 2015 - [info]  All other slaves should start replication from here. Statement should be: CHANGE MASTER TO MASTER_HOST='192.168.3.53', MASTER_PORT=3306, MASTER_LOG_FILE='mysql-bin.000001', MASTER_LOG_POS=1017, MASTER_USER='rep', MASTER_PASSWORD='xxx';

#在master上启动主从同步，密码为rep123
mysql> CHANGE MASTER TO MASTER_HOST='192.168.3.53', MASTER_PORT=3306, MASTER_LOG_FILE='mysql-bin.000001', MASTER_LOG_POS=1017, MASTER_USER='rep', MASTER_PASSWORD='rep123';
Query OK, 0 rows affected (0.03 sec)

mysql> start slave;
Query OK, 0 rows affected (0.00 sec)

mysql> show slave status\G
*************************** 1. row ***************************
Slave_IO_State: Waiting for master to send event
Master_Host: 192.168.3.53
Master_User: rep
Master_Port: 3306
Connect_Retry: 60
Master_Log_File: mysql-bin.000001
Read_Master_Log_Pos: 1017
Relay_Log_File: master-relay-bin.000002
Relay_Log_Pos: 253
Relay_Master_Log_File: mysql-bin.000001
Slave_IO_Running: Yes
Slave_SQL_Running: Yes
Replicate_Do_DB:
Replicate_Ignore_DB:
Replicate_Do_Table:
Replicate_Ignore_Table:
Replicate_Wild_Do_Table:
Replicate_Wild_Ignore_Table:
Last_Errno: 0
Last_Error:
Skip_Counter: 0
Exec_Master_Log_Pos: 1017
Relay_Log_Space: 410
Until_Condition: None
Until_Log_File:
Until_Log_Pos: 0
Master_SSL_Allowed: No
Master_SSL_CA_File:
Master_SSL_CA_Path:
Master_SSL_Cert:
Master_SSL_Cipher:
Master_SSL_Key:
Seconds_Behind_Master: 0
Master_SSL_Verify_Server_Cert: No
Last_IO_Errno: 0
Last_IO_Error:
Last_SQL_Errno: 0
Last_SQL_Error:
Replicate_Ignore_Server_Ids:
Master_Server_Id: 53
1 row in set (0.00 sec)

G.再次启动MHA的manager服务，并停止slave1

[root@manager ~]# nohup masterha_manager --conf=/etc/masterha/app1.cnf > /tmp/mha_manager.log 2>&1 &

#关闭slave1的mysql服务
[root@slave1 ~]# service mysqld stop
Shutting down MySQL... SUCCESS

#检查manager的日志
[root@manager ~]# cat /masterha/app1/manager.log
----- Failover Report -----

app1: MySQL Master failover 192.168.3.53(192.168.3.53:3306) to 192.168.3.52(192.168.3.52:3306) succeeded    #mysql的主服务器已从192.168.3.53切换到192.168.3.52上

Master 192.168.3.53(192.168.3.53:3306) is down!

Check MHA Manager logs at manager:/masterha/app1/manager.log for details.

Started automated(non-interactive) failover.
The latest slave 192.168.3.52(192.168.3.52:3306) has all relay logs for recovery.
Selected 192.168.3.52(192.168.3.52:3306) as a new master.
192.168.3.52(192.168.3.52:3306): OK: Applying all logs succeeded.
192.168.3.54(192.168.3.54:3306): This host has the latest relay log events.
Generating relay diff files from the latest slave succeeded.
192.168.3.54(192.168.3.54:3306): OK: Applying all logs succeeded. Slave started, replicating from 192.168.3.52(192.168.3.52:3306)
192.168.3.52(192.168.3.52:3306): Resetting slave info succeeded.
Master failover to 192.168.3.52(192.168.3.52:3306) completed successfully.

#在slave2上查看主从同步情况
[root@slave2 ~]# mysql -u root -plyao36843 -h 127.0.0.1 -e 'show slave status\G' |egrep 'Yes|Master_Host'
Master_Host: 192.168.3.52        #已更改主从同步地址
Slave_IO_Running: Yes
Slave_SQL_Running: Yes                 #主从正常

#恢复slave1的mysql服务，并启动slave
[root@slave1 ~]# service mysqld start
Starting MySQL.. SUCCESS!

#获取主从同步语句
[root@manager ~]# grep MASTER_HOST /masterha/app1/manager.log
Wed Jul  8 11:38:09 2015 - [info]  All other slaves should start replication from here. Statement should be: CHANGE MASTER TO MASTER_HOST='192.168.3.52', MASTER_PORT=3306, MASTER_LOG_FILE='mysql-bin.000003', MASTER_LOG_POS=107, MASTER_USER='rep', MASTER_PASSWORD='xxx';

#启动主从同步
mysql> CHANGE MASTER TO MASTER_HOST='192.168.3.52', MASTER_PORT=3306, MASTER_LOG_FILE='mysql-bin.000003', MASTER_LOG_POS=107, MASTER_USER='rep', MASTER_PASSWORD='rep123';
Query OK, 0 rows affected (0.02 sec)

mysql> start slave;
Query OK, 0 rows affected (0.00 sec)

mysql> show slave status\G
*************************** 1. row ***************************
Slave_IO_State: Waiting for master to send event
Master_Host: 192.168.3.52
Master_User: rep
Master_Port: 3306
Connect_Retry: 60
Master_Log_File: mysql-bin.000003
Read_Master_Log_Pos: 107
Relay_Log_File: slave1-relay-bin.000002
Relay_Log_Pos: 253
Relay_Master_Log_File: mysql-bin.000003
Slave_IO_Running: Yes
Slave_SQL_Running: Yes
Replicate_Do_DB:
Replicate_Ignore_DB:
Replicate_Do_Table:
Replicate_Ignore_Table:
Replicate_Wild_Do_Table:
Replicate_Wild_Ignore_Table:
Last_Errno: 0
Last_Error:
Skip_Counter: 0
Exec_Master_Log_Pos: 107
Relay_Log_Space: 410
Until_Condition: None
Until_Log_File:
Until_Log_Pos: 0
Master_SSL_Allowed: No
Master_SSL_CA_File:
Master_SSL_CA_Path:
Master_SSL_Cert:
Master_SSL_Cipher:
Master_SSL_Key:
Seconds_Behind_Master: 0
Master_SSL_Verify_Server_Cert: No
Last_IO_Errno: 0
Last_IO_Error:
Last_SQL_Errno: 0
Last_SQL_Error:
Replicate_Ignore_Server_Ids:
Master_Server_Id: 1
1 row in set (0.00 sec)

#最后恢复manager的服务
[root@manager ~]# rm -rf /masterha/app1/app1.failover.complete
[root@manager ~]# nohup masterha_manager --conf=/etc/masterha/app1.cnf > /tmp/mha_manager.log 2>&1 &
[1] 8146
[root@manager ~]# ps aux |grep manager
root      8146  3.0  1.8 194228 18648 pts/0    S    11:44   0:00 perl /usr/local/bin/masterha_manager --conf=/etc/masterha/app1.cnf

九.通过vip实现mysql的高可用
修改/etc/masterha/app1.cnf

[root@manager ~]# cat /etc/masterha/app1.cnf
[server default]
user=mha_rep
password=123456
manager_workdir=/masterha/app1
manager_log=/masterha/app1/manager.log
master_ip_failover_script=/scripts/master_ip_failover        #添加管理vip的脚本
ssh_user=root
repl_user=rep
repl_password=rep123
ping_interval=1

[server1]
hostname=192.168.3.52
candidate_master=1
master_binlog_dir=/data/mysql/data/

[server2]
hostname=192.168.3.53
candidate_master=1
master_binlog_dir=/data/mysql/data/

[server3]
hostname=192.168.3.54
master_binlog_dir=/data/mysql/data/
no_master=1

修改脚本master_ip_failover

[root@manager ~]# cat /scripts/master_ip_failover
#!/usr/bin/env perl
use strict;
use warnings FATAL => 'all';

use Getopt::Long;

my (
$command,          $ssh_user,        $orig_master_host, $orig_master_ip,
$orig_master_port, $new_master_host, $new_master_ip,    $new_master_port
);

my $vip = '192.168.3.55';            #vip地址
my $key = '1';
my $ssh_start_vip = "/sbin/ifconfig eth0:$key $vip";        #绑定在指定的网卡上面
my $ssh_stop_vip = "/sbin/ifconfig eth0:$key down";

GetOptions(
'command=s'          => \$command,
'ssh_user=s'         => \$ssh_user,
'orig_master_host=s' => \$orig_master_host,
'orig_master_ip=s'   => \$orig_master_ip,
'orig_master_port=i' => \$orig_master_port,
'new_master_host=s'  => \$new_master_host,
'new_master_ip=s'    => \$new_master_ip,
'new_master_port=i'  => \$new_master_port,
);

exit &main();

sub main {

print "\n\nIN SCRIPT TEST====$ssh_stop_vip==$ssh_start_vip===\n\n";

if ( $command eq "stop" || $command eq "stopssh" ) {

my $exit_code = 1;
eval {
print "Disabling the VIP on old master: $orig_master_host \n";
&stop_vip();
$exit_code = 0;
};
if ($@) {
warn "Got Error: $@\n";
exit $exit_code;
}
exit $exit_code;
}
elsif ( $command eq "start" ) {

my $exit_code = 10;
eval {
print "Enabling the VIP - $vip on the new master - $new_master_host \n";
&start_vip();
$exit_code = 0;
};
if ($@) {
warn $@;
exit $exit_code;
}
exit $exit_code;
}
elsif ( $command eq "status" ) {
print "Checking the Status of the script.. OK \n";
exit 0;
}
else {
&usage();
exit 1;
}
}
sub start_vip() {
`ssh $ssh_user\@$new_master_host \" $ssh_start_vip \"`;
}
# A simple system call that disable the VIP on the old_master
sub stop_vip() {
`ssh $ssh_user\@$orig_master_host \" $ssh_stop_vip \"`;
}

sub usage {
print
"Usage: master_ip_failover --command=start|stop|stopssh|status --orig_master_host=host --orig_master_ip=ip --orig_master_port=port --new_master_host=host --new_master_ip=ip --new_master_port=port\n";
}

进行切换

#停止master的mysql服务
[root@master ~]# service mysqld stop
Shutting down MySQL... SUCCESS!

#查看slave2的同步信息
[root@slave2 ~]# mysql -u root -plyao36843 -h 127.0.0.1 -e 'show slave status\G' |egrep 'Yes|Master_Host'
Master_Host: 192.168.3.53
Slave_IO_Running: Yes
Slave_SQL_Running: Yes

#查看slave1的IP信息
[root@slave1 ~]# ip a |grep eth0
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
inet 192.168.3.53/24 brd 192.168.3.255 scope global eth0
#这里能看到vip已自动添加
inet 192.168.3.55/24 brd 192.168.3.255 scope global secondary eth0:1

恢复master的mysql服务

#获取主从同步命令
[root@manager ~]# grep MASTER_HOST /masterha/app1/manager.log
Wed Jul  8 13:20:02 2015 - [info]  All other slaves should start replication from here. Statement should be: CHANGE MASTER TO MASTER_HOST='192.168.3.53', MASTER_PORT=3306, MASTER_LOG_FILE='mysql-bin.000002', MASTER_LOG_POS=107, MASTER_USER='rep', MASTER_PASSWORD='xxx';

#在master上启动主从同步
mysql> CHANGE MASTER TO MASTER_HOST='192.168.3.53', MASTER_PORT=3306, MASTER_LOG_FILE='mysql-bin.000002', MASTER_LOG_POS=107, MASTER_USER='rep', MASTER_PASSWORD='rep123';
Query OK, 0 rows affected (0.01 sec)

mysql> start slave;
Query OK, 0 rows affected (0.01 sec)

mysql> show slave status\G
*************************** 1. row ***************************
Slave_IO_State: Waiting for master to send event
Master_Host: 192.168.3.53
Master_User: rep
Master_Port: 3306
Connect_Retry: 60
Master_Log_File: mysql-bin.000002
Read_Master_Log_Pos: 107
Relay_Log_File: master-relay-bin.000002
Relay_Log_Pos: 253
Relay_Master_Log_File: mysql-bin.000002
Slave_IO_Running: Yes
Slave_SQL_Running: Yes
Replicate_Do_DB:
Replicate_Ignore_DB:
Replicate_Do_Table:
Replicate_Ignore_Table:
Replicate_Wild_Do_Table:
Replicate_Wild_Ignore_Table:
Last_Errno: 0
Last_Error:
Skip_Counter: 0
Exec_Master_Log_Pos: 107
Relay_Log_Space: 410
Until_Condition: None
Until_Log_File:
Until_Log_Pos: 0
Master_SSL_Allowed: No
Master_SSL_CA_File:
Master_SSL_CA_Path:
Master_SSL_Cert:
Master_SSL_Cipher:
Master_SSL_Key:
Seconds_Behind_Master: 0
Master_SSL_Verify_Server_Cert: No
Last_IO_Errno: 0
Last_IO_Error:
Last_SQL_Errno: 0
Last_SQL_Error:
Replicate_Ignore_Server_Ids:
Master_Server_Id: 53
1 row in set (0.00 sec)

#在manager上删除app1.failover.complete，并启动manager服务
[root@manager ~]# rm -rf /masterha/app1/app1.failover.complete
[root@manager ~]# nohup masterha_manager --conf=/etc/masterha/app1.cnf > /tmp/mha_manager.log 2>&1 &
[1] 13896
[root@manager ~]# ps aux |grep manager
root     13896  4.6  1.8 194240 18644 pts/0    S    13:30   0:00 perl /usr/local/bin/masterha_manager --conf=/etc/masterha/app1.cnf

#再次测试，停止slave1上的mysql服务
[root@slave1 ~]# service mysqld stop
Shutting down MySQL... SUCCESS!
[root@slave1 ~]# ip a |grep eth0
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
inet 192.168.3.53/24 brd 192.168.3.255 scope global eth0

#正常情况下，vip已经漂移到master上，主从同步也是指向的master
[root@master ~]# ip a |grep eth0
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
inet 192.168.3.52/24 brd 192.168.3.255 scope global eth0
inet 192.168.3.55/24 brd 192.168.3.255 scope global secondary eth0:1
[root@slave2 ~]# mysql -u root -plyao36843 -h 127.0.0.1 -e 'show slave status\G' |egrep 'Yes|Master_Host'
Master_Host: 192.168.3.52
Slave_IO_Running: Yes
Slave_SQL_Running: Yes

十.MHA日常维护命令

1.查看ssh登陆是否成功

masterha_check_ssh --conf=/etc/masterha/app1.cnf

2.查看复制是否建立好

masterha_check_repl --conf=/etc/masterha/app1.cnf

3.启动mha

nohup masterha_manager --conf=/etc/masterha/app1.cnf > /tmp/mha_manager.log  < /dev/null 2>&1 &

当有slave节点宕掉的情况是启动不了的，加上--ignore_fail_on_start即使有节点宕掉也能启动mha

nohup masterha_manager --conf=/etc/masterha/app1.cnf  --ignore_fail_on_start > /tmp/mha_manager.log  < /dev/null 2>&1 &

4.检查启动的状态

masterha_check_status --conf=/etc/masterha/app1.cnf

5.停止mha

masterha_stop  --conf=/etc/masterha/app1.cnf

6.failover后下次重启
每次failover切换后会在管理目录生成文件app1.failover.complete ，下次在切换的时候会发现有这个文件导致切换不成功，需要手动清理掉。

rm -rf /masterha/app1/app1.failover.complete

也可以加上参数--ignore_last_failover
7.手工failover
手工failover场景，master死掉，但是masterha_manager没有开启，可以通过手工failover：

masterha_master_switch --conf=/etc/masterha/app1.cnf --dead_master_host=192.168.3.54 --master_state=192.168.3.52 --new_master_host=192.168.3.53 --ignore_last_failover

8.masterha_manager是一种监视和故障转移的程序。另一方面,masterha_master_switch程序不监控主库。 masterha_master_switch可以用于主库故障转移,也可用于在线总开关。
9.手动在线切换

masterha_master_switch --conf=/etc/app1.cnf --master_state=alive --new_master_host=192.168.3.53 --orig_master_is_new_slave

或者

masterha_master_switch --conf=/etc/app1.cnf --master_state=alive --new_master_host=192.168.3.53 --orig_master_is_new_slave --running_updates_limit=10000

--orig_master_is_new_slave切换时加上此参数是将原master变为slave节点，如果不加此参数，原来的master将不启动
--running_updates_limit=10000 切换时候选master如果有延迟的话，mha切换不能成功，加上此参数表示延迟在此时间范围内都可切换（单位为s），但是切换的时间长短是由recover时relay日志的大小决定
手动在线切换mha，切换时需要将在运行的mha停掉后才能切换。
在备库先执行DDL，一般先stop slave，一般不记录mysql日志，可以通过set SQL_LOG_BIN = 0实现。然后进行一次主备切换操作，再在原来的主库上执行DDL。这种方法适用于增减索引，如果是增加字段就需要额外注意。
可以通过如下命令停止mha

masterha_stop --conf=/etc/app1.cnf

本文出自 “ly36843运维” 博客，请务必保留此出处http://ly36843.blog.51cto.com/3120113/1671718