linux下multipath实施rac的asm disk

点击打开链接

前篇blog中 http://www.dbaxiaoyu.com/archives/1130 小鱼说到了linux下多路径绑定单实例的asm，当然一般而言单实例下我们还是会尽可能选择文件系统，而如果考虑到rac环境由于需要共享存储，而共享存储更多的需要固化物理磁盘，此时我们可以选择的是asmlib、udev、多路径，一般而言linux上用的较多的是asmlib和udev绑定，多路径在unix上面较多。

下面小鱼介绍下如何用linux自带的多路径软件mulitpath绑定共享存储来创建asm disk：

1 获取共享存储的wwid唯一设备号。

[root@rac01 ~]# scsi_id -g -u -s /block/sdd

SATA_VBOX_HARDDISK_VB3e724211-d3c035ee_

[root@rac01 ~]# scsi_id -g -u -s /block/sde

SATA_VBOX_HARDDISK_VB4ba68a09-6456b28e_

[root@rac01 ~]# scsi_id -g -u -s /block/sdf

SATA_VBOX_HARDDISK_VB487de809-4febb72d_

[root@rac01 ~]# scsi_id -g -u -s /block/sdg

SATA_VBOX_HARDDISK_VB4659564f-3030047e_

blacklist {

devnode "sd[a-c]"

}

2 配置共享存储的多路径

[root@rac01 ~]# cat /etc/multipath.conf

blacklist {

devnode "sd[a-c]"

}

multipaths {

multipath {

wwid SATA_VBOX_HARDDISK_VB3e724211-d3c035ee_

alias data_mpath1

}

multipath {

wwid SATA_VBOX_HARDDISK_VB4ba68a09-6456b28e_

alias data_mpath2

}

multipath {

wwid SATA_VBOX_HARDDISK_VB487de809-4febb72d_

alias back_mpath1

}

multipath {

wwid SATA_VBOX_HARDDISK_VB4659564f-3030047e_

alias back_mpath2

}

}

拷贝多路径配置文件到节点2

[root@rac01 ~]# scp /etc/multipath.conf 192.168.56.103:/etc/multipath.conf

3 生成多路径磁盘

分别在两个节点执行下面的命令。

[root@rac01 ~]# multipath -F

[root@rac01 ~]# multipath -v2

create: data_mpath1 (SATA_VBOX_HARDDISK_VB3e724211-d3c035ee_) undef ATA,VBOX HARDDISK

size=4.0G features='0' hwhandler='0' wp=undef

`-+- policy='round-robin 0' prio=1 status=undef

`- 3:0:0:0 sdd 8:48 undef ready running

create: data_mpath2 (SATA_VBOX_HARDDISK_VB4ba68a09-6456b28e_) undef ATA,VBOX HARDDISK

size=4.0G features='0' hwhandler='0' wp=undef

`-+- policy='round-robin 0' prio=1 status=undef

`- 4:0:0:0 sde 8:64 undef ready running

create: back_mpath1 (SATA_VBOX_HARDDISK_VB487de809-4febb72d_) undef ATA,VBOX HARDDISK

size=4.0G features='0' hwhandler='0' wp=undef

`-+- policy='round-robin 0' prio=1 status=undef

`- 5:0:0:0 sdf 8:80 undef ready running

create: back_mpath2 (SATA_VBOX_HARDDISK_VB4659564f-3030047e_) undef ATA,VBOX HARDDISK

size=2.0G features='0' hwhandler='0' wp=undef

`-+- policy='round-robin 0' prio=1 status=undef

`- 6:0:0:0 sdg 8:96 undef ready running

[root@rac01 ~]# multipath -ll

back_mpath2 (SATA_VBOX_HARDDISK_VB4659564f-3030047e_) dm-3 ATA,VBOX HARDDISK

size=2.0G features='0' hwhandler='0' wp=rw

`-+- policy='round-robin 0' prio=1 status=active

`- 6:0:0:0 sdg 8:96 active ready running

back_mpath1 (SATA_VBOX_HARDDISK_VB487de809-4febb72d_) dm-2 ATA,VBOX HARDDISK

size=4.0G features='0' hwhandler='0' wp=rw

`-+- policy='round-robin 0' prio=1 status=active

`- 5:0:0:0 sdf 8:80 active ready running

data_mpath2 (SATA_VBOX_HARDDISK_VB4ba68a09-6456b28e_) dm-1 ATA,VBOX HARDDISK

size=4.0G features='0' hwhandler='0' wp=rw

`-+- policy='round-robin 0' prio=1 status=active

`- 4:0:0:0 sde 8:64 active ready running

data_mpath1 (SATA_VBOX_HARDDISK_VB3e724211-d3c035ee_) dm-0 ATA,VBOX HARDDISK

size=4.0G features='0' hwhandler='0' wp=rw

`-+- policy='round-robin 0' prio=1 status=active

`- 3:0:0:0 sdd 8:48 active ready running

4对多路径磁盘进行规划（这里可以选择分区也可以选择不分区，注意如果分区最好对/dev/mapper 下的目录进行分区，例如fdisk /dev/mapper/back_mpath1）

分区完成后分别在两个节点重启下iscsi服务和udev规则检验

[root@rac01 ~]# service iscsi restart

[root@rac01 ~]# start_udev

[root@rac01 ~]# ls -l /dev/mapper/

total 0

brw-rw---- 1 root disk 253, 2 Jun 27 22:35 back_mpath1

brw-rw---- 1 root disk 253, 5 Jun 27 22:35 back_mpath1p1

brw-rw---- 1 root disk 253, 3 Jun 27 22:35 back_mpath2

brw-rw---- 1 root disk 253, 4 Jun 27 22:35 back_mpath2p1

crw------- 1 root root 10, 62 Jun 27 21:10 control

brw-rw---- 1 root disk 253, 0 Jun 27 22:54 data_mpath1

brw-rw---- 1 root disk 253, 6 Jun 27 22:55 data_mpath1p1

brw-rw---- 1 root disk 253, 1 Jun 27 22:54 data_mpath2

brw-rw---- 1 root disk 253, 7 Jun 27 22:55 data_mpath2p1

5 改变磁盘的宿主权限。

[root@rac01 oracle]# cat /etc/udev/rules.d/99-asm-multipath.rules

PROGRAM="/bin/chown oracle:oinstall /dev/mapper/back_mpath1p1"

PROGRAM="/bin/chown oracle:oinstall /dev/mapper/back_mpath2p1"

PROGRAM="/bin/chown oracle:oinstall /dev/mapper/data_mpath1p1"

PROGRAM="/bin/chown oracle:oinstall /dev/mapper/data_mpath2p1"

[root@rac01 oracle]# ls -l /dev/mapper/

total 0

brw-rw---- 1 root disk 253, 2 Jun 27 22:36 back_mpath1

brw-rw---- 1 oracle oinstall 253, 5 Jun 27 22:36 back_mpath1p1

brw-rw---- 1 root disk 253, 3 Jun 27 22:36 back_mpath2

brw-rw---- 1 oracle oinstall 253, 4 Jun 27 22:36 back_mpath2p1

crw------- 1 root root 10, 62 Jun 27 21:11 control

brw-rw---- 1 root disk 253, 0 Jun 27 22:36 data_mpath1

brw-rw---- 1 oracle oinstall 253, 6 Jun 27 23:09 data_mpath1p1

brw-rw---- 1 root disk 253, 1 Jun 27 22:36 data_mpath2

brw-rw---- 1 oracle oinstall 253, 7 Jun 27 23:09 data_mpath2p1

拷贝规则文件到另一个节点

[root@rac01 oracle]# scp /etc/udev/rules.d/99-asm-multipath.rules 192.168.56.103:/etc/udev/rules.d/99-asm-multipath.rules

[root@rac02 oracle]# start_udev

[root@rac02 oracle]# ls -l /dev/mapper/

total 0

brw-rw---- 1 root disk 253, 2 Jun 27 22:36 back_mpath1

brw-rw---- 1 oracle oinstall 253, 5 Jun 27 22:36 back_mpath1p1

brw-rw---- 1 root disk 253, 3 Jun 27 22:36 back_mpath2

brw-rw---- 1 oracle oinstall 253, 4 Jun 27 22:36 back_mpath2p1

crw------- 1 root root 10, 62 Jun 27 21:11 control

brw-rw---- 1 root disk 253, 0 Jun 27 22:36 data_mpath1

brw-rw---- 1 oracle oinstall 253, 6 Jun 27 23:09 data_mpath1p1

brw-rw---- 1 root disk 253, 1 Jun 27 22:36 data_mpath2

brw-rw---- 1 oracle oinstall 253, 7 Jun 27 23:09 data_mpath2p1

此时asm disk所需的盘符已经规定好了，接下来可以直接进行asm配置，需要注意的是asm_diskstring需要设置/dev/mapper/*设备，而不要设置/dev/dm-*形式的设备，/dev/dm-*设备是不固定的而且也是供内部使用的。

Asm实例的信息如下：

SQL> show parameter asm;

NAME TYPE

------------------------------------ ----------------------

VALUE

------------------------------

asm_diskgroups string

DATA, BACK

asm_diskstring string

/dev/mapper/*

asm_power_limit integer

1

SQL> select disk_number,path,name from v$asm_disk;

DISK_NUMBER PATH NAME

----------- ---------------------------------------- --------------------

1 /dev/mapper/data_mpath2p1 DATA_0001

0 /dev/mapper/data_mpath1p1 DATA_0000

0 /dev/mapper/back_mpath1p1 BACK_0000

1 /dev/mapper/back_mpath2p1 BACK_0001