[root@node2 ~]# ifdown eth1 \\关闭某一个网卡，模拟单线路故障
[root@node2 ~]# multipath -ll
iscsi2 (1storage2_id) dm-1 IET,VIRTUAL-DISK
size=2.0G features='1 queue_if_no_path' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
|- 4:0:0:1 sde 8:64 failed faulty running  \\导出设备故障
`- 3:0:0:1 sdd 8:48 active ready  running
iscsi1 (1storage1_id) dm-0 IET,VIRTUAL-DISK
size=2.0G features='1 queue_if_no_path' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
|- 6:0:0:1 sdc 8:32 active ready  running
`- 5:0:0:1 sdb 8:16 failed faulty running \\导出设备故障
[root@node2 opt]# mount | grep opt
/dev/mapper/cvg0-clv0 on /opt type gfs2 (rw,relatime,hostdata=jid=0)
[root@node2 opt]# touch test  \\单线路故障并不影响集群文件系统正常使用
[root@node2 ~]# ifup eth1 \\恢复网卡
[root@node2 opt]# multipath –ll \\查看多路冗余是否恢复
iscsi2 (1storage2_id) dm-1 IET,VIRTUAL-DISK
size=2.0G features='1 queue_if_no_path' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
|- 4:0:0:1 sde 8:64 active ready running
`- 3:0:0:1 sdd 8:48 active ready running
iscsi1 (1storage1_id) dm-0 IET,VIRTUAL-DISK
size=2.0G features='1 queue_if_no_path' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
|- 6:0:0:1 sdc 8:32 active ready running
`- 5:0:0:1 sdb 8:16 active ready running

[root@manager ~]# for i in {1..3}; do ssh node$i "mount /dev/cvg0/clv0 /opt"; done
[root@node1 ~]# while :; do echo node1 >>/opt/test.txt;sleep 1; done \\节点1模拟向test.txt文件写入node1
[root@node2 ~]# while :; do echo node2 >>/opt/test.txt;sleep 1; done \\节点2模拟向test.txt文件写入node1
[root@node3 ~]# tail -f /opt/test.txt  \\节点3模拟读出节点1和节点2同时写入的数据
node1
node2
node1
node2

[root@node1 ~]# lvs \\mirror正常情况下的lv
LV   VG   Attr     LSize Pool Origin Data%  Move Log       Copy%  Convert
clv0 cvg0 mwi-a-m- 1.20g                         clv0_mlog 100.00
[root@storage1 ~]# ifdown eth1 && ifdown eth0 \\关闭storage1的两块网卡，相当于storage1宕机
[root@node2 opt]# lvs  \\ \mirror在storage1宕机正常情况下的lv
/dev/mapper/iscsi1: read failed after 0 of 4096 at 2150563840: Input/output error
/dev/mapper/iscsi1: read failed after 0 of 4096 at 2150637568: Input/output error
/dev/mapper/iscsi1: read failed after 0 of 4096 at 0: Input/output error
/dev/mapper/iscsi1: read failed after 0 of 4096 at 4096: Input/output error
/dev/sdb: read failed after 0 of 4096 at 0: Input/output error
/dev/sdb: read failed after 0 of 4096 at 2150563840: Input/output error
/dev/sdb: read failed after 0 of 4096 at 2150637568: Input/output error
/dev/sdb: read failed after 0 of 4096 at 4096: Input/output error
/dev/sdc: read failed after 0 of 4096 at 0: Input/output error
/dev/sdc: read failed after 0 of 4096 at 2150563840: Input/output error
/dev/sdc: read failed after 0 of 4096 at 2150637568: Input/output error
/dev/sdc: read failed after 0 of 4096 at 4096: Input/output error
Couldn't find device with uuid ziwJmg-Si56-l742-R3Nx-h0rK-KggJ-NdCigs.
LV   VG   Attr     LSize Pool Origin Data%  Move Log Copy%  Convert
clv0 cvg0 -wi-ao-- 1.20g
[root@node2 opt]# cp /var/log/messages .\\copy数据到挂载的目录，发现存储宕机一个并不影响读取和写入。
[root@node2 opt]# ll messages
-rw------- 1 root root 1988955 Jun  8 18:08 messages
[root@node2 opt]# dmsetup ls --tree
cvg0-clv0 (253:5)
└─iscsi2 (253:1)
├─ (8:48)
└─ (8:64)
iscsi1 (253:0)
├─ (8:16)
└─ (8:32)
[root@node2 opt]# vgs \\查看vgs情况
WARNING: Inconsistent metadata found for VG cvg0 - updating to use version 11
Missing device /dev/mapper/iscsi1 reappeared, updating metadata for VG cvg0 to version 11.
VG   #PV #LV #SN Attr   VSize VFree
cvg0   2   1   0 wz--nc 4.00g 2.80g
[root@node2 opt]# lvconvert -m 1 /dev/cvg0/clv0 /dev/mapper/iscsi1 \\恢复mirror
cvg0/clv0: Converted: 0.0%
cvg0/clv0: Converted: 8.5%
[root@node1 ~]# lvs
LV   VG   Attr     LSize Pool Origin Data%  Move Log       Copy%  Convert
clv0 cvg0 mwi-a-m- 1.20g                         clv0_mlog  77.45
[root@node1 ~]# lvs
LV   VG   Attr     LSize Pool Origin Data%  Move Log       Copy%  Convert
clv0 cvg0 mwi-a-m- 1.20g                         clv0_mlog  82.35
[root@node1 ~]# dmsetup ls --tree
cvg0-clv0 (253:5)
├─cvg0-clv0_mimage_1 (253:4)
│  └─iscsi1 (253:1)
│     ├─ (8:64)
│     └─ (8:48)
├─cvg0-clv0_mimage_0 (253:3)
│  └─iscsi2 (253:0)
│     ├─ (8:16)
│     └─ (8:32)
└─cvg0-clv0_mlog (253:2)
└─iscsi1 (253:1)
├─ (8:64)
└─ (8:48)
[root@node1 ~]# ll /opt/messages  \\可以看到数据还在
-rw------- 1 root root 1988955 Jun  8 18:08 /opt/messages