Storage – Linux Cache

GlusterFS分布式副本卷配置与本地挂载

5 月 272020

服务端

DigitalOcean/2Core/2G/60G+100G
165.227.27.221 server1
159.89.152.41 server2
159.89.151.236 server3
167.172.118.183 server4
167.172.126.43 server5
64.225.47.139 server6

客户端

DigitalOcean/2Core/2G/60G
64.225.47.123 server7

查看可用磁盘信息

[root@server1 ~]# df -h
Filesystem      Size  Used Avail Use% Mounted on
/dev/vda1        60G  901M   60G   2% /
devtmpfs        897M     0  897M   0% /dev
tmpfs           920M     0  920M   0% /dev/shm
tmpfs           920M   17M  903M   2% /run
tmpfs           920M     0  920M   0% /sys/fs/cgroup
/dev/sda        100G   33M  100G   1% /mnt/volume_sfo2_01
tmpfs           184M     0  184M   0% /run/user/0
[root@server1 ~]# 

[root@server2 ~]# df -h
Filesystem      Size  Used Avail Use% Mounted on
/dev/vda1        60G  901M   60G   2% /
devtmpfs        897M     0  897M   0% /dev
tmpfs           920M     0  920M   0% /dev/shm
tmpfs           920M   17M  903M   2% /run
tmpfs           920M     0  920M   0% /sys/fs/cgroup
/dev/sda        100G   33M  100G   1% /mnt/volume_sfo2_02
tmpfs           184M     0  184M   0% /run/user/0
[root@server2 ~]# 

[root@server3 ~]# df -h
Filesystem      Size  Used Avail Use% Mounted on
/dev/vda1        60G  901M   60G   2% /
devtmpfs        897M     0  897M   0% /dev
tmpfs           920M     0  920M   0% /dev/shm
tmpfs           920M   17M  903M   2% /run
tmpfs           920M     0  920M   0% /sys/fs/cgroup
/dev/sda        100G   33M  100G   1% /mnt/volume_sfo2_03
tmpfs           184M     0  184M   0% /run/user/0
[root@server3 ~]# 

[root@server4 ~]# df -h
Filesystem      Size  Used Avail Use% Mounted on
/dev/vda1        60G  901M   60G   2% /
devtmpfs        897M     0  897M   0% /dev
tmpfs           920M     0  920M   0% /dev/shm
tmpfs           920M   17M  903M   2% /run
tmpfs           920M     0  920M   0% /sys/fs/cgroup
/dev/sda        100G   33M  100G   1% /mnt/volume_sfo2_04
tmpfs           184M     0  184M   0% /run/user/0
[root@server4 ~]# 

[root@server5 ~]# df -h
Filesystem      Size  Used Avail Use% Mounted on
/dev/vda1        60G  974M   60G   2% /
devtmpfs        897M     0  897M   0% /dev
tmpfs           920M     0  920M   0% /dev/shm
tmpfs           920M   17M  903M   2% /run
tmpfs           920M     0  920M   0% /sys/fs/cgroup
/dev/sda        100G   33M  100G   1% /mnt/volume_sfo2_05
tmpfs           184M     0  184M   0% /run/user/0
[root@server5 ~]# 

[root@server6 ~]# df -h
Filesystem      Size  Used Avail Use% Mounted on
/dev/vda1        60G  973M   60G   2% /
devtmpfs        897M     0  897M   0% /dev
tmpfs           920M     0  920M   0% /dev/shm
tmpfs           920M   17M  903M   2% /run
tmpfs           920M     0  920M   0% /sys/fs/cgroup
/dev/sda        100G   33M  100G   1% /mnt/volume_sfo2_06
tmpfs           184M     0  184M   0% /run/user/0
[root@server6 ~]#

在服务端节点安装并启动GlusterFS服务

sed -i 's/^SELINUX=enforcing$/SELINUX=disabled/' /etc/selinux/config;
yum -y install centos-release-gluster;
yum -y install glusterfs-server;
systemctl enable glusterfsd;
systemctl start glusterfsd;

将节点加入受信存储池（Trusted Pool）

在受信存储池建立，节点间彼此建立通信连接后，只有受信成员节点可以将新节点加入池，新节点不可以直接操作已加入受信存储池的节点。

[root@server1 ~]# gluster peer probe server2
peer probe: success. 
[root@server1 ~]# gluster peer probe server3
peer probe: success. 
[root@server1 ~]# gluster peer probe server4
peer probe: success. 
[root@server1 ~]# gluster peer probe server5
peer probe: success. 
[root@server1 ~]# gluster peer probe server6
peer probe: success. 
[root@server1 ~]# gluster peer status
Number of Peers: 5

Hostname: server2
Uuid: 6231013f-07cc-4701-93b3-34d4c623a890
State: Peer in Cluster (Connected)

Hostname: server3
Uuid: aa808d87-4e7c-4ecd-bcf0-13ea03f844a8
State: Peer in Cluster (Connected)

Hostname: server4
Uuid: d153d847-ad46-4c85-8336-f8e553d5aab6
State: Peer in Cluster (Connected)

Hostname: server5
Uuid: a90c2969-67eb-4792-b5ce-6b4b3d782675
State: Peer in Cluster (Connected)

Hostname: server6
Uuid: 3ed5adc9-d3f7-40eb-8bbd-45f0882f55cd
State: Peer in Cluster (Connected)
[root@server1 ~]#

在节点上创建Brick目录

[root@server1 ~]# mkdir -p /mnt/volume_sfo2_01/brick1
[root@server2 ~]# mkdir -p /mnt/volume_sfo2_02/brick2
[root@server3 ~]# mkdir -p /mnt/volume_sfo2_03/brick3
[root@server4 ~]# mkdir -p /mnt/volume_sfo2_04/brick4
[root@server5 ~]# mkdir -p /mnt/volume_sfo2_05/brick5
[root@server6 ~]# mkdir -p /mnt/volume_sfo2_06/brick6

创建6节点3副本分布式副本卷

gluster volume create data-volume replica 3 transport tcp \
server1:/mnt/volume_sfo2_01/brick1 server2:/mnt/volume_sfo2_02/brick2 \
server3:/mnt/volume_sfo2_03/brick3 server4:/mnt/volume_sfo2_04/brick4 \
server5:/mnt/volume_sfo2_05/brick5 server6:/mnt/volume_sfo2_06/brick6

[root@server1 ~]# gluster volume create data-volume replica 3 transport tcp \
> server1:/mnt/volume_sfo2_01/brick1 server2:/mnt/volume_sfo2_02/brick2 \
> server3:/mnt/volume_sfo2_03/brick3 server4:/mnt/volume_sfo2_04/brick4 \
> server5:/mnt/volume_sfo2_05/brick5 server6:/mnt/volume_sfo2_06/brick6
volume create: data-volume: success: please start the volume to access data
[root@server1 ~]#

查看卷信息

[root@server1 ~]# gluster volume info

Volume Name: data-volume
Type: Distributed-Replicate
Volume ID: 2a2103ab-17e4-47b5-9d4c-96e460ac419c
Status: Created
Snapshot Count: 0
Number of Bricks: 2 x 3 = 6
Transport-type: tcp
Bricks:
Brick1: server1:/mnt/volume_sfo2_01/brick1
Brick2: server2:/mnt/volume_sfo2_02/brick2
Brick3: server3:/mnt/volume_sfo2_03/brick3
Brick4: server4:/mnt/volume_sfo2_04/brick4
Brick5: server5:/mnt/volume_sfo2_05/brick5
Brick6: server6:/mnt/volume_sfo2_06/brick6
Options Reconfigured:
transport.address-family: inet
storage.fips-mode-rchecksum: on
nfs.disable: on
performance.client-io-threads: off
[root@server1 ~]#

启动卷并查看卷信息和状态信息

[root@server1 ~]# gluster volume start data-volume
volume start: data-volume: success
[root@server1 ~]# gluster volume info
 
Volume Name: data-volume
Type: Distributed-Replicate
Volume ID: 2a2103ab-17e4-47b5-9d4c-96e460ac419c
Status: Started
Snapshot Count: 0
Number of Bricks: 2 x 3 = 6
Transport-type: tcp
Bricks:
Brick1: server1:/mnt/volume_sfo2_01/brick1
Brick2: server2:/mnt/volume_sfo2_02/brick2
Brick3: server3:/mnt/volume_sfo2_03/brick3
Brick4: server4:/mnt/volume_sfo2_04/brick4
Brick5: server5:/mnt/volume_sfo2_05/brick5
Brick6: server6:/mnt/volume_sfo2_06/brick6
Options Reconfigured:
transport.address-family: inet
storage.fips-mode-rchecksum: on
nfs.disable: on
performance.client-io-threads: off
[root@server1 ~]# gluster volume status
Status of volume: data-volume
Gluster process                             TCP Port  RDMA Port  Online  Pid
------------------------------------------------------------------------------
Brick server1:/mnt/volume_sfo2_01/brick1    49152     0          Y       9805 
Brick server2:/mnt/volume_sfo2_02/brick2    49152     0          Y       9843 
Brick server3:/mnt/volume_sfo2_03/brick3    49152     0          Y       9690 
Brick server4:/mnt/volume_sfo2_04/brick4    49152     0          Y       9734 
Brick server5:/mnt/volume_sfo2_05/brick5    49152     0          Y       10285
Brick server6:/mnt/volume_sfo2_06/brick6    49152     0          Y       10470
Self-heal Daemon on localhost               N/A       N/A        Y       9826 
Self-heal Daemon on server5                 N/A       N/A        Y       10306
Self-heal Daemon on server2                 N/A       N/A        Y       9864 
Self-heal Daemon on server6                 N/A       N/A        Y       10491
Self-heal Daemon on server3                 N/A       N/A        Y       9711 
Self-heal Daemon on server4                 N/A       N/A        Y       9755 
 
Task Status of Volume data-volume
------------------------------------------------------------------------------
There are no active volume tasks
 
[root@server1 ~]#

客户端安装GlusterFS必要组件

[root@server7 ~]# yum -y install centos-release-gluster
[root@server7 ~]# yum -y install glusterfs glusterfs-fuse glusterfs-rdma

挂载data-volume卷并查看磁盘信息（实际可用存储200GB）

[root@server7 ~]# mount -t glusterfs server6:/data-volume /mnt/
[root@server7 ~]# df -h
Filesystem            Size  Used Avail Use% Mounted on
/dev/vda1              60G 1003M   60G   2% /
devtmpfs              897M     0  897M   0% /dev
tmpfs                 920M     0  920M   0% /dev/shm
tmpfs                 920M   17M  903M   2% /run
tmpfs                 920M     0  920M   0% /sys/fs/cgroup
tmpfs                 184M     0  184M   0% /run/user/0
server6:/data-volume  200G  2.1G  198G   2% /mnt
[root@server7 ~]# 

[root@server7 ~]# mount |grep server6
server6:/data-volume on /mnt type fuse.glusterfs (rw,relatime,user_id=0,group_id=0,default_permissions,allow_other,max_read=131072)
[root@server7 ~]#

查看与服务端节点间通信状态

客户端随机写入文件

[root@server7 ~]# for i in `seq -w 1 20`; do cp -rp /var/log/messages /mnt/copy-test-$i; done
[root@server7 ~]# ls /mnt/
copy-test-01 copy-test-03 copy-test-05 copy-test-07 copy-test-09 copy-test-11 copy-test-13 copy-test-15 copy-test-17 copy-test-19
copy-test-02 copy-test-04 copy-test-06 copy-test-08 copy-test-10 copy-test-12 copy-test-14 copy-test-16 copy-test-18 copy-test-20
[root@server7 ~]#

在服务端节点查看随机写入文件的分布

[root@server1 ~]# ls /mnt/volume_sfo2_01/brick1/
copy-test-04 copy-test-05 copy-test-09 copy-test-15 copy-test-17 copy-test-18 copy-test-20
[root@server1 ~]#

[root@server2 ~]# ls /mnt/volume_sfo2_02/brick2/
copy-test-04 copy-test-05 copy-test-09 copy-test-15 copy-test-17 copy-test-18 copy-test-20
[root@server2 ~]#

[root@server3 ~]# ls /mnt/volume_sfo2_03/brick3/
copy-test-04 copy-test-05 copy-test-09 copy-test-15 copy-test-17 copy-test-18 copy-test-20
[root@server3 ~]#

[root@server4 ~]# ls /mnt/volume_sfo2_04/brick4/
copy-test-01 copy-test-03 copy-test-07 copy-test-10 copy-test-12 copy-test-14 copy-test-19
copy-test-02 copy-test-06 copy-test-08 copy-test-11 copy-test-13 copy-test-16
[root@server4 ~]#

[root@server5 ~]# ls /mnt/volume_sfo2_05/brick5/
copy-test-01 copy-test-03 copy-test-07 copy-test-10 copy-test-12 copy-test-14 copy-test-19
copy-test-02 copy-test-06 copy-test-08 copy-test-11 copy-test-13 copy-test-16
[root@server5 ~]#

[root@server6 ~]# ls /mnt/volume_sfo2_06/brick6/
copy-test-01 copy-test-03 copy-test-07 copy-test-10 copy-test-12 copy-test-14 copy-test-19
copy-test-02 copy-test-06 copy-test-08 copy-test-11 copy-test-13 copy-test-16
[root@server6 ~]#

CentOS 6.5 iSCSI存储及GFS2系统测试

存储 No Responses »

10 月 152014

1 配置iscsi服务器端

[root@localhost ~]# yum install scsi-target-utils
[root@localhost ~]# service tgtd start
 Starting SCSI target daemon: [ OK ]
[root@localhost ~]# chkconfig tgtd on
[root@localhost ~]#
[root@localhost ~]# mkdir /vdisk
[root@localhost ~]# dd if=/dev/zero of=/vdisk/vdisk.img bs=1M count=500
 500+0 records in
 500+0 records out
 524288000 bytes (524 MB) copied, 2.24159 s, 234 MB/s
[root@localhost ~]#

修改配置文件

[root@localhost ~]# vi /etc/tgt/targets.conf
 <target iqn.2014-10.com.example.target1>
 backing-store /vdisk/vdisk.img
 write-cache on
 </target>

重启服务

[root@localhost ~]# service tgtd restart
 Stopping SCSI target daemon: [ OK ]
 Starting SCSI target daemon: [ OK ]
[root@localhost ~]#

验证

[root@localhost ~]# tgt-admin --show target1
 Target 1: iqn.2014-10.com.example.target1
 System information:
 Driver: iscsi
 State: ready
 I_T nexus information:
 I_T nexus: 2
 Initiator: iqn.1994-05.com.redhat:f8c8fd7aac5c
 Connection: 0
 IP Address: 192.168.254.129
 I_T nexus: 4
 Initiator: iqn.1994-05.com.redhat:67f04a9b2426
 Connection: 0
 IP Address: 192.168.254.128
 LUN information:
 LUN: 0
 Type: controller
 SCSI ID: IET 00010000
 SCSI SN: beaf10
 Size: 0 MB, Block size: 1
 Online: Yes
 Removable media: No
 Prevent removal: No
 Readonly: No
 Backing store type: null
 Backing store path: None
 Backing store flags:
 LUN: 1
 Type: disk
 SCSI ID: IET 00010001
 SCSI SN: beaf11
 Size: 524 MB, Block size: 512
 Online: Yes
 Removable media: No
 Prevent removal: No
 Readonly: No
 Backing store type: rdwr
 Backing store path: /vdisk/vdisk.img
 Backing store flags:
 Account information:
 ACL information:
 ALL
 [root@localhost ~]#

2 配置iscsi客户端

安装，系统已默认安装
[root@localhost ~]# yum install iscsi-initiator-utils
Package iscsi-initiator-utils-6.2.0.873-10.el6.x86_64 already installed and latest version

发现

[root@localhost ~]# iscsiadm -m discovery -t sendtargets -p 192.168.254.130:3260
 Starting iscsid: [ OK ]
 192.168.254.130:3260,1 iqn.2014-10.com.example.target1
[root@localhost ~]#

[root@localhost ~]# iscsiadm -m node -T iqn.2014-10.com.example.target1 --login
 Logging in to [iface: default, target: iqn.2014-10.com.example.target1, portal: 192.168.254.130,3260] (multiple)
 Login to [iface: default, target: iqn.2014-10.com.example.target1, portal: 192.168.254.130,3260] successful.
[root@localhost ~]#

查看新增设备

[root@localhost ~]# fdisk -l
 Disk /dev/sdb: 524 MB, 524288000 bytes
 17 heads, 59 sectors/track, 1020 cylinders
 Units = cylinders of 1003 * 512 = 513536 bytes
 Sector size (logical/physical): 512 bytes / 512 bytes
 I/O size (minimum/optimal): 512 bytes / 512 bytes
 Disk identifier: 0x00000000

查看服务

[root@localhost ~]# service iscsid status
 iscsid (pid 1300) is running...
[root@localhost ~]# chkconfig --list iscsid
 iscsid 0:off 1:off 2:off 3:on 4:on 5:on 6:off
[root@localhost ~]#

3 配置multipath

[root@node1 ~]# rpm -q device-mapper-multipath
 device-mapper-multipath-0.4.9-72.el6.x86_64
[root@node1 ~]# service multipathd status
 multipathd is stopped
[root@node1 ~]# vi /etc/multipath.conf
[root@node1 ~]# service multipathd status
 multipathd is stopped
[root@node1 ~]# service multipathd start
 Starting multipathd daemon: [ OK ]
[root@node1 ~]# multipath -ll
 mpatha (1IET 00010001) dm-2 IET,VIRTUAL-DISK
 size=500M features='0' hwhandler='0' wp=rw
 `-+- policy='round-robin 0' prio=1 status=active
 `- 3:0:0:1 sdb 8:16 active ready running
[root@node1 ~]#

4，安装集群套件

[root@node1 ~]# yum install ricci
[root@node1 ~]# passwd ricci
[root@node1 ~]# chkconfig ricci on
[root@node1 ~]# service ricci start
[root@console ~]# yum install luci
[root@console ~]# chkconfig luci on

新建集群及增加节点并查看服务

[root@node2 ~]# service cman status
 cluster is running.
 [root@node2 ~]# service clvmd status
 clvmd (pid 3353) is running...
 Clustered Volume Groups: (none)
 Active clustered Logical Volumes: (none)
 [root@node2 ~]# service rgmanager status
 rgmanager (pid 3399) is running...
 [root@node2 ~]# service gfs2 status
 GFS2: no entries found in /etc/fstab
 [root@node2 ~]# service modclusterd status
 modclusterd (pid 3051) is running...
 [root@node2 ~]# clustat
 Cluster Status for htcluster @ Wed Oct 15 23:38:26 2014
 Member Status: Quorate

Member Name ID Status
 ------ ---- ---- ------
 node1 1 Online
 node2 2 Online, Local

[root@node2 ~]#

5 格式化文件系统

[root@node1 ~]# yum install gfs2-utils

[root@node1 ~]# mkfs.gfs2 -p lock_dlm -t htcluster:datastore -j 2 /dev/dm-2
This will destroy any data on /dev/dm-2.
It appears to contain: data

Are you sure you want to proceed? [y/n] y

Device: /dev/dm-2
Blocksize: 4096
Device Size 0.49 GB (128000 blocks)
Filesystem Size: 0.49 GB (127997 blocks)
Journals: 2
Resource Groups: 2
Locking Protocol: “lock_dlm”
Lock Table: “htcluster:datastore”
UUID: ea8bba63-df8c-9139-4e2e-e03a70f64891

[root@node1 ~]#
直接挂载（未启动集群
[root@node1 ~]# mount /dev/dm-2 /mnt/
gfs_controld join connect error: Connection refused
error mounting lockproto lock_dlm
[root@node1 ~]#

6，启动集群后，增加fstab配置文件手动挂载

方法1
[root@node2 ~]# vi /etc/fstab
/dev/dm-2 /mnt gfs2 _netdev 0 0
[root@node2 ~]# mount /dev/dm-2 /mnt/
[root@node2 ~]# service gfs2 status
Configured GFS2 mountpoints:
/mnt
Active GFS2 mountpoints:
/mnt
[root@node2 ~]#

方法2
[root@node1 ~]# vi /etc/fstab
[root@node1 ~]# service gfs2 status
GFS2: service is not running
[root@node1 ~]# service gfs2 start
Mounting GFS2 filesystem (/mnt): [ OK ]
[root@node1 ~]# service gfs2 status
Configured GFS2 mountpoints:
/mnt
Active GFS2 mountpoints:
/mnt
[root@node1 ~]#

查看当前的服务运行级别
[root@node2 ~]# chkconfig –list cman
cman 0:off 1:off 2:on 3:on 4:on 5:on 6:off
[root@node2 ~]# chkconfig –list rgmanager
rgmanager 0:off 1:off 2:on 3:on 4:on 5:on 6:off
[root@node2 ~]# chkconfig –list clvmd
clvmd 0:off 1:off 2:on 3:on 4:on 5:on 6:off
[root@node2 ~]# chkconfig –list modclusterd
modclusterd 0:off 1:off 2:on 3:on 4:on 5:on 6:off
[root@node2 ~]# chkconfig –list gfs2
gfs2 0:off 1:off 2:on 3:on 4:on 5:on 6:off
[root@node2 ~]#

验证
[root@node2 ~]# less /var/log/messages
[root@node2 ~]# touch /mnt/frm-n2

[root@node1 ~]# touch /mnt/frm-n1
[root@node1 ~]# ls /mnt/
frm-n1 frm-n2
[root@node1 ~]#
[root@node2 ~]# mount
/dev/mapper/VolGroup-lv_root on / type ext4 (rw)
proc on /proc type proc (rw)
sysfs on /sys type sysfs (rw)
devpts on /dev/pts type devpts (rw,gid=5,mode=620)
tmpfs on /dev/shm type tmpfs (rw)
/dev/sda1 on /boot type ext4 (rw)
none on /proc/sys/fs/binfmt_misc type binfmt_misc (rw)
none on /sys/kernel/config type configfs (rw)
/dev/mapper/mpatha on /mnt type gfs2 (rw,relatime,hostdata=jid=0)
[root@node2 ~]#
手动重启 node2和通过luci启动node1
通过
离开和加入集群产生的配置文件变化
离开后服务被停止，原配置文件无变化
[root@node1 ~]# clustat
Could not connect to CMAN: No such file or directory
[root@node1 ~]# service cman status
corosync is stopped
[root@node1 ~]#

增加resource，确认配置文件

[root@node2 ~]# cat /etc/cluster/cluster.conf
 <?xml version="1.0"?>
 <cluster config_version="3" name="htcluster">
 <clusternodes>
 <clusternode name="node1" nodeid="1"/>
 <clusternode name="node2" nodeid="2"/>
 </clusternodes>
 <cman expected_votes="1" two_node="1"/>
 <rm>
 <resources>
 <clusterfs device="/dev/dm-2" force_unmount="1" fsid="26886" fstype="gfs2" mountpoint="/mnt" name="htgfs2" options="_netdev" self_fence="1"/>
 </resources>
 </rm>
 </cluster>
 [root@node2 ~]#