https://docs.gluster.org/en/latest/Quick-Start-Guide/Quickstart/
https://wiki.centos.org/SpecialInterestGroup/Storage/gluster-Quickstart
可扩展的分布式文件系统，实现在唯一全局命名空间中聚合多个服务器节点的磁盘资源。
分布式文件系统节点
glusterfs-01 138.197.217.220 10.138.18.152
glusterfs-02 157.245.169.92 10.138.146.225
glusterfs-03 165.227.21.222 10.138.178.108
在所有节点配置hosts文件
[root@glusterfs-01 ~]# vi /etc/hosts
10.138.18.152 glusterfs-01
10.138.146.225 glusterfs-02
10.138.178.108 glusterfs-03
查看当前可用磁盘和分区信息
[root@glusterfs-01 ~]# fdisk -l
Disk /dev/vda: 64.4 GB, 64424509440 bytes, 125829120 sectors
Units = sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disk label type: dos
Disk identifier: 0x000b6061
Device Boot Start End Blocks Id System
/dev/vda1 * 2048 125829086 62913519+ 83 Linux
Disk /dev/vdb: 0 MB, 466944 bytes, 912 sectors
Units = sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disk /dev/sda: 107.4 GB, 107374182400 bytes, 209715200 sectors
Units = sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
[root@glusterfs-01 ~]#
创建分区
[root@glusterfs-01 ~]# fdisk /dev/sda
Welcome to fdisk (util-linux 2.23.2).
Changes will remain in memory only, until you decide to write them.
Be careful before using the write command.
Device does not contain a recognized partition table
Building a new DOS disklabel with disk identifier 0x99c4ee31.
Command (m for help): n
Partition type:
p primary (0 primary, 0 extended, 4 free)
e extended
Select (default p): e
Partition number (1-4, default 1):
First sector (2048-209715199, default 2048):
Using default value 2048
Last sector, +sectors or +size{K,M,G} (2048-209715199, default 209715199):
Using default value 209715199
Partition 1 of type Extended and of size 100 GiB is set
Command (m for help): n
Partition type:
p primary (0 primary, 1 extended, 3 free)
l logical (numbered from 5)
Select (default p): l
Adding logical partition 5
First sector (4096-209715199, default 4096):
Using default value 4096
Last sector, +sectors or +size{K,M,G} (4096-209715199, default 209715199):
Using default value 209715199
Partition 5 of type Linux and of size 100 GiB is set
Command (m for help): w
The partition table has been altered!
Calling ioctl() to re-read partition table.
Syncing disks.
[root@glusterfs-01 ~]#
查看当前可用磁盘和分区信息
[root@glusterfs-01 ~]# fdisk -l
Disk /dev/vda: 64.4 GB, 64424509440 bytes, 125829120 sectors
Units = sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disk label type: dos
Disk identifier: 0x000b6061
Device Boot Start End Blocks Id System
/dev/vda1 * 2048 125829086 62913519+ 83 Linux
Disk /dev/vdb: 0 MB, 466944 bytes, 912 sectors
Units = sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disk /dev/sda: 107.4 GB, 107374182400 bytes, 209715200 sectors
Units = sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disk label type: dos
Disk identifier: 0xbb370b51
Device Boot Start End Blocks Id System
/dev/sda1 2048 209715199 104856576 5 Extended
/dev/sda5 4096 209715199 104855552 83 Linux
[root@glusterfs-01 ~]#
在所有节点格式化并挂载数据盘
# mkfs.xfs -i size=512 /dev/sda5
# mkdir -p /data/brick1
# echo '/dev/sda5 /data/brick1 xfs defaults 1 2' >> /etc/fstab
# mount -a && mount
[root@glusterfs-01 ~]# mkfs.xfs -i size=512 /dev/sda5
meta-data=/dev/sda5 isize=512 agcount=4, agsize=6553472 blks
= sectsz=512 attr=2, projid32bit=1
= crc=1 finobt=0, sparse=0
data = bsize=4096 blocks=26213888, imaxpct=25
= sunit=0 swidth=0 blks
naming =version 2 bsize=4096 ascii-ci=0 ftype=1
log =internal log bsize=4096 blocks=12799, version=2
= sectsz=512 sunit=0 blks, lazy-count=1
realtime =none extsz=4096 blocks=0, rtextents=0
[root@glusterfs-01 ~]# mkdir -p /data/brick1
[root@glusterfs-01 ~]# echo '/dev/sda5 /data/brick1 xfs defaults 1 2' >> /etc/fstab
[root@glusterfs-01 ~]# mount -a && mount
sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime,seclabel)
proc on /proc type proc (rw,nosuid,nodev,noexec,relatime)
devtmpfs on /dev type devtmpfs (rw,nosuid,seclabel,size=917804k,nr_inodes=229451,mode=755)
securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime)
tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev,seclabel)
devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,seclabel,gid=5,mode=620,ptmxmode=000)
tmpfs on /run type tmpfs (rw,nosuid,nodev,seclabel,mode=755)
tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,seclabel,mode=755)
cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,seclabel,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd)
pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime)
cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,seclabel,hugetlb)
cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,seclabel,freezer)
cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,seclabel,cpuset)
cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,seclabel,perf_event)
cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,seclabel,pids)
cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,seclabel,net_prio,net_cls)
cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,seclabel,cpuacct,cpu)
cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,seclabel,memory)
cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,seclabel,blkio)
cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,seclabel,devices)
configfs on /sys/kernel/config type configfs (rw,relatime)
/dev/vda1 on / type xfs (rw,relatime,seclabel,attr2,inode64,noquota)
rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime)
selinuxfs on /sys/fs/selinux type selinuxfs (rw,relatime)
hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime,seclabel)
mqueue on /dev/mqueue type mqueue (rw,relatime,seclabel)
debugfs on /sys/kernel/debug type debugfs (rw,relatime)
systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=32,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=13335)
tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,seclabel,size=188220k,mode=700)
/dev/sda5 on /data/brick1 type xfs (rw,relatime,seclabel,attr2,inode64,noquota)
[root@glusterfs-01 ~]#
在所有节点安装GlusterFS软件
[root@glusterfs-01 ~]# yum -y install centos-release-gluster
[root@glusterfs-01 ~]# yum -y install glusterfs-server
[root@glusterfs-02 ~]# yum -y install centos-release-gluster
[root@glusterfs-02 ~]# yum -y install glusterfs-server
[root@glusterfs-03 ~]# yum -y install centos-release-gluster
[root@glusterfs-03 ~]# yum -y install glusterfs-server
在所有节点注册并启动glusterfsd系统服务
[root@glusterfs-01 ~]# systemctl enable glusterfsd
Created symlink from /etc/systemd/system/multi-user.target.wants/glusterfsd.service to /usr/lib/systemd/system/glusterfsd.service.
[root@glusterfs-01 ~]# systemctl start glusterfsd
[root@glusterfs-01 ~]# systemctl status glusterfsd
● glusterfsd.service - GlusterFS brick processes (stopping only)
Loaded: loaded (/usr/lib/systemd/system/glusterfsd.service; enabled; vendor preset: disabled)
Active: active (exited) since Tue 2020-05-26 07:28:17 UTC; 8s ago
Process: 10737 ExecStart=/bin/true (code=exited, status=0/SUCCESS)
Main PID: 10737 (code=exited, status=0/SUCCESS)
May 26 07:28:17 glusterfs-01 systemd[1]: Starting GlusterFS brick processes (stopping only)...
May 26 07:28:17 glusterfs-01 systemd[1]: Started GlusterFS brick processes (stopping only).
[root@glusterfs-01 ~]#
[root@glusterfs-02 ~]# systemctl enable glusterfsd
Created symlink from /etc/systemd/system/multi-user.target.wants/glusterfsd.service to /usr/lib/systemd/system/glusterfsd.service.
[root@glusterfs-02 ~]# systemctl start glusterfsd
[root@glusterfs-02 ~]# systemctl status glusterfsd
● glusterfsd.service - GlusterFS brick processes (stopping only)
Loaded: loaded (/usr/lib/systemd/system/glusterfsd.service; enabled; vendor preset: disabled)
Active: active (exited) since Tue 2020-05-26 07:29:21 UTC; 11s ago
Process: 18817 ExecStart=/bin/true (code=exited, status=0/SUCCESS)
Main PID: 18817 (code=exited, status=0/SUCCESS)
May 26 07:29:20 glusterfs-02 systemd[1]: Starting GlusterFS brick processes (stopping only)...
May 26 07:29:21 glusterfs-02 systemd[1]: Started GlusterFS brick processes (stopping only).
[root@glusterfs-02 ~]#
[root@glusterfs-03 ~]# systemctl enable glusterfsd
Created symlink from /etc/systemd/system/multi-user.target.wants/glusterfsd.service to /usr/lib/systemd/system/glusterfsd.service.
[root@glusterfs-03 ~]# systemctl start glusterfsd
[root@glusterfs-03 ~]# systemctl status glusterfsd
● glusterfsd.service - GlusterFS brick processes (stopping only)
Loaded: loaded (/usr/lib/systemd/system/glusterfsd.service; enabled; vendor preset: disabled)
Active: active (exited) since Tue 2020-05-26 07:30:27 UTC; 7s ago
Process: 18444 ExecStart=/bin/true (code=exited, status=0/SUCCESS)
Main PID: 18444 (code=exited, status=0/SUCCESS)
May 26 07:30:27 glusterfs-03 systemd[1]: Starting GlusterFS brick processes (stopping only)...
May 26 07:30:27 glusterfs-03 systemd[1]: Started GlusterFS brick processes (stopping only).
[root@glusterfs-03 ~]#
查看端口监听
[root@glusterfs-01 ~]# netstat -lntuop
Active Internet connections (only servers)
Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name Timer
tcp 0 0 127.0.0.1:25 0.0.0.0:* LISTEN 1195/master off (0.00/0/0)
tcp 0 0 0.0.0.0:24007 0.0.0.0:* LISTEN 1047/glusterd off (0.00/0/0)
tcp 0 0 0.0.0.0:111 0.0.0.0:* LISTEN 1/systemd off (0.00/0/0)
tcp 0 0 0.0.0.0:22 0.0.0.0:* LISTEN 1247/sshd off (0.00/0/0)
tcp6 0 0 ::1:25 :::* LISTEN 1195/master off (0.00/0/0)
tcp6 0 0 :::111 :::* LISTEN 1/systemd off (0.00/0/0)
tcp6 0 0 :::22 :::* LISTEN 1247/sshd off (0.00/0/0)
udp 0 0 0.0.0.0:111 0.0.0.0:* 1/systemd off (0.00/0/0)
udp 0 0 127.0.0.1:323 0.0.0.0:* 647/chronyd off (0.00/0/0)
udp 0 0 0.0.0.0:802 0.0.0.0:* 629/rpcbind off (0.00/0/0)
udp6 0 0 :::111 :::* 1/systemd off (0.00/0/0)
udp6 0 0 ::1:323 :::* 647/chronyd off (0.00/0/0)
udp6 0 0 :::802 :::* 629/rpcbind off (0.00/0/0)
[root@glusterfs-01 ~]#
查看版本信息
[root@glusterfs-01 ~]# glusterfs -V
glusterfs 7.5
Repository revision: git://git.gluster.org/glusterfs.git
Copyright (c) 2006-2016 Red Hat, Inc. <https://www.gluster.org/>
GlusterFS comes with ABSOLUTELY NO WARRANTY.
It is licensed to you under your choice of the GNU Lesser
General Public License, version 3 or any later version (LGPLv3
or later), or the GNU General Public License, version 2 (GPLv2),
in all cases as published by the Free Software Foundation.
[root@glusterfs-01 ~]#
将节点加入受信存储池
[root@glusterfs-01 ~]# gluster peer probe glusterfs-02
peer probe: success.
[root@glusterfs-01 ~]# gluster peer probe glusterfs-03
peer probe: success.
[root@glusterfs-01 ~]#
查看节点状态
[root@glusterfs-01 ~]# gluster peer status
Number of Peers: 2
Hostname: glusterfs-02
Uuid: 9375a552-1cce-414c-8850-997800dd1f6e
State: Peer in Cluster (Connected)
Hostname: glusterfs-03
Uuid: c490e4ee-03f7-4b83-9456-6cccd101020f
State: Peer in Cluster (Connected)
[root@glusterfs-01 ~]#
[root@glusterfs-02 ~]# gluster peer status
Number of Peers: 2
Hostname: glusterfs-01
Uuid: 605bacf2-abb4-4083-be2b-0d17c843bc68
State: Peer in Cluster (Connected)
Hostname: glusterfs-03
Uuid: c490e4ee-03f7-4b83-9456-6cccd101020f
State: Peer in Cluster (Connected)
[root@glusterfs-02 ~]#
[root@glusterfs-03 ~]# gluster peer status
Number of Peers: 2
Hostname: glusterfs-01
Uuid: 605bacf2-abb4-4083-be2b-0d17c843bc68
State: Peer in Cluster (Connected)
Hostname: glusterfs-02
Uuid: 9375a552-1cce-414c-8850-997800dd1f6e
State: Peer in Cluster (Connected)
[root@glusterfs-03 ~]#
创建一个存储卷（三副本）
[root@glusterfs-01 ~]# mkdir -p /data/brick1/gv0
[root@glusterfs-02 ~]# mkdir -p /data/brick1/gv0
[root@glusterfs-03 ~]# mkdir -p /data/brick1/gv0
[root@glusterfs-01 ~]# gluster volume create gv0 replica 3 glusterfs-01:/data/brick1/gv0 glusterfs-02:/data/brick1/gv0 glusterfs-03:/data/brick1/gv0
volume create: gv0: success: please start the volume to access data
[root@glusterfs-01 ~]#
[root@glusterfs-01 ~]# gluster volume start gv0
volume start: gv0: success
[root@glusterfs-01 ~]#
查看存储卷信息
[root@glusterfs-01 ~]# gluster volume info
Volume Name: gv0
Type: Replicate
Volume ID: aaa143ff-c7db-4b12-9d2f-4199c2cf76c9
Status: Started
Snapshot Count: 0
Number of Bricks: 1 x 3 = 3
Transport-type: tcp
Bricks:
Brick1: glusterfs-01:/data/brick1/gv0
Brick2: glusterfs-02:/data/brick1/gv0
Brick3: glusterfs-03:/data/brick1/gv0
Options Reconfigured:
transport.address-family: inet
storage.fips-mode-rchecksum: on
nfs.disable: on
performance.client-io-threads: off
[root@glusterfs-01 ~]#
查看卷状态信息
[root@glusterfs-01 ~]# gluster volume status
Status of volume: gv0
Gluster process TCP Port RDMA Port Online Pid
------------------------------------------------------------------------------
Brick glusterfs-01:/data/brick1/gv0 49152 0 Y 1580
Brick glusterfs-02:/data/brick1/gv0 49152 0 Y 10275
Brick glusterfs-03:/data/brick1/gv0 49152 0 Y 10248
Self-heal Daemon on localhost N/A N/A Y 1601
Self-heal Daemon on glusterfs-03 N/A N/A Y 10269
Self-heal Daemon on glusterfs-02 N/A N/A Y 10296
Task Status of Volume gv0
------------------------------------------------------------------------------
There are no active volume tasks
[root@glusterfs-01 ~]#
通过任意节点挂载已创建的三副本文件系统
在GlusterFS集群文件系统中，执行挂载命令时指定的服务器，仅用于获取卷的配置信息。随后客户端将直接与卷配置文件中的服务器进行通信（甚至不包括用于挂载的服务器）。
[root@glusterfs-01 ~]# mount -t glusterfs glusterfs-03:/gv0 /mnt
[root@glusterfs-01 ~]# mount
sysfs on /sys type sysfs (rw,nosuid,nodev,noexec,relatime)
proc on /proc type proc (rw,nosuid,nodev,noexec,relatime)
devtmpfs on /dev type devtmpfs (rw,nosuid,size=917804k,nr_inodes=229451,mode=755)
securityfs on /sys/kernel/security type securityfs (rw,nosuid,nodev,noexec,relatime)
tmpfs on /dev/shm type tmpfs (rw,nosuid,nodev)
devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000)
tmpfs on /run type tmpfs (rw,nosuid,nodev,mode=755)
tmpfs on /sys/fs/cgroup type tmpfs (ro,nosuid,nodev,noexec,mode=755)
cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd)
pstore on /sys/fs/pstore type pstore (rw,nosuid,nodev,noexec,relatime)
cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event)
cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio)
cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices)
cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids)
cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_prio,net_cls)
cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer)
cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpuacct,cpu)
cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset)
cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb)
cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory)
configfs on /sys/kernel/config type configfs (rw,relatime)
/dev/vda1 on / type xfs (rw,relatime,attr2,inode64,noquota)
rpc_pipefs on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw,relatime)
systemd-1 on /proc/sys/fs/binfmt_misc type autofs (rw,relatime,fd=24,pgrp=1,timeout=0,minproto=5,maxproto=5,direct,pipe_ino=12616)
hugetlbfs on /dev/hugepages type hugetlbfs (rw,relatime)
mqueue on /dev/mqueue type mqueue (rw,relatime)
debugfs on /sys/kernel/debug type debugfs (rw,relatime)
/dev/sda5 on /data/brick1 type xfs (rw,relatime,attr2,inode64,noquota)
tmpfs on /run/user/0 type tmpfs (rw,nosuid,nodev,relatime,size=188220k,mode=700)
glusterfs-03:/gv0 on /mnt type fuse.glusterfs (rw,relatime,user_id=0,group_id=0,default_permissions,allow_other,max_read=131072)
[root@glusterfs-01 ~]#
写入20个文件
[root@glusterfs-01 ~]# for i in `seq -w 1 20`; do cp -rp /var/log/messages /mnt/copy-test-$i; done
[root@glusterfs-01 ~]#
确认写入文件数量
[root@glusterfs-01 ~]# ls -lA /mnt/copy* | wc -l
20
[root@glusterfs-01 ~]#
在各个节点的本地挂载点查看写入的文件
[root@glusterfs-01 ~]# ls /data/brick1/gv0/
copy-test-01 copy-test-03 copy-test-05 copy-test-07 copy-test-09 copy-test-11 copy-test-13 copy-test-15 copy-test-17 copy-test-19
copy-test-02 copy-test-04 copy-test-06 copy-test-08 copy-test-10 copy-test-12 copy-test-14 copy-test-16 copy-test-18 copy-test-20
[root@glusterfs-01 ~]#
[root@glusterfs-02 ~]# ls /data/brick1/gv0/
copy-test-01 copy-test-03 copy-test-05 copy-test-07 copy-test-09 copy-test-11 copy-test-13 copy-test-15 copy-test-17 copy-test-19
copy-test-02 copy-test-04 copy-test-06 copy-test-08 copy-test-10 copy-test-12 copy-test-14 copy-test-16 copy-test-18 copy-test-20
[root@glusterfs-02 ~]#
[root@glusterfs-03 ~]# ls /data/brick1/gv0/
copy-test-01 copy-test-03 copy-test-05 copy-test-07 copy-test-09 copy-test-11 copy-test-13 copy-test-15 copy-test-17 copy-test-19
copy-test-02 copy-test-04 copy-test-06 copy-test-08 copy-test-10 copy-test-12 copy-test-14 copy-test-16 copy-test-18 copy-test-20
[root@glusterfs-03 ~]#