Greenplum6.0开源版本集群部署实战

1.环境说明

节点	系统版本	安装服务	Greenplum版本
node1	centos 7.6	master	gp13.0
node2	centos 7.8	segment	gp13.0
node3	centos 7.8	segment	gp13.0

2.安装文件下载

greenplum下载
上传到服务器

[root@test-cdh-01 tools]# ll
total 67740
drwxr-xr-x. 2 root root     4096 Oct 29  2019 20191029
drwxr-xr-x  2 root root     4096 Dec  3  2019 20191203
drwxr-xr-x  2 root root     4096 Dec  4  2019 20191204
-rw-r--r--. 1 root root    15945 Oct 28  2019 client.tgz
-rw-r--r--  1 root root 69333396 Jan 26 12:22 greenplum-db-6.13.0-rhel7-x86_64.rpm

3.1 修改hostname

[root@test-cdh-01 tools] cat /etc/hosts
127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
196.1.121.79 test-shuxi-02
196.1.121.80 test-cdh-01
196.1.121.81 test-cdh-02
196.1.121.82 test-cdh-03

3.2 配置root的免密登录

设置免密登录，从test-cdh-01到test-cdh02、test-cdh-03

# home下创建.ssh目录
[root@hnode1 ~] mkdir .ssh
[root@test-cdh-01 ~] cd .ssh/
[root@test-cdh-01 .ssh] pwd
/root/.ssh
# 生成秘钥
[root@test-cdh-01 .ssh] ssh-keygen -t rsa
Generating public/private rsa key pair.
Enter file in which to save the key (/root/.ssh/id_rsa): （回车）
Enter passphrase (empty for no passphrase):（回车）
Enter same passphrase again:（回车）
Your identification has been saved in /root/.ssh/id_rsa.
Your public key has been saved in /root/.ssh/id_rsa.pub.
The key fingerprint is:
SHA256:F1XLKMmnivY288gcK5IsY90xm0kawlM62D+uyGIeB0k root@test-cdh-01
The keys randomart image is:
+---[RSA 2048]----+
|            ...  |
|         . o o . |
| E        = o o  |
|. . .      =     
|| * o    S o      |
|. O . +. o             |
| . O *o*o             |
|oo* O.*+++         |
|=+.=.o o*+.         |
+----[SHA256]-----+
# 查看生成的文件
[root@hnode1 .ssh] ll
总用量 8
-rw-------. 1 root root 1675 3月  30 18:58 id_rsa
-rw-r--r--. 1 root root  393 3月  30 18:58 id_rsa.pub
[root@test-cdh-01 .ssh]
#将test-cdh-01的公钥拷贝到test-cdh-02和test-cdh-03
[root@test-cdh-01 .ssh] ssh-copy-id test-cdh-02
root@test-cdh-02 password: #(输入test-cdh-01的密码)# 尝试登陆test-cdh-02
[root@test-cdh-01 .ssh] ssh test-cdh-02
# 登出
[root@test-cdh-02 ~] exit
Connection to test-cdh-02 closed.
#将公钥拷贝到test-cdh-03
[root@test-cdh-01 .ssh] ssh-copy-id test-cdh-03
root@test-cdh-03s password: #(输入test-cdh-03的密码)
[root@test-cdh-01 .ssh] ssh test-cdh-03
# 登出
[root@test-cdh-03 ~] exit
Connection to test-cdh-03 closed.

3.3 配置时间同步

所有节点都需要配置

#安装时间同步服务（ntp）

[root@test-cdh-01 opt] yum install ntp
Loaded plugins: fastestmirror, refresh-packagekit, security
Loading mirror speeds from cached hostfile
 * base: ftp.sjtu.edu.cn
 * extras: ap.stykers.moe
 * updates: mirror.bit.edu.cn
# 强制从远程更新
[root@test-cdh-03 ~] sudo ntpdate 0.centos.pool.ntp.org
28 Apr 15:54:51 ntpdate[75627]: step time server 193.4.58.77 offset 2443633.848252 sec
# 启动服务
[root@test-cdh-01 opt] systemctl start ntpd.service
# 检查服务状态
[root@test-cdh-03 ~] systemctl status ntpd.service
# 设置开机启动
[root@test-cdh-03 ~] systemctl enable ntpd.service
# 检查开机启动是否设置成功
[root@test-cdh-03 ~] systemctl is-enabled ntpd
enabled

3.4 关闭防火墙

所有节点都要操作

关闭防火墙

[root@test-cdh-01 ~]# firewall-cmd --state
running
[root@test-cdh-01 ~]# systemctl stop firewall
[root@test-cdh-01 ~]# systemctl disable firewalld.service
[root@test-cdh-01 ~]# firewall-cmd --state
not running

关闭selinux

vim /etc/selinux/config
# 将 SELINUX=enforcing 修改为：
SELINUX=disabled
# 重启服务器后查看 SELINUX 状态
[root@test-cdh-01 ~]# sestatus
SELinux status:   disabled

3.5 修改内核参数

所有节点都要修改

vim /etc/sysctl.conf
# 末尾追加以下内容
kernel.shmmax = 500000000
kernel.shmmni = 4096
kernel.shmall = 4000000000
kernel.sem = 250 512000 100 2048
kernel.sysrq = 1
kernel.core_uses_pid = 1
kernel.msgmnb = 65536
kernel.msgmax = 65536
kernel.msgmni = 2048
net.ipv4.tcp_syncookies = 1
net.ipv4.ip_forward = 0
net.ipv4.conf.default.accept_source_route = 0
net.ipv4.tcp_tw_recycle = 1
net.ipv4.tcp_max_syn_backlog = 4096
net.ipv4.conf.all.arp_filter = 1
net.ipv4.ip_local_port_range = 10000 65535
net.core.netdev_max_backlog = 10000
net.core.rmem_max = 2097152
net.core.wmem_max = 2097152
vm.overcommit_memory = 2

kernerl.shmall和kernerl.shmmax参数的计算方式
kernerl.shmall ：echo $(expr $(getconf _PHYS_PAGES) / 2)
kernel.shmmax : echo $(expr $(getconf _PHYS_PAGES) / 2 * $(getconf PAGE_SIZE))
64GB内存以上主机的参数
vm.dirty_background_ratio = 0 vm.dirty_ratio = 0 vm.dirty_background_bytes = 1610612736 # 1.5GB vm.dirty_bytes = 4294967296 # 4GB
执行次命令设置 vm.min_free_kbytes (大内存时使用）
awk 'BEGIN {OFMT = "%.0f";} /MemTotal/ {print "vm.min_free_kbytes =", $2 * .03;}' /proc/meminfo >> /etc/sysctl.conf
64G内存以下得主机建议以下参数移除以下参数
vm.dirty_background_bytes
vm.dirty_bytes
设置以下得参数
vm.dirty_background_ratio = 3
vm.dirty_ratio = 10

每台机器都执行命令，使配置生效

[root@test-cdh-01 opt]# sysctl -p
kernel.shmmax = 500000000
kernel.shmmni = 4096
kernel.shmall = 4000000000
kernel.sem = 250 512000 100 2048
kernel.sysrq = 1
kernel.core_uses_pid = 1
kernel.msgmnb = 65536
kernel.msgmax = 65536
kernel.msgmni = 2048
net.ipv4.tcp_syncookies = 1
net.ipv4.ip_forward = 0
net.ipv4.conf.default.accept_source_route = 0
net.ipv4.tcp_tw_recycle = 1
net.ipv4.tcp_max_syn_backlog = 4096
net.ipv4.conf.all.arp_filter = 1
net.ipv4.ip_local_port_range = 10000 65535
net.core.netdev_max_backlog = 10000
net.core.rmem_max = 2097152
net.core.wmem_max = 2097152
vm.overcommit_memory = 2

3.6 配置资源限制参数

所有节点都需要进行配置

vim /etc/security/limits.conf
# 可打开的文件描述符的最大数,超过会警告(软限制)
* soft nofile 65536
# 可打开的文件描述符的最大数,超过会报错(硬限制)
* hard nofile 65536
# 单个用户可用的最大进程数量,超过会警告(软限制)
* soft nproc 131072
# 单个用户可用的最大进程数量,超过会报错(硬限制)
* hard nproc 131072

3.7 创建用户和用户组

所有节点都需要创建gpadmin用户及用户组，将其作为greenplum的操作用户

# 增加用户组
groupadd -g 530 gpadmin
# 增加用户
useradd -g 530 -u 530 -m -d /home/gpadmin -s /bin/bash gpadmin
# 对文件夹进行赋权
chown -R gpadmin:gpadmin /home/gpadmin/
# 为新用户创建密码
[root@test-cdh-02 ~]# passwd gpadmin
更改用户 gpadmin 的密码
新的 密码：(gpadmin)#密码提示太简单，连续输入两次即可。
无效的密码：密码少于 8 个字符
重新输入新的 密码：
passwd：所有的身份验证令牌已经成功更新

4 安装Greenplum

4.1 安装Greenplum rpm包

# 安装

[root@test-cdh-01 /]# cd /data/tools
[root@test-cdh-01 opt]#
[root@test-cdh-01 greenplum]# rpm -ivh greenplum-db-6.13.0-rhel7-x86_64.rpm# 查看安装好的文件
[root@test-cdh-01 opt]# cd /usr/local/
[root@test-cdh-01 local]# ll
总用量 0
lrwxrwxrwx   1 root root  29 6月   4 11:06 greenplum-db -> /usr/local/greenplum-db-6.13.0
drwxr-xr-x  12 root root 202 6月   4 11:06 greenplum-db-6.13.0
# 将文件分配给 gpadmin 用户
[root@test-cdh-01 local]#  chown -R gpadmin:gpadmin greenplum-db*# 将文件分发到 segment 节点
[root@test-cdh-01 local]# scp -r greenplum-db-6.13.0/ root@test-cdh-02:/usr/local
[root@test-cdh-01 local]# scp -r greenplum-db-6.13.0/ root@test-cdh-03:/usr/local# test-cdh-03 上面执行# 创建软连接
[root@test-cdh-02 local]# ln -s greenplum-db-6.13.0/ greenplum-db# 将文件分配给 gpadmin 用户
[root@test-cdh-02 local]# chown -R gpadmin:gpadmin greenplum-db*# test-cdh-03 上面执行
[root@test-cdh-03 local]# ln -s greenplum-db-6.13.0/ greenplum-db
[root@test-cdh-03 local]# chown -R gpadmin:gpadmin greenplum-db*

4.2 配置gpadmina用户的免密登陆

# 生成秘钥
[root@test-cdh-01 opt]# su gpadmin
[gpadmin@test-cdh-01 opt]$ ssh-keygen -t rsa
Generating public/private rsa key pair.
Enter file in which to save the key (/home/gpadmin/.ssh/id_rsa): #(回车)
Enter passphrase (empty for no passphrase):#(回车)
Enter same passphrase again:
Your identification has been saved in /home/gpadmin/.ssh/id_rsa.
Your public key has been saved in /home/gpadmin/.ssh/id_rsa.pub.
The key fingerprint is:
SHA256:TTr5ZrIMSa+VXwFJJiAoUPmYskWTR8DLiFpraX0+7dg gpadmin@test-cdh-01
The key s randomart image is:
+---[RSA 2048]----+
|ooo*o ... o      ||. B ..   + .     ||.= B      +      ||+ B .    = .     ||.= +  . S . .    ||o = ...o +   .   || o   oo.= + .    ||      oB.* .     ||      ooE .      |
+----[SHA256]-----+
[gpadmin@test-cdh-01 opt]$
[gpadmin@test-cdh-01 opt]$ cd ~
[gpadmin@test-cdh-01 .ssh]$ pwd
/home/gpadmin/.ssh
# 配置 test-cdh-01 免密登录
[gpadmin@test-cdh-01 .ssh]$ ssh-copy-id hnode1
gpadmin@test-cdh-01 password: #(输入gpadmin用户的密码)...# 尝试免密登录 test-cdh-01
[gpadmin@test-cdh-01 .ssh]$ ssh hnode1 
# 登录成功
Last login: Fri May 29 13:44:15 2020
# 退出登录
[gpadmin@test-cdh-01 ~]$ exit
登出
Connection to test-cdh-01 closed.
# 配置 hnode2 免密登录
ssh-copy-id test-cdh-02
# 配置 hnode3 免密登录
ssh-copy-id test-cdh-03

4.3 创建节点列表文件

在master节点配置

[gpadmin@test-cdh-01 .ssh]$ mkdir /home/gpadmin/conf
[gpadmin@test-cdh-01 .ssh]$
[gpadmin@test-cdh-01 .ssh]$
[gpadmin@test-cdh-01 .ssh]$ cd /home/gpadmin/conf
[gpadmin@test-cdh-01 conf]$ vim hostlist #添加 GreenPlum 集群的所有节点
[gpadmin@hnode1 conf]$ cat hostlist
test-cdh-01
test-cdh-02
test-cdh-03
[gpadmin@test-cdh-01 conf]$ vim seg_hosts #添加 GreenPlum 集群的所有segment节点
[gpadmin@hnode1 conf]$ cat seg_hosts
test-cdh-02
test-cdh-03

4.4 使用gpssh-exkeys连接所有服务器

在 master 节点使用 gpssh-exkeys 将所有机器的通道打开，这样在每台机器之间跳转，就不需要输入密码。

[gpadmin@test-cdh-01 conf]$ pwd
/home/gpadmin/conf
[gpadmin@test-cdh-01 conf]$ source /usr/local/greenplum-db/greenplum_path.sh
[gpadmin@test-cdh-01 conf]$ gpssh-exkeys -f hostlist
[STEP 1 of 5] create local ID and authorize on local host
  ... /home/gpadmin/.ssh/id_rsa file exists ... key generation skipped

[STEP 2 of 5] keyscan all hosts and update known_hosts file

[STEP 3 of 5] retrieving credentials from remote hosts
  ... send to test-cdh-02
  ... send to test-cdh-03

[STEP 4 of 5] determine common authentication file content

[STEP 5 of 5] copy authentication files to all remote hosts
  ... finished key exchange with test-cdh-02
  ... finished key exchange with test-cdh-03

[INFO] completed successfully
# 测试批量操作
[gpadmin@test-cdh-01 conf]$ gpssh -f hostlist
=> pwd
[test-cdh-01] /home/gpadmin
[test-cdh-02] /home/gpadmin
[test-cdh-03] /home/gpadmin
=> quit # 退出

4.5 创建数据存储空间

[gpadmin@test-cdh-01 ~]$ mkdir -p /data/gpadmin/data/master
#如果提示没有权限，先用root用户创建，然后使用chown -r gpadmin:gpadmin /gpadmin/data/master变更文件夹用户和组

[gpadmin@test-cdh-01 ~]$ source /usr/local/greenplum-db/greenplum_path.sh
[gpadmin@test-cdh-01 ~]$ gpssh -f /home/gpadmin/conf/seg_hosts
=> mkdir -p /data/gpadmin/data/primary
[test-cdh-02]
[test-cdh-03]
=> mkdir -p /data/gpadmin/data/mirror
[test-cdh-02]
[test-cdh-03]
=> ll /data/gpadmin/data/
[test-cdh-02] 总用量 0
[test-cdh-02] drwxrwxr-x 2 gpadmin gpadmin 6 5月  29 18:30 mirror
[test-cdh-02] drwxrwxr-x 2 gpadmin gpadmin 6 5月  29 18:30 primary
[test-cdh-03] 总用量 0
[test-cdh-03] drwxrwxr-x 2 gpadmin gpadmin 6 5月  29 18:30 mirror
[test-cdh-03] drwxrwxr-x 2 gpadmin gpadmin 6 5月  29 18:30 primary
=> quit

4.6配置.bash_profile环境变量

在master节点配置

[gpadmin@hnode1 ~]$ vim /home/gpadmin/.bash_profile

# 添加下面内容source /usr/local/greenplum-db/greenplum_path.sh 
# 这个路径对应安装 gp 的目录export MASTER_DATA_DIRECTORY=/data/gpadmin/data/master/gpseg-1
export PGPORT=5432
export PGUSER=gpadmin
export PGDATABASE=Greenplum

# 将刚刚配置的加到环境变量中
PATH=$PATH:$HOME/.local/bin:$HOME/bin:$MASTER_DATA_DIRECTORY

[gpadmin@hnode1 ~]$ source /home/gpadmgpin/.bash_profile
[gpadmin@hnode1 ~]$ echo $GPHOME
/usr/local/greenplum-db/.

如果初始化的时候提示MASTER_DATA_DIRECTORY不存在，则将MASTER_DATA_DIRECTORY=/data/gpadmin/data/master/gpseg-1添加到 /home/gpadmin/.bashrc中

[gpadmin@test-cdh-01 ~]$ pwd
/home/gpadmin
[gpadmin@test-cdh-01 ~]$ cat .bashrc 
# .bashrc
MASTER_DATA_DIRECTORY=/data/gpadmin/data/master/gpseg-1
export MASTER_DATA_DIRECTORY
# Source global definitionsif [ -f /etc/bashrc ]; then
        . /etc/bashrc
fi

4.7 初始化配置文件

在管理节点配置
配置模板为 $GPHOME/docs/cli_help/gpconfigs/gpinitsystem_config，将文件cp到/home/gpadmin/gpconfigs下进行配置

[gpadmin@test-cdh-01 ~]$ mkdir /home/gpadmin/gpconfigs
[gpadmin@test-cdh-01 ~]$ cd gpconfigs/
[gpadmin@test-cdh-01 gpconfigs]$ pwd
/home/gpadmin/gpconfigs

# 创建 节点列表文件 hostfile_gpinitsystem
[gpadmin@test-cdh-01 gpconfigs]$ touch hostfile_gpinitsystem

#将segment节点名称添加到该文件内
[gpadmin@test-cdh-01 gpconfigs]$ cat /home/gpadmin/gpconfigs/hostfile_gpinitsystem
hnode2
hnode3

# 复制配置文件模板
[gpadmin@test-cdh-01 ~]$ cd /home/gpadmin/gpconfigs/
[gpadmin@test-cdh-01 gpconfigs]$ cp $GPHOME/docs/cli_help/gpconfigs/gpinitsystem_config . #文件cp到当前目录

# 修改配置
[gpadmin@test-cdh-01 gpconfigs]$ vim gpinitsystem_config
# 配置文件文件内容如下#数据库代号
ARRAY_NAME="Greenplum"#segment前缀
SEG_PREFIX=gpseg
#primary segment 起始的端口号，如果初始化的时候启动不起来，可能是端口冲突，需要修改端口
PORT_BASE=6000
#指定primary segment的数据目录,网上写的是多个相同目录，多个目录表示一台机器有多个segmentdeclare -a DATA_DIRECTORY=(/data/gpadmin/data/primary)
#master所在机器的host name
MASTER_HOSTNAME=test-cdh-01
#master的数据目录
MASTER_DIRECTORY=/data/gpadmin/data/master
#master的端口
MASTER_PORT=5432
#指定bash的版本
TRUSTED_SHELL=/usr/bin/ssh
#将日志写入磁盘的间隔，每个段文件通常 =16MB < 2 * CHECK_POINT_SEGMENTS + 1
CHECK_POINT_SEGMENTS=8
#字符集
ENCODING=UNICODE
#mirror segment 起始的端口号
MIRROR_PORT_BASE=7000
# mirror的数据目录，和主数据一样，一个对一个，多个对多个declare -a MIRROR_DATA_DIRECTORY=(/home/gpadmin/data/mirror)
MACHINE_LIST_FILE=/home/gpadmin/gpconfigs/hostfile_gpinitsystem

5. 初始化数据库

5.1 初始化数据库

在master节点执行

[gpadmin@test-cdh-01 gpconfigs]$ cd ~
[gpadmin@test-cdh-01 ~]$ pwd
/home/gpadmin
[gpadmin@test-cdh-01 ~]$ source /usr/local/greenplum-db/greenplum_path.sh
[gpadmin@test-cdh-01 ~]$ gpinitsystem -c gpconfigs/gpinitsystem_config -h gpconfigs/hostfile_gpinitsystem
...
Continue with Greenplum creation Yy|Nn (default=N): #变量写Y
...
20200... gpstart:hnode1:gpadmin-[INFO]:-   Successful segment starts
...
20200... gpinitsystem:hnode1:gpadmin-[INFO]:-Greenplum Database instance successfully created

# GreenPlum 初始化成功

5.2 重新初始化

如果初始化失败。可以查看 less /home/gpadmin/gpAdminLogs/gpinitsystem_{初始化日期}.log来看是失败原因，解决后重新初始化。但重新初始化之前需要将失败的实例删掉，操作如下

[gpadmin@test-cdh-01 ~]$ rm -rf /home/gpadmin/data/master/gpseg-1

6. 连接数据库进行测试

gp常用命令参考

# 连接GreenPlum
[gpadmin@test-cdh-01 ~]$ psql -d postgres
psql (9.4.24)
Type "help" for help.

postgres=# select version();
                        version
----------------------------------------------------------
 PostgreSQL 9.4.24 (Greenplum Database 6.13.0 build ......
 (1 row)

# 查看现有database
postgres=# l
                               List of databases
| Name    |  Owner  | Encoding |  Collate   |   Ctype    |  Access privileges|
|-----------|---------|----------|------------|------------|---------------------|
 postgres  | gpadmin | UTF8     | en_US.utf8 | en_US.utf8 |
 template0 | gpadmin | UTF8     | en_US.utf8 | en_US.utf8 | =c/gpadmin         +
           |         |          |            |                               | gpadmin=CTc/gpadmin
 template1 | gpadmin | UTF8     | en_US.utf8 | en_US.utf8 | =c/gpadmin         +
           |         |          |            |            | gpadmin=CTc/gpadmin
(3 rows)

# 新建database
postgres=# create database test;
CREATE DATABASE

# 查看 database 是否创建成功
postgres=# l
                               List of databases
   Name    |  Owner  | Encoding |  Collate   |   Ctype    |  Access privileges
-----------+---------+----------+------------+------------+---------------------
 postgres  | gpadmin | UTF8     | en_US.utf8 | en_US.utf8 |
 template0 | gpadmin | UTF8     | en_US.utf8 | en_US.utf8 | =c/gpadmin         +
           |         |          |            |            | gpadmin=CTc/gpadmin
 template1 | gpadmin | UTF8     | en_US.utf8 | en_US.utf8 | =c/gpadmin         +
           |         |          |            |            | gpadmin=CTc/gpadmin
 test      | gpadmin | UTF8     | en_US.utf8 | en_US.utf8 |(4 rows)

# 退出命令行
postgres=# q
[gpadmin@test-cdh-01 ~]$

7.开启远程登陆

7.1 为gp内的gpadmin用户创建密码

[gpadmin@test-cdh-01 ~]$ psql -d postgres
psql (9.4.24)
Type "help" for help.

postgres=#
postgres=#
postgres=# alter role gpadmin with password 'gpadmin';
ALTER ROLE
postgres=#

7.2 修改授权文件

[gpadmin@test-cdh-01 ~]$ vim /home/gpadmin/data/master/gpseg-1/pg_hba.conf
#在末尾追加如下内容

host     all         gpadmin         196.1.121.1/32   trust

# 重启 GreenPlum
[gpadmin@test-cdh-01 ~]$ gpstop -M fast -a
[gpadmin@test-cdh-01 ~]$ gpstart -a

配置说明

连接方式	允许连接的数据库	连接的用户	要连接gp的主机IP	认证方式
host	all	gpadmin	196.1.121.1/32	trust

8. 常用命令与日志

说明	命令
启动	gpstart
关闭	gpstop
状态	gpstate
日志地址	/home/gpadmin/gpAdminLogs
日志地址	/home/gpadmin/data/mirror/gpseg1/pg_log
日志地址	/home/gpadmin/data/primary/gpseg0/pg_log