Postgres-XL-9.5 集群部署测试

一、部署环境
集群环境如下
CentOS6.6_x86_64
Postgres-XL 9.5

主机名	IP	角色	端口	nodename	数据目录
pgxlsrv1	192.168.1.41	GTM	6666	gtm	$PGHOME/data/gtm
Coordinator	5301	coord1	$PGHOME/data/coord1
Coordinator	5302	coord2	$PGHOME/data/coord2
pgxlsrv2	192.168.1.42	Datanode	5401	dn1	$PGHOME/data/dn1
Datanode	5402	dn2	$PGHOME/data/dn2
pgxlsrv3	192.168.1.43	Datanode	5401	dn3	$PGHOME/data/dn3
Datanode	5402	dn4	$PGHOME/data/dn4

pgxlsrv1作为gtm与主节点Coordinator协调器
pgxlsrv2，pgxlsrv3 作为数据节点存储数据

二、服务器配置
1、环境变量 (3个节点)
export PGHOME=/home/postgres/pgxl9.5
export LD_LIBRARY_PATH=$PGHOME/lib:$LD_LIBRARY_PATH
export PATH=$PGHOME/bin:$PATH

2、安装依赖软件包 (3个节点)
yum -y install openssh-clients
yum -y install flex bison readline-devel zlib-devel openjade docbook-style-dsssl
yum -y install gcc

3、配置无密钥登录
------- root 用户 -----------------------------------------
--- pgxlsrv1 节点
[root @pgxlsrv1 ~]# ssh-keygen
[root @pgxlsrv1 ~]# ssh-copy-id -i ~/.ssh/id_rsa.pub root@pgxlsrv1
[root @pgxlsrv1 ~]# ssh-copy-id -i ~/.ssh/id_rsa.pub root@pgxlsrv2
[root @pgxlsrv1 ~]# ssh-copy-id -i ~/.ssh/id_rsa.pub root@pgxlsrv3

--- pgxlsrv2 节点
[root @pgxlsrv2 ~]# ssh-keygen
[root @pgxlsrv2 ~]# ssh-copy-id -i ~/.ssh/id_rsa.pub root@pgxlsrv2
[root @pgxlsrv2 ~]# ssh-copy-id -i ~/.ssh/id_rsa.pub root@pgxlsrv1
[root @pgxlsrv2 ~]# ssh-copy-id -i ~/.ssh/id_rsa.pub root@pgxlsrv3

--- pgxlsrv3 节点
[root @pgxlsrv3 ~]# ssh-keygen
[root @pgxlsrv3 ~]# ssh-copy-id -i ~/.ssh/id_rsa.pub root@pgxlsrv3
[root @pgxlsrv3 ~]# ssh-copy-id -i ~/.ssh/id_rsa.pub root@pgxlsrv1
[root @pgxlsrv3 ~]# ssh-copy-id -i ~/.ssh/id_rsa.pub root@pgxlsrv2

-------- 用户 postgres ------------------------------------
--- pgxlsrv1 节点
[postgres @pgxlsrv1 ~]$ ssh-keygen
[postgres @pgxlsrv1 ~]$ ssh-copy-id -i ~/.ssh/id_rsa.pub postgres@pgxlsrv1
[postgres @pgxlsrv1 ~]$ ssh-copy-id -i ~/.ssh/id_rsa.pub postgres@pgxlsrv2
[postgres @pgxlsrv1 ~]$ ssh-copy-id -i ~/.ssh/id_rsa.pub postgres@pgxlsrv3

--- pgxlsrv2 节点
[postgres @pgxlsrv2 ~]$ ssh-keygen
[postgres @pgxlsrv2 ~]$ ssh-copy-id -i ~/.ssh/id_rsa.pub postgres@pgxlsrv2
[postgres @pgxlsrv2 ~]$ ssh-copy-id -i ~/.ssh/id_rsa.pub postgres@pgxlsrv1
[postgres @pgxlsrv2 ~]$ ssh-copy-id -i ~/.ssh/id_rsa.pub postgres@pgxlsrv3

--- pgxlsrv3 节点
[postgres @pgxlsrv3 ~]$ ssh-keygen
[postgres @pgxlsrv3 ~]$ ssh-copy-id -i ~/.ssh/id_rsa.pub postgres@pgxlsrv3
[postgres @pgxlsrv3 ~]$ ssh-copy-id -i ~/.ssh/id_rsa.pub postgres@pgxlsrv1
[postgres @pgxlsrv3 ~]$ ssh-copy-id -i ~/.ssh/id_rsa.pub postgres@pgxlsrv2

三、 Postgres-XL安装
1、在3个节点分别安装postgrex-xl软件
-- pgxlsrv1 节点
[root @pgxlsrv1 ~]# cd /mytmp
[root @pgxlsrv1 mytmp]# tar -xzvf postgres-xl-9.5r1.5.tar.gz
[root @pgxlsrv1 mytmp]# chown -R postgres:postgres postgres-xl-9.5r1.5
[root @pgxlsrv1 mytmp]# su - postgres
[postgres @pgxlsrv1 ~]$ cd /mytmp/postgres-xl-9.5r1.5
[postgres @pgxlsrv1 postgres-xl-9.5r1.5]$ ./configure --prefix=/home/postgres/pgxl9.5
[postgres @pgxlsrv1 postgres-xl-9.5r1.5]$ make
[postgres @pgxlsrv1 postgres-xl-9.5r1.5]$ make install

-- pgxlsrv2 节点
[root @pgxlsrv2 ~]# cd /mytmp
[root @pgxlsrv2 mytmp]# tar -xzvf postgres-xl-9.5r1.5.tar.gz
[root @pgxlsrv2 mytmp]# chown -R postgres:postgres postgres-xl-9.5r1.5
[root @pgxlsrv2 mytmp]# su - postgres
[postgres @pgxlsrv2 ~]$ cd /mytmp/postgres-xl-9.5r1.5
[postgres @pgxlsrv2 postgres-xl-9.5r1.5]$ ./configure --prefix=/home/postgres/pgxl9.5
[postgres @pgxlsrv2 postgres-xl-9.5r1.5]$ make
[postgres @pgxlsrv2 postgres-xl-9.5r1.5]$ make install

-- pgxlsrv3 节点
[root @pgxlsrv3 ~]# cd /mytmp
[root @pgxlsrv3 mytmp]# tar -xzvf postgres-xl-9.5r1.5.tar.gz
[root @pgxlsrv3 mytmp]# chown -R postgres:postgres postgres-xl-9.5r1.5
[root @pgxlsrv3 mytmp]# su - postgres
[postgres @pgxlsrv3 ~]$ cd /mytmp/postgres-xl-9.5r1.5
[postgres @pgxlsrv3 postgres-xl-9.5r1.5]$ ./configure --prefix=/home/postgres/pgxl9.5
[postgres @pgxlsrv3 postgres-xl-9.5r1.5]$ make
[postgres @pgxlsrv3 postgres-xl-9.5r1.5]$ make install

2、创建 gtm、coordinator、datanode 相关目录
-- pgxlsrv1 节点
[postgres @pgxlsrv1 ~]$ cd $PGHOME
[postgres @pgxlsrv1 pgxl9.5]$ mkdir data
[postgres @pgxlsrv1 pgxl9.5]$ cd data
[postgres @pgxlsrv1 data]$ mkdir gtm
[postgres @pgxlsrv1 data]$ mkdir coord1
[postgres @pgxlsrv1 data]$ mkdir coord2

-- pgxlsrv2 节点
[postgres @pgxlsrv2 ~]$ cd $PGHOME
[postgres @pgxlsrv2 pgxl9.5]$ mkdir data
[postgres @pgxlsrv2 pgxl9.5]$ cd data
[postgres @pgxlsrv2 data]$ mkdir dn1
[postgres @pgxlsrv2 data]$ mkdir dn2

-- pgxlsrv3 节点
[postgres @pgxlsrv3 ~]$ cd $PGHOME
[postgres @pgxlsrv3 pgxl9.5]$ mkdir data
[postgres @pgxlsrv3 pgxl9.5]$ cd data
[postgres @pgxlsrv3 data]$ mkdir dn3
[postgres @pgxlsrv3 data]$ mkdir dn4

3、gtm、coordinator 初始化及配置
-- 初始化gtm
[postgres @pgxlsrv1 ~]$ initgtm -Z gtm -D /home/postgres/pgxl9.5/data/gtm

--初始化coord1
[postgres @pgxlsrv1 ~]$ initdb -D /home/postgres/pgxl9.5/data/coord1 --nodename coord1 -E UTF8 --locale=C -U postgres -W

--初始化coord2
[postgres @pgxlsrv1 ~]$ initdb -D /home/postgres/pgxl9.5/data/coord2 --nodename coord2 -E UTF8 --locale=C -U postgres -W

-- 配置gtm
[postgres @pgxlsrv1 ~]$ vi /home/postgres/pgxl9.5/data/gtm/gtm.conf
nodename = 'gtm'
listen_addresses = '*'
port =6666
startup = ACT

-- 配置 coord1、coord2
[postgres @pgxlsrv1 ~]$ vi /home/postgres/pgxl9.5/data/coord1/postgresql.conf
# - Connection Settings -
listen_addresses = '*'
port = 5301 #coord2这个端口改为5302
max_connections = 100

# DATA NODES AND CONNECTION POOLING
#----------------------------------
pooler_port = 6701 #coord2这个端口改为6702
max_pool_size = 100

# GTM CONNECTION
#--------------------------
gtm_host = '192.168.1.41' #即pgxlsrv1，也就是gtm所在的主机地址
gtm_port = 6666 #gtm配置中，gtm端口号配置为6666
pgxc_node_name = 'coord1'

------- 配置文件 pg_hba.conf
# "local" is for Unix domain socket connections only
local all all trust
# IPv4 local connections:
host all all 0.0.0.0/0 trust
# IPv6 local connections:
host all all ::1/128 trust

4、dn1、dn2、dn3、dn4 初始化及配置
---------- pgxlsrv2 节点 ----------------------------------
--dn1初始化
[postgres @pgxlsrv2 ~]$ initdb -D /home/postgres/pgxl9.5/data/dn1 --nodename dn1 -E UTF8 --locale=C -U postgres -W

--dn2初始化
[postgres @pgxlsrv2 ~]$ initdb -D /home/postgres/pgxl9.5/data/dn2 --nodename dn2 -E UTF8 --locale=C -U postgres -W

----------- pgxlsrv3 节点 ---------------------------------
--dn3初始化
[postgres @pgxlsrv3 ~]$ initdb -D /home/postgres/pgxl9.5/data/dn3 --nodename dn3 -E UTF8 --locale=C -U postgres -W

--dn4初始化
[postgres @pgxlsrv3 ~]$ initdb -D /home/postgres/pgxl9.5/data/dn4 --nodename dn4 -E UTF8 --locale=C -U postgres -W

---------- pgxlsrv2 节点 ---------------------------------
-- dn1 配置
[postgres @pgxlsrv2 ~]$ vi /home/postgres/pgxl9.5/data/dn1/postgresql.conf
CONNECTIONS AND AUTHENTICATION
#------------------------------------
listen_addresses = '*'
port =5401 #dn2，dn3,dn4配置文件分别改为5402、5401、5402
max_connections = 100

# DATA NODES AND CONNECTION POOLING
#----------------------------------------------
pooler_port = 6801 #dn2，dn3,dn4配置文件分别改为6802、6801、6802
#同一台机器要使用不同的端口，如dn1，dn2在同一台机器pgxlsrv2上分别是6801,6802
#dn3，dn4在pgxlsrv3上端口号也是6801,6802。就是同一机器端口号不同即可。
max_pool_size = 100

# GTM CONNECTION
#-----------------------------
gtm_host = '192.168.1.41' #gtm所在的pgxlsrv1的ip地址
gtm_port = 6666 #gtm端口号
pgxc_node_name = 'dn1'

5、启动Postgres-XL
--启动gtm (pgxlsrv1)
[postgres @pgxlsrv1 ~]$ gtm_ctl start -Z gtm -D /home/postgres/pgxl9.5/data/gtm

--启动datanode (pgxlsrv2)
[postgres @pgxlsrv2 ~]$ pg_ctl start -Z datanode -D /home/postgres/pgxl9.5/data/dn1
[postgres @pgxlsrv2 ~]$ pg_ctl start -Z datanode -D /home/postgres/pgxl9.5/data/dn2

--启动datanode (pgxlsrv3)
[postgres @pgxlsrv3 ~]$ pg_ctl start -Z datanode -D /home/postgres/pgxl9.5/data/dn3
[postgres @pgxlsrv3 ~]$ pg_ctl start -Z datanode -D /home/postgres/pgxl9.5/data/dn4

--启动coordinator (pgxlsrv1)
[postgres @pgxlsrv1 ~]$ pg_ctl start -Z coordinator -D /home/postgres/pgxl9.5/data/coord1
[postgres @pgxlsrv1 ~]$ pg_ctl start -Z coordinator -D /home/postgres/pgxl9.5/data/coord2

6、Postgres-XL集群配置
-- 集群配置 --
--配置coord1节点，以postgres用户下进入psql。coord1配置端口号是5301，直接以psql加端口号进入配置
[postgres@pgxlsrv1 ~]$ psql -p 5301
psql (PGXL 9.5r1.5, based on PG 9.5.6 (Postgres-XL 9.5r1.5))
Type "help" for help.

postgres=# select * from pgxc_node;

postgres=# alter node coord1 with (type=coordinator,host='192.168.1.41', port=5301);
postgres=# create node coord2 with (type=coordinator,host='192.168.1.41', port=5302);

postgres=# create node dn1 with (type=datanode, host='192.168.1.42',port=5401,primary,preferred);
postgres=# create node dn2 with (type=datanode, host='192.168.1.42',port=5402);
postgres=# create node dn3 with (type=datanode, host='192.168.1.43',port=5401);
postgres=# create node dn4 with (type=datanode, host='192.168.1.43',port=5402);

postgres=# select pgxc_pool_reload();
postgres=# select * from pgxc_node;

--配置coord2节点，以postgres用户下进入psql。coord2配置端口号是5302，直接以psql加端口号进入配置
[postgres@pgxlsrv1 ~]$ psql -p 5302
psql (PGXL 9.5r1.5, based on PG 9.5.6 (Postgres-XL 9.5r1.5))
Type "help" for help.

postgres=# select * from pgxc_node;
postgres=# alter node coord2 with (type=coordinator,host='192.168.1.41', port=5302);
postgres=# create node coord1 with (type=coordinator,host='192.168.1.41', port=5301);

postgres=# create node dn1 with (type=datanode, host='192.168.1.42',port=5401,primary,preferred);
postgres=# create node dn2 with (type=datanode, host='192.168.1.42',port=5402);
postgres=# create node dn3 with (type=datanode, host='192.168.1.43',port=5401);
postgres=# create node dn4 with (type=datanode, host='192.168.1.43',port=5402);

postgres=# select pgxc_pool_reload();
postgres=# select * from pgxc_node;

-- 配置数据节点(pgxlsrv2) --
--dn1
[postgres@pgxlsrv2 ~]$ psql -p 5401
psql (PGXL 9.5r1.5, based on PG 9.5.6 (Postgres-XL 9.5r1.5))
Type "help" for help.

postgres=# select * from pgxc_node;

postgres=# create node coord1 with (type=coordinator,host='192.168.1.41', port=5301);
postgres=# create node coord2 with (type=coordinator,host='192.168.1.41', port=5302);

postgres=# alter node dn1 with (type=datanode, host='192.168.1.42',port=5401,primary,preferred);
postgres=# create node dn2 with (type=datanode, host='192.168.1.42',port=5402);
postgres=# create node dn3 with (type=datanode, host='192.168.1.43',port=5401);
postgres=# create node dn4 with (type=datanode, host='192.168.1.43',port=5402);

postgres=# select pgxc_pool_reload();
postgres=# select * from pgxc_node;

--dn2
[postgres@pgxlsrv2 ~]$ psql -p 5402
psql (PGXL 9.5r1.5, based on PG 9.5.6 (Postgres-XL 9.5r1.5))
Type "help" for help.

postgres=# select * from pgxc_node;

postgres=# create node coord1 with (type=coordinator,host='192.168.1.41', port=5301);
postgres=# create node coord2 with (type=coordinator,host='192.168.1.41', port=5302);

postgres=# alter node dn2 with (type=datanode, host='192.168.1.42',port=5402);
postgres=# create node dn1 with (type=datanode, host='192.168.1.42',port=5401,primary,preferred);
postgres=# create node dn3 with (type=datanode, host='192.168.1.43',port=5401);
postgres=# create node dn4 with (type=datanode, host='192.168.1.43',port=5402);

postgres=# select pgxc_pool_reload();
postgres=# select * from pgxc_node;

-- 配置数据节点(pgxlsrv3) --
--dn3
[postgres@pgxlsrv3 ~]$ psql -p 5401
psql (PGXL 9.5r1.5, based on PG 9.5.6 (Postgres-XL 9.5r1.5))
Type "help" for help.

postgres=# select * from pgxc_node;

postgres=# create node coord1 with (type=coordinator,host='192.168.1.41', port=5301);
postgres=# create node coord2 with (type=coordinator,host='192.168.1.41', port=5302);

postgres=# alter node dn3 with (type=datanode, host='192.168.1.43',port=5401);
postgres=# create node dn1 with (type=datanode, host='192.168.1.42',port=5401,primary,preferred);
postgres=# create node dn2 with (type=datanode, host='192.168.1.42',port=5402);
postgres=# create node dn4 with (type=datanode, host='192.168.1.43',port=5402);

postgres=# select pgxc_pool_reload();
postgres=# select * from pgxc_node;

--dn4
[postgres@pgxlsrv3 ~]$ psql -p 5402
psql (PGXL 9.5r1.5, based on PG 9.5.6 (Postgres-XL 9.5r1.5))
Type "help" for help.

postgres=# select * from pgxc_node;

postgres=# create node coord1 with (type=coordinator,host='192.168.1.41', port=5301);
postgres=# create node coord2 with (type=coordinator,host='192.168.1.41', port=5302);

postgres=# alter node dn4 with (type=datanode, host='192.168.1.43',port=5402);
postgres=# create node dn1 with (type=datanode, host='192.168.1.42',port=5401,primary,preferred);
postgres=# create node dn2 with (type=datanode, host='192.168.1.42',port=5402);
postgres=# create node dn3 with (type=datanode, host='192.168.1.43',port=5401);

postgres=# select pgxc_pool_reload();
postgres=# select * from pgxc_node;

四、验证测试
1、 创建分布模式的测试表并插入数据
[postgres@pgxlsrv1]$ psql -p 5301
psql (PGXL 9.5r1.5, based on PG 9.5.6 (Postgres-XL 9.5r1.5))
Type "help" for help.
postgres=# create table t1(id integer, name varchar(20));
postgres=# insert into t1 values(1,'测试1');
postgres=# insert into t1 values(2,'测试2');
postgres=# insert into t1 values(3,'测试3');
postgres=# insert into t1 values(4,'测试4');
postgres=# insert into t1 values(5,'测试5');
postgres=# insert into t1 values(6,'测试6');
postgres=# insert into t1 values(7,'测试7');
postgres=# insert into t1 values(8,'测试8');

-- 插入记录验证 --
-- 在 coord1 和 coord2 上都能查询到所有记录
[postgres@pgxlsrv1]$ psql -p 5301
psql (PGXL 9.5r1.5, based on PG 9.5.6 (Postgres-XL 9.5r1.5))
Type "help" for help.
postgres=# select * from t1;
id | name
----+-------
5 | 测试5
6 | 测试6
8 | 测试8
3 | 测试3
4 | 测试4
7 | 测试7
1 | 测试1
2 | 测试2
(8 rows)

[postgres@pgxlsrv1]$ psql -p 5302
psql (PGXL 9.5r1.5, based on PG 9.5.6 (Postgres-XL 9.5r1.5))
Type "help" for help.
postgres=# select * from t1;
id | name
----+-------
5 | 测试5
6 | 测试6
8 | 测试8
3 | 测试3
4 | 测试4
7 | 测试7
1 | 测试1
2 | 测试2
(8 rows)

--在数据节点 dn1, dn2, dn3, dn4 分别查询
[postgres@pgxlsrv2]$ psql -p 5401
psql (PGXL 9.5r1.5, based on PG 9.5.6 (Postgres-XL 9.5r1.5))
Type "help" for help.
postgres=# select * from t1;
id | name
----+-------
5 | 测试5
6 | 测试6
8 | 测试8
(3 rows)

[postgres@pgxlsrv2]$ psql -p 5402
psql (PGXL 9.5r1.5, based on PG 9.5.6 (Postgres-XL 9.5r1.5))
Type "help" for help.
postgres=# select * from t1;
id | name
----+-------
(0 rows)

[postgres@pgxlsrv3]$ psql -p 5401
psql (PGXL 9.5r1.5, based on PG 9.5.6 (Postgres-XL 9.5r1.5))
Type "help" for help.
postgres=# select * from t1;
id | name
----+-------
1 | 测试1
2 | 测试2
(2 rows)

[postgres@pgxlsrv3]$ psql -p 5402
psql (PGXL 9.5r1.5, based on PG 9.5.6 (Postgres-XL 9.5r1.5))
Type "help" for help.
postgres=# select * from t1;
id | name
----+-------
3 | 测试3
4 | 测试4
7 | 测试7
(3 rows)

2、 创建复制模式的测试表并插入数据
[postgres@pgxlsrv1]$ psql -p 5301
psql (PGXL 9.5r1.5, based on PG 9.5.6 (Postgres-XL 9.5r1.5))
Type "help" for help.
postgres=# create table t2(id integer, name varchar(20)) distribute by replication;
postgres=# insert into t2 values(1,'测试1');
postgres=# insert into t2 values(2,'测试2');
postgres=# insert into t2 values(3,'测试3');
postgres=# insert into t2 values(4,'测试4');
postgres=# insert into t2 values(5,'测试5');
postgres=# insert into t2 values(6,'测试6');
postgres=# insert into t2 values(7,'测试7');
postgres=# insert into t2 values(8,'测试8');

-- 插入记录验证 --
-- 在 coord1 和 coord2 以及 dn1、dn2、dn3、dn4 上都能查询到所有记录

-- coord1 --
[postgres@pgxlsrv1]$ psql -p 5301
psql (PGXL 9.5r1.5, based on PG 9.5.6 (Postgres-XL 9.5r1.5))
Type "help" for help.
postgres=# select * from t2;
id | name
----+-------
1 | 测试1
2 | 测试2
3 | 测试3
4 | 测试4
5 | 测试5
6 | 测试6
7 | 测试7
8 | 测试8
(8 rows)

-- coord2 --
[postgres@pgxlsrv1]$ psql -p 5302
psql (PGXL 9.5r1.5, based on PG 9.5.6 (Postgres-XL 9.5r1.5))
Type "help" for help.
postgres=# select * from t2;
id | name
----+-------
1 | 测试1
2 | 测试2
3 | 测试3
4 | 测试4
5 | 测试5
6 | 测试6
7 | 测试7
8 | 测试8
(8 rows)

-- dn1 --
[postgres@pgxlsrv2]$ psql -p 5401
psql (PGXL 9.5r1.5, based on PG 9.5.6 (Postgres-XL 9.5r1.5))
Type "help" for help.
postgres=# select * from t2;
id | name
----+-------
1 | 测试1
2 | 测试2
3 | 测试3
4 | 测试4
5 | 测试5
6 | 测试6
7 | 测试7
8 | 测试8
(8 rows)

-- dn2 --
[postgres@pgxlsrv2]$ psql -p 5402
psql (PGXL 9.5r1.5, based on PG 9.5.6 (Postgres-XL 9.5r1.5))
Type "help" for help.
postgres=# select * from t2;
id | name
----+-------
1 | 测试1
2 | 测试2
3 | 测试3
4 | 测试4
5 | 测试5
6 | 测试6
7 | 测试7
8 | 测试8
(8 rows)

-- dn3 --
[postgres@pgxlsrv3]$ psql -p 5401
psql (PGXL 9.5r1.5, based on PG 9.5.6 (Postgres-XL 9.5r1.5))
Type "help" for help.
postgres=# select * from t2;
id | name
----+-------
1 | 测试1
2 | 测试2
3 | 测试3
4 | 测试4
5 | 测试5
6 | 测试6
7 | 测试7
8 | 测试8
(8 rows)

-- dn4 --
[postgres@pgxlsrv3]$ psql -p 5402
psql (PGXL 9.5r1.5, based on PG 9.5.6 (Postgres-XL 9.5r1.5))
Type "help" for help.
postgres=# select * from t2;
id | name
----+-------
1 | 测试1
2 | 测试2
3 | 测试3
4 | 测试4
5 | 测试5
6 | 测试6
7 | 测试7
8 | 测试8
(8 rows)

----------------------------- 测试完毕 -----------------------------------