mongodb sharding集群配置

环境：

192.168.10.91:[10000-10002] mongodb-shard-1

192.168.10.92:[10000-10002] mongodb-shard-2

192.168.10.93:[10000-10002] mongodb-shard-3

192.168.10.93:20000 configs

192.168.10.94:20000 configs

192.168.10.95:20000 configs

192.168.10.94:30000 mongos

#三台mongodb-shard服务器
# mkdir -p /data/mongodb/{shard1,shard2,shard3}

# mongod --shardsvr --replSet shard1 --port 10000 --dbpath /data/mongodb/shard1 --oplogSize 100 --logpath /data/mongodb/shard1/shard1.log --logappend --fork

# mongod --shardsvr --replSet shard2 --port 10001 --dbpath /data/mongodb/shard2 --oplogSize 100 --logpath /data/mongodb/shard2/shard2.log --logappend --fork

# mongod --shardsvr --replSet shard3 --port 10002 --dbpath /data/mongodb/shard3 --oplogSize 100 --logpath /data/mongodb/shard3/shard3.log --logappend --fork

#### 配置shard1
# mongo --port 10000
MongoDB shell version: 2.4.14
connecting to: 127.0.0.1:10000/test
Welcome to the MongoDB shell.
For interactive help, type "help".
For more comprehensive documentation, see http://docs.mongodb.org/ Questions? Try the support group http://groups.google.com/group/mongodb-user > config={_id:'shard1',members:[{_id:0,host:'192.168.10.91:10000'},{_id:1,host:'192.168.10.92:10000'},{_id:2,host:'192.168.10.93:10000'}]}
{
"_id" : "shard1",
"members" : [
{
"_id" : 0,
"host" : "192.168.10.91:10000"
},
{
"_id" : 1,
"host" : "192.168.10.92:10000"
},
{
"_id" : 2,
"host" : "192.168.10.93:10000"
}
]
}
> rs.initiate(config)
{
"info" : "Config now saved locally.  Should come online in about a minute.",
"ok" : 1
}
> rs.status()
{
"set" : "shard1",
"date" : ISODate("2016-04-05T10:08:08Z"),
"myState" : 1,
"members" : [
{
"_id" : 0,
"name" : "192.168.10.91:10000",
"health" : 1,
"state" : 1,
"stateStr" : "PRIMARY",
"uptime" : 623,
"optime" : Timestamp(1459850855, 1),
"optimeDate" : ISODate("2016-04-05T10:07:35Z"),
"self" : true
},
{
"_id" : 1,
"name" : "192.168.10.92:10000",
"health" : 1,
"state" : 2,
"stateStr" : "SECONDARY",
"uptime" : 32,
"optime" : Timestamp(1459850855, 1),
"optimeDate" : ISODate("2016-04-05T10:07:35Z"),
"lastHeartbeat" : ISODate("2016-04-05T10:08:08Z"),
"lastHeartbeatRecv" : ISODate("2016-04-05T10:08:07Z"),
"pingMs" : 0,
"syncingTo" : "192.168.10.91:10000"
},
{
"_id" : 2,
"name" : "192.168.10.93:10000",
"health" : 1,
"state" : 2,
"stateStr" : "SECONDARY",
"uptime" : 32,
"optime" : Timestamp(1459850855, 1),
"optimeDate" : ISODate("2016-04-05T10:07:35Z"),
"lastHeartbeat" : ISODate("2016-04-05T10:08:08Z"),
"lastHeartbeatRecv" : ISODate("2016-04-05T10:08:08Z"),
"pingMs" : 0,
"syncingTo" : "192.168.10.91:10000"
}
],
"ok" : 1
}

同理配置shard2 和shard3

# mongo --port 10001
MongoDB shell version: 2.4.14
connecting to: 127.0.0.1:10001/test
> config={_id:'shard2',members:[{_id:0,host:'192.168.10.91:10001'},{_id:1,host:'192.168.10.92:10001'},{_id:2,host:'192.168.10.93:10001'}]}
{
"_id" : "shard2",
"members" : [
{
"_id" : 0,
"host" : "192.168.10.91:10001"
},
{
"_id" : 1,
"host" : "192.168.10.92:10001"
},
{
"_id" : 2,
"host" : "192.168.10.93:10001"
}
]
}
> rs.initiate(config)
{
"info" : "Config now saved locally.  Should come online in about a minute.",
"ok" : 1
}

# mongo --port 10002
MongoDB shell version: 2.4.14
connecting to: 127.0.0.1:10002/test
> config={_id:'shard3',members:[{_id:0,host:'192.168.10.91:10002'},{_id:1,host:'192.168.10.92:10002'},{_id:2,host:'192.168.10.93:10002'}]}
{
"_id" : "shard3",
"members" : [
{
"_id" : 0,
"host" : "192.168.10.91:10002"
},
{
"_id" : 1,
"host" : "192.168.10.92:10002"
},
{
"_id" : 2,
"host" : "192.168.10.93:10002"
}
]
}

####配置configs
# mkdir -p /data/mongodb/config1
# mongod --configsvr --dbpath /data/mongodb/config1 --port 20000 --logpath /data/mongodb/config1/config1.log --logappend --fork
\

####配置mongos
# mongos --configdb 192.168.10.93:20000,192.168.10.94:20000,192.168.10.95:20000 --port 30000 --chunkSize 1 --logpath /data/mongodb/mongos.log --logappend --fork

# mongo --port 30000
MongoDB shell version: 2.4.14
connecting to: 127.0.0.1:30000/test
Welcome to the MongoDB shell.
For interactive help, type "help".
For more comprehensive documentation, see http://docs.mongodb.org/ Questions? Try the support group http://groups.google.com/group/mongodb-user mongos> show dbs
admin   (empty)
config  0.046875GB
mongos> use admin
switched to db admin
mongos> db.runCommand({addshard:'shard1/192.168.10.91:10000,192.168.10.92:10000,192.168.10.93:10000'})     #依次添加shard
{ "shardAdded" : "shard1", "ok" : 1 }
mongos> db.runCommand({addshard:'shard2/192.168.10.91:10001,192.168.10.92:10001,192.168.10.93:10001'})
{ "shardAdded" : "shard2", "ok" : 1 }
mongos> db.runCommand({addshard:'shard3/192.168.10.91:10002,192.168.10.92:10002,192.168.10.93:10002'})
{ "shardAdded" : "shard3", "ok" : 1 }
mongos> db.runCommand({listshards:1})      #列出shard
{
"shards" : [
{
"_id" : "shard1",
"host" : "shard1/192.168.10.91:10000,192.168.10.92:10000,192.168.10.93:10000"
},
{
"_id" : "shard2",
"host" : "shard2/192.168.10.91:10001,192.168.10.92:10001,192.168.10.93:10001"
},
{
"_id" : "shard3",
"host" : "shard3/192.168.10.91:10002,192.168.10.92:10002,192.168.10.93:10002"
}
],
"ok" : 1
}
mongos> db.runCommand({enablesharding:"testdb"})     #激活testdb的分片
{ "ok" : 1 }
mongos> db.runCommand({shardcollection:"testdb.table1",key:{_id:1}})    #激活testdb中tables1的分片，只有_id可为唯一索引
{ "collectionsharded" : "testdb.table1", "ok" : 1 }
mongos> printShardingStatus()         #查看分片信息
--- Sharding Status ---
sharding version: {
"_id" : 1,
"version" : 3,
"minCompatibleVersion" : 3,
"currentVersion" : 4,
"clusterId" : ObjectId("570394b40d1b67f7f38c434a")
}
shards:
{  "_id" : "shard1",  "host" : "shard1/192.168.10.91:10000,192.168.10.92:10000,192.168.10.93:10000" }
{  "_id" : "shard2",  "host" : "shard2/192.168.10.91:10001,192.168.10.92:10001,192.168.10.93:10001" }
{  "_id" : "shard3",  "host" : "shard3/192.168.10.91:10002,192.168.10.92:10002,192.168.10.93:10002" }
databases:
{  "_id" : "admin",  "partitioned" : false,  "primary" : "config" }
{  "_id" : "test",  "partitioned" : false,  "primary" : "shard3" }
{  "_id" : "testdb",  "partitioned" : true,  "primary" : "shard1" }
testdb.table1
shard key: { "_id" : 1 }
chunks:
shard1  1
{ "_id" : { "$minKey" : 1 } } -->> { "_id" : { "$maxKey" : 1 } } on : shard1 Timestamp(1, 0)

####插入数据测试分片

mongos> use testdb
switched to db testdb
mongos> for (var i=1;i<=500000;i++) db.table1.save({name:'test',id:123})
mongos> db.table1.stats()
{
"sharded" : true,
"ns" : "testdb.table1",
"count" : 518405,
"numExtents" : 19,
"size" : 29030680,
"storageSize" : 51773440,
"totalIndexSize" : 16858912,
"indexSizes" : {
"_id_" : 16858912
},
"avgObjSize" : 56,
"nindexes" : 1,
"nchunks" : 6,
"shards" : {
"shard1" : {
"ns" : "testdb.table1",
"count" : 10440,
"size" : 584640,
"avgObjSize" : 56,
"storageSize" : 2793472,
"numExtents" : 5,
"nindexes" : 1,
"lastExtentSize" : 2097152,
"paddingFactor" : 1,
"systemFlags" : 1,
"userFlags" : 0,
"totalIndexSize" : 351568,
"indexSizes" : {
"_id_" : 351568
},
"ok" : 1
},
"shard2" : {
"ns" : "testdb.table1",
"count" : 41362,
"size" : 2316272,
"avgObjSize" : 56,
"storageSize" : 11182080,
"numExtents" : 6,
"nindexes" : 1,
"lastExtentSize" : 8388608,
"paddingFactor" : 1,
"systemFlags" : 1,
"userFlags" : 0,
"totalIndexSize" : 1349040,
"indexSizes" : {
"_id_" : 1349040
},
"ok" : 1
},
"shard3" : {
"ns" : "testdb.table1",
"count" : 466603,
"size" : 26129768,
"avgObjSize" : 56,
"storageSize" : 37797888,
"numExtents" : 8,
"nindexes" : 1,
"lastExtentSize" : 15290368,
"paddingFactor" : 1,
"systemFlags" : 1,
"userFlags" : 0,
"totalIndexSize" : 15158304,
"indexSizes" : {
"_id_" : 15158304
},
"ok" : 1
}
},
"ok" : 1
}
mongos> printShardingStatus()
--- Sharding Status ---
sharding version: {
"_id" : 1,
"version" : 3,
"minCompatibleVersion" : 3,
"currentVersion" : 4,
"clusterId" : ObjectId("570394b40d1b67f7f38c434a")
}
shards:
{  "_id" : "shard1",  "host" : "shard1/192.168.10.91:10000,192.168.10.92:10000,192.168.10.93:10000" }
{  "_id" : "shard2",  "host" : "shard2/192.168.10.91:10001,192.168.10.92:10001,192.168.10.93:10001" }
{  "_id" : "shard3",  "host" : "shard3/192.168.10.91:10002,192.168.10.92:10002,192.168.10.93:10002" }
databases:
{  "_id" : "admin",  "partitioned" : false,  "primary" : "config" }
{  "_id" : "test",  "partitioned" : false,  "primary" : "shard3" }
{  "_id" : "testdb",  "partitioned" : true,  "primary" : "shard1" }
testdb.table1
shard key: { "_id" : 1 }
chunks:
shard1  2
shard2  2
shard3  2
{ "_id" : { "$minKey" : 1 } } -->> { "_id" : ObjectId("57039cd208bca3c16785e081") } on : shard1 Timestamp(2, 1)
{ "_id" : ObjectId("57039cd208bca3c16785e081") } -->> { "_id" : ObjectId("57039d9708bca3c167860949") } on : shard1 Timestamp(1, 3)
{ "_id" : ObjectId("57039d9708bca3c167860949") } -->> { "_id" : ObjectId("57039d9e08bca3c167865f9e") } on : shard2 Timestamp(3, 1)
{ "_id" : ObjectId("57039d9e08bca3c167865f9e") } -->> { "_id" : ObjectId("57039da408bca3c16786aadb") } on : shard2 Timestamp(4, 0)
{ "_id" : ObjectId("57039da408bca3c16786aadb") } -->> { "_id" : ObjectId("57039da508bca3c16786f705") } on : shard3 Timestamp(4, 1)
{ "_id" : ObjectId("57039da508bca3c16786f705") } -->> { "_id" : { "$maxKey" : 1 } } on : shard3 Timestamp(3, 5)

####模拟集群故障

192.168.10.91：停止shard1的primary

[root@ops1 gcc-5.3.0]# ps aux|grep mon
root     13431  0.3  2.5 1365448 49308 ?       Sl   17:57   0:51 mongod --shardsvr --replSet shard1 --port 10000 --dbpath /data/mongodb/shard1 --oplogSize 100 --logpath /data/mongodb/shard1/shard1.log --logappend --fork
root     13477  0.4  2.9 1374716 56892 ?       Sl   17:58   0:59 mongod --shardsvr --replSet shard2 --port 10001 --dbpath /data/mongodb/shard2 --oplogSize 100 --logpath /data/mongodb/shard2/shard2.log --logappend --fork
root     13535  0.5  8.2 1370600 158352 ?      Sl   17:58   1:24 mongod --shardsvr --replSet shard3 --port 10002 --dbpath /data/mongodb/shard3 --oplogSize 100 --logpath /data/mongodb/shard3/shard3.log --logappend --fork
root     22462  0.0  0.0 103248   872 pts/1    S+   22:04   0:00 grep mon
[root@ops1 gcc-5.3.0]# kill -9 13431

192.168.10.92：查看shard1集群的状态,发现10.92已成为primary

# mongo --port 10000
MongoDB shell version: 2.4.14
connecting to: 127.0.0.1:10000/test
Welcome to the MongoDB shell.
For interactive help, type "help".
For more comprehensive documentation, see http://docs.mongodb.org/ Questions? Try the support group http://groups.google.com/group/mongodb-user shard1:PRIMARY> rs.status()
{
"set" : "shard1",
"date" : ISODate("2016-04-05T14:06:40Z"),
"myState" : 1,
"members" : [
{
"_id" : 0,
"name" : "192.168.10.91:10000",
"health" : 0,
"state" : 8,
"stateStr" : "(not reachable/healthy)",
"uptime" : 0,
"optime" : Timestamp(1459854751, 1),
"optimeDate" : ISODate("2016-04-05T11:12:31Z"),
"lastHeartbeat" : ISODate("2016-04-05T14:06:38Z"),
"lastHeartbeatRecv" : ISODate("2016-04-05T14:05:33Z"),
"pingMs" : 0
},
{
"_id" : 1,
"name" : "192.168.10.92:10000",
"health" : 1,
"state" : 1,
"stateStr" : "PRIMARY",
"uptime" : 14850,
"optime" : Timestamp(1459854751, 1),
"optimeDate" : ISODate("2016-04-05T11:12:31Z"),
"self" : true
},
{
"_id" : 2,
"name" : "192.168.10.93:10000",
"health" : 1,
"state" : 2,
"stateStr" : "SECONDARY",
"uptime" : 14339,
"optime" : Timestamp(1459854751, 1),
"optimeDate" : ISODate("2016-04-05T11:12:31Z"),
"lastHeartbeat" : ISODate("2016-04-05T14:06:38Z"),
"lastHeartbeatRecv" : ISODate("2016-04-05T14:06:39Z"),
"pingMs" : 0,
"lastHeartbeatMessage" : "syncing to: 192.168.10.92:10000",
"syncingTo" : "192.168.10.92:10000"
}
],
"ok" : 1
}
shard1:PRIMARY>

恢复10.91上的shard1
# mongod --shardsvr --replSet shard1 --port 10000 --dbpath /data/mongodb/shard1 --oplogSize 100 --logpath /data/mongodb/shard1/shard1.log --logappend --fork

查看集群状态，10.91恢复成为slave

shard1:PRIMARY> rs.status()
{
"set" : "shard1",
"date" : ISODate("2016-04-05T14:08:48Z"),
"myState" : 1,
"members" : [
{
"_id" : 0,
"name" : "192.168.10.91:10000",
"health" : 1,
"state" : 2,
"stateStr" : "SECONDARY",
"uptime" : 11,
"optime" : Timestamp(1459854751, 1),
"optimeDate" : ISODate("2016-04-05T11:12:31Z"),
"lastHeartbeat" : ISODate("2016-04-05T14:08:47Z"),
"lastHeartbeatRecv" : ISODate("2016-04-05T14:08:46Z"),
"pingMs" : 0,
"lastHeartbeatMessage" : "syncing to: 192.168.10.92:10000",
"syncingTo" : "192.168.10.92:10000"
},
{
"_id" : 1,
"name" : "192.168.10.92:10000",
"health" : 1,
"state" : 1,
"stateStr" : "PRIMARY",
"uptime" : 14978,
"optime" : Timestamp(1459854751, 1),
"optimeDate" : ISODate("2016-04-05T11:12:31Z"),
"self" : true
},
{
"_id" : 2,
"name" : "192.168.10.93:10000",
"health" : 1,
"state" : 2,
"stateStr" : "SECONDARY",
"uptime" : 14467,
"optime" : Timestamp(1459854751, 1),
"optimeDate" : ISODate("2016-04-05T11:12:31Z"),
"lastHeartbeat" : ISODate("2016-04-05T14:08:46Z"),
"lastHeartbeatRecv" : ISODate("2016-04-05T14:08:47Z"),
"pingMs" : 0,
"syncingTo" : "192.168.10.92:10000"
}
],
"ok" : 1
}