ceph源码分析--monitor leader选举

本篇博客主要分为三个部分

1.概述

2.选举的时机

3.选举的过程

1.概述

monitor在运行的过程中，必须存在一个leader节点来。众多的更新操作都是由leader节点来完成，写的命令也会经由peon转发到leader节点进行。

leader的选择是根据rank值来的，rank值小的为leader，而rank值跟IP地址有关。

2.选举的时机

在ceph中有大致三处会引发选举

1.收到quorum exit/enter

2.收到选举消息

3.bootstrap之后

之所以这样排序是因为从简单到复杂。

1）收到quorum exit/enter命令

在这个过程中participating会被改变，导致该monitor不再参与选举或者参与选举

void Monitor::handle_command(MonOpRequestRef op)
{
····
else if (prefix == "quorum") {
string quorumcmd;
cmd_getval(g_ceph_context, cmdmap, "quorumcmd", quorumcmd);
//quorum exit发起选举
if (quorumcmd == "exit") {
start_election();
elector.stop_participating();
rs = "stopped responding to quorum, initiated new election";
r = 0;
}
//quorum enter发起选举
else if (quorumcmd == "enter") {
elector.start_participating();
start_election();
rs = "started responding to quorum, initiated new election";
r = 0;
}
····
}

2）收到选举消息

monitor收到选举消息后

void Monitor::dispatch_op(MonOpRequestRef op)
{
···
case MSG_MON_ELECTION:
op->set_type_election();
//check privileges here for simplicity
if (!op->get_session()->is_capable("mon", MON_CAP_X)) {
dout(0) << "MMonElection received from entity without enough caps!"
<< op->get_session()->caps << dendl;
break;
}
if (!is_probing() && !is_synchronizing()) {
elector.dispatch(op);
}
break;
···
}

调用了void Elector::dispatch(MonOpRequestRef op)

这时收到的肯定是OP_PROPOSE

void Elector::dispatch(MonOpRequestRef op)
{
···
case MMonElection::OP_PROPOSE:
handle_propose(op);
return;
···
}

进入void Elector::handle_propose(MonOpRequestRef op)

void Elector::handle_propose(MonOpRequestRef op)
{
···
//如果收到一个“旧”的epoch，但是它并不在quorum里，发起选举
if (m->epoch < epoch) {
// got an "old" propose,
if (epoch % 2 == 0 &&    // in a non-election cycle
mon->quorum.count(from) == 0) {  // from someone outside the quorum
// a mon just started up, call a new election so they can rejoin!
dout(5) << " got propose from old epoch, quorum is " << mon->quorum
<< ", " << m->get_source() << " must have just started" << dendl;
// we may be active; make sure we reset things in the monitor appropriately.
mon->start_election();
} else {
dout(5) << " ignoring old propose" << dendl;
return;
}
···
//如果自身的rank值比发消息的monitor rank值要小，这时候要分两种情况
//如果已经回复过别人，说明已经有更小的rank值，则不回复，若没有回复过别人则自身rank值较小应发起选举
if (mon->rank < from) {
// i would win over them.
if (leader_acked >= 0) {        // we already acked someone
assert(leader_acked < from);  // and they still win, of course
dout(5) << "no, we already acked " << leader_acked << dendl;
} else {
// wait, i should win!
if (!electing_me) {
mon->start_election();
}
}
}
···
}

3）bootstrap之后

在bootstrap中也有两种情况

第一种：单monitor集群

void Monitor::win_standalone_election()

void Monitor::win_standalone_election()
{
dout(1) << "win_standalone_election" << dendl;

// bump election epoch, in case the previous epoch included other
// monitors; we need to be able to make the distinction.
elector.init();
elector.advance_epoch();

rank = monmap->get_rank(name);
assert(rank == 0);
set<int> q;
q.insert(rank);

map<int,Metadata> metadata;
collect_metadata(&metadata[0]);

win_election(elector.get_epoch(), q,
CEPH_FEATURES_ALL,
ceph::features::mon::get_supported(),
metadata);
}

第二种：先probe其他monitor

博客中主要讲解第二种先probe其他monitor

void Monitor::bootstrap()
{
dout(10) << "bootstrap" << dendl;
····
//单monitor集群选举
// singleton monitor?
if (monmap->size() == 1 && rank == 0) {
win_standalone_election();
return;
}

reset_probe_timeout();

// i'm outside the quorum
if (monmap->contains(name))
outside_quorum.insert(name);

// probe monitors
dout(10) << "probing other monitors" << dendl;
for (unsigned i = 0; i < monmap->size(); i++) {
if ((int)i != rank)
messenger->send_message(new MMonProbe(monmap->fsid, MMonProbe::OP_PROBE, name, has_ever_joined),
monmap->get_inst(i));
}
for (set<entity_addr_t>::iterator p = extra_probe_peers.begin();
p != extra_probe_peers.end();
++p) {
if (*p != messenger->get_myaddr()) {
entity_inst_t i;
i.name = entity_name_t::MON(-1);
i.addr = *p;
messenger->send_message(new MMonProbe(monmap->fsid, MMonProbe::OP_PROBE, name, has_ever_joined), i);
}
}
}

接收方

void Monitor::handle_probe(MonOpRequestRef op)
{
MMonProbe *m = static_cast<MMonProbe*>(op->get_req());
dout(10) << "handle_probe " << *m << dendl;
//必须有相同的fsid
if (m->fsid != monmap->fsid) {
dout(0) << "handle_probe ignoring fsid " << m->fsid << " != " << monmap->fsid << dendl;
return;
}

switch (m->op) {
//处理probe
case MMonProbe::OP_PROBE:
handle_probe_probe(op);
break;

case MMonProbe::OP_REPLY:
handle_probe_reply(op);
break;

case MMonProbe::OP_MISSING_FEATURES:
derr << __func__ << " missing features, have " << CEPH_FEATURES_ALL
<< ", required " << m->required_features
<< ", missing " << (m->required_features & ~CEPH_FEATURES_ALL)
<< dendl;
break;
}
}

void Monitor::handle_probe_probe(MonOpRequestRef op)
{
···
if (!is_probing() && !is_synchronizing()) {
// If the probing mon is way ahead of us, we need to re-bootstrap.
// Normally we capture this case when we initially bootstrap, but
// it is possible we pass those checks (we overlap with
// quorum-to-be) but fail to join a quorum before it moves past
// us.  We need to be kicked back to bootstrap so we can
// synchonize, not keep calling elections.
//from端数据比自己新，重新bootstrap
if (paxos->get_version() + 1 < m->paxos_first_version) {
dout(1) << " peer " << m->get_source_addr() << " has first_committed "
<< "ahead of us, re-bootstrapping" << dendl;
bootstrap();
goto out;

}
}

MMonProbe *r;
//
r = new MMonProbe(monmap->fsid, MMonProbe::OP_REPLY, name, has_ever_joined);
r->name = name;
r->quorum = quorum;
monmap->encode(r->monmap_bl, m->get_connection()->get_features());
r->paxos_first_version = paxos->get_first_committed();
r->paxos_last_version = paxos->get_version();
m->get_connection()->send_message(r);

// did we discover a peer here?
if (!monmap->contains(m->get_source_addr())) {
dout(1) << " adding peer " << m->get_source_addr()
<< " to list of hints" << dendl;
extra_probe_peers.insert(m->get_source_addr());
}

out:
return;
}

处理probe_reply消息，在这期间会同步monitor的一些数据

void Monitor::handle_probe_reply(MonOpRequestRef op)
{
MMonProbe *m = static_cast<MMonProbe*>(op->get_req());
dout(10) << "handle_probe_reply " << m->get_source_inst() << *m << dendl;
dout(10) << " monmap is " << *monmap << dendl;

// discover name and addrs during probing or electing states.
if (!is_probing() && !is_electing()) {
return;
}

// newer map, or they've joined a quorum and we haven't?
bufferlist mybl;
monmap->encode(mybl, m->get_connection()->get_features());
// make sure it's actually different; the checks below err toward
// taking the other guy's map, which could cause us to loop.
if (!mybl.contents_equal(m->monmap_bl)) {
MonMap *newmap = new MonMap;
newmap->decode(m->monmap_bl);
if (m->has_ever_joined && (newmap->get_epoch() > monmap->get_epoch() ||
!has_ever_joined)) {
dout(10) << " got newer/committed monmap epoch " << newmap->get_epoch()
<< ", mine was " << monmap->get_epoch() << dendl;
delete newmap;
monmap->decode(m->monmap_bl);
//远端的monmap更新，重新进行bootstrap
bootstrap();
return;
}
delete newmap;
}

// rename peer?
string peer_name = monmap->get_name(m->get_source_addr());
if (monmap->get_epoch() == 0 && peer_name.compare(0, 7, "noname-") == 0) {
dout(10) << " renaming peer " << m->get_source_addr() << " "
<< peer_name << " -> " << m->name << " in my monmap"
<< dendl;
monmap->rename(peer_name, m->name);

if (is_electing()) {
bootstrap();
return;
}
} else {
dout(10) << " peer name is " << peer_name << dendl;
}

// new initial peer?
if (monmap->get_epoch() == 0 &&
monmap->contains(m->name) &&
monmap->get_addr(m->name).is_blank_ip()) {
dout(1) << " learned initial mon " << m->name << " addr " << m->get_source_addr() << dendl;
monmap->set_addr(m->name, m->get_source_addr());

bootstrap();
return;
}

// end discover phase
if (!is_probing()) {
return;
}

assert(paxos != NULL);

if (is_synchronizing()) {
dout(10) << " currently syncing" << dendl;
return;
}

entity_inst_t other = m->get_source_inst();

if (m->paxos_last_version < sync_last_committed_floor) {
dout(10) << " peer paxos versions [" << m->paxos_first_version
<< "," << m->paxos_last_version << "] < my sync_last_committed_floor "
<< sync_last_committed_floor << ", ignoring"
<< dendl;
} else {
if (paxos->get_version() < m->paxos_first_version &&
m->paxos_first_version > 1) {  // no need to sync if we're 0 and they start at 1.
dout(10) << " peer paxos first versions [" << m->paxos_first_version
<< "," << m->paxos_last_version << "]"
<< " vs my version " << paxos->get_version()
<< " (too far ahead)"
<< dendl;
cancel_probe_timeout();
sync_start(other, true);
return;
}
if (paxos->get_version() + g_conf->paxos_max_join_drift < m->paxos_last_version) {
dout(10) << " peer paxos last version " << m->paxos_last_version
<< " vs my version " << paxos->get_version()
<< " (too far ahead)"
<< dendl;
cancel_probe_timeout();
sync_start(other, false);
return;
}
}

// is there an existing quorum?
if (m->quorum.size()) {
dout(10) << " existing quorum " << m->quorum << dendl;

dout(10) << " peer paxos version " << m->paxos_last_version
<< " vs my version " << paxos->get_version()
<< " (ok)"
<< dendl;

if (monmap->contains(name) &&
!monmap->get_addr(name).is_blank_ip()) {
// i'm part of the cluster; just initiate a new election
//已经在monmap,直接发起选举
start_election();
} else {
dout(10) << " ready to join, but i'm not in the monmap or my addr is blank, trying to join" << dendl;
messenger->send_message(new MMonJoin(monmap->fsid, name, messenger->get_myaddr()),
monmap->get_inst(*m->quorum.begin()));
}
} else {
if (monmap->contains(m->name)) {
dout(10) << " mon." << m->name << " is outside the quorum" << dendl;
outside_quorum.insert(m->name);
} else {
dout(10) << " mostly ignoring mon." << m->name << ", not part of monmap" << dendl;
return;
}

unsigned need = monmap->size() / 2 + 1;
dout(10) << " outside_quorum now " << outside_quorum << ", need " << need << dendl;
if (outside_quorum.size() >= need) {
if (outside_quorum.count(name)) {
//集群必须有半数以上的monitor可用，才能发起选举
dout(10) << " that's enough to form a new quorum, calling election" << dendl;
start_election();
} else {
dout(10) << " that's enough to form a new quorum, but it does not include me; waiting" << dendl;
}
} else {
dout(10) << " that's not yet enough for a new quorum, waiting" << dendl;
}
}
}

经过这个些步骤之后，开始选举

3.选举的过程

选举从void Monitor::start_election()开始

void Monitor::start_election()
{
dout(10) << "start_election" << dendl;
wait_for_paxos_write();
_reset();
state = STATE_ELECTING;

logger->inc(l_mon_num_elections);
logger->inc(l_mon_election_call);

clog->info() << "mon." << name << " calling new monitor election";
elector.call_election();
}

void call_election()

void call_election() {
start();
}

void Elector::start()

void Elector::start()
{
//ceph quorum exit退出之后这个值会置成false
if (!participating) {
dout(0) << "not starting new election -- not participating" << dendl;
return;
}
dout(5) << "start -- can i be leader?" << dendl;
//清空ack的集合
acked_me.clear();
init();

// start by trying to elect me
if (epoch % 2 == 0) {
bump_epoch(epoch+1);  // odd == election cycle
} else {
// do a trivial db write just to ensure it is writeable.
auto t(std::make_shared<MonitorDBStore::Transaction>());
t->put(Monitor::MONITOR_NAME, "election_writeable_test", rand());
int r = mon->store->apply_transaction(t);
assert(r >= 0);
}
start_stamp = ceph_clock_now();
electing_me = true;
acked_me[mon->rank].cluster_features = CEPH_FEATURES_ALL;
acked_me[mon->rank].mon_features = ceph::features::mon::get_supported();
mon->collect_metadata(&acked_me[mon->rank].metadata);
leader_acked = -1;

// bcast to everyone else
for (unsigned i=0; i<mon->monmap->size(); ++i) {
if ((int)i == mon->rank) continue;
//发起选举消息
MMonElection *m =
new MMonElection(MMonElection::OP_PROPOSE, epoch, mon->monmap);
m->mon_features = ceph::features::mon::get_supported();
mon->messenger->send_message(m, mon->monmap->get_inst(i));
}

reset_timer();
}

void Elector::init()

void Elector::init()
{
//从db中取出epoch号+1，如果没有从1开始
epoch = mon->store->get(Monitor::MONITOR_NAME, "election_epoch");
if (!epoch) {
dout(1) << "init, first boot, initializing epoch at 1 " << dendl;
epoch = 1;
} else if (epoch % 2) {
dout(1) << "init, last seen epoch " << epoch
<< ", mid-election, bumping" << dendl;
++epoch;
auto t(std::make_shared<MonitorDBStore::Transaction>());
t->put(Monitor::MONITOR_NAME, "election_epoch", epoch);
mon->store->apply_transaction(t);
} else {
dout(1) << "init, last seen epoch " << epoch << dendl;
}
}

void Elector::handle_propose(MonOpRequestRef op)

void Elector::handle_propose(MonOpRequestRef op)
{
op->mark_event("elector:handle_propose");
MMonElection *m = static_cast<MMonElection*>(op->get_req());
dout(5) << "handle_propose from " << m->get_source() << dendl;
int from = m->get_source().num();

assert(m->epoch % 2 == 1); // election
uint64_t required_features = mon->get_required_features();
mon_feature_t required_mon_features = mon->get_required_mon_features();

dout(10) << __func__ << " required features " << required_features
<< " " << required_mon_features
<< ", peer features " << m->get_connection()->get_features()
<< " " << m->mon_features
<< dendl;

if ((required_features ^ m->get_connection()->get_features()) &
required_features) {
dout(5) << " ignoring propose from mon" << from
<< " without required features" << dendl;
nak_old_peer(op);
return;
} else if (!m->mon_features.contains_all(required_mon_features)) {
// all the features in 'required_mon_features' not in 'm->mon_features'
mon_feature_t missing = required_mon_features.diff(m->mon_features);
dout(5) << " ignoring propose from mon." << from
<< " without required mon_features " << missing
<< dendl;
nak_old_peer(op);
} else if (m->epoch > epoch) {
bump_epoch(m->epoch);
} else if (m->epoch < epoch) {
// got an "old" propose,
if (epoch % 2 == 0 &&    // in a non-election cycle
//收到集群外节点发来的选举消息，发起选举
mon->quorum.count(from) == 0) {  // from someone outside the quorum
// a mon just started up, call a new election so they can rejoin!
dout(5) << " got propose from old epoch, quorum is " << mon->quorum
<< ", " << m->get_source() << " must have just started" << dendl;
// we may be active; make sure we reset things in the monitor appropriately.
mon->start_election();
} else {
dout(5) << " ignoring old propose" << dendl;
return;
}
}
//如果本方的rank小于对方的rank值，如果回复过别人则不做处理，如果未回复过别人则发起选举。如果本方的rank大于对方的rank值，且之前回复的rank值较小，则不回复本次，如果本次的rank值较小则调用defer发ack。
if (mon->rank < from) {
// i would win over them.
if (leader_acked >= 0) {        // we already acked someone
assert(leader_acked < from);  // and they still win, of course
dout(5) << "no, we already acked " << leader_acked << dendl;
} else {
// wait, i should win!
if (!electing_me) {
mon->start_election();
}
}
} else {
// they would win over me
if (leader_acked < 0 ||      // haven't acked anyone yet, or
leader_acked > from ||   // they would win over who you did ack, or
leader_acked == from) {  // this is the guy we're already deferring to
defer(from);
} else {
// ignore them!
dout(5) << "no, we already acked " << leader_acked << dendl;
}
}
}

void Elector::defer(int who)

void Elector::defer(int who)
{
dout(5) << "defer to " << who << dendl;

if (electing_me) {
// drop out
acked_me.clear();
electing_me = false;
}

// ack them
leader_acked = who;
ack_stamp = ceph_clock_now();
MMonElection *m = new MMonElection(MMonElection::OP_ACK, epoch, mon->monmap);
m->mon_features = ceph::features::mon::get_supported();
mon->collect_metadata(&m->metadata);

// This field is unused completely in luminous, but jewel uses it to
// determine whether we are a dumpling mon due to some crufty old
// code.  It only needs to see this buffer non-empty, so put
// something useless there.
m->sharing_bl = mon->get_local_commands_bl(mon->get_required_mon_features());

mon->messenger->send_message(m, mon->monmap->get_inst(who));

// set a timer
reset_timer(1.0);  // give the leader some extra time to declare victory
}

选举不超时的情况下

void Elector::handle_ack(MonOpRequestRef op)

void Elector::handle_ack(MonOpRequestRef op)
{
op->mark_event("elector:handle_ack");
MMonElection *m = static_cast<MMonElection*>(op->get_req());
dout(5) << "handle_ack from " << m->get_source() << dendl;
int from = m->get_source().num();

assert(m->epoch % 2 == 1); // election
if (m->epoch > epoch) {
dout(5) << "woah, that's a newer epoch, i must have rebooted.  bumping and re-starting!" << dendl;
bump_epoch(m->epoch);
start();
return;
}
assert(m->epoch == epoch);
uint64_t required_features = mon->get_required_features();
if ((required_features ^ m->get_connection()->get_features()) &
required_features) {
dout(5) << " ignoring ack from mon" << from
<< " without required features" << dendl;
return;
}

mon_feature_t required_mon_features = mon->get_required_mon_features();
if (!m->mon_features.contains_all(required_mon_features)) {
mon_feature_t missing = required_mon_features.diff(m->mon_features);
dout(5) << " ignoring ack from mon." << from
<< " without required mon_features " << missing
<< dendl;
return;
}

if (electing_me) {
// thanks
acked_me[from].cluster_features = m->get_connection()->get_features();
acked_me[from].mon_features = m->mon_features;
acked_me[from].metadata = m->metadata;
dout(5) << " so far i have {";
for (map<int, elector_info_t>::const_iterator p = acked_me.begin();
p != acked_me.end();
++p) {
if (p != acked_me.begin())
*_dout << ",";
*_dout << " mon." << p->first << ":"
<< " features " << p->second.cluster_features
<< " " << p->second.mon_features;
}
*_dout << " }" << dendl;

// is that _everyone_?
//获得所有的ack之后
if (acked_me.size() == mon->monmap->size()) {
// if yes, shortcut to election finish
victory();
}
} else {
// ignore, i'm deferring already.
assert(leader_acked >= 0);
}
}

选举超时的情况下

void Elector::expire()
{
dout(5) << "election timer expired" << dendl;
//超时情况下半数以上monitor回复即可
// did i win?
if (electing_me &&
acked_me.size() > (unsigned)(mon->monmap->size() / 2)) {
// i win
victory();
} else {
// whoever i deferred to didn't declare victory quickly enough.
if (mon->has_ever_joined)
start();
else
mon->bootstrap();
}
}

如果赢得选举

void Elector::victory()

void Elector::victory()
{
leader_acked = -1;
electing_me = false;

uint64_t cluster_features = CEPH_FEATURES_ALL;
mon_feature_t mon_features = ceph::features::mon::get_supported();
set<int> quorum;
map<int,Metadata> metadata;
for (map<int, elector_info_t>::iterator p = acked_me.begin();
p != acked_me.end();
++p) {
quorum.insert(p->first);
cluster_features &= p->second.cluster_features;
mon_features &= p->second.mon_features;
metadata[p->first] = p->second.metadata;
}

cancel_timer();

assert(epoch % 2 == 1);  // election
bump_epoch(epoch+1);     // is over!

// tell everyone!
for (set<int>::iterator p = quorum.begin();
p != quorum.end();
++p) {
if (*p == mon->rank) continue;
MMonElection *m = new MMonElection(MMonElection::OP_VICTORY, epoch,
mon->monmap);
m->quorum = quorum;
m->quorum_features = cluster_features;
m->mon_features = mon_features;
m->sharing_bl = mon->get_local_commands_bl(mon_features);
mon->messenger->send_message(m, mon->monmap->get_inst(*p));
}

// tell monitor
//赢得选举，把自己标为leader
mon->win_election(epoch, quorum,
cluster_features, mon_features, metadata);
}

void Elector::handle_victory(MonOpRequestRef op)

void Elector::handle_victory(MonOpRequestRef op)
{
op->mark_event("elector:handle_victory");
MMonElection *m = static_cast<MMonElection*>(op->get_req());
dout(5) << "handle_victory from " << m->get_source()
<< " quorum_features " << m->quorum_features
<< " " << m->mon_features
<< dendl;
int from = m->get_source().num();

assert(from < mon->rank);
assert(m->epoch % 2 == 0);

leader_acked = -1;

// i should have seen this election if i'm getting the victory.
if (m->epoch != epoch + 1) {
dout(5) << "woah, that's a funny epoch, i must have rebooted.  bumping and re-starting!" << dendl;
bump_epoch(m->epoch);
start();
return;
}

bump_epoch(m->epoch);

// they win
//输掉选举，把自己标成peon
mon->lose_election(epoch, m->quorum, from,
m->quorum_features, m->mon_features);

// cancel my timer
cancel_timer();

// stash leader's commands
assert(m->sharing_bl.length());
vector<MonCommand> new_cmds;
bufferlist::iterator bi = m->sharing_bl.begin();
MonCommand::decode_vector(new_cmds, bi);
mon->set_leader_commands(new_cmds);
}

初始化leader

void Monitor::win_election(epoch_t epoch, set& active, uint64_t features,const mon_feature_t& mon_features,const map

void Monitor::win_election(epoch_t epoch, set<int>& active, uint64_t features,const mon_feature_t& mon_features,const map<int,Metadata>& metadata)
{
dout(10) << __func__ << " epoch " << epoch << " quorum " << active
<< " features " << features
<< " mon_features " << mon_features
<< dendl;
assert(is_electing());
state = STATE_LEADER;
leader_since = ceph_clock_now();
leader = rank;
quorum = active;
quorum_con_features = features;
quorum_mon_features = mon_features;
pending_metadata = metadata;
outside_quorum.clear();

clog->info() << "mon." << name << "@" << rank
<< " won leader election with quorum " << quorum;

set_leader_commands(get_local_commands(mon_features));

paxos->leader_init();
// NOTE: tell monmap monitor first.  This is important for the
// bootstrap case to ensure that the very first paxos proposal
// codifies the monmap.  Otherwise any manner of chaos can ensue
// when monitors are call elections or participating in a paxos
// round without agreeing on who the participants are.
monmon()->election_finished();
_finish_svc_election();
health_monitor->start(epoch);

logger->inc(l_mon_election_win);

// inject new metadata in first transaction.
{
// include previous metadata for missing mons (that aren't part of
// the current quorum).
map<int,Metadata> m = metadata;
for (unsigned rank = 0; rank < monmap->size(); ++rank) {
if (m.count(rank) == 0 &&
mon_metadata.count(rank)) {
m[rank] = mon_metadata[rank];
}
}

// FIXME: This is a bit sloppy because we aren't guaranteed to submit
// a new transaction immediately after the election finishes.  We should
// do that anyway for other reasons, though.
MonitorDBStore::TransactionRef t = paxos->get_pending_transaction();
bufferlist bl;
::encode(m, bl);
t->put(MONITOR_STORE_PREFIX, "last_metadata", bl);
}

finish_election();
if (monmap->size() > 1 &&
monmap->get_epoch() > 0) {
timecheck_start();
health_tick_start();
do_health_to_clog_interval();
scrub_event_start();
}
}

输掉选举，把自己初始化成peon

void Monitor::lose_election(epoch_t epoch, set &q, int l,uint64_t features,const mon_feature_t& mon_features)

void Monitor::lose_election(epoch_t epoch, set<int> &q, int l,
uint64_t features,
const mon_feature_t& mon_features)
{
state = STATE_PEON;
leader_since = utime_t();
leader = l;
quorum = q;
outside_quorum.clear();
quorum_con_features = features;
quorum_mon_features = mon_features;
dout(10) << "lose_election, epoch " << epoch << " leader is mon" << leader
<< " quorum is " << quorum << " features are " << quorum_con_features
<< " mon_features are " << quorum_mon_features
<< dendl;

paxos->peon_init();
_finish_svc_election();
health_monitor->start(epoch);

logger->inc(l_mon_election_lose);

finish_election();

if ((quorum_con_features & CEPH_FEATURE_MON_METADATA) &&
!HAVE_FEATURE(quorum_con_features, SERVER_LUMINOUS)) {
// for pre-luminous mons only
Metadata sys_info;
collect_metadata(&sys_info);
messenger->send_message(new MMonMetadata(sys_info),
monmap->get_inst(get_leader()));
}
}

选举结束void Monitor::finish_election()

void Monitor::finish_election()
{
apply_quorum_to_compatset_features();
apply_monmap_to_compatset_features();
timecheck_finish();
exited_quorum = utime_t();
finish_contexts(g_ceph_context, waitfor_quorum);
finish_contexts(g_ceph_context, maybe_wait_for_quorum);
resend_routed_requests();
update_logger();
register_cluster_logger();

// am i named properly?
string cur_name = monmap->get_name(messenger->get_myaddr());
if (cur_name != name) {
dout(10) << " renaming myself from " << cur_name << " -> " << name << dendl;
messenger->send_message(new MMonJoin(monmap->fsid, name, messenger->get_myaddr()),
monmap->get_inst(*quorum.begin()));
}
}

选举的基本流程已经基本结束

关于选举的部分还遗留着几个问题

1.handle_nak没有介绍

2.完成选举之后的初始化（paxos）

3.为什么收到集群外的选举需要重新发起选举？