epoll + 线程池 回应服务器(Linux) －－－可以支持上万个连接

[cpp] view
plaincopy

//ptypes

#include "ptime.h"

#include "pinet.h"

#include "ptypes.h"

#include "pasync.h"

#include "logfile.h"

//network

#include <sys/socket.h>

#include <sys/epoll.h>

#include <netinet/in.h>

#include <arpa/inet.h>

#include <fcntl.h>

#include <unistd.h>

#include <stdio.h>

#include <errno.h>

#include <iostream>

#include <set>

using namespace std;

USING_PTYPES

#define MAX_EVENTS 10000

#define maxtoken 1024

const int maxthreads = 64;

const int MSG_MYJOB = 100;

#define BUFF_MAX 1 //现在不用到事件里的缓存区，先设定最小值

struct myevent_s

{

int fd;

void (*call_back)(int fd, int events, void *arg);

int events;

void *arg;

int status; // 1: in epoll wait list, 0 not in

char buff[BUFF_MAX]; // recv data buffer

int len;

long last_active; // last active time

};

//以Socket为关键字的集合.

std::set<int> m_set;

//增加锁定。

mutex m_mxLock;

//把socket插入到集合中,如果存在返回false

bool InsertSocket(int nSocket)

{

scopelock lock(m_mxLock);

if (m_set.end() == m_set.find(nSocket))

{

//没有找到插入并返回false；

m_set.insert(nSocket);

return true;

}

else

{

//已经存在不插入返回false

return false;

}

}

//重集合中删除socket;

bool EraseSocket(int nSocket)

{

scopelock lock(m_mxLock);

//如果集合中存在socket,把Socket从集合里删除

if (m_set.end() != m_set.find(nSocket))

{

//找到再删除；

m_set.erase(nSocket);

return true;

}

else

{

//不存在不删除

return false;

}

}

// set event

void EventSet(myevent_s *ev, int fd, void (*call_back)(int, int, void*), void *arg)

{

ev->fd = fd;

ev->call_back = call_back;

ev->events = 0;

ev->arg = arg;

ev->status = 0;

ev->last_active = time(NULL);

}

// add/mod an event to epoll

void EventAdd(int epollFd, int events, myevent_s *ev)

{

struct epoll_event epv = {0, {0}};

int op;

epv.data.ptr = ev;

epv.events = ev->events = events;

if(ev->status == 1){

op = EPOLL_CTL_MOD;

}

else{

op = EPOLL_CTL_ADD;

ev->status = 1;

}

if(epoll_ctl(epollFd, op, ev->fd, &epv) < 0)

{

printf("Event Add failed[fd=%d], status:%d\n", ev->fd, ev->status);

CLogFile::Instance()->LOG_WriteLine(LOG_INFO_ERROR, "%s: Event Add failed[fd=%d], status:%d error=%d %s\n", __func__, ev->fd, ev->status,errno, strerror(errno));

}

else

{

printf("Event Add OK[fd=%d]\n", ev->fd);

CLogFile::Instance()->LOG_WriteLine(LOG_INFO_HIT, "%s: Event Add OK[fd=%d]\n", __func__, ev->fd);

}

}

// delete an event from epoll

void EventDel(int epollFd, myevent_s *ev)

{

struct epoll_event epv = {0, {0}};

if(ev->status != 1) return;

epv.data.ptr = ev;

ev->status = 0;

epoll_ctl(epollFd, EPOLL_CTL_DEL, ev->fd, &epv);

}

int g_epollFd;

myevent_s g_Events[MAX_EVENTS+1]; // g_Events[MAX_EVENTS] is used by listen fd

void RecvData(int fd, int events, void *arg);

void SendData(int fd, int events, void *arg);

// accept new connections from clients

void AcceptConn(int fd, int events, void *arg)

{

struct sockaddr_in sin;

socklen_t len = sizeof(struct sockaddr_in);

int nfd, i;

// accept

if((nfd = accept(fd, (struct sockaddr*)&sin, &len)) == -1)

{

if(errno != EAGAIN && errno != EINTR)

{

printf("%s: bad accept", __func__);

CLogFile::Instance()->LOG_WriteLine(LOG_INFO_ERROR, "%s: bad accept error = %d %s", __func__, errno, strerror(errno) );

}

return;

}

do

{

for(i = 0; i < MAX_EVENTS; i++)

{

if(g_Events[i].status == 0)

{

break;

}

}

if(i == MAX_EVENTS)

{

printf("%s:max connection limit[%d].", __func__, MAX_EVENTS);

CLogFile::Instance()->LOG_WriteLine(LOG_INFO_ERROR, "%s: max connection limit[%d]; error = %d %s", __func__, MAX_EVENTS, errno, strerror(errno) );

break;

}

// set nonblocking

if(fcntl(nfd, F_SETFL, O_NONBLOCK) < 0) break;

// add a read event for receive data

EventSet(&g_Events[i], nfd, RecvData, &g_Events[i]);

EventAdd(g_epollFd, EPOLLIN|EPOLLET, &g_Events[i]);

printf("new conn[%s:%d][time:%d]\n", inet_ntoa(sin.sin_addr), ntohs(sin.sin_port), g_Events[i].last_active);

CLogFile::Instance()->LOG_WriteLine(LOG_INFO_HIT, "%s: new conn[%s:%d][time:%d]", __func__, inet_ntoa(sin.sin_addr), ntohs(sin.sin_port), g_Events[i].last_active);

}while(0);

}

// receive data

void RecvData(int fd, int events, void *arg)

{

struct myevent_s *ev = (struct myevent_s*)arg;

int len;

// receive data

len = recv(fd, ev->buff, sizeof(ev->buff)-1, 0);

CLogFile::Instance()->LOG_WriteLine(LOG_INFO_HIT, "%s: recv len:%d", __func__, len);

EventDel(g_epollFd, ev);

if(len > 0)

{

ev->len = len;

ev->buff[len] = '\0';

printf("C[%d]:%s\n", fd, ev->buff);

// change to send event

EventSet(ev, fd, SendData, ev);

EventAdd(g_epollFd, EPOLLOUT|EPOLLET, ev);

}

else if(len == 0)

{

close(ev->fd);

printf("[fd=%d] closed gracefully.\n", fd);

CLogFile::Instance()->LOG_WriteLine(LOG_INFO_HIT, "%s: closed gracefully", __func__);

}

else

{

close(ev->fd);

printf("recv[fd=%d] error[%d]:%s\n", fd, errno, strerror(errno));

CLogFile::Instance()->LOG_WriteLine(LOG_INFO_ERROR, "%s: recv[fd=%d] error[%d]:%s", __func__, fd, errno, strerror(errno));

}

}

// send data

void SendData(int fd, int events, void *arg)

{

struct myevent_s *ev = (struct myevent_s*)arg;

int len;

// send data

len = send(fd, ev->buff, ev->len, 0);

CLogFile::Instance()->LOG_WriteLine(LOG_INFO_HIT, "%s: send len:%d", __func__, len);

ev->len = 0;

EventDel(g_epollFd, ev);

if(len > 0)

{

// change to receive event

EventSet(ev, fd, RecvData, ev);

EventAdd(g_epollFd, EPOLLIN|EPOLLET, ev);

}

else

{

close(ev->fd);

printf("send[fd=%d] error[%d]\n", fd, errno);

CLogFile::Instance()->LOG_WriteLine(LOG_INFO_HIT, " %s: send[fd=%d] error[%d]:%s", __func__, fd, errno, strerror(errno));

}

}

void InitListenSocket(int epollFd, short port)

{

int listenFd = socket(AF_INET, SOCK_STREAM, 0);

fcntl(listenFd, F_SETFL, O_NONBLOCK); // set non-blocking

printf("server listen fd=%d/n", listenFd);

EventSet(&g_Events[MAX_EVENTS], listenFd, AcceptConn, &g_Events[MAX_EVENTS]);

// add listen socket

//EventAdd(epollFd, EPOLLIN|EPOLLET, &g_Events[MAX_EVENTS]);

EventAdd(epollFd, EPOLLIN, &g_Events[MAX_EVENTS]);

// bind & listen

sockaddr_in sin;

bzero(&sin, sizeof(sin));

sin.sin_family = AF_INET;

sin.sin_addr.s_addr = INADDR_ANY;

sin.sin_port = htons(port);

bind(listenFd, (const sockaddr*)&sin, sizeof(sin));

listen(listenFd, SOMAXCONN);

}

//线程池函数

class myjobthread: public thread

{

protected:

int id;

jobqueue* jq;

virtual void execute();

public:

myjobthread(int iid, jobqueue* ijq)

: thread(false), id(iid), jq(ijq) {}

~myjobthread() { waitfor(); }

};

class myjob: public message

{

public:

myevent_s* m_pSocketInfo;

myjob(myevent_s* pSocketInfo)

: message(MSG_MYJOB), m_pSocketInfo(pSocketInfo) {}

~myjob()

{

//delete m_pSocketInfo;

}

};

void myjobthread::execute()

{

bool quit = false;

while (!quit)

{

// get the next message from the queue

message* msg = jq->getmessage();

try

{

switch (msg->id)

{

case MSG_MYJOB:

{

pout.putf("%t,线程：%d\n", now(), id);

myevent_s *ev = ((myjob*)msg)->m_pSocketInfo;

if(ev->events&EPOLLIN) // read event

{

ev->call_back(ev->fd, ev->events, ev->arg);

}

if(ev->events&EPOLLOUT) // write event

{

ev->call_back(ev->fd, ev->events, ev->arg);

}

//pout.putf("%t,%d返回:%d字节\n",now(), id, nSend);

}

break;

case MSG_QUIT:

// MSG_QUIT is not used in our example

quit = true;

break;

}

}

catch(pt::exception*)

{

// the message object must be freed!

delete msg;

throw;

}

delete msg;

}

}

int main(int argc, char **argv)

{

short port = 8108; // default port

if(argc == 2){

port = atoi(argv[1]);

}

CLogFile::Instance()->LOG_Start("./log/");

//启动线程池

jobqueue jq;

tobjlist<myjobthread> threads(true);

// create the thread pool

for(int n = 0; n < maxthreads; n++)

{

myjobthread* j = new myjobthread(n + 1, &jq);

j->start();

threads.add(j);

}

// create epoll

g_epollFd = epoll_create(MAX_EVENTS);

if(g_epollFd <= 0)

{

CLogFile::Instance()->LOG_WriteLine(LOG_INFO_ERROR, "create epoll failed.");

printf("create epoll failed.%d\n", g_epollFd);

}

// create & bind listen socket, and add to epoll, set non-blocking

InitListenSocket(g_epollFd, port);

// event loop

struct epoll_event events[MAX_EVENTS];

printf("server running:port[%d]\n", port);

int checkPos = 0;

while(1)

{

// a simple timeout check here, every time 100, better to use a mini-heap, and add timer event

long now = time(NULL);

for(int k = 0; k < MAX_EVENTS; k++, checkPos++) // doesn't check listen fd

{

if(checkPos == MAX_EVENTS) checkPos = 0; // recycle

if(g_Events[checkPos].status != 1) continue;

long duration = now - g_Events[checkPos].last_active;

if(duration >= 600) // 600s timeout

{

closesocket(g_Events[checkPos].fd);

printf("[fd=%d] timeout[%d--%d].\n", g_Events[checkPos].fd, g_Events[checkPos].last_active, now);

CLogFile::Instance()->LOG_WriteLine(LOG_INFO_WARN, "超时Socket：[fd=%d] timeout[%d--%d].", g_Events[checkPos].fd, g_Events[checkPos].last_active, now);

EventDel(g_epollFd, &g_Events[checkPos]);

}

}

// wait for events to happen

int fds = epoll_wait(g_epollFd, events, MAX_EVENTS, -1);

if(fds < 0)

{

printf("epoll_wait error, exit/n");

CLogFile::Instance()->LOG_WriteLine(LOG_INFO_ERROR, "epoll_wait error.");

break;

}

CLogFile::Instance()->LOG_WriteLine(LOG_INFO_HIT, "EPOLL_WAIT FDS counts:%d", fds);

for(int i = 0; i < fds; i++)

{

myevent_s *ev = (struct myevent_s*)events[i].data.ptr;

jq.post(new myjob(ev));

}

}

// free resource

return 0;

}