页面访问数据统计[初步]

问题：项目中加一个数据统计模块，主要统计页面访问信息，用户uv,pv,ip等做趋势分析，然后定时将访问数据批量存储到数据库。第一次写，一些代码或者说想法不太好，主要是实现缓存数据定时存储这个过程。

涉及：JFinal框架、quartz、ehcache、cookie

解决办法：

1、config中：

//1.配置ehcache缓存
EhCachePlugin ecp = new EhCachePlugin();
me.add(ecp);
//2.定时任务
QuartzPlugin quartzPlugin = new QuartzPlugin("jobs.properties");
me.add(quartzPlugin);

//数据库连接等...

2、jobs.properties：
定义一个工具任务 job为执行任务所在路径
                                 cron为定义工作执行时间 一般是定义到凌晨执行 可参照：http://www.cnblogs.com/sunjie9606/archive/2012/03/15/2397626.html
                                enable是否执行
job=com.cloud.main.stats.model.HelloJob
cron=0 0/30 * * * ?
enable=true3、ehcache.xml

缓存配置文件

<ehcache name="lingshi-protal">
<diskStore path="java.io.tmpdir/lingshi-protal-ehcache"/>

<!-- 默认缓存，不指定缓存名称时使用 -->
<defaultCache
maxElementsInMemory="10000"
eternal="false"
timeToIdleSeconds="120"
timeToLiveSeconds="120"
overflowToDisk="false"
diskPersistent="false"
diskExpiryThreadIntervalSeconds="120">
</defaultCache>

<!-- 数据统计缓存 -->
<cache name="stats"
maxEntriesLocalHeap="10000"
maxEntriesLocalDisk="1000"
eternal="false"
overflowToDisk="true"
diskSpoolBufferSizeMB="20"
timeToIdleSeconds="300"
timeToLiveSeconds="100"
memoryStoreEvictionPolicy="LFU"
transactionalMode="off"
/>

</ehcache>

4、statsController

获取数据

package com.cloud.main.stats.controller;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.UUID;
import javax.servlet.http.Cookie;
import org.apache.commons.lang3.StringUtils;
import com.cloud.core.BaseController;
import com.jfinal.plugin.activerecord.Record;
import com.jfinal.plugin.ehcache.CacheKit;
import com.jfinal.plugin.ehcache.IDataLoader;
/**
* 页面数据统计：
* 将页面访问数据置于记录中 ，添加到缓存
* @author MN
*/
public class StatsController extends BaseController {

//客户段唯一标识
private static final String CUSTOMER_IDENTIFICATION = "CusID";
private String accessorid ;
//页面id
private String pid;
//页面路径
private String referer;
//访问者ip
private String accessor;
//获取访问设备
private String device;

public void index(){
//当前页码
pid=getPara("pid");
//当前访问路径
referer=getRequest().getHeader("Referer");
// 判断 Referer      PropKit.get("domain")
if(referer!=null&&(referer.trim().startsWith("http://localhost:8080/"))){
//当前访问者ip
accessor=getAccessIp();
//当前访问设备
device=checkDevice();
//获取cookie
Cookie[] cookies = getRequest().getCookies();
List<Record> pageval = null;
if(cookies==null){
//缓存中添加一笔访问记录
pageval=new ArrayList<Record>();
//设置唯一标识cookie【cusID】
accessorid = UUID.randomUUID().toString().replaceAll("[-]", "");
Cookie cidCookie = new Cookie(CUSTOMER_IDENTIFICATION, accessorid);
cidCookie.setMaxAge(24*60*60);
cidCookie.setPath("/");
getResponse().addCookie(cidCookie);
}else{
for(Cookie cookie:cookies){
//取得用户唯一标识码
if(StringUtils.equals(cookie.getName(),CUSTOMER_IDENTIFICATION)){
accessorid =cookie.getValue();
pageval =CacheKit.get("stats",accessorid,new IDataLoader(){
@Override
public Object load() {
return new ArrayList<Record>();
}
});
break;
}
}
}
pageval.add(new Record()
.set("pid", pid)
.set("time", new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS").format(new Date()))
.set("device", device)
.set("accessor", accessor));
CacheKit.put("stats", accessorid, pageval);
}
//避免出现模板加载失败错误
renderNull();
}
/**
* 获取访问ip
*/
public String getAccessIp(){
String ip=getRequest().getHeader("X-Real-IP");
if (!StringUtils.isBlank(ip) && !"unknown".equalsIgnoreCase(ip)) {
return ip;
}
ip = getRequest().getHeader("X-Forwarded-For");
if (!StringUtils.isBlank(ip) && !"unknown".equalsIgnoreCase(ip)) {
// 多次反向代理后会有多个IP值，第一个为真实IP。
int index = ip.indexOf(',');
if (index != -1) {
return ip.substring(0, index);
} else {
return ip;
}
} else {
return getRequest().getRemoteAddr() ;
}
}
/**
* 判断设备类型
* 这里写的很简单 网上有正则表达式验证的准确
*/
public String  checkDevice(){
String device=getRequest().getHeader("USER-AGENT").toLowerCase().substring(0,7);
if(device.equals("mozilla")){
return "PC";
}
return "手持设备";
}

}

5、quartzPlugin

/*
* All content copyright Terracotta, Inc., unless otherwise indicated. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
* use this file except in compliance with the License. You may obtain a copy
* of the License at
*
*   http://www.apache.org/licenses/LICENSE-2.0 *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*
*/

package com.cloud.main.stats.model;

import java.io.IOException;

import java.io.InputStream;
import java.util.Enumeration;
import java.util.Properties;
import static org.quartz.CronScheduleBuilder.cronSchedule;
import static org.quartz.JobBuilder.newJob;
import static org.quartz.TriggerBuilder.newTrigger;
import org.quartz.CronTrigger;
import org.quartz.JobDetail;
import org.quartz.Scheduler;

import org.quartz.SchedulerException;
import org.quartz.SchedulerFactory;
import org.quartz.impl.StdSchedulerFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.jfinal.plugin.IPlugin;

/**
* 复写QuartzPlugin
* @author MN
*
*/
public class QuartzPlugin implements IPlugin {

private Logger logger = LoggerFactory.getLogger(getClass());

private static SchedulerFactory sf = null ;

private static Scheduler sched=null;

private String config = "jobs.properties";

private Properties properties;

public QuartzPlugin(String config) {
this.config = config;
}
public QuartzPlugin() {

}
@Override
public boolean start() {

sf = new  StdSchedulerFactory();
try {
sched = sf.getScheduler();
} catch (SchedulerException e1) {
e1.printStackTrace();
}
loadProperties();

Enumeration enums = properties.keys();

while (enums.hasMoreElements()) {
String key = enums.nextElement() + "";
if (!key.endsWith("job")) {
continue ;
}
String cronKey = key.substring(0, key.indexOf("job")) + "cron";
String enable = key.substring(0, key.indexOf("job")) + "enable";
if (isDisableJob(enable)) {

continue;

}

String jobClassName = properties.get(key) + "";
String jobCronExp = properties.getProperty(cronKey) + "";
Class clazz;
try {
clazz = Class.forName(
4000
jobClassName);

} catch (ClassNotFoundException e) {

throw new RuntimeException(e);

}
//向Scheduler加入Job
JobDetail job = newJob(clazz)
.withIdentity(jobClassName,jobClassName)
.build();
CronTrigger trigger = newTrigger()
.withIdentity(jobClassName, jobClassName)
.withSchedule(cronSchedule(jobCronExp))
.build();
try {
sched.scheduleJob(job, trigger);
sched.start();

} catch (SchedulerException e) {
e.printStackTrace();
}
logger.info(job.getKey()
+ " has been scheduled to run at: "
+ " and repeat based on expression: "
+ trigger.getCronExpression());

}
return true;

}

private boolean isDisableJob(String enable) {
return Boolean.valueOf(properties.get(enable) + "") == false;

}

private void loadProperties() {

properties = new Properties();

InputStream is = QuartzPlugin.class.getClassLoader()

.getResourceAsStream(config);

try {

properties.load(is);

} catch (IOException e) {

throw new RuntimeException(e);

}

}

@Override

public boolean stop() {

try {

sched.shutdown();

} catch (SchedulerException e) {

logger.error("shutdown error", e);

return false;

}

return true;

}
public static void main(String []args)  {
QuartzPlugin plugin = new QuartzPlugin();
plugin.start();
System.out.println("执行成功！！！");

}

}

6、HelloJob

package com.cloud.main.stats.model;

import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.jfinal.plugin.activerecord.Db;
import com.jfinal.plugin.activerecord.Record;
import com.jfinal.plugin.ehcache.CacheKit;
import org.quartz.Job;
import org.quartz.JobExecutionContext;
import org.quartz.JobExecutionException;

/**
* jobs.properties
* 0/30 * * * * ?
* 0 0/30 * * * ?
* @author lixiang
*/
public class HelloJob implements Job {

private static Logger _log = LoggerFactory.getLogger(HelloJob.class);
private static boolean executing = false;

public HelloJob() {

}
public void execute(JobExecutionContext context)  throws JobExecutionException {
if(executing)return;
executing = true;
_log.info("\n"+"-------------------"+"\n"+"Test " + new Date());

@SuppressWarnings("unchecked")
List<String>  list=CacheKit.getKeys("stats");
List<Record> saveval= new ArrayList<Record>();
saveval.add(new Record().set("pid", 0).set("count", 0));
for (String  accessorid: list) {
List<Record> pageval =CacheKit.get("stats",accessorid);
String  startTime = pageval.get(0).get("time");
String  accessor=pageval.get(0).get("accessor");
String device=pageval.get(0).get("device");
HashMap<String, Integer> hs = new HashMap<String, Integer>();
for (Record page : pageval) {
Integer count = 1;
if(hs.get(page.get("pid")) != null) {
count = hs.get(page.get("pid")) + 1;
}
hs.put(page.get("pid"), count);
}
Iterator iter = hs.entrySet().iterator();
List<String> sqlList=new ArrayList<String>();
while (iter.hasNext()) {
Map.Entry entry = (Map.Entry) iter.next();
String key = (String) entry.getKey();
int val = (int) entry.getValue();
String sql="INSERT INTO	t_stats_accesslog"
+ "(accessor,startTime,pid,pagecount,accessorid,device)"
+ "VALUES("+"'"+accessor+"'"+","
+"'"+startTime+"'"+","
+key+","
+val+","
+"'"+accessorid+"'"+","
+"'"+device+"'"+")";
sqlList.add(sql);
}
Db.batch(sqlList, sqlList.size());
try{
Thread.sleep(1000);
CacheKit.remove("stats", accessorid);
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
executing = false;
}
}
}

最后附上quartz包和官方的例子 http://download.csdn.net/detail/qq_21544619/9805944