使用job自动抓取网页数据发送邮件,以springboot服务启动

Quartz-spring.xml：

<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd"> 
<!--监控recket mq未消费记录的job配置-->
<bean id="scoutNoConsumeWorker" class="com.lppz.job.work.ScoutNoConsumeWorker" />
<bean id="scoutNoConsumeJob"
class="org.springframework.scheduling.quartz.MethodInvokingJobDetailFactoryBean">
<property name="targetObject" ref="scoutNoConsumeWorker" />
<property name="targetMethod" value="execute" />
<property name="concurrent" value="false" />
</bean>

<!--监控recket mq未消费记录调度的配置 -->
<bean id="scoutNoConsumeTrigger"
class="org.springframework.scheduling.quartz.CronTriggerFactoryBean">
<property name="jobDetail">
<ref bean="scoutNoConsumeJob" />
</property>
<property name="cronExpression">
<value>${job.cronExpression}</value>
</property>
<property name="group" value="initTriggerGroup" />
</bean>

<!-- 所有启动触发器的配置开始 -->
<bean name="startQuertz" lazy-init="false" autowire="no"
class="org.springframework.scheduling.quartz.SchedulerFactoryBean">
<property name="triggers">
<list>
<ref bean="scoutNoConsumeTrigger"/>
</list>
</property>
</bean>
</beans>

application.properties:

job.cronExpression=0 0/2 * * * ?

maxnum=1000

test=test@qq.com

logging-config.xml:

<configuration>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<Pattern>%d [%-5level][%t][%c][%X{tenant}][%X{requestId}] %m%n
</Pattern>
</encoder>
</appender>

<appender name="logfile"
class="ch.qos.logback.core.rolling.RollingFileAppender">
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
<FileNamePattern>${LOG_PATH}/${LOG_FILE}-%d{yyyy-MM-dd}.log
</FileNamePattern>
</rollingPolicy>
<layout class="ch.qos.logback.classic.PatternLayout">
<pattern>[%d{HH:mm:ss:SSS}][%5p][%c:%L] %m%n</pattern>
</layout>
</appender>

<root level="info">
<appender-ref ref="logfile" />
</root>
</configuration>

QuartzConfig.java

package com.lppz.job.config;

import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.ImportResource;

@Configuration
@ImportResource("classpath:/META-INF/scout-quartz-spring.xml")
public class QuartzConfig {

}

CatchDataUtils.java

package com.lppz.job.utils;

import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.NameValuePair;
import org.apache.http.client.HttpClient;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

/**
* Created by hlcen on 2017/7/28.
*/
public class CatchDataUtils {

public static Integer catchData(String url){
int noConsumeNum = 0;
try {

//创建client实例
HttpClient client = HttpClients.createDefault();
//创建httpget实例
HttpPost httpPost = new HttpPost(url);
List<NameValuePair> list = new ArrayList<NameValuePair>();
list.add(new BasicNameValuePair("groupName","memberGroup"));
UrlEncodedFormEntity entity = new UrlEncodedFormEntity(list,"UTF-8");
httpPost.setEntity(entity);
//执行get请求
HttpResponse response = client.execute(httpPost);
String result = "";
if (response  !=null) {
HttpEntity resEntity = response.getEntity();
if(resEntity != null){
result = EntityUtils.toString(resEntity,"UTF-8");
}
Document doc = Jsoup.parse(result);
Elements ps=doc.select("p");//选择器，选取特征信息
String data = ps.get(1).toString();
noConsumeNum = Integer.valueOf(data.substring(data.indexOf("=")+1,data.lastIndexOf(":")));
}

} catch (IOException e) {
e.printStackTrace();
}
return noConsumeNum;
}
}

DateTime.java (getCurrentDateString)

public static String getCurrentDateString() {
Date currentTime = new Date();
SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
formatter.setTimeZone(TimeZone.getTimeZone("GMT+8"));
String dateString = formatter.format(currentTime);
return dateString;
}

Email.java

package com.lppz.job.utils;

import java.util.List;

public class Email {

static public String sendEmll(String time, String noConsumeNum,String maxNoconsumeNum, List<String> list){
//时间
long timestamp = System.currentTimeMillis();
//签名
String requestSign = "NOTIFY" + "30003" + timestamp + "66963c3a81ecf2c9b5660c814f40792c";

StringBuffer s = new StringBuffer();
//拼接参数信息
s.append("{\"ContractRoot\":{\"BusCont\":{\"dep\":\"自动发送\",\"messages\":[");//{\"content\":\"");
if(list !=null && list.size()>0){
for (String email : list) {
s.append("{\"content\":\"");
s.append("<h3><b>【<font color='red'>请注意</font>】！</b></h3>");
s.append("<br>");
s.append("      ");
s.append("xxxxx！");
s.append("<br>");
s.append("\",\"msgId\":\"0\",\"subPort\":\"*\",\"target\":\"");
//接收人的邮件地址
s.append(email);
s.append("\",\"title\":\"");
s.append("【xxx！！！】");
s.append("\"},");
}
}
s.deleteCharAt(s.length() - 1);
s.append("],\"sys\":\"type\":\"YT\",\"user\":\"");
String user = "zheng";
s.append(user);
s.append("\"},\"HeadCont\":{\"ReqTime\":\"");
s.append(timestamp);
s.append("\",\"SrcSysId\":\"30003\",\"SrcSysSign\":\"");
s.append(MD5Util.MD5(requestSign));
s.append("\"}}}");
String str = HttpUtil.sendJson("xxx.jhtml",  s.toString());//邮件接口
return str;
}

}

HttpUtil.java

package com.lppz.job.utils;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.io.UnsupportedEncodingException;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.util.List;
import java.util.Map;

public class HttpUtil {

/**
* 向指定URL发送GET方法的请求
*
* @param url
*            发送请求的URL
* @param param
*            请求参数，请求参数应该是 name1=value1&name2=value2 的形式。
* @return URL 所代表远程资源的响应结果
* @throws Exception
*/
public static String sendGet(String url, String param) throws Exception {
String result = "";
BufferedReader in = null;
try {
String urlNameString = url + "?" + param;
URL realUrl = new URL(urlNameString);
// 打开和URL之间的连接
URLConnection connection = realUrl.openConnection();
// 设置通用的请求属性
connection.setRequestProperty("accept", "*/*");
connection.setRequestProperty("connection", "Keep-Alive");
connection.setRequestProperty("user-agent",
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1;SV1)");
// 建立实际的连接
connection.connect();
// 获取所有响应头字段
Map<String, List<String>> map = connection.getHeaderFields();
// 遍历所有的响应头字段
for (String key : map.keySet()) {
System.out.println(key + "--->" + map.get(key));
}
// 定义 BufferedReader输入流来读取URL的响应
in = new BufferedReader(new InputStreamReader(
connection.getInputStream()));
String line;
while ((line = in.readLine()) != null) {
result += line;
}
} catch (Exception e) {
e.printStackTrace();
throw e;
}
// 使用finally块来关闭输入流
finally {
try {
if (in != null) {
in.close();
}
} catch (Exception e2) {
e2.printStackTrace();
}
}
return result;
}

/**
* 向指定 URL 发送POST方法的请求
*
* @param url
*            发送请求的 URL
* @param param
*            请求参数，请求参数应该是 name1=value1&name2=value2 的形式。
* @return 所代表远程资源的响应结果
* @throws Exception
*/
public static String sendPost(String url, String param) throws Exception {
PrintWriter out = null;
BufferedReader in = null;
String result = "";
try {
URL realUrl = new URL(url);
// 打开和URL之间的连接
URLConnection conn = realUrl.openConnection();
// 设置通用的请求属性
conn.setRequestProperty("accept", "*/*");
conn.setRequestProperty("connection", "Keep-Alive");
conn.setRequestProperty("user-agent",
"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1;SV1)");
// 发送POST请求必须设置如下两行
conn.setDoOutput(true);
conn.setDoInput(true);
// 获取URLConnection对象对应的输出流
//out = new PrintWriter(conn.getOutputStream());
out=new PrintWriter(new OutputStreamWriter(conn.getOutputStream(),"UTF-8"));
// 发送请求参数
out.print(param);
// flush输出流的缓冲
out.flush();
// 定义BufferedReader输入流来读取URL的响应
in = new BufferedReader(
new InputStreamReader(conn.getInputStream()));
String line;
while ((line = in.readLine()) != null) {
result += line;
}
} catch (Exception e) {
e.printStackTrace();
throw e;
}
//使用finally块来关闭输出流、输入流
finally{
try{
if(out!=null){
out.close();
}
if(in!=null){
in.close();
}
}
catch(IOException ex){
ex.printStackTrace();
}
}
return result;
}

public static String sendJson(String urlStr, String jsonStr){
PrintWriter out = null;
BufferedReader reader = null;
String result = "";
try {
URL url = new URL(urlStr);
HttpURLConnection connection = (HttpURLConnection)url.openConnection();
connection.setDoInput(true);
connection.setDoOutput(true);
connection.setRequestMethod("POST");
connection.setUseCaches(false);
connection.setInstanceFollowRedirects(true);
connection.setRequestProperty("Content-Type","application/json");
connection.connect();
//out = new DataOutputStream(connection.getOutputStream());

//out.writeBytes(jsonStr);
out=new PrintWriter(new OutputStreamWriter(connection.getOutputStream(),"UTF-8"));
// 发送请求参数
out.print(jsonStr);
out.flush();
out.close();

reader = new BufferedReader(new InputStreamReader(connection.getInputStream(),"UTF-8"));
String lines;
StringBuffer sbf = new StringBuffer();
while ((lines = reader.readLine()) != null) {
lines = new String(lines.getBytes());
sbf.append(lines);
}
reader.close();
// 断开连接
connection.disconnect();
result = sbf.toString();
} catch (MalformedURLException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}finally{
try{
if(out!=null){
out.close();
}
if(reader!=null){
reader.close();
}
}
catch(IOException ex){
ex.printStackTrace();
}
}
return result;
}

}

MD5Util.java

package com.lppz.job.utils;

import java.security.MessageDigest;

import org.apache.commons.codec.binary.Base64;

public class MD5Util {
public static String WM_KEY="ABCDEFGHT";

private static String byteArrayToHexString(byte b[]) {
StringBuffer resultSb = new StringBuffer();
for (int i = 0; i < b.length; i++)
resultSb.append(byteToHexString(b[i]));

return resultSb.toString();
}

private static String byteToHexString(byte b) {
int n = b;
if (n < 0)
n += 256;
int d1 = n / 16;
int d2 = n % 16;
return hexDigits[d1] + hexDigits[d2];
}

public static String MD5(String origin) {
String resultString = null;
try {
resultString = new String(origin);
MessageDigest md = MessageDigest.getInstance("MD5");
resultString = byteArrayToHexString(md.digest(resultString
.getBytes()));
} catch (Exception exception) {
}
return resultString;
}

public static String MD5Encode(String origin, String charsetname) {
String resultString = null;
try {
resultString = new String(origin);
MessageDigest md = MessageDigest.getInstance("MD5");
if (charsetname == null || "".equals(charsetname))
resultString = byteArrayToHexString(md.digest(resultString
.getBytes()));
else
resultString = byteArrayToHexString(md.digest(resultString
.getBytes(charsetname)));
} catch (Exception exception) {
}
return resultString;
}

private static final String hexDigits[] = { "0", "1", "2", "3", "4", "5",
"6", "7", "8", "9", "a", "b", "c", "d", "e", "f" };

/**
* 解密 直接返回memberId
*
* @param 从cookie里取出的加密字符串
* @return
* @see
*/
@SuppressWarnings("static-access")
public static String decodeStr(String cookies)
{
String memberId="";
try {
Base64 base64 = new Base64();
byte[] debytes = base64.decodeBase64(new String(cookies).getBytes());
String decode = new String(debytes);//解密后字符串
String[] arg = decode.split("-");
if(arg!=null&&arg.length>0){
memberId = new String(base64.decodeBase64(new String(arg[0]).getBytes()));//再次解密
}
} catch (Exception e) {
e.printStackTrace();
}
return memberId;
}

/**
* 加密 方式 memeerId+"-"+加密key
*
* @param memberId
* @return
* @see
*/
@SuppressWarnings("static-access")
public static String encodeStr(String memberId)
{
try {
Base64 base64 = new Base64();
String newId = new String(base64.encodeBase64Chunked(memberId.getBytes()));//一次加密
memberId=newId+"-"+WM_KEY;
byte[] enbytes = base64.encodeBase64Chunked(memberId.getBytes()); //二次加密
return new String(enbytes);
} catch (Exception e) {
e.printStackTrace();
}
return "";
}

}

ScoutNoConsumeWorker.java

package com.lppz.job.work;

import com.lppz.job.utils.CatchDataUtils;
import com.lppz.job.utils.DateTime;
import com.lppz.job.utils.Email;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;

import java.util.ArrayList;
import java.util.List;

public class ScoutNoConsumeWorker {

private static final Logger log = LoggerFactory.getLogger(ScoutNoConsumeWorker.class);

final  String no_consume_url = "xxx.do";

@Value("${maxnum}")
private String maxnum ;

@Value("${test}")
private String test;

public void execute() {
List<String> list = new ArrayList<String>();
list.add(test);
int noConsumeNum  = CatchDataUtils.catchData(no_consume_url);
if(noConsumeNum>=Integer.valueOf(maxnum)){
String time = DateTime.getCurrentDateString();
Email.sendEmll(time,String.valueOf(noConsumeNum),maxnum,list);
}
}
}

Application.java

package com.lppz.job;

import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;

@SpringBootApplication
public class Application {

public static void main(String[] args){
//程序启动入口，启动嵌入式的tomcat，初始化spring环境和组件
SpringApplication.run(Application.class,args);
}

}

Springboot-pom.xml

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> 
<modelVersion>4.0.0</modelVersion>
<packaging>jar</packaging>
<artifactId>lppz-scout</artifactId>

<!-- Spring Boot 启动父依赖 -->
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>1.5.4.RELEASE</version>
</parent>

<dependencies>

<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.5.3</version>
</dependency>

<dependency>
<groupId>org.jsoup</groupId>
<artifactId>jsoup</artifactId>
<version>1.7.2</version>
</dependency>

<!-- Spring Boot Web 依赖 -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>

<!-- Spring Boot Test 依赖 -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>

<!-- Junit -->
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.12</version>
</dependency>
<dependency>
<groupId>org.quartz-scheduler</groupId>
<artifactId>quartz</artifactId>
<version>2.2.1</version>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-context-support</artifactId>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-tx</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-actuator</artifactId>
</dependency>
</dependencies>

<build>
<finalName>${project.artifactId}</finalName>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
</project>