[置顶] Java断点续传下载视频
2015-06-09 11:58
399 查看
控制层类：
package com.grab.video.controller;
import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.io.Writer;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLDecoder;
import java.net.URLEncoder;
import java.nio.charset.Charset;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Scanner;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import javax.servlet.ServletContext;
import javax.servlet.ServletOutputStream;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.servlet.http.HttpSession;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.HttpEntity;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.ServletRequestUtils;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.servlet.ModelAndView;
import org.springframework.web.servlet.view.RedirectView;
import com.fasterxml.jackson.core.JsonGenerationException;
import com.fasterxml.jackson.databind.JsonMappingException;
import com.fasterxml.jackson.databind.ObjectMapper;
@Controller
public class GrabVideoController {
private static final Logger LOG = LoggerFactory.getLogger(GrabVideoController.class);
//private static String filePath="D:\\logs\\video";
private static String filePath = "/home/grabVideo/";
@Autowired
ServletContext context;
/**
* 输入userid可以使用
*
* @param request
* @param response
* @return
*/
@RequestMapping(value = "/grab/login", method = { RequestMethod.GET })
public ModelAndView login(HttpServletRequest request, HttpServletResponse response) {
String userId = ServletRequestUtils.getStringParameter(request, "userId", "");
String ts = ServletRequestUtils.getStringParameter(request, "ts", "");
String sign = ServletRequestUtils.getStringParameter(request, "sign", "");
ModelAndView mav = new ModelAndView();
// 身份验证
if (StringUtils.isNotBlank(userId)) {
String encryptedSign = EncryptionUtils.md5Hex(ts + userId + "grab");
if (sign.equals(encryptedSign)) {
HttpSession session = request.getSession();
session.setMaxInactiveInterval(5*24*60*60);//秒为单位,设置session周期为5天
session.setAttribute("userId", userId);// 把userId存放到session
String url = "/grab/import";
mav.setView(new RedirectView(url));
return mav;
}
}
mav.setViewName("video/error");
return mav;
}
/**
* 导入文件
*
* @return
*/
@RequestMapping(value = "/grab/import", method = { RequestMethod.GET })
public ModelAndView importFile(HttpServletRequest request, HttpServletResponse response) {
// String userId = ServletRequestUtils.getStringParameter(request,
// "userId", null);
ModelAndView mav = new ModelAndView();
HttpSession session = request.getSession();
String userId = null;
if (session.getAttribute("userId") != null) {
userId = (String) session.getAttribute("userId");
SqlFileList sqlFileList = new SqlFileList();
List<FileListModel> list = new ArrayList<FileListModel>();
try {
list = sqlFileList.selectDate(userId);
} catch (SQLException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
// 从POLYV的API获取目录
mav.addObject("list", list);
mav.addObject("userId", userId);
mav.setViewName("video/import");
return mav;
}
mav.setViewName("video/login");
return mav;
}
/**
* 删除文件
*
* @param request
* @param response
* @return
*/
@RequestMapping(value = "/grab/delete/file", method = { RequestMethod.GET })
public ResponseEntity<AjaxPostResponse> deleteFile(HttpServletRequest request,
HttpServletResponse response) {
String fileId = ServletRequestUtils.getStringParameter(request, "fileId", null);
MediaType mediaType = new MediaType("application", "json", Charset.forName("UTF-8"));
HttpHeaders headers = new HttpHeaders();
headers.setContentType(mediaType);
SqlFileList sqlFileList = new SqlFileList();
try {
sqlFileList.deleteDate(fileId);
} catch (SQLException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
AjaxPostResponse resp = new AjaxPostResponse("yes");
return new ResponseEntity<AjaxPostResponse>(resp, headers, HttpStatus.OK);
}
/**
* 解析文件
*
* @return
* @throws UnsupportedEncodingException
*/
@RequestMapping(value = "/grab/analysis", method = { RequestMethod.GET })
public ResponseEntity<AjaxPostResponse> analysisFile(HttpServletRequest request,
HttpServletResponse response) throws UnsupportedEncodingException {
Integer fileId = ServletRequestUtils.getIntParameter(request, "fileId", 0);
String fileUrl = ServletRequestUtils.getStringParameter(request, "fileUrl", "");
String classifyId = ServletRequestUtils.getStringParameter(request, "classifyId",
"classifyId");
String classifyName = ServletRequestUtils.getStringParameter(request, "classifyName", "");
String userId = ServletRequestUtils.getStringParameter(request, "userId", null);
MediaType mediaType = new MediaType("application", "json", Charset.forName("UTF-8"));
HttpHeaders headers = new HttpHeaders();
headers.setContentType(mediaType);
String errorStr="";
System.out.println("==========="+userId);
List<String> urlList = new ArrayList<String>();
List<String> titleList = new ArrayList<String>();
try {
System.out.println("file============"+fileUrl);
CloseableHttpClient httpclient = HttpClients.createDefault();
HttpGet httpGet = new HttpGet(fileUrl);
try {
CloseableHttpResponse response2 = httpclient.execute(httpGet);
InputStream is = null;
Scanner sc = null;
Writer os = null;
if (response2.getStatusLine().getStatusCode() == org.apache.http.HttpStatus.SC_OK) {
try {
// 2、获取response的entity。
HttpEntity entity = response2.getEntity();
is = entity.getContent();
//sc = new Scanner(is);
BufferedReader reader = new BufferedReader(new InputStreamReader(
is, "UTF-8"));
int n=0;
String line = null;
while ((line = reader.readLine()) != null) {
n++;
try {
String str = line;
//
String urlstr = str.substring(0, str.indexOf(","));
String title = str.substring(str.lastIndexOf(",") + 1, str.length());
//
urlList.add(urlstr);
titleList.add(title);
} catch (Exception e) {
// TODO Auto-generated catch block
System.out.println("解析失败"+n);
errorStr=errorStr+","+n;
e.printStackTrace();
}
}
errorStr=errorStr+"行格式原因";
}catch(Exception e){
System.out.println("解析失败");
} finally {
if (sc != null) {
sc.close();
}
if (is != null) {
is.close();
}
if (os != null) {
os.close();
}
if (response2 != null) {
response2.close();
}
}
}
}catch(Exception e){
System.out.println("解析失败");
}
} catch (Exception e) {
// TODO Auto-generated catch block
LOG.info("文件解析失败：" + e);
e.printStackTrace();
AjaxPostResponse resp = new AjaxPostResponse(errorStr+":解析失败");
return new ResponseEntity<AjaxPostResponse>(resp, headers, HttpStatus.OK);
}
// 更新状态
SqlFileList sqlFileList = new SqlFileList();
FileListModel file = new FileListModel();
file.setFileId(fileId);
file.setStatus("Y");
try {
sqlFileList.updateDate(file);
} catch (SQLException e1) {
// TODO Auto-generated catch block
LOG.info("文件状态修改成功：" + e1);
e1.printStackTrace();
}
// LOG.info("00000"+classifyName);
classifyName = classifyName.replace("-", "");
// LOG.info(classifyName);
// 添加数据
Date date = new Date();
Timestamp timeStamp = new Timestamp(date.getTime());
GetRandomString randomStr = new GetRandomString();
for (int i = 0; i < urlList.size(); i++) {
VideoListModel video = new VideoListModel();
video.setUserId(userId);
video.setUrl(urlList.get(i));// 视频源地址
video.setTitle(titleList.get(i));// 视频标题
String urlstr = urlList.get(i);
// String path=urlstr.substring(0, urlstr.indexOf("?"));
// String format=path.substring(path.lastIndexOf("."),
// path.length());//视频格式
// String baseName = FilenameUtils.getBaseName(urlstr);
String extendname = FilenameUtils.getExtension(urlstr);
if(extendname.contains("?")){
extendname=extendname.substring(0,extendname.indexOf("?"));
}
if (StringUtils.isBlank(extendname)) {
extendname = "mp4";
}
String trueName = randomStr.generateRandomString(15);
String filename = trueName + "." + extendname;
video.setTrueName(filename);// 用于下载使用的视频名称
video.setClassifyId(classifyId);
//video.setClassifyName(classifyName.trim());
video.setClassifyName(URLDecoder.decode(classifyName.trim(),"GBK"));
video.setStatus(VideoStatus.NO.getValue());// 等待、
video.setVid("");
video.setCreateTime(timeStamp);
video.setLastDate(timeStamp);
SqlVideoList sqlvideo = new SqlVideoList();
try {
sqlvideo.insertDate(video);// 添加数据库
} catch (SQLException e) {
// TODO Auto-generated catch block
LOG.info("添加数据库:" + e);
e.printStackTrace();
AjaxPostResponse resp = new AjaxPostResponse("no");
return new ResponseEntity<AjaxPostResponse>(resp, headers, HttpStatus.OK);
}
}
AjaxPostResponse resp = new AjaxPostResponse("yes");
return new ResponseEntity<AjaxPostResponse>(resp, headers, HttpStatus.OK);
}
/**
* 获取下载进度
*
* @return
*/
@RequestMapping(value = "/grab/download/progress", method = { RequestMethod.POST,
RequestMethod.GET })
public ResponseEntity<AjaxPostResponse> getProgress(HttpServletRequest request,
HttpServletResponse response) {
Integer id = ServletRequestUtils.getIntParameter(request, "videoId", 0);
String userId = ServletRequestUtils.getStringParameter(request, "userId", "test");
String urlstr = ServletRequestUtils.getStringParameter(request, "url", "");
String trueName = ServletRequestUtils.getStringParameter(request, "trueName", "");
MediaType mediaType = new MediaType("application", "json", Charset.forName("UTF-8"));
HttpHeaders headers = new HttpHeaders();
headers.setContentType(mediaType);
// LOG.info("--id---"+id+"---u---"+userId);
int content = 1;
int length = 1;
int progress = 1;
HttpSession session = request.getSession();
// LOG.info("-------ccccc4------------"+session.getAttribute("fileSize"+id));
if (session.getAttribute("fileSize" + String.valueOf(id)) == null) {
// 文件大小还没存进session中
List<TaskQueue> list = new ArrayList<TaskQueue>();
SqlTaskQueue stq = new SqlTaskQueue();
try {
list = stq.selectDateOne(String.valueOf(id));
} catch (SQLException e2) {
// TODO Auto-generated catch block
LOG.info("查询文件大小" + e2);
e2.printStackTrace();
}
if (list.size() > 0) {
TaskQueue tQueue = list.get(0);
content = tQueue.getFileSize();
session.setAttribute("fileSize" + String.valueOf(id), content);// 存进session
} else {
URL url = null;
try {
url = new URL(urlstr);
HttpURLConnection connection = (HttpURLConnection) url.openConnection();// 进行连接握手
connection.setRequestMethod("GET");// 请求方式
content = (int) connection.getContentLength();
session.setAttribute("fileSize" + String.valueOf(id), content);// 存进session
// LOG.info("-------content------"+content);
} catch (Exception e1) {
// TODO Auto-generated catch block
LOG.info("链接失败" + e1);
e1.printStackTrace();
}
}
} else {
// 文件大少在session中
String contentString = String.valueOf(session.getAttribute("fileSize"
+ String.valueOf(id)));
// LOG.info("-------ccccc------------"+contentString);
content = Integer.parseInt(contentString.trim());
}
// 文件存储位置、文件命名处理
try {
// String path=urlstr.substring(0, urlstr.indexOf("?"));
// String name=path.substring(path.lastIndexOf("/")+1,
// path.length());
// String filename=name.trim();
String filename = trueName;
File file = new File(filePath, filename);
if (!file.exists()) {
progress = (Integer) session.getAttribute(userId + id);// 将当前下载进度存放到session中。
} else {
length = (int) file.length();
progress = length * 100 / content;
// 将当前下载进度存放到session中。
session.setAttribute(userId + id, progress);
LOG.info(id + "-------progress------" + progress);
}
} catch (Exception e) {
LOG.info("不能解析的路径：" + e);
AjaxPostResponse resp = new AjaxPostResponse(progress);
return new ResponseEntity<AjaxPostResponse>(resp, headers, HttpStatus.OK);
}
AjaxPostResponse resp = new AjaxPostResponse(progress);
return new ResponseEntity<AjaxPostResponse>(resp, headers, HttpStatus.OK);
}
/**
* 批量抓取视频(下载视频模块--根据视频源地址去抓取视频)管理
*
* @return
* @throws MalformedURLException
*/
@RequestMapping(value = "/grab/download/manage", method = { RequestMethod.POST })
public ModelAndView grabDownloadVideo(HttpServletRequest request, HttpServletResponse response) {
int[] id = ServletRequestUtils.getIntParameters(request, "videoId");
String userId = ServletRequestUtils.getStringParameter(request, "userId", "test");
String[] urlstr = ServletRequestUtils.getStringParameters(request, "url");
String[] trueName = ServletRequestUtils.getStringParameters(request, "trueName");
int len = id.length;
List<TaskQueue> taskQueues = new ArrayList<TaskQueue>();
for (int i = 0; i < len; i++) {
TaskQueue tq = new TaskQueue();
tq.setTaskId(String.valueOf(id[i]));
tq.setVideoId(id[i]);
tq.setUserId(userId);
tq.setVideoUrl(urlstr[i]);
tq.setTrueName(trueName[i]);
taskQueues.add(tq);
}
// 把任务队列添加进数据库
if (taskQueues.size() > 0) {
// 存在有任务
for (int i = 0; i < taskQueues.size(); i++) {
TaskQueue task = taskQueues.get(i);
List<TaskQueue> taskList = new ArrayList<TaskQueue>();// 查询任务是否已存在
try {
SqlTaskQueue stq = new SqlTaskQueue();
taskList = stq.selectDateOne(task.getTaskId());
} catch (SQLException e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
}
if (taskList.size() > 0) {
// 该任务已存在
} else {
task.setFileSize(0);
task.setProgress(0);
task.setStatus("N");
Date date = new Date();
Timestamp timeStamp = new Timestamp(date.getTime());
task.setCreateTime(timeStamp);
try {
SqlTaskQueue stq = new SqlTaskQueue();
stq.insertDate(task);
} catch (SQLException e) {
// TODO Auto-generated catch block
LOG.info("下载任务添加失败！" + e);
e.printStackTrace();
}
}
}
}
// 获取所有的任务队列
List<TaskQueue> workQueues = new ArrayList<TaskQueue>();
try {
SqlTaskQueue stq = new SqlTaskQueue();
workQueues = stq.selectDate(userId);
} catch (SQLException e) {
// TODO Auto-generated catch block
LOG.info("获取下载任务失败" + e);
e.printStackTrace();
}
// ExecutorService pool = Executors.newFixedThreadPool(3);
if (workQueues.size() > 0) {
for (int i = 0; i < workQueues.size(); i++) {
String taskId = workQueues.get(i).getTaskId();
String urltxt = workQueues.get(i).getVideoUrl();
String filename = workQueues.get(i).getTrueName();
File saveFile = new File(filePath, filename);// 文件保存的位置
File fileDir = new File(filePath);
if (!fileDir.exists()) {
fileDir.mkdirs();// 目录不存在创建目录
}
URL url = null;
try {
url = new URL(workQueues.get(i).getVideoUrl());
} catch (MalformedURLException e) {
// TODO Auto-generated catch block
LOG.info("握手失败" + e);
e.printStackTrace();
}
if (url != null) {
// 将下载任务线程，放入线程池中执行
ExecutorService executor = (ExecutorService) context
.getAttribute("DOWNLOAD_EXECUTOR");
executor.submit(new DownloadVideo(url, saveFile, taskId));
// pool.execute(new
// DownloadVideo(url,saveFile,taskId));////////////////////////////////////
VideoListModel vlm = new VideoListModel();
vlm.setId(workQueues.get(i).getVideoId());
vlm.setUserId(userId);
vlm.setStatus(VideoStatus.WAIT.getValue());// 将状态改为等待
try {
SqlVideoList svl = new SqlVideoList();
svl.updateDate(vlm);
} catch (SQLException e) {
// TODO Auto-generated catch block
LOG.info("更改下载状态失败" + e);
e.printStackTrace();
}
}
}
}
// 关闭线程池
// pool.shutdown();
// 重新查询视频列表
List<VideoListModel> list = new ArrayList<VideoListModel>();
try {
SqlVideoList svl = new SqlVideoList();
list = svl.selectDate(userId);
} catch (SQLException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
ModelAndView mav = new ModelAndView();
mav.addObject("videolist", list);
mav.addObject("userId", userId);
mav.setViewName("video/download");
return mav;
}
/**
* 获取视频的下载进度()
* @param request
*/
@RequestMapping(value="/grab/download/status",method = {RequestMethod.GET,RequestMethod.POST})
public @ResponseBody
ResponseEntity<String> downloadStatus(HttpServletRequest request, HttpServletResponse response)
throws JsonGenerationException, JsonMappingException, IOException {
HttpSession session = request.getSession();
if (session.getAttribute("userId") != null) {
String userId = (String) session.getAttribute("userId");
List<VideoListModel> list = new ArrayList<VideoListModel>();
try {
SqlVideoList sqlVideoList = new SqlVideoList();
list = sqlVideoList.selectAllDate(userId);
//list = sqlVideoList.selectExecuteDate(userId);
for (int i = 0; i < list.size(); i++) {
VideoListModel model = list.get(i);
String filename = model.getTrueName();
File file = new File(filePath, filename);
if (file.exists()) {
int downloaded = (int) file.length();
if (model.getFileSize() != 0) {
//System.out.println(model.getId()+"===n==="+model.getTrueName()+"===d==="+downloaded+"===s==="+model.getFileSize()+"===="+(long)downloaded * 100 /(long) model.getFileSize());
model.setPercent((int) ((long)downloaded * 100 /(long) model.getFileSize()));
}
}
}
ObjectMapper objectMapper = new ObjectMapper();
String result = objectMapper.writeValueAsString(list);
MediaType mediaType = new MediaType("application", "json", Charset.forName("UTF-8"));
HttpHeaders headers = new HttpHeaders();
headers.setContentType(mediaType);
return new ResponseEntity<String>(result, headers, HttpStatus.OK);
} catch (SQLException e) {
// TODO Auto-generated catch block
System.out.println("获取下载进度出现异常！");
e.printStackTrace();
}
}
return null;
}
@RequestMapping(value = "/grab/download/manage", method = { RequestMethod.GET })
public ModelAndView grabVideo(HttpServletRequest request, HttpServletResponse response) {
// String userId = ServletRequestUtils.getStringParameter(request,
// "userId", "");
String userId = null;
ModelAndView mav = new ModelAndView();
HttpSession session = request.getSession();
if (session.getAttribute("userId") != null) {
userId = (String) session.getAttribute("userId");
SqlVideoList sqlVideoList = new SqlVideoList();
List<VideoListModel> list = new ArrayList<VideoListModel>();
try {
list = sqlVideoList.selectDate(userId);
} catch (SQLException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
mav.addObject("videolist", list);
mav.addObject("userId", userId);
mav.setViewName("video/download");
return mav;
}
mav.setViewName("video/login");
return mav;
}
/**
* 暂停下载
*/
@RequestMapping(value = "/grab/video/stop", method = { RequestMethod.GET })
public ResponseEntity<AjaxPostResponse> downloadStop(HttpServletRequest request,
HttpServletResponse response) {
int id = ServletRequestUtils.getIntParameter(request, "videoId",0);
String userId = ServletRequestUtils.getStringParameter(request, "userId", "test");
//String urlstr = ServletRequestUtils.getStringParameter(request, "url","");
//String trueName = ServletRequestUtils.getStringParameter(request, "trueName","");
MediaType mediaType = new MediaType("application", "json", Charset.forName("UTF-8"));
HttpHeaders headers = new HttpHeaders();
headers.setContentType(mediaType);
try {
SqlTaskQueue stq = new SqlTaskQueue();
stq.deleteDate(String.valueOf(id));
} catch (SQLException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
// 获取所有的任务队列
List<TaskQueue> workQueues = new ArrayList<TaskQueue>();
try {
SqlTaskQueue stq = new SqlTaskQueue();
workQueues = stq.selectDate(userId);
} catch (SQLException e) {
// TODO Auto-generated catch block
LOG.info("获取下载任务失败" + e);
e.printStackTrace();
}
// ExecutorService pool = Executors.newFixedThreadPool(3);
if (workQueues.size() > 0) {
for (int i = 0; i < workQueues.size(); i++) {
String taskId = workQueues.get(i).getTaskId();
String urltxt = workQueues.get(i).getVideoUrl();
String filename = workQueues.get(i).getTrueName();
File saveFile = new File(filePath, filename);// 文件保存的位置
File fileDir = new File(filePath);
if (!fileDir.exists()) {
fileDir.mkdirs();// 目录不存在创建目录
}
URL url = null;
try {
url = new URL(workQueues.get(i).getVideoUrl());
} catch (MalformedURLException e) {
// TODO Auto-generated catch block
LOG.info("握手失败" + e);
e.printStackTrace();
}
if (url != null) {
// 将下载任务线程，放入线程池中执行
ExecutorService executor = (ExecutorService) context
.getAttribute("DOWNLOAD_EXECUTOR");
executor.submit(new DownloadVideo(url, saveFile, taskId));
// pool.execute(new
// DownloadVideo(url,saveFile,taskId));////////////////////////////////////
VideoListModel vlm = new VideoListModel();
vlm.setId(workQueues.get(i).getVideoId());
vlm.setUserId(userId);
vlm.setStatus(VideoStatus.WAIT.getValue());// 将状态改为等待
try {
SqlVideoList svl = new SqlVideoList();
svl.updateDate(vlm);
} catch (SQLException e) {
// TODO Auto-generated catch block
LOG.info("更改下载状态失败" + e);
e.printStackTrace();
}
}
}
}
System.out.println("停止下载！");
AjaxPostResponse resp = new AjaxPostResponse("yes");
return new ResponseEntity<AjaxPostResponse>(resp, headers, HttpStatus.OK);
}
/**
* 导出下载成功的视频
*
* @return
*/
@RequestMapping(value = "/grab/export", method = { RequestMethod.GET })
public ModelAndView exportVideo(HttpServletRequest request, HttpServletResponse response) {
// String userId = ServletRequestUtils.getStringParameter(request,
// "userId", "");
String userId = null;
ModelAndView mav = new ModelAndView();
HttpSession session = request.getSession();
if (session.getAttribute("userId") != null) {
userId = (String) session.getAttribute("userId");
SqlVideoList sqlVideoList = new SqlVideoList();
List<VideoListModel> list = new ArrayList<VideoListModel>();
try {
list = sqlVideoList.selectSuccessDate(userId);
} catch (SQLException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
mav.addObject("videolist", list);
mav.addObject("userId", userId);
mav.setViewName("video/export");
return mav;
}
mav.setViewName("video/login");
return mav;
}
/**
* export导出文件
*/
@RequestMapping(value = "/grab/export/csv", method = { RequestMethod.GET })
public void exportCsv(HttpServletRequest request, HttpServletResponse response) {
String userId = ServletRequestUtils.getStringParameter(request, "userId", "");
if (StringUtils.isNotBlank(userId)) {
SqlVideoList sqlVideoList = new SqlVideoList();
List<VideoListModel> list = new ArrayList<VideoListModel>();
try {
list = sqlVideoList.selectSuccessDate(userId);
} catch (SQLException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
// 导出txt文件
//response.setContentType("text/plain");
response.setContentType("text/csv; charset=UTF-8");
String fileName = "videolist";
try {
fileName = URLEncoder.encode("videolist", "UTF-8");
} catch (UnsupportedEncodingException e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
}
response.setHeader("Content-Disposition", "attachment; filename=" + fileName + ".csv");
BufferedOutputStream buff = null;
StringBuffer write = new StringBuffer();
String enter = "\r\n";
ServletOutputStream outSTr = null;
try {
outSTr = response.getOutputStream(); // 建立
buff = new BufferedOutputStream(outSTr);
// 把内容写入文件
if (list.size() > 0) {
for (int i = 0; i < list.size(); i++) {
write.append(list.get(i).getUrl());
write.append(",");
write.append(list.get(i).getTitle());
write.append(",");
write.append(list.get(i).getVid());
write.append(",");
write.append(list.get(i).getLastDate());
write.append(enter);
}
}
buff.write(write.toString().getBytes("GBK"));
buff.flush();
buff.close();
} catch (Exception e) {
e.printStackTrace();
} finally {
try {
buff.close();
outSTr.close();
} catch (Exception e) {
e.printStackTrace();
}
}
}
}
/**
* 清空导出视频列表
* /grab/export/clean
*/
@RequestMapping(value = "/grab/export/clean", method = { RequestMethod.GET })
public ResponseEntity<AjaxPostResponse> cleanVideo(HttpServletRequest request,
HttpServletResponse response) {
MediaType mediaType = new MediaType("application", "json", Charset.forName("UTF-8"));
HttpHeaders headers = new HttpHeaders();
headers.setContentType(mediaType);
HttpSession session = request.getSession();
if (session.getAttribute("userId") != null) {
String userId=(String) session.getAttribute("userId");
SqlVideoList sqlVideoList=new SqlVideoList();
try {
sqlVideoList.cleanDate(userId);
} catch (SQLException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
//删除成功
System.out.println("删除成功！");
AjaxPostResponse resp = new AjaxPostResponse("yes");
return new ResponseEntity<AjaxPostResponse>(resp, headers, HttpStatus.OK);
}else{
//删除失败
System.out.println("删除失败！");
AjaxPostResponse resp = new AjaxPostResponse("no");
return new ResponseEntity<AjaxPostResponse>(resp, headers, HttpStatus.OK);
}
}
/**
* 删除视频
* /grab/export/clean
*/
@RequestMapping(value = "/grab/delete/videoId", method = { RequestMethod.GET })
public ResponseEntity<AjaxPostResponse> deleteVideo(HttpServletRequest request,
HttpServletResponse response) {
String videoId = ServletRequestUtils.getStringParameter(request, "videoId", null);
MediaType mediaType = new MediaType("application", "json", Charset.forName("UTF-8"));
HttpHeaders headers = new HttpHeaders();
headers.setContentType(mediaType);
if (StringUtils.isNotBlank(videoId)) {
SqlVideoList sqlVideoList=new SqlVideoList();
try {
SqlTaskQueue stqSe=new SqlTaskQueue();
List<TaskQueue> list=stqSe.selectDateOne(videoId);
if(list.size()>0){
SqlTaskQueue stq=new SqlTaskQueue();
stq.deleteDate(videoId);//删除任务
}
sqlVideoList.deleteDate(videoId);//删除视频
} catch (SQLException e) {
// TODO Auto-generated catch block
System.out.println("删除失败"+e);
e.printStackTrace();
}
//删除成功
AjaxPostResponse resp = new AjaxPostResponse("yes");
return new ResponseEntity<AjaxPostResponse>(resp, headers, HttpStatus.OK);
}else{
//删除失败
AjaxPostResponse resp = new AjaxPostResponse("no");
return new ResponseEntity<AjaxPostResponse>(resp, headers, HttpStatus.OK);
}
}
/***
* 获取文件内容
* @param url
* @return
*/
public String httpGetFile(String url) {
String contentStr="";
CloseableHttpClient httpclient = HttpClients.createDefault();
HttpGet httpGet = new HttpGet(url);
try {
CloseableHttpResponse response = httpclient.execute(httpGet);
InputStream is = null;
Scanner sc = null;
Writer os = null;
if (response.getStatusLine().getStatusCode() == org.apache.http.HttpStatus.SC_OK) {
try {
// 2、获取response的entity。
HttpEntity entity = response.getEntity();
is = entity.getContent();
sc = new Scanner(is);
while (sc.hasNext()) {
contentStr=contentStr+sc.nextLine();
}
} catch (ClientProtocolException e) {
e.printStackTrace();
} finally {
if (sc != null) {
sc.close();
}
if (is != null) {
is.close();
}
if (os != null) {
os.close();
}
if (response != null) {
response.close();
}
}
}
} catch (ClientProtocolException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
return contentStr;
}
}
下载视频主线程类：
package com.grab.video.controller;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.io.Writer;
import java.net.HttpURLConnection;
import java.net.ProtocolException;
import java.net.URL;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Scanner;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.HttpEntity;
import org.apache.http.HttpStatus;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
public class DownloadVideo implements Runnable {
private static final Logger LOG = LoggerFactory.getLogger(DownloadVideo.class);
private static String TAG = "Downloader";
private HttpURLConnection connection;
private URL url;
private File saveFile;
private long fileLength;// 文件总大少
private int progress;// 当前进度
private long downloaded = 0;
private boolean stop=false;
private String taskId;
private static String fileUrl = "http://grab.polyv.net/video/";
private boolean downloadStatus = false;
public DownloadVideo(URL url, File saveFile, String taskId) {
this.url = url;
this.saveFile = saveFile;
this.taskId = taskId;
}
public DownloadVideo(URL url, File saveFile, String taskId,boolean stop) {
this.url = url;
this.saveFile = saveFile;
this.taskId = taskId;
this.stop = stop;
}
public DownloadVideo(String vid, File saveFile) {
// 通过vid,获取视频TODO
}
public URL getUrl() {
return url;
}
public void setUrl(URL url) {
this.url = url;
}
public void setStop(boolean stop) {
this.stop = stop;
}
public long getFileLength() {
return fileLength;
}
public int getProgress() {
return progress;
}
public void setProgress(int progress) {
this.progress = progress;
}
public long getDownloaded() {
return downloaded;
}
public int getPercent() {
if (fileLength == 0) {
return 0;
}
return (int) (downloaded * 100 / fileLength);
}
public void stop() {
stop = true;
}
public void start() {
stop = false;
}
// 开始执行,实现run方法
public void run() {
VideoListModel vlm = new VideoListModel();
vlm.setId(Integer.valueOf(taskId));
vlm.setStatus(VideoStatus.EXECUTE.getValue());// 将状态改为正在执行
try {
SqlVideoList sqlVideoList = new SqlVideoList();
System.out.println("更新下载中状态。。。。");
sqlVideoList.updateDate(vlm);
} catch (SQLException e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
}
try {
BufferedInputStream in = null;
FileOutputStream fos = null;
BufferedOutputStream bout = null;
print("start download:" + url);
connection = (HttpURLConnection) url.openConnection();// 进行连接握手
connection.setRequestProperty("User-Agent", "Polyv");
Map<String, List<String>> map = connection.getRequestProperties();
print(map.toString());// 输出参数
connection.setRequestMethod("GET");// 请求方式
if (saveFile.exists()) {
downloaded = saveFile.length();
connection.setRequestProperty("Range", "bytes=" + downloaded + "-");
} else {
downloaded = 0;// 重新开始下载
}
// connection.getResponseCode() == 200 206?
int code = connection.getResponseCode();// 获取状态码
print("code=" + code + ", downloaded =" + downloaded);
printResponseHeader(connection);
// http200状态,重新开始下载，206状态续点下载
if (connection.getResponseCode() == 206 || connection.getResponseCode() == 200) {
String range = "";
HashMap<String, String> header = (HashMap<String, String>) getHttpResponseHeader(connection);
for (Map.Entry<String, String> entry : header.entrySet()) {
String key = entry.getKey();
String value = entry.getValue();
System.out.println(key+"============="+value);
if (key.equals("Content-Range")) {
range = entry.getValue();
}
}
System.out.println("-----range-----"+range);
long content = (long) connection.getContentLength();
fileLength = range.equals("") ? content : Long.valueOf(range.split("/")[1]);
System.out.println("文件===c==="+content+"===f==="+fileLength+"======"+(long)(content+downloaded));
try {
SqlVideoList sqlVideoList = new SqlVideoList();
sqlVideoList.updateFileSize(Integer.valueOf(taskId),content+downloaded);//获取文件的大少(合计才是文件的大少,content是还有多少下载的，downloaded已下载的大少)
} catch (SQLException e2) {
// TODO Auto-generated catch block
System.out.println("获取文件大少失败");
e2.printStackTrace();
}
in = new BufferedInputStream(connection.getInputStream());
fos = (downloaded == 0) ? new FileOutputStream(saveFile) : new FileOutputStream(
saveFile, true);
bout = new BufferedOutputStream(fos, 1024);
byte[] data = new byte[1024];
int x = 0;
boolean p = false;
while (!stop && (x = in.read(data, 0, 1024)) >= 0) {
bout.write(data, 0, x);
downloaded += x;
}
bout.close();
System.out.println("下载完毕{}");
downloadStatus = false;// 代表下载成功，不用去更新下载失败的状态
String vid = null;
// 获取vid,(根据taskId即videoId获取视频的title,cataId)
List<VideoListModel> videoList = new ArrayList<VideoListModel>();
try {
SqlVideoList sqlVideoList = new SqlVideoList();
videoList = sqlVideoList.selectDateOne(Integer.valueOf(taskId));
} catch (SQLException e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
}
if (videoList.size() > 0) {
VideoListModel video = videoList.get(0);
String title = video.getTitle().replace(" ", "");
String cataid = video.getClassifyId();
String userid = video.getUserId();
String trueName = video.getTrueName();
long ts = System.currentTimeMillis();
String sign = EncryptionUtils.md5Hex(ts + userid + "grab");
String url = fileUrl + trueName;
String pathStr="http://v.polyv.net/uc/services/rest?method=uploadForDownloader&fileUrl="+url+"&userid="+userid+"&title="+title+"&cataid="+cataid+"&ts="+ts+"&sign="+sign;
System.out.println("----url-----"+pathStr);
vid=httpGetVid(pathStr);//获取vid
if(vid==null){
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
vid=httpGetVid(pathStr);//获取vid
}
//第二次
if(vid==null){
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
vid=httpGetVid(pathStr);//获取vid
}
System.out.println("------下载完成获取vid---------" + vid);
}
// 下载完成，更改视频状态为SUCCESS
VideoListModel video = new VideoListModel();
video.setId(Integer.valueOf(taskId));
video.setStatus(VideoStatus.SUCCESS.getValue());
video.setVid(vid);
Date date = new Date();
Timestamp timeStamp = new Timestamp(date.getTime());
video.setLastDate(timeStamp);
try {
SqlVideoList sqlVideoList = new SqlVideoList();
sqlVideoList.updateDateVid(video);
} catch (SQLException e) {
// TODO Auto-generated catch block
System.out.println("下载完成,更改视频状态失败" + e);
e.printStackTrace();
}
// 下载完成，从任务队列移除该任务
try {
SqlTaskQueue stq = new SqlTaskQueue();
stq.deleteDate(taskId);
} catch (SQLException e) {
// TODO Auto-generated catch block
System.out.println("任务移除失败" + e);
e.printStackTrace();
}
}else if (connection.getResponseCode() == 416) {
// 416- 请求长度超出范围
System.out.println("已经下载完了======"+connection.getResponseCode());
downloadStatus = false;// 代表下载失败，要去更新下载失败的状态
}else {
downloadStatus = true;// 代表下载失败，要去更新下载失败的状态
}
} catch (NumberFormatException e) {
// TODO Auto-generated catch block
downloadStatus = true;// 代表下载失败，要去更新下载失败的状态
e.printStackTrace();
} catch (ProtocolException e) {
// TODO Auto-generated catch block
downloadStatus = true;// 代表下载失败，要去更新下载失败的状态
e.printStackTrace();
} catch (FileNotFoundException e) {
// TODO Auto-generated catch block
downloadStatus = true;// 代表下载失败，要去更新下载失败的状态
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
downloadStatus = true;// 代表下载失败，要去更新下载失败的状态
e.printStackTrace();
}
if (downloadStatus) {
// 下载失败，更改视频状态
VideoListModel video = new VideoListModel();
video.setId(Integer.valueOf(taskId));
video.setStatus(VideoStatus.FAIL.getValue());
try {
SqlVideoList sqlVideoList = new SqlVideoList();
sqlVideoList.updateDate(video);
} catch (SQLException e) {
// TODO Auto-generated catch block
System.out.println("下载失败,更改视频状态失败" + e);
e.printStackTrace();
}
}else{
// 下载完成，从任务队列移除该任务
try {
SqlTaskQueue stq = new SqlTaskQueue();
stq.deleteDate(taskId);
} catch (SQLException e) {
// TODO Auto-generated catch block
System.out.println("任务移除失败" + e);
e.printStackTrace();
}
VideoListModel video = new VideoListModel();
video.setId(Integer.valueOf(taskId));
video.setStatus(VideoStatus.SUCCESS.getValue());
Date date = new Date();
Timestamp timeStamp = new Timestamp(date.getTime());
video.setLastDate(timeStamp);
try {
SqlVideoList sqlVideoList = new SqlVideoList();
sqlVideoList.updateSuccessDate(video);
} catch (SQLException e) {
// TODO Auto-generated catch block
System.out.println("下载好了,更改视频状态失败" + e);
e.printStackTrace();
}
}
}
/**
* 获取相应头部
*
* @param http
* @return
*/
public static Map<String, String> getHttpResponseHeader(HttpURLConnection http) {
Map<String, String> header = new LinkedHashMap<String, String>();
for (int i = 0;; i++) {
String mine = http.getHeaderField(i);
String key = http.getHeaderFieldKey(i);
System.out.println(key+"=======test======="+mine);
if (key == null || mine == null) {
break;
} else {
header.put(http.getHeaderFieldKey(i), mine);
}
}
return header;
}
/**
* 输出信息
*
* @param msg
*/
private static void print(String msg) {
// Log.i(TAG, msg);
System.out.println(TAG + "TAG{}," + msg);
}
/**
* 输出相应头信息
*
* @param http
*/
public static void printResponseHeader(HttpURLConnection http) {
Map<String, String> header = getHttpResponseHeader(http);
for (Map.Entry<String, String> entry : header.entrySet()) {
String key = entry.getKey() != null ? entry.getKey() + ":" : "";
print(key + entry.getValue());
}
}
/**
* 发送请求，获取API数据
*
* @param userId
* @return
* http://beta.polyv.net/uc/services/rest?url=http://grap.polyv.net/xxx * .mp4&title=filename&cataid=xxx&ts=&userid=&sign=
*/
public String httpGetVid(String url) {
String contentStr = "";
String vid = null;
String urlStr = url;
CloseableHttpClient httpclient = HttpClients.createDefault();
HttpGet httpGet = new HttpGet(urlStr);
try {
CloseableHttpResponse response = httpclient.execute(httpGet);
InputStream is = null;
Scanner sc = null;
Writer os = null;
if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
try {
// 2、获取response的entity。
HttpEntity entity = response.getEntity();
is = entity.getContent();
sc = new Scanner(is);
while (sc.hasNext()) {
contentStr = contentStr + sc.nextLine();
}
} catch (ClientProtocolException e) {
e.printStackTrace();
} finally {
if (sc != null) {
sc.close();
}
if (is != null) {
is.close();
}
if (os != null) {
os.close();
}
if (response != null) {
response.close();
}
}
}
} catch (ClientProtocolException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
if (StringUtils.isNotBlank(contentStr)) {
System.out.println("====="+contentStr);
Pattern pattern = Pattern.compile("\"vid\":\"([0-9a-z_]{34})\"");
Matcher matcher = pattern.matcher(contentStr);
if(matcher.find()){
vid = matcher.group(1);
System.out.println(vid);
}
}
if (vid != null) {
return vid;
} else {
return "";
}
}
}
线程池类：
package com.grab.video.listener;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadFactory;
/**
* Hands out threads from the wrapped threadfactory with setDeamon(true), so the
* threads won't keep the JVM alive when it should otherwise exit.
*/
public class DaemonThreadFactory implements ThreadFactory {
private final ThreadFactory factory;
/**
* Construct a ThreadFactory with setDeamon(true) using
* Executors.defaultThreadFactory()
*/
public DaemonThreadFactory() {
this(Executors.defaultThreadFactory());
}
/**
* Construct a ThreadFactory with setDeamon(true) wrapping the given factory
*
* @param thread
* factory to wrap
*/
public DaemonThreadFactory(ThreadFactory factory) {
if (factory == null)
throw new NullPointerException("factory cannot be null");
this.factory = factory;
}
public Thread newThread(Runnable r) {
final Thread t = factory.newThread(r);
t.setDaemon(true);
return t;
}
}任务执行线程池：
package com.grab.video.listener;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadFactory;
import javax.servlet.ServletContext;
import javax.servlet.ServletContextEvent;
import javax.servlet.ServletContextListener;
public class ExecutorContextListener implements ServletContextListener {
private ExecutorService executor;
public void contextInitialized(ServletContextEvent arg0) {
ServletContext context = arg0.getServletContext();
int nr_executors = 3;
ThreadFactory daemonFactory = new DaemonThreadFactory();
try {
nr_executors = Integer.parseInt(context.getInitParameter("nr-executors"));
} catch (NumberFormatException ignore) {
}
if (nr_executors <= 1) {
executor = Executors.newSingleThreadExecutor(daemonFactory);
} else {
executor = Executors.newFixedThreadPool(nr_executors, daemonFactory);
}
context.setAttribute("DOWNLOAD_EXECUTOR", executor);
}
public void contextDestroyed(ServletContextEvent arg0) {
ServletContext context = arg0.getServletContext();
executor.shutdownNow(); // or process/wait until all pending jobs are
// done
}
}
web.xml(项目启动时，启动任务线程类)
<?xml version="1.0" encoding="UTF-8"?>
<web-app xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns="http://java.sun.com/xml/ns/javaee" xmlns:web="http://java.sun.com/xml/ns/javaee"
xsi:schemaLocation="http://java.sun.com/xml/ns/javaee http://java.sun.com/xml/ns/javaee/web-app_2_5.xsd" id="school" version="2.5">
<display-name>Archetype Created Web Application</display-name>
<context-param>
<param-name>contextConfigLocation</param-name>
<param-value>
classpath:applicationContext.xml
</param-value>
</context-param>
<listener>
<listener-class>org.springframework.web.context.ContextLoaderListener</listener-class>
</listener>
<listener>
<listener-class>com.grab.video.listener.ExecutorContextListener</listener-class>
</listener>
<filter>
<filter-name>encodingFilter</filter-name>
<filter-class>org.springframework.web.filter.CharacterEncodingFilter</filter-class>
<init-param>
<param-name>encoding</param-name>
<param-value>UTF-8</param-value>
</init-param>
<init-param>
<param-name>forceEncoding</param-name>
<param-value>true</param-value>
</init-param>
</filter>
<filter-mapping>
<filter-name>encodingFilter</filter-name>
<url-pattern>/*</url-pattern>
</filter-mapping>
<servlet>
<servlet-name>dispatcher</servlet-name>
<servlet-class>org.springframework.web.servlet.DispatcherServlet</servlet-class>
<load-on-startup>1</load-on-startup>
</servlet>
<servlet-mapping>
<servlet-name>dispatcher</servlet-name>
<url-pattern>/</url-pattern>
</servlet-mapping>
<session-config>
<session-timeout>120</session-timeout>
</session-config>
<welcome-file-list>
<welcome-file>index.jsp</welcome-file>
</welcome-file-list>
<error-page>
<error-code>403</error-code>
<location>/error/403</location>
</error-page>
<error-page>
<error-code>404</error-code>
<location>/error/404</location>
</error-page>
</web-app>
下载页面：
<%@ page language="java" import="java.util.*" pageEncoding="UTF-8"%>
<%
String path = request.getContextPath();
String basePath = request.getScheme()+"://"+request.getServerName()+":"+request.getServerPort()+path+"/";
%>
<%@ page language="java"
import="java.util.List"
import="com.grab.video.controller.VideoListModel"
%>
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
<html>
<head>
<base href="<%=basePath%>">
<title>视频</title>
<meta http-equiv="pragma" content="no-cache">
<meta http-equiv="cache-control" content="no-cache">
<meta http-equiv="expires" content="0">
<meta http-equiv="keywords" content="keyword1,keyword2,keyword3">
<meta http-equiv="description" content="This is my page">
<!--
<link rel="stylesheet" type="text/css" href="styles.css">
-->
<link href="/resources/bootstrap/css/bootstrap.css" rel="stylesheet" />
<script src="/resources/js/jquery-1.7.2.js"></script>
<script type="text/javascript">
var userId;
$(document).ready(function(){
userId=$("#userId").val();
var timeId = setInterval(function () {
$.ajax({
type: "POST",
url: "/grab/download/status", /* 注意后面的名字对应CS的方法名称 */
data: "{}", /* 注意参数的格式和名称 */
contentType: "application/json; charset=utf-8",
dataType: "json",
success: function (data) {
//console.log(data);
for(var i in data) {
var percent = data[i].percent;
console.log(percent);
if(percent>0 && percent<100){
$("#progress"+data[i].id).html("<div class='progress'><div class='progress-bar' role='progressbar' aria-valuenow='60' aria-valuemin='0' aria-valuemax='100' style='width:"+ percent+"%;'>"+percent+"%</div></div>");
$("#downloadTd"+data[i].id).html("下载中...");
}else if(percent>100){
percent=100;
$("#progress"+data[i].id).html("<div class='progress'><div class='progress-bar' role='progressbar' aria-valuenow='60' aria-valuemin='0' aria-valuemax='100' style='width:"+ percent+"%;'>"+percent+"%</div></div>");
$("#downloadTd"+data[i].id).html("下载完");
}
if(data[i].status=="SUCCESS"){
$("#Tr"+data[i].id).remove();
}else if(data[i].status=="FAIL"){
$("#progress"+data[i].id).html("失败");
//$("#downloadTd"+data[i].id).html("失败");
}
}
}
});
},4000);
});
//下载单个文件
function downvideo(obj){
var num="0%";
var htmlstr="<div class='progress'><div class='progress-bar' role='progressbar' aria-valuenow='60' aria-valuemin='0' aria-valuemax='100' style='width:"+ num+";'>"+num+"</div></div>";
$(obj).parents(".downloadTd").siblings(".progressStatus").html(htmlstr);
$(obj).parents(".downloadTd").html("下载中...");
var videoId=$(obj).attr("videoId");
var url=$(obj).attr("dataUrl");
//下载
$.post(url,{},function(data){});
var test = setTimeout(function(){
//location.href ="/grab/download/manage?userId="+userId;
},1000);
}
</script>
</head>
<body>
<div class="container">
<div class="col-md-12">
<div class="page-header clearfix">
<h3 class="pull-left">视频管理</h3>
</div>
<div>
<ul class="nav nav-tabs">
<li><a href="/grab/import">导入任务</a></li>
<li class="active"><a href="/grab/download/manage">待抓视频</a></li>
<li><a href="/grab/export">已抓视频</a></li>
</ul>
</div>
<input type="hidden" id="userId" name="userId" value="${userId}"/>
<%List<VideoListModel> ls = (List) request.getAttribute("videolist");
if(ls.size()>0){%>
<form action="/grab/download/manage" method="post">
<button type="submit" class="btn btn-default btn-info pull-right" style="margin-top:10px;">全部下载</button>
<input type="hidden" id="userId" class="userId" name="userId" value="<%=ls.get(0).getUserId() %>" />
<div class="row">
<table id="guanggao-table" class="table table-hover">
<thead>
<tr>
<th>序号</th>
<th>视频URL</th>
<th>标题</th>
<th>视频分类</th>
<th width="15%">状态</th>
<th>操作1</th>
<th>操作2</th>
</tr>
</thead>
<tbody>
<%for(int i=0;i<ls.size();i++){ %>
<tr id="Tr<%=ls.get(i).getId() %>" pUrl="/grab/download/progress?videoId=<%=ls.get(i).getId() %>&trueName=<%=ls.get(i).getTrueName() %>&userId=<%=ls.get(i).getUserId() %>&url=<%=ls.get(i).getUrl() %>" >
<td> <%=i+1 %>
<input type="hidden" id="videoId" class="videoId" name="videoId" value="<%=ls.get(i).getId() %>" />
<input type="hidden" id="url" class="url" name="url" value="<%=ls.get(i).getUrl() %>" />
<input type="hidden" id="trueName" class="trueName" name="trueName" value="<%=ls.get(i).getTrueName() %>" />
</td>
<td>
<a href="<%=ls.get(i).getUrl() %>" title="<%=ls.get(i).getUrl() %>"><%String str=ls.get(i).getUrl();String pathstr=str.substring(0, 20);%>
<%=pathstr %>...
</a>
</td>
<td><%=ls.get(i).getTitle() %></td>
<td><%=ls.get(i).getClassifyName() %></td>
<td class="progressStatus" id="progress<%=ls.get(i).getId() %>">
<% if("NO".equals(ls.get(i).getStatus().trim())){%>
<div>待抓取</div>
<%}else if("WAIT".equals(ls.get(i).getStatus().trim())){ %>
<div>等待</div>
<%}else if("TRANSCODING".equals(ls.get(i).getStatus().trim())){ %>
<div class="progress">
<div class="progress-bar" role="progressbar" aria-valuenow="60" aria-valuemin="0" aria-valuemax="100" style="width:1%;">1%</div>
</div>
<%}else if("EXECUTE".equals(ls.get(i).getStatus().trim())){ %>
<%}else if("SUCCESS".equals(ls.get(i).getStatus().trim())){ %>
<div>成功</div>
<%}else{ %>
<div>失败 </div>
<%} %>
</td>
<td class="downloadTd" id="downloadTd<%=ls.get(i).getId() %>">
<% if("EXECUTE".equals(ls.get(i).getStatus().trim())){%>
下载中...<div class="execute" videoId="<%=ls.get(i).getId() %>" style="display:none;"></div>
<%}else if("WAIT".equals(ls.get(i).getStatus().trim())){ %>
等待
<%}else{%>
<button type="button" class="btn btn-default btn-info" videoId="<%=ls.get(i).getId() %>" dataUrl="/grab/download/manage?videoId=<%=ls.get(i).getId() %>&trueName=<%=ls.get(i).getTrueName() %>&userId=<%=ls.get(i).getUserId() %>&url=<%=ls.get(i).getUrl() %>" onclick="downvideo(this);">抓取</button>
<%} %>
</td>
<td>
<!--
<button type="button" class="btn btn-default" videoId="<%=ls.get(i).getId() %>" dataUrl="/grab/video/stop?videoId=<%=ls.get(i).getId() %>&trueName=<%=ls.get(i).getTrueName() %>&userId=<%=ls.get(i).getUserId() %>&url=<%=ls.get(i).getUrl() %>" onclick="stopVideo(this);">暂停</button>
-->
<button type="button" class="btn btn-default" videoId="<%=ls.get(i).getId() %>" dataUrl="/grab/delete/videoId?videoId=<%=ls.get(i).getId() %>" onclick="deleteVideo(this);">删除</button>
</td>
</tr>
<%} %>
</tbody>
</table>
</div>
</form>
<%}else{ %>
<div class="col-md-6" style="padding-top:30px;">暂无视频可以抓取</div>
<%} %>
</div>
</div>
</body>
</html>
<script>
//删除文件
function deleteVideo(obj){
var url=$(obj).attr("dataUrl");
if (!confirm('真的要永久删除该内容吗？')) {
return ;
}else{
$.ajax({
type: "GET",
url: url, /* 注意后面的名字对应CS的方法名称 */
data: "{}", /* 注意参数的格式和名称 */
contentType: "application/json; charset=utf-8",
dataType: "json",
success: function (data) {
console.log(data);
var mark=data.result;
if(mark=="yes"){
// window.location.reload();
location.href ="/grab/download/manage?userId="+userId;
}else{
alert("删除失败！");
}
}
});
}
}
//暂停下载
function stopVideo(obj){
var url=$(obj).attr("dataUrl");
if (!confirm('暂停下载')) {
return ;
}else{
$.ajax({
type: "GET",
url: url, /* 注意后面的名字对应CS的方法名称 */
data: "{}", /* 注意参数的格式和名称 */
contentType: "application/json; charset=utf-8",
dataType: "json",
success: function (data) {
console.log(data);
var mark=data.result;
if(mark=="yes"){
// window.location.reload();
location.href ="/grab/download/manage?userId="+userId;
}else{
alert("操作失败！");
}
}
});
}
}
</script>
package com.grab.video.controller;
import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.io.Writer;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLDecoder;
import java.net.URLEncoder;
import java.nio.charset.Charset;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Scanner;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import javax.servlet.ServletContext;
import javax.servlet.ServletOutputStream;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.servlet.http.HttpSession;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.HttpEntity;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.ServletRequestUtils;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.servlet.ModelAndView;
import org.springframework.web.servlet.view.RedirectView;
import com.fasterxml.jackson.core.JsonGenerationException;
import com.fasterxml.jackson.databind.JsonMappingException;
import com.fasterxml.jackson.databind.ObjectMapper;
@Controller
public class GrabVideoController {
private static final Logger LOG = LoggerFactory.getLogger(GrabVideoController.class);
//private static String filePath="D:\\logs\\video";
private static String filePath = "/home/grabVideo/";
@Autowired
ServletContext context;
/**
* 输入userid可以使用
*
* @param request
* @param response
* @return
*/
@RequestMapping(value = "/grab/login", method = { RequestMethod.GET })
public ModelAndView login(HttpServletRequest request, HttpServletResponse response) {
String userId = ServletRequestUtils.getStringParameter(request, "userId", "");
String ts = ServletRequestUtils.getStringParameter(request, "ts", "");
String sign = ServletRequestUtils.getStringParameter(request, "sign", "");
ModelAndView mav = new ModelAndView();
// 身份验证
if (StringUtils.isNotBlank(userId)) {
String encryptedSign = EncryptionUtils.md5Hex(ts + userId + "grab");
if (sign.equals(encryptedSign)) {
HttpSession session = request.getSession();
session.setMaxInactiveInterval(5*24*60*60);//秒为单位,设置session周期为5天
session.setAttribute("userId", userId);// 把userId存放到session
String url = "/grab/import";
mav.setView(new RedirectView(url));
return mav;
}
}
mav.setViewName("video/error");
return mav;
}
/**
* 导入文件
*
* @return
*/
@RequestMapping(value = "/grab/import", method = { RequestMethod.GET })
public ModelAndView importFile(HttpServletRequest request, HttpServletResponse response) {
// String userId = ServletRequestUtils.getStringParameter(request,
// "userId", null);
ModelAndView mav = new ModelAndView();
HttpSession session = request.getSession();
String userId = null;
if (session.getAttribute("userId") != null) {
userId = (String) session.getAttribute("userId");
SqlFileList sqlFileList = new SqlFileList();
List<FileListModel> list = new ArrayList<FileListModel>();
try {
list = sqlFileList.selectDate(userId);
} catch (SQLException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
// 从POLYV的API获取目录
mav.addObject("list", list);
mav.addObject("userId", userId);
mav.setViewName("video/import");
return mav;
}
mav.setViewName("video/login");
return mav;
}
/**
* 删除文件
*
* @param request
* @param response
* @return
*/
@RequestMapping(value = "/grab/delete/file", method = { RequestMethod.GET })
public ResponseEntity<AjaxPostResponse> deleteFile(HttpServletRequest request,
HttpServletResponse response) {
String fileId = ServletRequestUtils.getStringParameter(request, "fileId", null);
MediaType mediaType = new MediaType("application", "json", Charset.forName("UTF-8"));
HttpHeaders headers = new HttpHeaders();
headers.setContentType(mediaType);
SqlFileList sqlFileList = new SqlFileList();
try {
sqlFileList.deleteDate(fileId);
} catch (SQLException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
AjaxPostResponse resp = new AjaxPostResponse("yes");
return new ResponseEntity<AjaxPostResponse>(resp, headers, HttpStatus.OK);
}
/**
* 解析文件
*
* @return
* @throws UnsupportedEncodingException
*/
@RequestMapping(value = "/grab/analysis", method = { RequestMethod.GET })
public ResponseEntity<AjaxPostResponse> analysisFile(HttpServletRequest request,
HttpServletResponse response) throws UnsupportedEncodingException {
Integer fileId = ServletRequestUtils.getIntParameter(request, "fileId", 0);
String fileUrl = ServletRequestUtils.getStringParameter(request, "fileUrl", "");
String classifyId = ServletRequestUtils.getStringParameter(request, "classifyId",
"classifyId");
String classifyName = ServletRequestUtils.getStringParameter(request, "classifyName", "");
String userId = ServletRequestUtils.getStringParameter(request, "userId", null);
MediaType mediaType = new MediaType("application", "json", Charset.forName("UTF-8"));
HttpHeaders headers = new HttpHeaders();
headers.setContentType(mediaType);
String errorStr="";
System.out.println("==========="+userId);
List<String> urlList = new ArrayList<String>();
List<String> titleList = new ArrayList<String>();
try {
System.out.println("file============"+fileUrl);
CloseableHttpClient httpclient = HttpClients.createDefault();
HttpGet httpGet = new HttpGet(fileUrl);
try {
CloseableHttpResponse response2 = httpclient.execute(httpGet);
InputStream is = null;
Scanner sc = null;
Writer os = null;
if (response2.getStatusLine().getStatusCode() == org.apache.http.HttpStatus.SC_OK) {
try {
// 2、获取response的entity。
HttpEntity entity = response2.getEntity();
is = entity.getContent();
//sc = new Scanner(is);
BufferedReader reader = new BufferedReader(new InputStreamReader(
is, "UTF-8"));
int n=0;
String line = null;
while ((line = reader.readLine()) != null) {
n++;
try {
String str = line;
//
String urlstr = str.substring(0, str.indexOf(","));
String title = str.substring(str.lastIndexOf(",") + 1, str.length());
//
urlList.add(urlstr);
titleList.add(title);
} catch (Exception e) {
// TODO Auto-generated catch block
System.out.println("解析失败"+n);
errorStr=errorStr+","+n;
e.printStackTrace();
}
}
errorStr=errorStr+"行格式原因";
}catch(Exception e){
System.out.println("解析失败");
} finally {
if (sc != null) {
sc.close();
}
if (is != null) {
is.close();
}
if (os != null) {
os.close();
}
if (response2 != null) {
response2.close();
}
}
}
}catch(Exception e){
System.out.println("解析失败");
}
} catch (Exception e) {
// TODO Auto-generated catch block
LOG.info("文件解析失败：" + e);
e.printStackTrace();
AjaxPostResponse resp = new AjaxPostResponse(errorStr+":解析失败");
return new ResponseEntity<AjaxPostResponse>(resp, headers, HttpStatus.OK);
}
// 更新状态
SqlFileList sqlFileList = new SqlFileList();
FileListModel file = new FileListModel();
file.setFileId(fileId);
file.setStatus("Y");
try {
sqlFileList.updateDate(file);
} catch (SQLException e1) {
// TODO Auto-generated catch block
LOG.info("文件状态修改成功：" + e1);
e1.printStackTrace();
}
// LOG.info("00000"+classifyName);
classifyName = classifyName.replace("-", "");
// LOG.info(classifyName);
// 添加数据
Date date = new Date();
Timestamp timeStamp = new Timestamp(date.getTime());
GetRandomString randomStr = new GetRandomString();
for (int i = 0; i < urlList.size(); i++) {
VideoListModel video = new VideoListModel();
video.setUserId(userId);
video.setUrl(urlList.get(i));// 视频源地址
video.setTitle(titleList.get(i));// 视频标题
String urlstr = urlList.get(i);
// String path=urlstr.substring(0, urlstr.indexOf("?"));
// String format=path.substring(path.lastIndexOf("."),
// path.length());//视频格式
// String baseName = FilenameUtils.getBaseName(urlstr);
String extendname = FilenameUtils.getExtension(urlstr);
if(extendname.contains("?")){
extendname=extendname.substring(0,extendname.indexOf("?"));
}
if (StringUtils.isBlank(extendname)) {
extendname = "mp4";
}
String trueName = randomStr.generateRandomString(15);
String filename = trueName + "." + extendname;
video.setTrueName(filename);// 用于下载使用的视频名称
video.setClassifyId(classifyId);
//video.setClassifyName(classifyName.trim());
video.setClassifyName(URLDecoder.decode(classifyName.trim(),"GBK"));
video.setStatus(VideoStatus.NO.getValue());// 等待、
video.setVid("");
video.setCreateTime(timeStamp);
video.setLastDate(timeStamp);
SqlVideoList sqlvideo = new SqlVideoList();
try {
sqlvideo.insertDate(video);// 添加数据库
} catch (SQLException e) {
// TODO Auto-generated catch block
LOG.info("添加数据库:" + e);
e.printStackTrace();
AjaxPostResponse resp = new AjaxPostResponse("no");
return new ResponseEntity<AjaxPostResponse>(resp, headers, HttpStatus.OK);
}
}
AjaxPostResponse resp = new AjaxPostResponse("yes");
return new ResponseEntity<AjaxPostResponse>(resp, headers, HttpStatus.OK);
}
/**
* 获取下载进度
*
* @return
*/
@RequestMapping(value = "/grab/download/progress", method = { RequestMethod.POST,
RequestMethod.GET })
public ResponseEntity<AjaxPostResponse> getProgress(HttpServletRequest request,
HttpServletResponse response) {
Integer id = ServletRequestUtils.getIntParameter(request, "videoId", 0);
String userId = ServletRequestUtils.getStringParameter(request, "userId", "test");
String urlstr = ServletRequestUtils.getStringParameter(request, "url", "");
String trueName = ServletRequestUtils.getStringParameter(request, "trueName", "");
MediaType mediaType = new MediaType("application", "json", Charset.forName("UTF-8"));
HttpHeaders headers = new HttpHeaders();
headers.setContentType(mediaType);
// LOG.info("--id---"+id+"---u---"+userId);
int content = 1;
int length = 1;
int progress = 1;
HttpSession session = request.getSession();
// LOG.info("-------ccccc4------------"+session.getAttribute("fileSize"+id));
if (session.getAttribute("fileSize" + String.valueOf(id)) == null) {
// 文件大小还没存进session中
List<TaskQueue> list = new ArrayList<TaskQueue>();
SqlTaskQueue stq = new SqlTaskQueue();
try {
list = stq.selectDateOne(String.valueOf(id));
} catch (SQLException e2) {
// TODO Auto-generated catch block
LOG.info("查询文件大小" + e2);
e2.printStackTrace();
}
if (list.size() > 0) {
TaskQueue tQueue = list.get(0);
content = tQueue.getFileSize();
session.setAttribute("fileSize" + String.valueOf(id), content);// 存进session
} else {
URL url = null;
try {
url = new URL(urlstr);
HttpURLConnection connection = (HttpURLConnection) url.openConnection();// 进行连接握手
connection.setRequestMethod("GET");// 请求方式
content = (int) connection.getContentLength();
session.setAttribute("fileSize" + String.valueOf(id), content);// 存进session
// LOG.info("-------content------"+content);
} catch (Exception e1) {
// TODO Auto-generated catch block
LOG.info("链接失败" + e1);
e1.printStackTrace();
}
}
} else {
// 文件大少在session中
String contentString = String.valueOf(session.getAttribute("fileSize"
+ String.valueOf(id)));
// LOG.info("-------ccccc------------"+contentString);
content = Integer.parseInt(contentString.trim());
}
// 文件存储位置、文件命名处理
try {
// String path=urlstr.substring(0, urlstr.indexOf("?"));
// String name=path.substring(path.lastIndexOf("/")+1,
// path.length());
// String filename=name.trim();
String filename = trueName;
File file = new File(filePath, filename);
if (!file.exists()) {
progress = (Integer) session.getAttribute(userId + id);// 将当前下载进度存放到session中。
} else {
length = (int) file.length();
progress = length * 100 / content;
// 将当前下载进度存放到session中。
session.setAttribute(userId + id, progress);
LOG.info(id + "-------progress------" + progress);
}
} catch (Exception e) {
LOG.info("不能解析的路径：" + e);
AjaxPostResponse resp = new AjaxPostResponse(progress);
return new ResponseEntity<AjaxPostResponse>(resp, headers, HttpStatus.OK);
}
AjaxPostResponse resp = new AjaxPostResponse(progress);
return new ResponseEntity<AjaxPostResponse>(resp, headers, HttpStatus.OK);
}
/**
* 批量抓取视频(下载视频模块--根据视频源地址去抓取视频)管理
*
* @return
* @throws MalformedURLException
*/
@RequestMapping(value = "/grab/download/manage", method = { RequestMethod.POST })
public ModelAndView grabDownloadVideo(HttpServletRequest request, HttpServletResponse response) {
int[] id = ServletRequestUtils.getIntParameters(request, "videoId");
String userId = ServletRequestUtils.getStringParameter(request, "userId", "test");
String[] urlstr = ServletRequestUtils.getStringParameters(request, "url");
String[] trueName = ServletRequestUtils.getStringParameters(request, "trueName");
int len = id.length;
List<TaskQueue> taskQueues = new ArrayList<TaskQueue>();
for (int i = 0; i < len; i++) {
TaskQueue tq = new TaskQueue();
tq.setTaskId(String.valueOf(id[i]));
tq.setVideoId(id[i]);
tq.setUserId(userId);
tq.setVideoUrl(urlstr[i]);
tq.setTrueName(trueName[i]);
taskQueues.add(tq);
}
// 把任务队列添加进数据库
if (taskQueues.size() > 0) {
// 存在有任务
for (int i = 0; i < taskQueues.size(); i++) {
TaskQueue task = taskQueues.get(i);
List<TaskQueue> taskList = new ArrayList<TaskQueue>();// 查询任务是否已存在
try {
SqlTaskQueue stq = new SqlTaskQueue();
taskList = stq.selectDateOne(task.getTaskId());
} catch (SQLException e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
}
if (taskList.size() > 0) {
// 该任务已存在
} else {
task.setFileSize(0);
task.setProgress(0);
task.setStatus("N");
Date date = new Date();
Timestamp timeStamp = new Timestamp(date.getTime());
task.setCreateTime(timeStamp);
try {
SqlTaskQueue stq = new SqlTaskQueue();
stq.insertDate(task);
} catch (SQLException e) {
// TODO Auto-generated catch block
LOG.info("下载任务添加失败！" + e);
e.printStackTrace();
}
}
}
}
// 获取所有的任务队列
List<TaskQueue> workQueues = new ArrayList<TaskQueue>();
try {
SqlTaskQueue stq = new SqlTaskQueue();
workQueues = stq.selectDate(userId);
} catch (SQLException e) {
// TODO Auto-generated catch block
LOG.info("获取下载任务失败" + e);
e.printStackTrace();
}
// ExecutorService pool = Executors.newFixedThreadPool(3);
if (workQueues.size() > 0) {
for (int i = 0; i < workQueues.size(); i++) {
String taskId = workQueues.get(i).getTaskId();
String urltxt = workQueues.get(i).getVideoUrl();
String filename = workQueues.get(i).getTrueName();
File saveFile = new File(filePath, filename);// 文件保存的位置
File fileDir = new File(filePath);
if (!fileDir.exists()) {
fileDir.mkdirs();// 目录不存在创建目录
}
URL url = null;
try {
url = new URL(workQueues.get(i).getVideoUrl());
} catch (MalformedURLException e) {
// TODO Auto-generated catch block
LOG.info("握手失败" + e);
e.printStackTrace();
}
if (url != null) {
// 将下载任务线程，放入线程池中执行
ExecutorService executor = (ExecutorService) context
.getAttribute("DOWNLOAD_EXECUTOR");
executor.submit(new DownloadVideo(url, saveFile, taskId));
// pool.execute(new
// DownloadVideo(url,saveFile,taskId));////////////////////////////////////
VideoListModel vlm = new VideoListModel();
vlm.setId(workQueues.get(i).getVideoId());
vlm.setUserId(userId);
vlm.setStatus(VideoStatus.WAIT.getValue());// 将状态改为等待
try {
SqlVideoList svl = new SqlVideoList();
svl.updateDate(vlm);
} catch (SQLException e) {
// TODO Auto-generated catch block
LOG.info("更改下载状态失败" + e);
e.printStackTrace();
}
}
}
}
// 关闭线程池
// pool.shutdown();
// 重新查询视频列表
List<VideoListModel> list = new ArrayList<VideoListModel>();
try {
SqlVideoList svl = new SqlVideoList();
list = svl.selectDate(userId);
} catch (SQLException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
ModelAndView mav = new ModelAndView();
mav.addObject("videolist", list);
mav.addObject("userId", userId);
mav.setViewName("video/download");
return mav;
}
/**
* 获取视频的下载进度()
* @param request
*/
@RequestMapping(value="/grab/download/status",method = {RequestMethod.GET,RequestMethod.POST})
public @ResponseBody
ResponseEntity<String> downloadStatus(HttpServletRequest request, HttpServletResponse response)
throws JsonGenerationException, JsonMappingException, IOException {
HttpSession session = request.getSession();
if (session.getAttribute("userId") != null) {
String userId = (String) session.getAttribute("userId");
List<VideoListModel> list = new ArrayList<VideoListModel>();
try {
SqlVideoList sqlVideoList = new SqlVideoList();
list = sqlVideoList.selectAllDate(userId);
//list = sqlVideoList.selectExecuteDate(userId);
for (int i = 0; i < list.size(); i++) {
VideoListModel model = list.get(i);
String filename = model.getTrueName();
File file = new File(filePath, filename);
if (file.exists()) {
int downloaded = (int) file.length();
if (model.getFileSize() != 0) {
//System.out.println(model.getId()+"===n==="+model.getTrueName()+"===d==="+downloaded+"===s==="+model.getFileSize()+"===="+(long)downloaded * 100 /(long) model.getFileSize());
model.setPercent((int) ((long)downloaded * 100 /(long) model.getFileSize()));
}
}
}
ObjectMapper objectMapper = new ObjectMapper();
String result = objectMapper.writeValueAsString(list);
MediaType mediaType = new MediaType("application", "json", Charset.forName("UTF-8"));
HttpHeaders headers = new HttpHeaders();
headers.setContentType(mediaType);
return new ResponseEntity<String>(result, headers, HttpStatus.OK);
} catch (SQLException e) {
// TODO Auto-generated catch block
System.out.println("获取下载进度出现异常！");
e.printStackTrace();
}
}
return null;
}
@RequestMapping(value = "/grab/download/manage", method = { RequestMethod.GET })
public ModelAndView grabVideo(HttpServletRequest request, HttpServletResponse response) {
// String userId = ServletRequestUtils.getStringParameter(request,
// "userId", "");
String userId = null;
ModelAndView mav = new ModelAndView();
HttpSession session = request.getSession();
if (session.getAttribute("userId") != null) {
userId = (String) session.getAttribute("userId");
SqlVideoList sqlVideoList = new SqlVideoList();
List<VideoListModel> list = new ArrayList<VideoListModel>();
try {
list = sqlVideoList.selectDate(userId);
} catch (SQLException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
mav.addObject("videolist", list);
mav.addObject("userId", userId);
mav.setViewName("video/download");
return mav;
}
mav.setViewName("video/login");
return mav;
}
/**
* 暂停下载
*/
@RequestMapping(value = "/grab/video/stop", method = { RequestMethod.GET })
public ResponseEntity<AjaxPostResponse> downloadStop(HttpServletRequest request,
HttpServletResponse response) {
int id = ServletRequestUtils.getIntParameter(request, "videoId",0);
String userId = ServletRequestUtils.getStringParameter(request, "userId", "test");
//String urlstr = ServletRequestUtils.getStringParameter(request, "url","");
//String trueName = ServletRequestUtils.getStringParameter(request, "trueName","");
MediaType mediaType = new MediaType("application", "json", Charset.forName("UTF-8"));
HttpHeaders headers = new HttpHeaders();
headers.setContentType(mediaType);
try {
SqlTaskQueue stq = new SqlTaskQueue();
stq.deleteDate(String.valueOf(id));
} catch (SQLException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
// 获取所有的任务队列
List<TaskQueue> workQueues = new ArrayList<TaskQueue>();
try {
SqlTaskQueue stq = new SqlTaskQueue();
workQueues = stq.selectDate(userId);
} catch (SQLException e) {
// TODO Auto-generated catch block
LOG.info("获取下载任务失败" + e);
e.printStackTrace();
}
// ExecutorService pool = Executors.newFixedThreadPool(3);
if (workQueues.size() > 0) {
for (int i = 0; i < workQueues.size(); i++) {
String taskId = workQueues.get(i).getTaskId();
String urltxt = workQueues.get(i).getVideoUrl();
String filename = workQueues.get(i).getTrueName();
File saveFile = new File(filePath, filename);// 文件保存的位置
File fileDir = new File(filePath);
if (!fileDir.exists()) {
fileDir.mkdirs();// 目录不存在创建目录
}
URL url = null;
try {
url = new URL(workQueues.get(i).getVideoUrl());
} catch (MalformedURLException e) {
// TODO Auto-generated catch block
LOG.info("握手失败" + e);
e.printStackTrace();
}
if (url != null) {
// 将下载任务线程，放入线程池中执行
ExecutorService executor = (ExecutorService) context
.getAttribute("DOWNLOAD_EXECUTOR");
executor.submit(new DownloadVideo(url, saveFile, taskId));
// pool.execute(new
// DownloadVideo(url,saveFile,taskId));////////////////////////////////////
VideoListModel vlm = new VideoListModel();
vlm.setId(workQueues.get(i).getVideoId());
vlm.setUserId(userId);
vlm.setStatus(VideoStatus.WAIT.getValue());// 将状态改为等待
try {
SqlVideoList svl = new SqlVideoList();
svl.updateDate(vlm);
} catch (SQLException e) {
// TODO Auto-generated catch block
LOG.info("更改下载状态失败" + e);
e.printStackTrace();
}
}
}
}
System.out.println("停止下载！");
AjaxPostResponse resp = new AjaxPostResponse("yes");
return new ResponseEntity<AjaxPostResponse>(resp, headers, HttpStatus.OK);
}
/**
* 导出下载成功的视频
*
* @return
*/
@RequestMapping(value = "/grab/export", method = { RequestMethod.GET })
public ModelAndView exportVideo(HttpServletRequest request, HttpServletResponse response) {
// String userId = ServletRequestUtils.getStringParameter(request,
// "userId", "");
String userId = null;
ModelAndView mav = new ModelAndView();
HttpSession session = request.getSession();
if (session.getAttribute("userId") != null) {
userId = (String) session.getAttribute("userId");
SqlVideoList sqlVideoList = new SqlVideoList();
List<VideoListModel> list = new ArrayList<VideoListModel>();
try {
list = sqlVideoList.selectSuccessDate(userId);
} catch (SQLException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
mav.addObject("videolist", list);
mav.addObject("userId", userId);
mav.setViewName("video/export");
return mav;
}
mav.setViewName("video/login");
return mav;
}
/**
* export导出文件
*/
@RequestMapping(value = "/grab/export/csv", method = { RequestMethod.GET })
public void exportCsv(HttpServletRequest request, HttpServletResponse response) {
String userId = ServletRequestUtils.getStringParameter(request, "userId", "");
if (StringUtils.isNotBlank(userId)) {
SqlVideoList sqlVideoList = new SqlVideoList();
List<VideoListModel> list = new ArrayList<VideoListModel>();
try {
list = sqlVideoList.selectSuccessDate(userId);
} catch (SQLException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
// 导出txt文件
//response.setContentType("text/plain");
response.setContentType("text/csv; charset=UTF-8");
String fileName = "videolist";
try {
fileName = URLEncoder.encode("videolist", "UTF-8");
} catch (UnsupportedEncodingException e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
}
response.setHeader("Content-Disposition", "attachment; filename=" + fileName + ".csv");
BufferedOutputStream buff = null;
StringBuffer write = new StringBuffer();
String enter = "\r\n";
ServletOutputStream outSTr = null;
try {
outSTr = response.getOutputStream(); // 建立
buff = new BufferedOutputStream(outSTr);
// 把内容写入文件
if (list.size() > 0) {
for (int i = 0; i < list.size(); i++) {
write.append(list.get(i).getUrl());
write.append(",");
write.append(list.get(i).getTitle());
write.append(",");
write.append(list.get(i).getVid());
write.append(",");
write.append(list.get(i).getLastDate());
write.append(enter);
}
}
buff.write(write.toString().getBytes("GBK"));
buff.flush();
buff.close();
} catch (Exception e) {
e.printStackTrace();
} finally {
try {
buff.close();
outSTr.close();
} catch (Exception e) {
e.printStackTrace();
}
}
}
}
/**
* 清空导出视频列表
* /grab/export/clean
*/
@RequestMapping(value = "/grab/export/clean", method = { RequestMethod.GET })
public ResponseEntity<AjaxPostResponse> cleanVideo(HttpServletRequest request,
HttpServletResponse response) {
MediaType mediaType = new MediaType("application", "json", Charset.forName("UTF-8"));
HttpHeaders headers = new HttpHeaders();
headers.setContentType(mediaType);
HttpSession session = request.getSession();
if (session.getAttribute("userId") != null) {
String userId=(String) session.getAttribute("userId");
SqlVideoList sqlVideoList=new SqlVideoList();
try {
sqlVideoList.cleanDate(userId);
} catch (SQLException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
//删除成功
System.out.println("删除成功！");
AjaxPostResponse resp = new AjaxPostResponse("yes");
return new ResponseEntity<AjaxPostResponse>(resp, headers, HttpStatus.OK);
}else{
//删除失败
System.out.println("删除失败！");
AjaxPostResponse resp = new AjaxPostResponse("no");
return new ResponseEntity<AjaxPostResponse>(resp, headers, HttpStatus.OK);
}
}
/**
* 删除视频
* /grab/export/clean
*/
@RequestMapping(value = "/grab/delete/videoId", method = { RequestMethod.GET })
public ResponseEntity<AjaxPostResponse> deleteVideo(HttpServletRequest request,
HttpServletResponse response) {
String videoId = ServletRequestUtils.getStringParameter(request, "videoId", null);
MediaType mediaType = new MediaType("application", "json", Charset.forName("UTF-8"));
HttpHeaders headers = new HttpHeaders();
headers.setContentType(mediaType);
if (StringUtils.isNotBlank(videoId)) {
SqlVideoList sqlVideoList=new SqlVideoList();
try {
SqlTaskQueue stqSe=new SqlTaskQueue();
List<TaskQueue> list=stqSe.selectDateOne(videoId);
if(list.size()>0){
SqlTaskQueue stq=new SqlTaskQueue();
stq.deleteDate(videoId);//删除任务
}
sqlVideoList.deleteDate(videoId);//删除视频
} catch (SQLException e) {
// TODO Auto-generated catch block
System.out.println("删除失败"+e);
e.printStackTrace();
}
//删除成功
AjaxPostResponse resp = new AjaxPostResponse("yes");
return new ResponseEntity<AjaxPostResponse>(resp, headers, HttpStatus.OK);
}else{
//删除失败
AjaxPostResponse resp = new AjaxPostResponse("no");
return new ResponseEntity<AjaxPostResponse>(resp, headers, HttpStatus.OK);
}
}
/***
* 获取文件内容
* @param url
* @return
*/
public String httpGetFile(String url) {
String contentStr="";
CloseableHttpClient httpclient = HttpClients.createDefault();
HttpGet httpGet = new HttpGet(url);
try {
CloseableHttpResponse response = httpclient.execute(httpGet);
InputStream is = null;
Scanner sc = null;
Writer os = null;
if (response.getStatusLine().getStatusCode() == org.apache.http.HttpStatus.SC_OK) {
try {
// 2、获取response的entity。
HttpEntity entity = response.getEntity();
is = entity.getContent();
sc = new Scanner(is);
while (sc.hasNext()) {
contentStr=contentStr+sc.nextLine();
}
} catch (ClientProtocolException e) {
e.printStackTrace();
} finally {
if (sc != null) {
sc.close();
}
if (is != null) {
is.close();
}
if (os != null) {
os.close();
}
if (response != null) {
response.close();
}
}
}
} catch (ClientProtocolException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
return contentStr;
}
}
下载视频主线程类：
package com.grab.video.controller;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.io.Writer;
import java.net.HttpURLConnection;
import java.net.ProtocolException;
import java.net.URL;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Scanner;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.HttpEntity;
import org.apache.http.HttpStatus;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
public class DownloadVideo implements Runnable {
private static final Logger LOG = LoggerFactory.getLogger(DownloadVideo.class);
private static String TAG = "Downloader";
private HttpURLConnection connection;
private URL url;
private File saveFile;
private long fileLength;// 文件总大少
private int progress;// 当前进度
private long downloaded = 0;
private boolean stop=false;
private String taskId;
private static String fileUrl = "http://grab.polyv.net/video/";
private boolean downloadStatus = false;
public DownloadVideo(URL url, File saveFile, String taskId) {
this.url = url;
this.saveFile = saveFile;
this.taskId = taskId;
}
public DownloadVideo(URL url, File saveFile, String taskId,boolean stop) {
this.url = url;
this.saveFile = saveFile;
this.taskId = taskId;
this.stop = stop;
}
public DownloadVideo(String vid, File saveFile) {
// 通过vid,获取视频TODO
}
public URL getUrl() {
return url;
}
public void setUrl(URL url) {
this.url = url;
}
public void setStop(boolean stop) {
this.stop = stop;
}
public long getFileLength() {
return fileLength;
}
public int getProgress() {
return progress;
}
public void setProgress(int progress) {
this.progress = progress;
}
public long getDownloaded() {
return downloaded;
}
public int getPercent() {
if (fileLength == 0) {
return 0;
}
return (int) (downloaded * 100 / fileLength);
}
public void stop() {
stop = true;
}
public void start() {
stop = false;
}
// 开始执行,实现run方法
public void run() {
VideoListModel vlm = new VideoListModel();
vlm.setId(Integer.valueOf(taskId));
vlm.setStatus(VideoStatus.EXECUTE.getValue());// 将状态改为正在执行
try {
SqlVideoList sqlVideoList = new SqlVideoList();
System.out.println("更新下载中状态。。。。");
sqlVideoList.updateDate(vlm);
} catch (SQLException e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
}
try {
BufferedInputStream in = null;
FileOutputStream fos = null;
BufferedOutputStream bout = null;
print("start download:" + url);
connection = (HttpURLConnection) url.openConnection();// 进行连接握手
connection.setRequestProperty("User-Agent", "Polyv");
Map<String, List<String>> map = connection.getRequestProperties();
print(map.toString());// 输出参数
connection.setRequestMethod("GET");// 请求方式
if (saveFile.exists()) {
downloaded = saveFile.length();
connection.setRequestProperty("Range", "bytes=" + downloaded + "-");
} else {
downloaded = 0;// 重新开始下载
}
// connection.getResponseCode() == 200 206?
int code = connection.getResponseCode();// 获取状态码
print("code=" + code + ", downloaded =" + downloaded);
printResponseHeader(connection);
// http200状态,重新开始下载，206状态续点下载
if (connection.getResponseCode() == 206 || connection.getResponseCode() == 200) {
String range = "";
HashMap<String, String> header = (HashMap<String, String>) getHttpResponseHeader(connection);
for (Map.Entry<String, String> entry : header.entrySet()) {
String key = entry.getKey();
String value = entry.getValue();
System.out.println(key+"============="+value);
if (key.equals("Content-Range")) {
range = entry.getValue();
}
}
System.out.println("-----range-----"+range);
long content = (long) connection.getContentLength();
fileLength = range.equals("") ? content : Long.valueOf(range.split("/")[1]);
System.out.println("文件===c==="+content+"===f==="+fileLength+"======"+(long)(content+downloaded));
try {
SqlVideoList sqlVideoList = new SqlVideoList();
sqlVideoList.updateFileSize(Integer.valueOf(taskId),content+downloaded);//获取文件的大少(合计才是文件的大少,content是还有多少下载的，downloaded已下载的大少)
} catch (SQLException e2) {
// TODO Auto-generated catch block
System.out.println("获取文件大少失败");
e2.printStackTrace();
}
in = new BufferedInputStream(connection.getInputStream());
fos = (downloaded == 0) ? new FileOutputStream(saveFile) : new FileOutputStream(
saveFile, true);
bout = new BufferedOutputStream(fos, 1024);
byte[] data = new byte[1024];
int x = 0;
boolean p = false;
while (!stop && (x = in.read(data, 0, 1024)) >= 0) {
bout.write(data, 0, x);
downloaded += x;
}
bout.close();
System.out.println("下载完毕{}");
downloadStatus = false;// 代表下载成功，不用去更新下载失败的状态
String vid = null;
// 获取vid,(根据taskId即videoId获取视频的title,cataId)
List<VideoListModel> videoList = new ArrayList<VideoListModel>();
try {
SqlVideoList sqlVideoList = new SqlVideoList();
videoList = sqlVideoList.selectDateOne(Integer.valueOf(taskId));
} catch (SQLException e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
}
if (videoList.size() > 0) {
VideoListModel video = videoList.get(0);
String title = video.getTitle().replace(" ", "");
String cataid = video.getClassifyId();
String userid = video.getUserId();
String trueName = video.getTrueName();
long ts = System.currentTimeMillis();
String sign = EncryptionUtils.md5Hex(ts + userid + "grab");
String url = fileUrl + trueName;
String pathStr="http://v.polyv.net/uc/services/rest?method=uploadForDownloader&fileUrl="+url+"&userid="+userid+"&title="+title+"&cataid="+cataid+"&ts="+ts+"&sign="+sign;
System.out.println("----url-----"+pathStr);
vid=httpGetVid(pathStr);//获取vid
if(vid==null){
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
vid=httpGetVid(pathStr);//获取vid
}
//第二次
if(vid==null){
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
vid=httpGetVid(pathStr);//获取vid
}
System.out.println("------下载完成获取vid---------" + vid);
}
// 下载完成，更改视频状态为SUCCESS
VideoListModel video = new VideoListModel();
video.setId(Integer.valueOf(taskId));
video.setStatus(VideoStatus.SUCCESS.getValue());
video.setVid(vid);
Date date = new Date();
Timestamp timeStamp = new Timestamp(date.getTime());
video.setLastDate(timeStamp);
try {
SqlVideoList sqlVideoList = new SqlVideoList();
sqlVideoList.updateDateVid(video);
} catch (SQLException e) {
// TODO Auto-generated catch block
System.out.println("下载完成,更改视频状态失败" + e);
e.printStackTrace();
}
// 下载完成，从任务队列移除该任务
try {
SqlTaskQueue stq = new SqlTaskQueue();
stq.deleteDate(taskId);
} catch (SQLException e) {
// TODO Auto-generated catch block
System.out.println("任务移除失败" + e);
e.printStackTrace();
}
}else if (connection.getResponseCode() == 416) {
// 416- 请求长度超出范围
System.out.println("已经下载完了======"+connection.getResponseCode());
downloadStatus = false;// 代表下载失败，要去更新下载失败的状态
}else {
downloadStatus = true;// 代表下载失败，要去更新下载失败的状态
}
} catch (NumberFormatException e) {
// TODO Auto-generated catch block
downloadStatus = true;// 代表下载失败，要去更新下载失败的状态
e.printStackTrace();
} catch (ProtocolException e) {
// TODO Auto-generated catch block
downloadStatus = true;// 代表下载失败，要去更新下载失败的状态
e.printStackTrace();
} catch (FileNotFoundException e) {
// TODO Auto-generated catch block
downloadStatus = true;// 代表下载失败，要去更新下载失败的状态
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
downloadStatus = true;// 代表下载失败，要去更新下载失败的状态
e.printStackTrace();
}
if (downloadStatus) {
// 下载失败，更改视频状态
VideoListModel video = new VideoListModel();
video.setId(Integer.valueOf(taskId));
video.setStatus(VideoStatus.FAIL.getValue());
try {
SqlVideoList sqlVideoList = new SqlVideoList();
sqlVideoList.updateDate(video);
} catch (SQLException e) {
// TODO Auto-generated catch block
System.out.println("下载失败,更改视频状态失败" + e);
e.printStackTrace();
}
}else{
// 下载完成，从任务队列移除该任务
try {
SqlTaskQueue stq = new SqlTaskQueue();
stq.deleteDate(taskId);
} catch (SQLException e) {
// TODO Auto-generated catch block
System.out.println("任务移除失败" + e);
e.printStackTrace();
}
VideoListModel video = new VideoListModel();
video.setId(Integer.valueOf(taskId));
video.setStatus(VideoStatus.SUCCESS.getValue());
Date date = new Date();
Timestamp timeStamp = new Timestamp(date.getTime());
video.setLastDate(timeStamp);
try {
SqlVideoList sqlVideoList = new SqlVideoList();
sqlVideoList.updateSuccessDate(video);
} catch (SQLException e) {
// TODO Auto-generated catch block
System.out.println("下载好了,更改视频状态失败" + e);
e.printStackTrace();
}
}
}
/**
* 获取相应头部
*
* @param http
* @return
*/
public static Map<String, String> getHttpResponseHeader(HttpURLConnection http) {
Map<String, String> header = new LinkedHashMap<String, String>();
for (int i = 0;; i++) {
String mine = http.getHeaderField(i);
String key = http.getHeaderFieldKey(i);
System.out.println(key+"=======test======="+mine);
if (key == null || mine == null) {
break;
} else {
header.put(http.getHeaderFieldKey(i), mine);
}
}
return header;
}
/**
* 输出信息
*
* @param msg
*/
private static void print(String msg) {
// Log.i(TAG, msg);
System.out.println(TAG + "TAG{}," + msg);
}
/**
* 输出相应头信息
*
* @param http
*/
public static void printResponseHeader(HttpURLConnection http) {
Map<String, String> header = getHttpResponseHeader(http);
for (Map.Entry<String, String> entry : header.entrySet()) {
String key = entry.getKey() != null ? entry.getKey() + ":" : "";
print(key + entry.getValue());
}
}
/**
* 发送请求，获取API数据
*
* @param userId
* @return
* http://beta.polyv.net/uc/services/rest?url=http://grap.polyv.net/xxx * .mp4&title=filename&cataid=xxx&ts=&userid=&sign=
*/
public String httpGetVid(String url) {
String contentStr = "";
String vid = null;
String urlStr = url;
CloseableHttpClient httpclient = HttpClients.createDefault();
HttpGet httpGet = new HttpGet(urlStr);
try {
CloseableHttpResponse response = httpclient.execute(httpGet);
InputStream is = null;
Scanner sc = null;
Writer os = null;
if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
try {
// 2、获取response的entity。
HttpEntity entity = response.getEntity();
is = entity.getContent();
sc = new Scanner(is);
while (sc.hasNext()) {
contentStr = contentStr + sc.nextLine();
}
} catch (ClientProtocolException e) {
e.printStackTrace();
} finally {
if (sc != null) {
sc.close();
}
if (is != null) {
is.close();
}
if (os != null) {
os.close();
}
if (response != null) {
response.close();
}
}
}
} catch (ClientProtocolException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
if (StringUtils.isNotBlank(contentStr)) {
System.out.println("====="+contentStr);
Pattern pattern = Pattern.compile("\"vid\":\"([0-9a-z_]{34})\"");
Matcher matcher = pattern.matcher(contentStr);
if(matcher.find()){
vid = matcher.group(1);
System.out.println(vid);
}
}
if (vid != null) {
return vid;
} else {
return "";
}
}
}
线程池类：
package com.grab.video.listener;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadFactory;
/**
* Hands out threads from the wrapped threadfactory with setDeamon(true), so the
* threads won't keep the JVM alive when it should otherwise exit.
*/
public class DaemonThreadFactory implements ThreadFactory {
private final ThreadFactory factory;
/**
* Construct a ThreadFactory with setDeamon(true) using
* Executors.defaultThreadFactory()
*/
public DaemonThreadFactory() {
this(Executors.defaultThreadFactory());
}
/**
* Construct a ThreadFactory with setDeamon(true) wrapping the given factory
*
* @param thread
* factory to wrap
*/
public DaemonThreadFactory(ThreadFactory factory) {
if (factory == null)
throw new NullPointerException("factory cannot be null");
this.factory = factory;
}
public Thread newThread(Runnable r) {
final Thread t = factory.newThread(r);
t.setDaemon(true);
return t;
}
}任务执行线程池：
package com.grab.video.listener;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadFactory;
import javax.servlet.ServletContext;
import javax.servlet.ServletContextEvent;
import javax.servlet.ServletContextListener;
public class ExecutorContextListener implements ServletContextListener {
private ExecutorService executor;
public void contextInitialized(ServletContextEvent arg0) {
ServletContext context = arg0.getServletContext();
int nr_executors = 3;
ThreadFactory daemonFactory = new DaemonThreadFactory();
try {
nr_executors = Integer.parseInt(context.getInitParameter("nr-executors"));
} catch (NumberFormatException ignore) {
}
if (nr_executors <= 1) {
executor = Executors.newSingleThreadExecutor(daemonFactory);
} else {
executor = Executors.newFixedThreadPool(nr_executors, daemonFactory);
}
context.setAttribute("DOWNLOAD_EXECUTOR", executor);
}
public void contextDestroyed(ServletContextEvent arg0) {
ServletContext context = arg0.getServletContext();
executor.shutdownNow(); // or process/wait until all pending jobs are
// done
}
}
web.xml(项目启动时，启动任务线程类)
<?xml version="1.0" encoding="UTF-8"?>
<web-app xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns="http://java.sun.com/xml/ns/javaee" xmlns:web="http://java.sun.com/xml/ns/javaee"
xsi:schemaLocation="http://java.sun.com/xml/ns/javaee http://java.sun.com/xml/ns/javaee/web-app_2_5.xsd" id="school" version="2.5">
<display-name>Archetype Created Web Application</display-name>
<context-param>
<param-name>contextConfigLocation</param-name>
<param-value>
classpath:applicationContext.xml
</param-value>
</context-param>
<listener>
<listener-class>org.springframework.web.context.ContextLoaderListener</listener-class>
</listener>
<listener>
<listener-class>com.grab.video.listener.ExecutorContextListener</listener-class>
</listener>
<filter>
<filter-name>encodingFilter</filter-name>
<filter-class>org.springframework.web.filter.CharacterEncodingFilter</filter-class>
<init-param>
<param-name>encoding</param-name>
<param-value>UTF-8</param-value>
</init-param>
<init-param>
<param-name>forceEncoding</param-name>
<param-value>true</param-value>
</init-param>
</filter>
<filter-mapping>
<filter-name>encodingFilter</filter-name>
<url-pattern>/*</url-pattern>
</filter-mapping>
<servlet>
<servlet-name>dispatcher</servlet-name>
<servlet-class>org.springframework.web.servlet.DispatcherServlet</servlet-class>
<load-on-startup>1</load-on-startup>
</servlet>
<servlet-mapping>
<servlet-name>dispatcher</servlet-name>
<url-pattern>/</url-pattern>
</servlet-mapping>
<session-config>
<session-timeout>120</session-timeout>
</session-config>
<welcome-file-list>
<welcome-file>index.jsp</welcome-file>
</welcome-file-list>
<error-page>
<error-code>403</error-code>
<location>/error/403</location>
</error-page>
<error-page>
<error-code>404</error-code>
<location>/error/404</location>
</error-page>
</web-app>
下载页面：
<%@ page language="java" import="java.util.*" pageEncoding="UTF-8"%>
<%
String path = request.getContextPath();
String basePath = request.getScheme()+"://"+request.getServerName()+":"+request.getServerPort()+path+"/";
%>
<%@ page language="java"
import="java.util.List"
import="com.grab.video.controller.VideoListModel"
%>
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
<html>
<head>
<base href="<%=basePath%>">
<title>视频</title>
<meta http-equiv="pragma" content="no-cache">
<meta http-equiv="cache-control" content="no-cache">
<meta http-equiv="expires" content="0">
<meta http-equiv="keywords" content="keyword1,keyword2,keyword3">
<meta http-equiv="description" content="This is my page">
<!--
<link rel="stylesheet" type="text/css" href="styles.css">
-->
<link href="/resources/bootstrap/css/bootstrap.css" rel="stylesheet" />
<script src="/resources/js/jquery-1.7.2.js"></script>
<script type="text/javascript">
var userId;
$(document).ready(function(){
userId=$("#userId").val();
var timeId = setInterval(function () {
$.ajax({
type: "POST",
url: "/grab/download/status", /* 注意后面的名字对应CS的方法名称 */
data: "{}", /* 注意参数的格式和名称 */
contentType: "application/json; charset=utf-8",
dataType: "json",
success: function (data) {
//console.log(data);
for(var i in data) {
var percent = data[i].percent;
console.log(percent);
if(percent>0 && percent<100){
$("#progress"+data[i].id).html("<div class='progress'><div class='progress-bar' role='progressbar' aria-valuenow='60' aria-valuemin='0' aria-valuemax='100' style='width:"+ percent+"%;'>"+percent+"%</div></div>");
$("#downloadTd"+data[i].id).html("下载中...");
}else if(percent>100){
percent=100;
$("#progress"+data[i].id).html("<div class='progress'><div class='progress-bar' role='progressbar' aria-valuenow='60' aria-valuemin='0' aria-valuemax='100' style='width:"+ percent+"%;'>"+percent+"%</div></div>");
$("#downloadTd"+data[i].id).html("下载完");
}
if(data[i].status=="SUCCESS"){
$("#Tr"+data[i].id).remove();
}else if(data[i].status=="FAIL"){
$("#progress"+data[i].id).html("失败");
//$("#downloadTd"+data[i].id).html("失败");
}
}
}
});
},4000);
});
//下载单个文件
function downvideo(obj){
var num="0%";
var htmlstr="<div class='progress'><div class='progress-bar' role='progressbar' aria-valuenow='60' aria-valuemin='0' aria-valuemax='100' style='width:"+ num+";'>"+num+"</div></div>";
$(obj).parents(".downloadTd").siblings(".progressStatus").html(htmlstr);
$(obj).parents(".downloadTd").html("下载中...");
var videoId=$(obj).attr("videoId");
var url=$(obj).attr("dataUrl");
//下载
$.post(url,{},function(data){});
var test = setTimeout(function(){
//location.href ="/grab/download/manage?userId="+userId;
},1000);
}
</script>
</head>
<body>
<div class="container">
<div class="col-md-12">
<div class="page-header clearfix">
<h3 class="pull-left">视频管理</h3>
</div>
<div>
<ul class="nav nav-tabs">
<li><a href="/grab/import">导入任务</a></li>
<li class="active"><a href="/grab/download/manage">待抓视频</a></li>
<li><a href="/grab/export">已抓视频</a></li>
</ul>
</div>
<input type="hidden" id="userId" name="userId" value="${userId}"/>
<%List<VideoListModel> ls = (List) request.getAttribute("videolist");
if(ls.size()>0){%>
<form action="/grab/download/manage" method="post">
<button type="submit" class="btn btn-default btn-info pull-right" style="margin-top:10px;">全部下载</button>
<input type="hidden" id="userId" class="userId" name="userId" value="<%=ls.get(0).getUserId() %>" />
<div class="row">
<table id="guanggao-table" class="table table-hover">
<thead>
<tr>
<th>序号</th>
<th>视频URL</th>
<th>标题</th>
<th>视频分类</th>
<th width="15%">状态</th>
<th>操作1</th>
<th>操作2</th>
</tr>
</thead>
<tbody>
<%for(int i=0;i<ls.size();i++){ %>
<tr id="Tr<%=ls.get(i).getId() %>" pUrl="/grab/download/progress?videoId=<%=ls.get(i).getId() %>&trueName=<%=ls.get(i).getTrueName() %>&userId=<%=ls.get(i).getUserId() %>&url=<%=ls.get(i).getUrl() %>" >
<td> <%=i+1 %>
<input type="hidden" id="videoId" class="videoId" name="videoId" value="<%=ls.get(i).getId() %>" />
<input type="hidden" id="url" class="url" name="url" value="<%=ls.get(i).getUrl() %>" />
<input type="hidden" id="trueName" class="trueName" name="trueName" value="<%=ls.get(i).getTrueName() %>" />
</td>
<td>
<a href="<%=ls.get(i).getUrl() %>" title="<%=ls.get(i).getUrl() %>"><%String str=ls.get(i).getUrl();String pathstr=str.substring(0, 20);%>
<%=pathstr %>...
</a>
</td>
<td><%=ls.get(i).getTitle() %></td>
<td><%=ls.get(i).getClassifyName() %></td>
<td class="progressStatus" id="progress<%=ls.get(i).getId() %>">
<% if("NO".equals(ls.get(i).getStatus().trim())){%>
<div>待抓取</div>
<%}else if("WAIT".equals(ls.get(i).getStatus().trim())){ %>
<div>等待</div>
<%}else if("TRANSCODING".equals(ls.get(i).getStatus().trim())){ %>
<div class="progress">
<div class="progress-bar" role="progressbar" aria-valuenow="60" aria-valuemin="0" aria-valuemax="100" style="width:1%;">1%</div>
</div>
<%}else if("EXECUTE".equals(ls.get(i).getStatus().trim())){ %>
<%}else if("SUCCESS".equals(ls.get(i).getStatus().trim())){ %>
<div>成功</div>
<%}else{ %>
<div>失败 </div>
<%} %>
</td>
<td class="downloadTd" id="downloadTd<%=ls.get(i).getId() %>">
<% if("EXECUTE".equals(ls.get(i).getStatus().trim())){%>
下载中...<div class="execute" videoId="<%=ls.get(i).getId() %>" style="display:none;"></div>
<%}else if("WAIT".equals(ls.get(i).getStatus().trim())){ %>
等待
<%}else{%>
<button type="button" class="btn btn-default btn-info" videoId="<%=ls.get(i).getId() %>" dataUrl="/grab/download/manage?videoId=<%=ls.get(i).getId() %>&trueName=<%=ls.get(i).getTrueName() %>&userId=<%=ls.get(i).getUserId() %>&url=<%=ls.get(i).getUrl() %>" onclick="downvideo(this);">抓取</button>
<%} %>
</td>
<td>
<!--
<button type="button" class="btn btn-default" videoId="<%=ls.get(i).getId() %>" dataUrl="/grab/video/stop?videoId=<%=ls.get(i).getId() %>&trueName=<%=ls.get(i).getTrueName() %>&userId=<%=ls.get(i).getUserId() %>&url=<%=ls.get(i).getUrl() %>" onclick="stopVideo(this);">暂停</button>
-->
<button type="button" class="btn btn-default" videoId="<%=ls.get(i).getId() %>" dataUrl="/grab/delete/videoId?videoId=<%=ls.get(i).getId() %>" onclick="deleteVideo(this);">删除</button>
</td>
</tr>
<%} %>
</tbody>
</table>
</div>
</form>
<%}else{ %>
<div class="col-md-6" style="padding-top:30px;">暂无视频可以抓取</div>
<%} %>
</div>
</div>
</body>
</html>
<script>
//删除文件
function deleteVideo(obj){
var url=$(obj).attr("dataUrl");
if (!confirm('真的要永久删除该内容吗？')) {
return ;
}else{
$.ajax({
type: "GET",
url: url, /* 注意后面的名字对应CS的方法名称 */
data: "{}", /* 注意参数的格式和名称 */
contentType: "application/json; charset=utf-8",
dataType: "json",
success: function (data) {
console.log(data);
var mark=data.result;
if(mark=="yes"){
// window.location.reload();
location.href ="/grab/download/manage?userId="+userId;
}else{
alert("删除失败！");
}
}
});
}
}
//暂停下载
function stopVideo(obj){
var url=$(obj).attr("dataUrl");
if (!confirm('暂停下载')) {
return ;
}else{
$.ajax({
type: "GET",
url: url, /* 注意后面的名字对应CS的方法名称 */
data: "{}", /* 注意参数的格式和名称 */
contentType: "application/json; charset=utf-8",
dataType: "json",
success: function (data) {
console.log(data);
var mark=data.result;
if(mark=="yes"){
// window.location.reload();
location.href ="/grab/download/manage?userId="+userId;
}else{
alert("操作失败！");
}
}
});
}
}
</script>
内容来自用户分享和网络整理，不保证内容的准确性，如有侵权内容，可联系管理员处理 
相关文章推荐
- JAVASE学习笔记：第八章 常用类Util工具包之日期类、数字类
- Spring 3整合Quartz 2实现定时任务三：动态暂停 恢复 修改和删除任务
- spring配置文件出错
- java面向对象学习笔记
- 解决org.hibernate.LazyInitializationException: could not initialize proxy - no Session问题
- java 获取当期时间之前几小时的时间
- Java HashTable的Cloneable实现
- 实际中遇到的JSON与JAVA的转换问题
- spring拦截器的使用
- springmvc+mybatis
- Java String类型数据的字节长度
- mac下卸载jdk1.7
- 手写Java框架（一）-----理想的开源框架与设计原则
- Java泛型 继承中类型变量的关系
- 文件上传下载—struts2实现
- Spring <context:annotation-config/> 解说
- IDEA和Eclipse常用快捷键（Win Mac）
- Java的GUI练习之——留言板实现
- java集合框架详解（二）、set接口
- java基础复习之四：注释，标识符，关键字, 变量，常量