httpclient爬虫爬取电影信息和下载地址实例
2019-09-16 15:27
197 查看
本次更新主要解决了老旧页面下载链接可能是迅雷和ftp格式的，还有就是去重，因为每一页有一个推荐列表，里面也会有相应的详情链接，还有兼容了另外的页面格式，更新了两个方法：
public static void spider(int pa) {
List<String> page = getPage(pa);
String[] abc = "http://www.***.net/ys/20170620/37704.htm, http://www.***.net/ys/20170727/38028.htm, http://www.***.net/ys/20170810/38113.htm, http://www.***.net/ys/20170703/37769.htm, http://www.***.net/ys/20170615/37680.htm, http://www.***.net/ys/20170615/37678.htm, http://www.***.net/ys/20170727/38027.htm, http://www.***.net/ys/20170802/38060.htm, http://www.***.net/ys/20170515/37385.htm, http://www.***.net/ys/20170725/38001.htm, http://www.***.net/ys/20170608/37614.htm, http://www.***.net/ys/20170802/38059.htm, http://www.***.net/ys/20170629/37742.htm, http://www.***.net/ys/20170512/37323.htm, http://www.***.net/ys/20170426/37219.htm, http://www.***.net/ys/20170727/38026.htm, http://www.***.net/ys/20170730/38046.htm, http://www.***.net/ys/20170804/38082.htm, http://www.***.net/ys/20170714/37848.htm, http://www.***.net/ys/20180819/40982.htm, http://www.***.net/ys/20180819/40981.htm, http://www.***.net/ys/20180818/40980.htm, http://www.***.net/ys/20180818/40979.htm, http://www.***.net/ys/20180818/40978.htm, http://www.***.net/ys/20180818/40977.htm, http://www.***.net/ys/20180817/40975.htm, http://www.***.net/ys/20180817/40974.htm".split(", ");
List<String> list = Arrays.asList(abc);
page.removeAll(list);
output(page.size());
Set<String> truelist = new HashSet<>();
page.forEach(l -> truelist.add(l));
truelist.forEach(p -> {
try {
getMovieInfo(p);
sleep(getRandomInt(3) + 3);
} catch (Exception e) {
output(p);
}
});
}
public static void spider(String text) {
List<String> page = getPage(text);
Set<String> truelist = new HashSet<>();
page.forEach(l -> truelist.add(l));
truelist.forEach(p -> {
try {
getMovieInfo(p);
sleep(getRandomInt(3));
} catch (Exception e) {
output(p);
}
});
}
public static List<String> getPage(int page) {
String url = "http://www.***.net/ys/index_" + page + ".htm";
if (page == 1) url = "http://www.***.net/ys/";
output(url);
HttpGet httpGet = getHttpGet(url);
JSONObject response = getHttpResponse(httpGet);
String content = response.getString("content");
byte[] bytes = content.getBytes(UTF_8);
String all = new String(bytes, UTF_8);
List<String> list = regexAll(all, "http://www.***.net/ys/\\d+/\\d+.htm");
return list;
}
public static List<String> getPage(String page) {
String content = page;
byte[] bytes = content.getBytes(UTF_8);
String all = new String(bytes, UTF_8);
List<String> list = regexAll(all, "http://www.***.net/ys/\\d+/\\d+.htm");
return list;
}
public static boolean getMovieInfo(int day, int index) {
// String url = "http://www.***.net/ys/20180819/40981.htm";
String url = "http://www.***.net/ys/" + day + "/" + index + ".htm";
getMovieInfo(url);
return true;
}
public static boolean getMovieInfo(String url) {
HttpGet httpGet = getHttpGet(url);
JSONObject response = getHttpResponse(httpGet);
String s = response.getString("content");
if (s.contains("您查询的内容不存在，请返回首页重新搜索")) return false;
byte[] bytes = s.getBytes(UTF_8);
String all = new String(bytes, UTF_8);
String name = EMPTY, tname = EMPTY, year = EMPTY, language = EMPTY, date = EMPTY, score = EMPTY, length = EMPTY, author = EMPTY;
if (all.contains("◎")) {
int i = all.indexOf("◎");
int i1 = all.indexOf("<hr");
String info = s.substring(i, i1);
name = getInfo(info, "片 名 ");
tname = getInfo(info, "译 名 ");
year = getInfo(info, "年 代 ");
language = getInfo(info, "语 言 ");
date = getInfo(info, "上映日期 ");
score = getInfo(info, "豆瓣评分 ");
length = getInfo(info, "片 长 ");
author = getInfo(info, "导 演 ");
} else {
name = getInfo(all, "<title>");
if (name.contains("_")) name = name.substring(0, name.indexOf("_"));
length = getInfo(all, "片长: ");
date = getInfo(all, "上映日期: ");
author = getInfo(all, "导演: ");
language = getInfo(all, "语言: ");
}
List<String> magnets = regexAll(all, "magnet:.+?>");
List<String> ed2ks = regexAll(all, "ed2k:.+?>");
if (ed2ks.size() == 0) ed2ks = regexAll(all, "ftp://.+?>");
if (ed2ks.size() == 0) ed2ks = regexAll(all, "thunder://.+?>");
List<String> pans = regexAll(all, "http(s)*://pan.baidu.com/.+?</td>");
String sql = "INSERT INTO movie (name,tname,year,language,date,score,length,author,magnet,ed2k,pan) VALUES(\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\");";
sql = String.format(sql, name, tname, year, language, date, score, length, author, magnets.toString().replace("\"", EMPTY), ed2ks.toString().replace("\"", EMPTY), pans.toString().replace("\"", EMPTY));
if (ed2ks.size() != 0) MySqlTest.sendWork(sql);
output(magnets.toString().length(), ed2ks.toString().length(), pans.toString().length());
output(sql);
return true;
}
public static String getInfo(String text, String start) {
String value = EMPTY;
List<String> nameinfo = regexAll(text, start + ".+?<");
if (nameinfo.size() > 0) value = nameinfo.get(0).replace(start, EMPTY).replace("<", EMPTY);
return value;
}
----------------------------------分割线-------------------------------------------------
本人使用httpclient爬虫过程中，想爬取关注的一个电影网站的下载地址。在经过尝试之后，终于成功爬取了几百部热门电影的信息和下载地址（电驴和磁力链接）。中间遇到了编码格式，正则匹配不一致，还有重复链接过滤等问题，也都一一搞定。附上代码，供大家参考。
关键信息隐藏，思路供大家参考：先去访问列表页，拿到详情页的链接，去重之后去访问详情页，拿到相关信息和下载地址，存储到数据库中。
public class MyTest extends ApiLibrary {
public static void main(String[] args) {
DEFAULT_CHARSET = GB2312;
for (int i = 0; i < 10; i++) {
spider(1);
}
testOver();
}
public static void spider(int pa) {
String[] abc = "http://www.***.net/ys/20170620/37704.htm, http://www.***.net/ys/20170727/38028.htm, http://www.***.net/ys/20170810/38113.htm, http://www.***.net/ys/20170703/37769.htm, http://www.***.net/ys/20170615/37680.htm, http://www.***.net/ys/20170615/37678.htm, http://www.***.net/ys/20170727/38027.htm, http://www.***.net/ys/20170802/38060.htm, http://www.***.net/ys/20170515/37385.htm, http://www.***.net/ys/20170725/38001.htm, http://www.***.net/ys/20170608/37614.htm, http://www.***.net/ys/20170802/38059.htm, http://www.***.net/ys/20170629/37742.htm, http://www.***.net/ys/20170512/37323.htm, http://www.***.net/ys/20170426/37219.htm, http://www.***.net/ys/20170727/38026.htm, http://www.***.net/ys/20170730/38046.htm, http://www.***.net/ys/20170804/38082.htm, http://www.***.net/ys/20170714/37848.htm, http://www.***.net/ys/20180819/40982.htm, http://www.***.net/ys/20180819/40981.htm, http://www.***.net/ys/20180818/40980.htm, http://www.***.net/ys/20180818/40979.htm, http://www.***.net/ys/20180818/40978.htm, http://www.***.net/ys/20180818/40977.htm, http://www.***.net/ys/20180817/40975.htm, http://www.***.net/ys/20180817/40974.htm".split(", ");
List<String> list = Arrays.asList(abc);
page.removeAll(list);
Set<String> truelist = new HashSet<>();
page.forEach(l -> truelist.add(l));
truelist.forEach(p -> {
try {
getMovieInfo(p);
sleep(getRandomInt(3));
} catch (Exception e) {
output(p);
}
});
}
public static List<String> getPage(int page) {
String url = "http://www.***.net/ys/index_" + page + ".htm";
if (page == 1) url = "http://www.***.net/ys/";
output(url);
HttpGet httpGet = getHttpGet(url);
JSONObject response = getHttpResponse(httpGet);
String content = response.getString("content");
output(content);
byte[] bytes = content.getBytes(UTF_8);
String all = new String(bytes, UTF_8);
List<String> list = regexAll(all, "http://www.***.net/ys/\\d+/\\d+.htm");
return list;
}
public static boolean getMovieInfo(int day, int index) {
// String url = "http://www.***.net/ys/20180819/40981.htm";
String url = "http://www.***.net/ys/" + day + "/" + index + ".htm";
HttpGet httpGet = getHttpGet(url);
JSONObject response = getHttpResponse(httpGet);
String s = response.getString("content");
if (s.contains("您查询的内容不存在，请返回首页重新搜索")) return false;
byte[] bytes = s.getBytes(UTF_8);
String all = new String(bytes, UTF_8);
int i = all.indexOf("◎");
int i1 = all.indexOf("<hr");
String info = s.substring(i, i1);
String name = getInfo(info, "片 名 ");
String tname = getInfo(info, "译 名 ");
String year = getInfo(info, "年 代 ");
String language = getInfo(info, "语 言 ");
String date = getInfo(info, "上映日期 ");
String score = getInfo(info, "豆瓣评分 ");
String length = getInfo(info, "片 长 ");
String author = getInfo(info, "导 演 ");
List<String> magnets = regexAll(all, "magnet:.+?>");
List<String> ed2ks = regexAll(all, "ed2k:.+?>");
List<String> pans = regexAll(all, "http(s)*://pan.baidu.com/.+?</td>");
String sql = "INSERT INTO movie (name,tname,year,language,date,score,length,author,magnet,ed2k,pan) VALUES(\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\");";
sql = String.format(sql, name, tname, year, language, date, score, length, author, magnets.toString().replace("\"", EMPTY), ed2ks.toString().replace("\"", EMPTY), pans.toString().replace("\"", EMPTY));
MySqlTest.sendWork(sql);
return true;
}
public static boolean getMovieInfo(String url) {
HttpGet httpGet = getHttpGet(url);
JSONObject response = getHttpResponse(httpGet);
String s = response.getString("content");
if (s.contains("您查询的内容不存在，请返回首页重新搜索")) return false;
byte[] bytes = s.getBytes(UTF_8);
String all = new String(bytes, UTF_8);
int i = all.indexOf("◎");
int i1 = all.indexOf("<hr");
String info = s.substring(i, i1);
String name = getInfo(info, "片 名 ");
String tname = getInfo(info, "译 名 ");
String year = getInfo(info, "年 代 ");
String language = getInfo(info, "语 言 ");
String date = getInfo(info, "上映日期 ");
String score = getInfo(info, "豆瓣评分 ");
String length = getInfo(info, "片 长 ");
String author = getInfo(info, "导 演 ");
List<String> magnets = regexAll(all, "magnet:.+?>");
List<String> ed2ks = regexAll(all, "ed2k:.+?>");
List<String> pans = regexAll(all, "http(s)*://pan.baidu.com/.+?</td>");
String sql = "INSERT INTO movie (name,tname,year,language,date,score,length,author,magnet,ed2k,pan) VALUES(\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\");";
sql = String.format(sql, name, tname, year, language, date, score, length, author, magnets.toString().replace("\"", EMPTY), ed2ks.toString().replace("\"", EMPTY), pans.toString().replace("\"", EMPTY));
MySqlTest.sendWork(sql);
output(magnets.toString().length(), ed2ks.toString().length(), pans.toString().length());
output(sql);
return true;
}
public static String getInfo(String text, String start) {
String value = EMPTY;
List<String> nameinfo = regexAll(text, start + ".+?<");
if (nameinfo.size() > 0) value = nameinfo.get(0).replace(start, EMPTY).replace("<", EMPTY);
return value;
}
}
下面是数据库存储的截图：
技术类文章精选
- java一行代码打印心形
- Linux性能监控软件netdata中文汉化版
- 接口测试代码覆盖率（jacoco）方案分享
- 性能测试框架
- 如何在Linux命令行界面愉快进行性能测试
- 图解HTTP脑图
- 如何测试概率型业务接口
- httpclient处理多用户同时在线
- 将swagger文档自动变成测试代码
- 五行代码构建静态博客
- httpclient如何处理302重定向
- 基于java的直线型接口测试框架初探
- Tcloud 云测平台--集大成者
非技术文章精选
- 为什么选择软件测试作为职业道路?
- 成为杰出Java开发人员的10个步骤
- 写给所有人的编程思维
- 自动化测试的障碍
- 自动化测试的问题所在
- 测试之《代码不朽》脑图
- 成为优秀自动化测试工程师的7个步骤
- 优秀软件开发人员的态度
- 如何正确执行功能API测试
点击查看公众号地图
内容来自用户分享和网络整理，不保证内容的准确性，如有侵权内容，可联系管理员处理 
相关文章推荐
- 不写代码开发信息管理系统-配置实现 下载地址：http://121.18.78.216
- java爬虫框架——jsoup的简单使用（爬取电影天堂的所有电影的信息，包括下载的链接）
- 电影无极下载地址（HTTP下载）
- JAVA编写网络爬虫笔记（第二部分:httpClient下载页面）
- python爬虫（实例1）:requests+xpath 爬取并下载豆瓣电影的演员照片
- Step05：爬虫小项目，爬取最新电影迅雷下载地址
- memcache实战之三 ：客户端 enyim.com Memcached Client 的安装与调试以及实例，2012年下载地址
- httpclient 多线程爬虫实例
- httpclient 多线程爬虫实例
- HttpClient入门实例之简单的pdf文件爬虫
- httpclient4文档及下载地址
- httpclient爬虫爬取汉字拼音等信息
- 原文：一套HTML网站后台信息管理静态网页模版下载 源代码下载地址：http://www.zuidaima.com/share/1821271068036096.htm 体验地址:http://18
- android通过okhttpClient下载网页内容的实例代码
- java resteasy restful webservice教程(六):实现文件下载功能实例文章来源:爱上123 原文地址：http://www.ishang123.com/jishubowen/
- httpclient 爬虫实例——爬取中学名（总计6万+）
- HttpClient下载图片和向服务器提交数据实例
- RxAndroid 与OkHttpClient打造下载实例
- 用php实现一个简单的爬虫，抓取电影网站的视频下载地址
- HttpClient下载图片和向服务器提交数据实例