网络爬虫Java实现抓取网页内容
2016-06-05 16:13
615 查看
package 抓取网页;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpException;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.httpclient.NameValuePair;
import org.apache.commons.httpclient.methods.PostMethod;
public class RetrivePage {
private static HttpClient httpClient = new HttpClient();
public static void main(String[] args) {
//抓取猎兔的首页,并且输出出来
try {
RetrivePage.downloadPage("http://www.lietu.com");
} catch (HttpException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
}
private static void downloadPage(String path) throws HttpException, IOException {
System.out.println("123123");
InputStream input = null;
OutputStream output = null;
//得到post方法
PostMethod postMethod = new PostMethod(path);
//设置post方法的参数
NameValuePair[] postData = new NameValuePair[2];
postData[0] = new NameValuePair("name","lietu");
postData[1] = new NameValuePair("password","*****");
//把参数添加到请求路径上去
postMethod.addParameters(postData);
//执行,返回状态码
int statusCode = httpClient.executeMethod(postMethod);
System.out.println(statusCode);
if (statusCode == HttpStatus.SC_OK) {
input = postMethod.getResponseBodyAsStream();
//得到文件的名字
String fileName = path.substring(path.lastIndexOf('/')+1);
//获得文件的输出流
System.out.println(fileName);
output = new FileOutputStream(fileName);
//输出到文件中
int tempByte = -1;
while ((tempByte = input.read()) > 0) {
output.write(tempByte);
}
//关闭资源
if (input != null) {
input.close();
}
if (output != null) {
output.close();
}
}
}
}
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpException;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.httpclient.NameValuePair;
import org.apache.commons.httpclient.methods.PostMethod;
public class RetrivePage {
private static HttpClient httpClient = new HttpClient();
public static void main(String[] args) {
//抓取猎兔的首页,并且输出出来
try {
RetrivePage.downloadPage("http://www.lietu.com");
} catch (HttpException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
}
private static void downloadPage(String path) throws HttpException, IOException {
System.out.println("123123");
InputStream input = null;
OutputStream output = null;
//得到post方法
PostMethod postMethod = new PostMethod(path);
//设置post方法的参数
NameValuePair[] postData = new NameValuePair[2];
postData[0] = new NameValuePair("name","lietu");
postData[1] = new NameValuePair("password","*****");
//把参数添加到请求路径上去
postMethod.addParameters(postData);
//执行,返回状态码
int statusCode = httpClient.executeMethod(postMethod);
System.out.println(statusCode);
if (statusCode == HttpStatus.SC_OK) {
input = postMethod.getResponseBodyAsStream();
//得到文件的名字
String fileName = path.substring(path.lastIndexOf('/')+1);
//获得文件的输出流
System.out.println(fileName);
output = new FileOutputStream(fileName);
//输出到文件中
int tempByte = -1;
while ((tempByte = input.read()) > 0) {
output.write(tempByte);
}
//关闭资源
if (input != null) {
input.close();
}
if (output != null) {
output.close();
}
}
}
}
相关文章推荐
- libnet、libnids、libpcap轻松搭建Linux网络入侵检测系统
- 计算机网络(一)带宽
- OSI七层与TCP/IP五层网络架构详解
- 在JAVA中使用NIO进行网络编程
- 大型网络游戏服务器的框架设计
- note05-计算机网络
- 使用httpclient发送post请求与get请求
- HttpServletResponse对象(二)
- 刷新神经网络新深度:ImageNet计算机视觉挑战赛微软中国研究员夺冠
- 关于HTTP中的keep-alive
- Linux下的TCP/IP编程----多播和广播的实现
- Nginx tcp反向代理工作原理简述
- Nginx 实现tcp反向代理(Linux上)
- Linux下基于http的小型web服务器编写
- 计算机网络简介
- HttpSession与Hibernate中Session的区别
- 网络编程课程复习
- 制作本地yun源 http方式的详解
- httpd之禁止解析php
- note04-计算机网络