HttpClient学习笔记

HttpClient 是 Apache Jakarta Common 下的子项目，用来提供高效的、最新的、功能丰富的支持 HTTP 协议的客户端编程工具包，并且它支持 HTTP 协议最新的版本和建议。HttpClient 已经应用在很多的项目中。爬虫主要是用HttpClient模拟浏览器请求第三方站点url，然后响应，获取网页数据，然后用Jsoup来提取我们需要的信息；

jsoup 是一款Java 的HTML解析器，可直接解析某个URL地址、HTML文本内容。它提供了一套非常省力的API，可通过DOM，CSS以及类似于jQuery的操作方法来取出和操作数据。

package com.httpclient.jsoup;

import java.io.IOException;

import org.apache.http.HttpEntity;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

public class JsoupDemo {

public static void main(String[] args) throws ClientProtocolException, IOException {
CloseableHttpClient HttpClient = HttpClients.createDefault();

HttpGet get = new HttpGet("http://www.xicidaili.com/");
/*HttpHost proxy = new HttpHost("182.112.228.38",80);

RequestConfig config = RequestConfig.custom().setProxy(proxy).setConnectTimeout(10000).setSocketTimeout(10000).build();
get.setConfig(config );*/
get.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:54.0) Gecko/20100101 Firefox/54.0");
CloseableHttpResponse respose = HttpClient.execute(get);
System.out.println("状态:"+respose.getStatusLine().getStatusCode());
HttpEntity entity = respose.getEntity();
System.out.println("Content-Type:"+entity.getContentType().getValue());
String content = EntityUtils.toString(entity, "utf-8");
System.out.println("内容:"+content);
respose.close();

Document doc = Jsoup.parse(content);

Elements elements= doc.getElementsByTag("title");
System.out.println("网页标题："+elements.get(0).text());

Element element2=doc.getElementById("header");
String navTop=element2.text();
System.out.println("id 为header的内容："+navTop);

Elements postItemElements=doc.getElementsByClass("site_name"); // 根据样式名称来查询DOM
for(Element e:postItemElements){
System.out.println("根据样式名称来查询DOM"+e.html());
}

Elements widthElements=doc.getElementsByAttribute("width"); // 根据属性名来查询DOM
for(Element e:widthElements){
System.out.println("根据属性名来查询DOM"+e.toString());
}

Elements targetElements=doc.getElementsByAttributeValue("width", "11%");  // 根据属性名和属性值来查询DOM
for(Element e:targetElements){
System.out.println("根据属性名和属性值来查询DOM"+e.toString());
}

Elements linkElements=doc.select(".subtitle .country");
for(Element e:linkElements){
System.out.println("根据class："+e.text());
}

Elements hrefElements=doc.select("a[href]"); //
for(Element e:hrefElements){
System.out.println("带有href属性的a元素"+e.toString());
}

Elements imgElements=doc.select("img[src$=.png]"); //
for(Element e:imgElements){
System.out.println("查找扩展名为.png的图片DOM节点"+e.toString());
}
}

}