网络爬虫基础！

网络爬虫最基础的就是下载文件或页面，然后利用正则表达式提取需要的数据！

假如有如下一个mail.txt文件:需要从里面提取有用的邮箱地址，该怎么做呢？

skdajflksadjflk> abc0@sina.com<djfsa;lkdfjksa;ljfs;alfjsa;lkjfsa;lkfjska;ljfsafsd
skdajflksadjflkdjfsa;lkdfjksa;ljfs;alfjsa;lkjfsa;lkfjska;ljfsafsd
skdajfl> abc1@sina.com<ksadjflkdjfsa;lkdfjksa;ljfs;alfjsa;lkjfsa;lkfjska;ljfsafsd
skdajflksadjflkdjfsa;lkdfjksa;ljfs;alfjsa;lkjfsa;lkfjska;ljfsafsd
skdajfl> abc2@sina.com<sadjflkdjfsa;lkdfjks> abc3@sina.com<a;ljfs;alfjsa;lkjfsa;lkfjska;ljfsafsd
skdajflksadjflkdjfsa;lkdfjksa;ljfs;alfjsa;lkjfsa;lkfjska;ljfsafsd
skdajflksadjflkdjfsa;lkv> ab4c@sina.com<dfjksa;ljfs;alfjsa;lkjfsa;lkfjska;ljfsafsd
skdajf> abc5@sina.com<ksadjflkdjfsa;lkdfjksa;ljfs;alfjsa;lkjfsa;lkfjska;ljfsafsd
skdajflksadjflkdjfsa;lkdfjksa;ljfs;alfjsa;lkjfsa;lkfjska;ljfsafsd
skdajflksadjf> abc6@sina.com<lkdjfsa;lkdfjksa;ljfs;alfjsa;lkjfsa;lkfjska;ljfsafsd
skdajflksadjflkdjfsa;lkd> abc7@sina.com<fjksa;ljfs;alfj> abc8@sina.com<sa;lkjfsa;lkfjska;ljfsafsd
skdajflksadjflkdjfsa;lkdfjksa;ljfs;alfjsa;lkjfsa;lkfjska;ljfsafsd
skdajflks> abc11@sina.com<adjflkdjfsa;> abc9@sina.com<lkdfjksa;ljfs;alfjsa;lkjfsa;lkfjska;ljfsafsd
skdajflksadjflkdjfsa;lkdfjksa;ljfs;alfjsa;lkjfsa;lkfjska;ljfsafsd
skdajflksadjf> abc12@sina.com<lkdjfsa;lk> abc55@sina.com<dfjksa;ljfs;alfjsa;lkjfsa;lkfjska;ljfsafsd
skdajflksadjflk> abc123@sina.com<djfsa;lkdfjksa;ljfs;alfjsa;lkjfs> abc1234@sina.com<a;lkfjska;ljfsafsd
sdfsakfjask;lfjsak;lfj qqq@sina.com aslk;fjsafsafsafs

我们只需要下载这个页面，然后解析其中的邮箱地址即可！

public class TestRegex {
//获取指定文档中的邮件地址
public static void getMails() throws IOException {
BufferedReader bufr = new BufferedReader(new FileReader("D:\\mail.txt"));
String line = null;
String mailReg = "\\w+@\\w+(\\.\\w)+";
Pattern p = Pattern.compile(mailReg);

while((line = bufr.readLine()) != null) {
Matcher m = p.matcher(line);
while(m.find()) {
System.out.println(m.group());
}
}

}

public static void main(String[] args) throws IOException {
getMails();
}
}

好了，一个简单的网络爬虫就是这样的了！

如果要下载web页面呢？可以使用urlconnection进行下载：

public class TestRegex2 {
public static void getHtml() throws IOException {
URL url = new URL("http://s.taobao.com/search?q=%E8%A1%AC%E8%A1%AB&commend=all&ssid=s5-e&search_type=item&sourceId=tb.index&spm=1.7274553.1997520841.1&initiative_id=tbindexz_20150603");
URLConnection conn = url.openConnection();
BufferedReader bufferedReader = new BufferedReader(
new InputStreamReader(conn.getInputStream()));
String line = null;
// String mailReg = "g_page_config = ";
// Pattern p = Pattern.compile(mailReg);

while((line = bufferedReader.readLine()) != null) {
// Matcher m = p.matcher(line);
// while(m.find())
// {
// System.out.println(m.group());
// }
System.out.println(line);
}
}

public static void main(String[] args) throws IOException {
getHtml();
}
}