Jsoup抓取数据实现为一个网站做第三方Android客户端

[b]一前言[/b]

学Android有很大一部分的乐趣其实就是做出一个网站客户端，比官方广告少，速度快，不臃肿，受到众多网友追捧。

由于学校图书馆网站没有APP，网站也没有移动版的，所以说体验相当差，用的实在是太烦就做了这个小应用，先下看下效果。



二Jsoup实现抓取书名

Jsoup是一个Java的一个工具包，百度一搜一大堆，不对Jsoup做过多介绍，先来看看对http://222.188.3.137:8080/opac/search.php的抓取。

这里抓取到数据只要往适配器里填充数据就好了。

package com.wyf.newlibrary;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.IOException;

public class BookNameJsoup {
String url;

String[] bName;
String[] bLink;
String nextPage;

public BookNameJsoup(String  link) {

url = link;
bName = new String[20];
bLink = new String[20];

}

public void init() {
try {
Document doc = Jsoup.connect(url).get();
int j = 0;
Elements bookName = doc.getElementsByTag("h3").select("a");
for (Element i : bookName) {
bName[j] = i.text().trim();
bLink[j++] = i.attr("abs:href");
}

Elements next=doc.getElementsByAttributeValue("class", "blue");
for(Element i:next)
{
if(i.text().contains("下一页"))
{
nextPage=i.attr("abs:href");
break;
}

}

} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}

/***************
* 得到LINK
********************/
public String[] getBookName() {
return bName;
}

/*******************
* 得到LINK
******************************/

public String[] getLink() {
return bLink;
}

public  String getNextPage()
{
return  nextPage;
}

/***
* 判断搜索结果是否为空
*/

}

三 MainActivity(书名列表界面)

在MainActivity中有一个ListView，在用Jsoup抓取到的数据往里面填充，ListView只用极其简单的布局，一看就能明白。

package com.wyf.newlibrary;

import android.content.Intent;
import android.os.AsyncTask;
import android.os.Bundle;
import android.support.v7.app.AppCompatActivity;
import android.util.Log;
import android.view.KeyEvent;
import android.view.LayoutInflater;
import android.view.View;
import android.view.inputmethod.EditorInfo;
import android.widget.AbsListView;
import android.widget.AdapterView;
import android.widget.ArrayAdapter;
import android.widget.EditText;
import android.widget.ImageButton;
import android.widget.ListView;
import android.widget.ProgressBar;
import android.widget.TextView;
import android.widget.Toast;

import com.umeng.analytics.MobclickAgent;
import com.umeng.update.UmengUpdateAgent;

import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLEncoder;
import java.util.ArrayList;

public class MainActivity extends AppCompatActivity  {

EditText edit_search;
String url="我去",key;
ListView list_bookname;
ArrayList<String> bookName;  //得到的书名
ArrayList<String> bookLink;   //书名链接
ArrayAdapter adapter;
GetBookName get;
View foot,complete,fail;
ProgressBar progressBar;
boolean firstLoad=false;
ImageButton ibtn_clear;

@Override
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
getSupportActionBar().hide();
setContentView(R.layout.activity_main);

init();
initEvent();
list_bookname.setAdapter(adapter);

}

private void initEvent() {

edit_search.setOnEditorActionListener(new TextView.OnEditorActionListener() {
@Override
public boolean onEditorAction(TextView v, int actionId, KeyEvent event) {

if(actionId== EditorInfo.IME_ACTION_SEARCH)
{
url=edit_search.getText().toString().trim();
try {
url= URLEncoder.encode(url,"UTF-8");
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
}
bookName.clear();
bookLink.clear();
boolean  loadNothing=true;
if(url!=null)
{
firstLoad=true;
new GetBookName(url).execute();
}
else{
Toast.makeText(MainActivity.this, "不能为空", Toast.LENGTH_SHORT).show();
}

}

return false;
}
});

list_bookname.setOnScrollListener(new AbsListView.OnScrollListener() {
//AbsListView view 这个view对象就是listview
int lastItem;
@Override
public void onScrollStateChanged(AbsListView view, int scrollState) {
if (scrollState == AbsListView.OnScrollListener.SCROLL_STATE_IDLE) {
if (view.getLastVisiblePosition() == view.getCount() - 1) {
if(url!=null)
{
list_bookname.addFooterView(foot);
new GetBookName().execute();
}

else if(bookName.isEmpty()){
list_bookname.addFooterView(fail);

}
else {
list_bookname.addFooterView(complete);
}

}
}
}
@Override
public void onScroll(AbsListView view, int firstVisibleItem,
int visibleItemCount, int totalItemCount) {
lastItem = firstVisibleItem + visibleItemCount - 1 ;
}
});
/*************************************ListView每个Item设置监听，转到这本书具体信息的Activity*************************************************/
list_bookname.setOnItemClickListener(new AdapterView.OnItemClickListener() {
@Override
public void onItemClick(AdapterView<?> parent, View view, int position, long id) {
Intent intent=new Intent(MainActivity.this,BookInfoActivity.class);
intent.putExtra("href",bookLink.get(position));
startActivity(intent);

}
});
}

private void init() {

// ibtn_clear= (ImageButton) findViewById(R.id.ibtn_clear);
edit_search= (EditText) findViewById(R.id.edit_search);
list_bookname= (ListView)findViewById(R.id.list_bookname);
bookName=new ArrayList<String>();
bookLink=new ArrayList<String>();
adapter=new ArrayAdapter(MainActivity.this,android.R.layout.simple_list_item_1,bookName);
foot= LayoutInflater.from(MainActivity.this).inflate(R.layout.layout_foot,null);
fail= LayoutInflater.from(MainActivity.this).inflate(R.layout.layout_fail,null);
complete= LayoutInflater.from(MainActivity.this).inflate(R.layout.layout_complete,null);
progressBar= (ProgressBar) findViewById(R.id.progressBar);

UmengUpdateAgent.update(this);

}

/***
* 获取图书馆书名数据
*/
class GetBookName extends AsyncTask {

BookNameJsoup jsoup;
URL href;

public GetBookName(String keyword)  {
super();
url="http://222.188.3.137:8080/opac/openlink.php?strSearchType=title&match_flag=forward&historyCount=1&strText="+keyword+
"&doctype=ALL&with_ebook=on&displaypg=20&showmode=list&sort=CATA_DATE&orderby=desc&dept=ALL&page=1";

try {
href=new URL(url);
url=href.toString();

} catch (MalformedURLException e) {
e.printStackTrace();
}

}
public GetBookName()
{
}

@Override
protected void onPreExecute() {
super.onPreExecute();
Log.d("TAG", "onPreExecute: "+url);
if(firstLoad)
{
progressBar.setVisibility(View.VISIBLE);
firstLoad=false;
}

}

@Override
protected Object doInBackground(Object[] params) {

jsoup=new BookNameJsoup(url);
jsoup.init();
return null;
}

@Override
protected void onPostExecute(Object o) {
super.onPostExecute(o);
progressBar.setVisibility(View.GONE);

String[] book = jsoup.getBookName();
String[] link = jsoup.getLink();

for (int i = 0; i < 20; i++) {
if (book[i] != null) {
bookName.add(book[i]);
Log.d("TAG", "onPostExecute: " + book[i]);
} else {
break;
}
if (link != null) {
bookLink.add(link[i]);
}
}
if(bookName.isEmpty())
{
list_bookname.addFooterView(fail);
}else {
list_bookname.removeFooterView(fail);
}

url = jsoup.getNextPage();
list_bookname.removeFooterView(foot);
adapter.notifyDataSetChanged();
}
}

}

四 书的详细信息

这里展现的是书的信息。如图

package com.wyf.newlibrary;

import android.os.AsyncTask;
import android.os.Bundle;
import android.support.v7.app.AppCompatActivity;
import android.util.Log;
import android.view.MenuItem;
import android.view.View;
import android.widget.ImageView;
import android.widget.ScrollView;
import android.widget.TextView;

import com.android.volley.RequestQueue;
import com.android.volley.Response;
import com.android.volley.VolleyError;
import com.android.volley.toolbox.StringRequest;
import com.android.volley.toolbox.Volley;
import com.umeng.analytics.MobclickAgent;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.UnsupportedEncodingException;

public class BookInfoActivity extends AppCompatActivity {

TextView text_bookName,text_douban,text_position;
ImageView image_logo;
String url,logoUrl;
ScrollView scrollView;
@Override
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
setContentView(R.layout.activity_book_info);
getSupportActionBar().setDisplayHomeAsUpEnabled(true);
setTitle("图书信息");
url=getIntent().getStringExtra("href");

init();

RequestQueue queue= Volley.newRequestQueue(BookInfoActivity.this);
StringRequest stringRequest=new StringRequest(url, new Response.Listener<String>() {
@Override
public void onResponse(String s) {
try {
s=new String(s.getBytes("ISO-8859-1"),"utf-8");
// Toast.makeText(BookInfoActivity.this, s, Toast.LENGTH_SHORT).show();
new  GetDetail(s).execute();
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
}
}
}, new Response.ErrorListener() {
@Override
public void onErrorResponse(VolleyError volleyError) {
Log.d("TAG", "onErrorResponse: "+"i fail");
}
});
queue.add(stringRequest);

}

private void init() {
text_bookName= (TextView) findViewById(R.id.text_bookname);
text_douban= (TextView) findViewById(R.id.text_douban);
text_position= (TextView) findViewById(R.id.text_position);
scrollView= (ScrollView) findViewById(R.id.scroll_position);

}

@Override
public boolean onOptionsItemSelected(MenuItem item) {

if(item.getItemId()==android.R.id.home)
{
finish();
return true;
}
return super.onOptionsItemSelected(item);
}

/*****************异步请求拿到数据**********************/
class GetDetail extends AsyncTask{
String response,name,douBan="110",position="图书位置：\n";

public GetDetail(String response) {
super();
this.response=response;
}

@Override
protected Object doInBackground(Object[] params) {
Document doc= Jsoup.parse(response);
name=doc.getElementsByAttributeValue("class","booklist").first().text();
name=name.substring(name.indexOf(":")+1);
//  douBan=doc.getElementsByAttributeValue("id","douban_content").select("p").text();

Elements a=doc.select("dl.booklist");
String temp;
for(Element i:a)
{

System.out.println(i.lastElementSibling().text());
if(i.text().contains("提要文摘附注")&&i.text().length()>8)
{
temp=i.text();

douBan=temp;

}
}

//douBan=doc.select("intro").text();
logoUrl=doc.select("img#book_img").attr("src");
Elements posi=doc.getElementsByAttributeValue("align","left");
posi=posi.select("tr.whitetext");

for(Element i:posi)
{
if(i.text()!=null)
{
//temp=i.text();
// temp=temp.substring(0,temp.indexOf(' '))+temp.substring(temp.indexOf("-"));
//Log.d("TAG", "doInBackground: "+temp);
position+=i.text()+"\n";

}

}
return null;
}

@Override
protected void onPostExecute(Object o) {
super.onPostExecute(o);
text_bookName.setText(name);
text_douban.setText(douBan);
text_position.setText(position);

if(douBan.equals("110"))
{
text_douban.setVisibility(View.GONE);
}

scrollView.setVisibility(View.VISIBLE);

}
}

}

五 总结

这个Demo其实并不难，很容易理解，但要对Jsoup和异步请求有所了解。你也可以做出自己第三方客户端。

附上源码http://pan.baidu.com/s/1c1B1VIo