分享一个天气历史数据的采集脚本

最近一个项目中需要用到过往的天气数据, 我找到了天气后报这个网站 (www.tianqihoubao.com), 并在SS中完成了相关采集, 和大家分享一下.

首先分析这个网站提供了两种信息:

1. 省市关系

2. 天气记录

对应的我们创建数据结构:

[Serializable]
public class Province
{
public string ProvinceName;
public string ProvinceUrl;
}

[Serializable]
public class City
{
public Province Province;
public string CityName;
public string CityUrl;
}

[Serializable]
public class WeatherDataSet
{
public City City;
public string Title;
public string Url;
}

[Serializable]
public class WeatherData
{
public WeatherDataSet DataSet;
public string Date;
public string TextWeather;
public string Temp;
public string Wind;
}

>> 网站一共提供34个直辖市/省/特区的天气信息, 完整的列表在: http://www.tianqihoubao.com/lishi/index.htm

public void Run()
{
Logger.ClearAll();
Default.ScriptErrorsSuppressed = true;

var pl = GetProvinceList();
foreach(var p in pl)
{
Logger.Log(p.ProvinceName);
Logger.Log(p.ProvinceUrl);
}
var cl = GetCityList(pl[1]);
foreach(var c in cl)
{
Logger.Log(c.Province.ProvinceName);
Logger.Log(c.Province.ProvinceUrl);
Logger.Log(c.CityName);
Logger.Log(c.CityUrl);
}
var ds = GetWeatherDataSet(cl[1]);
foreach(var d in ds)
{
Logger.Log(d.City.CityName);
Logger.Log(d.Title);
Logger.Log(d.Url);
}
var dl = GetWeatherData(ds[0]);
foreach(var d in dl)
{
Logger.Log(d.DataSet.Title);
Logger.Log(d.Date);
Logger.Log(d.TextWeather);
Logger.Log(d.Temp);
Logger.Log(d.Wind);
}
}

public List<Province> GetProvinceList()
{
Default.Navigate("http://www.tianqihoubao.com/lishi/index.htm");
Default.Ready("#content DT");
var list = Default.SelectNodes("#content DT a");
var result = new List<Province>();
foreach(var item in list)
{
var p = new Province();
p.ProvinceName = item.Text();
p.ProvinceUrl = item.Attr("href");
p.ProvinceUrl = new Uri(Default.Url, p.ProvinceUrl).ToString();
result.Add(p);
}
return result;
}

public List<City> GetCityList(Province province)
{
Default.Navigate(province.ProvinceUrl);
Default.Ready("#content DD");
var list = Default.SelectNodes("#content DD a");
var result = new List<City>();
foreach(var item in list)
{
var c = new City();
c.Province = province;
c.CityName = item.Text();
c.CityUrl = item.Attr("href");
c.CityUrl = new Uri(Default.Url, c.CityUrl).ToString();
result.Add(c);
}
return result;
}

public List<WeatherDataSet> GetWeatherDataSet(City city)
{
Default.Navigate(city.CityUrl);
Default.Ready("#content>div.pcity");
var list = Default.SelectNodes("#content>div.pcity a");
var result = new List<WeatherDataSet>();
foreach(var item in list)
{
var ds = new WeatherDataSet();
ds.Title = item.Text();
ds.Url = item.Attr("href");
ds.Url = new Uri(Default.Url, ds.Url).ToString();
ds.City = city;
result.Add(ds);
}
return result;
}

public List<WeatherData> GetWeatherData(WeatherDataSet ds)
{
Default.Navigate(ds.Url);
Default.Ready("#content>table.b");
var list = Default.SelectNodes("#content>table.b tr:gt(0)");
var result = new List<WeatherData>();
foreach(var item in list)
{
var d = new WeatherData();
d.DataSet = ds;
d.Date = item.SelectSingleNode("td:eq(0)").Text();
d.TextWeather = item.SelectSingleNode("td:eq(1)").Text();
d.Temp = item.SelectSingleNode("td:eq(2)").Text();
d.Wind = item.SelectSingleNode("td:eq(3)").Text();
result.Add(d);
}
return result;
}

[Serializable]
public class Province
{
public string ProvinceName;
public string ProvinceUrl;
}

[Serializable]
public class City
{
public Province Province;
public string CityName;
public string CityUrl;
}

[Serializable]
public class WeatherDataSet
{
public City City;
public string Title;
public string Url;
}

[Serializable]
public class WeatherData
{
public WeatherDataSet DataSet;
public string Date;
public string TextWeather;
public string Temp;
public string Wind;
}

View Code

>> 运行效果: