代码写于一年前，一直没有用在实际系统中。不知道针对现在的天气预报网站是不是有效，不过对各位应该又很大的参考价值
使用BeautifulSoup做HTML分析。
抓取最近的5天数据，并保存到mysql数据库中。
如果出现处理失败，会向指定的邮件地址，发送报警。这是一个比较完善的天气预报抓取程序。
代码： 全选
#! /usr/bin/env python
# -*- coding: utf-8 -*-
"""
版权：www.lalfa.com ealpha@gmail.com
转载请不要删除
todo ：设置一个字段表示，是否成功更新，一旦成功则记录ID,系统运行结束，进行update。否则发送短信。保证只发送一次
"""
import os,urllib2,re,MySQLdb,datetime,time,smtplib
from BeautifulSoup import BeautifulSoup
from StringIO import StringIO
from email.mime.text import MIMEText
USER_AGENT = ' Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9.0.1) Gecko/2008070208 Firefox/3.0.1'
BASE_URL_BEGIN= 'http://www.weather.com.cn/html/weather/'
BASE_URL_END = '.shtml'
conn = MySQLdb.connect(host="localhost", user="fun", passwd="fun", db="fun",use_unicode=1, charset='utf8')
#Mail 接收方邮件
mailto_list=["XXXXX@XXXXX.com"]
#SMTP 服务器，用户名、口令以及邮箱的后缀
mail_host="XXXX.com"
mail_user="XXXX"
mail_pass="XXXX"
mail_postfix="imichat.com"
#失败的WID,记录用来判断，是否发送邮件
faultwid = []
#失败重试次数
dotime = 0
def send_mail(to_list,sub,content):
'''
to_list:发给谁
sub:主题
content:内容
send_mail("aaa@126.com","sub","content")
'''
me=mail_user+"<"+mail_user+"@"+mail_postfix+">"
msg = MIMEText(content)
msg['Subject'] = sub
msg['From'] = me
msg['To'] = ";".join(to_list)
try:
s = smtplib.SMTP()
s.connect(mail_host)
s.login(mail_user,mail_pass)
s.sendmail(me, to_list, msg.as_string())
s.close()
return True
except Exception, e:
print str(e)
return False
def getFiveDayWeather(wid,pageid,agent=USER_AGENT):
"""
将需要的数据表格从整个网页取出来
"""
url = BASE_URL_BEGIN + pageid + BASE_URL_END
#print '*************************'
#print url
request = urllib2.Request(url)
request.add_header('User-Agent', agent)
opener = urllib2.build_opener()
allhtml = StringIO(str((opener.open(request)).read()))
soup = BeautifulSoup(allhtml, fromEncoding="utf-8")
html = soup.find('div', id='dd_0').parent.contents
getWeatherList(wid,html)
return html
def getWeatherList(wid,html):
"""
取得最后发布时间，已经5天数据
"""
soup1 = BeautifulSoup(str(html))
time = soup1.find('h2')
update_time = ''
for t in time:
update_time = t
#print '\r'
#print update_time
#print '\r'
html2 = soup1.findAll('div', { "class" : "fut_weatherbox" })
dayid = 0
for dayweather in html2:
dayid += 1
getOneDayWeather(wid,dayid,update_time,dayweather)
pass
def getOneDayWeather(wid,dayid,update_time,html):
"""
分析一天的天气预报信息
"""
soup = BeautifulSoup(StringIO(str(html)), fromEncoding="UTF-8")
day = soup.findAll('h3')
imgs = soup.findAll('img')
t00 = soup.findAll('h4', { "class" : "temp00_dn" })
t01 = soup.findAll('h4', { "class" : "temp01_dn" })
t02 = soup.findAll('h4', { "class" : "temp02_dn" })
t03 = soup.findAll('h4', { "class" : "temp03_dn" })
#print '----------------------'
soup_h3 = BeautifulSoup(StringIO(str(day)), fromEncoding="UTF-8")
day_value = soup_h3.h3.renderContents()
#for img in imgs: 为了确定值，不使用循环
soup_img = BeautifulSoup(StringIO(str(imgs[0])), fromEncoding="UTF-8")
imgsrc = soup_img.first('img')['src']
d_pic_value = imgsrc.split('/')[-1].split('.')[-2]
soup_img = BeautifulSoup(StringIO(str(imgs[1])), fromEncoding="UTF-8")
imgsrc = soup_img.first('img')['src']
n_pic_value = imgsrc.split('/')[-1].split('.')[-2]
soup_t00 = BeautifulSoup(StringIO(str(t00)), fromEncoding="UTF-8")
weather_value = soup_t00.h4.renderContents()
soup_t01 = BeautifulSoup(StringIO(str(t01)), fromEncoding="UTF-8")
max_temp = soup_t01.h4.renderContents()
soup_t02 = BeautifulSoup(StringIO(str(t02)), fromEncoding="UTF-8")
min_temp = soup_t02.h4.renderContents()
soup_t03 = BeautifulSoup(StringIO(str(t03)), fromEncoding="UTF-8")
wind = soup_t03.h4.renderContents()
insertDB(wid,dayid,update_time,day_value,d_pic_value,n_pic_value,weather_value,max_temp,min_temp,wind )
def insertDB(wid,dayid,update_time,day_value,d_pic_value,n_pic_value,weather_value,max_temp,min_temp,wind ):
"""
插入数据库，此处要修改，5天数据一次commit(),异常rollback()
"""
cursor_uodate=conn.cursor()
sql="INSERT INTO weatherdetail( wid, dayid, lastupdate, currdate, dpic, npic,weather, maxtemp, mintemp, wind) VALUES( %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"
param = (wid,dayid,update_time ,day_value,d_pic_value,n_pic_value,weather_value,max_temp,min_temp,wind)
#print param
n=cursor_uodate.execute(sql,param)
conn.commit()
#print n
def sendMonitor():
"""
处理失败，发送报警邮件或短信
"""
if len(faultwid) <> 0 :
if send_mail(mailto_list,"Error: Get Weather Error "+str(datetime.datetime.now()),str(faultwid)):
print "监控邮件发送成功."
else:
print "监控邮件发送失败."
pass
def doworking(dotime,wid,pageid):
"""
业务处理入口
"""
try:
getFiveDayWeather(wid,pageid)
except (NameError,Exception),e:
print "has one error on %s %s , then do it again , waiting five secs." % (wid,pageid)
time.sleep(5)
if dotime < 3 :
doworking(dotime + 1,wid,pageid)
else:
faultwid.append(wid)
pass
if __name__ == "__main__":
"""
入口函数
"""
starttime = datetime.datetime.now()
print "Start."+str(starttime)
cursor = conn.cursor()
cursor.execute("SELECT id,weather_com_cn_pageid FROM weather")
result = cursor.fetchall()
for record in result:
# 将 dotime 恢复到 0 ，代表本次请求首次执行
doworking(0,str(record[0]),record[1])
#time.sleep(2)
print '\r'
endtime = datetime.datetime.now()
print "End."+str(endtime)
print "-------------------------------------------------"
sendMonitor()
print (endtime - starttime).seconds
