[分享]抓取网页上的图片,练习用的东西,见笑见笑

Python/PHP/Perl 开发与设计
回复
gmslinux
帖子: 24
注册时间: 2007-02-03 21:58
送出感谢: 0
接收感谢: 0

[分享]抓取网页上的图片,练习用的东西,见笑见笑

#1

帖子 gmslinux » 2007-04-05 11:09

代码: 全选

# -*- coding: gb2312 -*-
"""
some function by gms,2007-04-03,copyleft
version 0.3
"""
import urllib, httplib, urlparse
import re
import random

"""judge url exists or not,by others"""
def httpExists(url):
    host, path = urlparse.urlsplit(url)[1:3]
    if ':' in host:
        # port specified, try to use it
        host, port = host.split(':', 1)
        try:
            port = int(port)
        except ValueError:
            print 'invalid port number %r' % (port,)
            return False
    else:
        # no port specified, use default port
        port = None
    try:
        connection = httplib.HTTPConnection(host, port=port)
        connection.request("HEAD", path)
        resp = connection.getresponse( )
        if resp.status == 200:       # normal 'found' status
            found = True
        elif resp.status == 302:     # recurse on temporary redirect
            found = httpExists(urlparse.urljoin(url,resp.getheader('location', '')))
        else:                        # everything else -> not found
            print "Status %d %s : %s" % (resp.status, resp.reason, url)
            found = False
    except Exception, e:
        print e.__class__, e, url
        found = False
    return found

"""get html src,return lines[]"""
def gGetHtmlLines(url):
    if url==None : return
    if not httpExists(url): return 
    try:
        page = urllib.urlopen(url)   
        html = page.readlines()
        page.close()
        return html
    except:
        print "gGetHtmlLines() error!"
        return

"""根据url获取文件名"""
def gGetFileName(url):
    if url==None: return None
    if url=="" : return ""
    arr=url.split("/")
    return arr[-1]

"""生成随机文件名"""
def gRandFileName(fileType):
    fname = ''
    for i in range(16):
        fname = fname + chr(random.randint(65,90))
        fname = fname + chr(random.randint(48,57))
    return fname + '.' + fileType

"""根据url和其上的link,得到link的绝对地址"""
def gGetAbslLink(url,link):
    if url==None or link == None : return 
    if url=='' or link=='' : return url 
    addr = '' 
    if link[0] == '/' : 
        addr = gGetHttpAddr(url) + link 
    elif len(link)>3 and link[0:4] == 'http':
        addr =  link 
    elif len(link)>2 and link[0:2] == '..':
        addr = gGetHttpAddrFatherAssign(url,link)
    else:
        addr = gGetHttpAddrFather(url) + link 
    return addr 
"""根据url取主站地址"""
def gGetHttpAddr(url):
    if url== '' : return ''
    arr=url.split("/")
    return arr[0]+"//"+arr[2]
"""根据url取上级目录"""
def gGetHttpAddrFather(url):
    if url=='' : return ''
    arr=url.split("/")
    addr = arr[0]+'//'+arr[2]+ '/'
    if len(arr)-1>3 :
        for i in range(3,len(arr)-1):
            addr = addr + arr[i] + '/'
    return addr

"""根据url和上级的link取link的绝对地址"""
def gGetHttpAddrFatherAssign(url,link):
    if url=='' : return ''
    if link=='': return ''
    linkArray=link.split("/")
    urlArray = url.split("/")
    partLink =''
    partUrl = ''
    for i in range(len(linkArray)):        
        if linkArray[i]=='..': 
            numOfFather = i + 1    #上级数
        else:
            partLink = partLink + '/'  + linkArray[i]
    for i in range(len(urlArray)-1-numOfFather):
        partUrl = partUrl + urlArray[i] 
        if i < len(urlArray)-1-numOfFather -1 : 
            partUrl = partUrl + '/'
    return  partUrl + partLink

"""根据输入的lines,匹配正则表达式,返回list"""
def gGetRegList(linesList,regx):
    if linesList==None : return 
    rtnList=[]
    for line in linesList:
        matchs = re.search(regx, line, re.IGNORECASE)
        if matchs!=None:
            allGroups = matchs.groups()
            for foundStr in allGroups:
                if foundStr not in rtnList:
                    rtnList.append(foundStr)
    return rtnList

"""根据url下载文件,文件名参数指定"""
def gDownload(url,savePath,file):
    #参数检查,现忽略
    try:
        urlopen=urllib.URLopener()
        fp = urlopen.open(url)
        data = fp.read()
        fp.close()
        file=open(savePath + file,'w+b')
        file.write(data)
        file.close()
    except IOError:
        print "download error!"+ url
        
"""根据某网页的url,下载该网页的jpg"""
def gDownloadHtmlJpg(downloadUrl,savePath):
    lines= gGetHtmlLines(downloadUrl)
    regx = r"""src\s*="?(\S+)\.jpg"""
    lists =gGetRegList(lines,regx)
    if lists==None: return 
    for jpg in lists:
        jpg = gGetAbslLink(downloadUrl,jpg) + '.jpg'
        fileName =gRandFileName('jpg')
        gDownload(jpg,savePath,fileName)
        print gGetFileName(jpg)

"""根据url获取其上的相关htm、html链接,返回list"""
def gGetHtmlLink(url):
    #参数检查,现忽略
    rtnList=[]
    lines=gGetHtmlLines(url)
    regx = r"""href="?(\S+)\.htm"""
    for link in gGetRegList(lines,regx):
        link = gGetAbslLink(url,link) + '.htm'
        if link not in rtnList:
            rtnList.append(link)
    return rtnList

"""根据url,抓取其上的jpg和其链接htm上的jpg"""
def gDownloadAllJpg(url,savePath):
    #参数检查,现忽略
    gDownloadHtmlJpg(url,savePath)
    #抓取link上的jpg
    links=gGetHtmlLink(url)
    for link in links:
        gDownloadHtmlJpg(link,savePath)

"""test"""
def uTest():
    u='http://www.gs.xinhuanet.com/news/2007-01/31/content_9188207_1.htm'
    save='d:\\desktop\\'
    lst = gGetHtmlLink(u)
uTest()

fireflycn
帖子: 18
注册时间: 2006-07-09 19:22
送出感谢: 0
接收感谢: 0

#2

帖子 fireflycn » 2007-09-10 20:21

相当不错。
用用试试先
cgk100
帖子: 60
注册时间: 2007-04-22 10:03
来自: china
送出感谢: 0
接收感谢: 0
联系:

请问python可以抓去网页的html么

#3

帖子 cgk100 » 2007-10-24 21:53

请问python可以抓去网页的html么
头像
anticlockwise
帖子: 2393
注册时间: 2007-03-01 20:46
来自: 湖南长沙
送出感谢: 0
接收感谢: 0
联系:

Re: 请问python可以抓去网页的html么

#4

帖子 anticlockwise » 2007-10-25 1:10

cgk100 写了:请问python可以抓去网页的html么
当然可以~~Python就是另一门高级语言而已
jonlee1979
帖子: 2
注册时间: 2010-05-10 15:41
送出感谢: 0
接收感谢: 0

Re: [分享]抓取网页上的图片,练习用的东西,见笑见笑

#5

帖子 jonlee1979 » 2010-06-03 20:42

如果能有更详细的说明对新手来说就更好了,顶顶顶!!!!
aerofox
帖子: 1453
注册时间: 2008-05-24 8:30
送出感谢: 2 次
接收感谢: 31 次

Re: [分享]抓取网页上的图片,练习用的东西,见笑见笑

#6

帖子 aerofox » 2010-06-03 21:47

顶一个!
是否可以考虑用 BeautifulSoup?用这个来分析 html 感觉挺方便的。
回复

回到 “Python/Php/Perl”