python百度贴吧爬虫

qq123 2021年02月16日编程语言 409 0

# -*- coding: utf-8 -*- 
#coding=utf-8 
 
import urllib 
import urllib2 
import re 
import thread 
import time 
 
class BDTB: 
    def __init__(self,baseurl,seeLz): 
        self.baseUrl=baseurl 
        self.seeLz='?see_lz='+str(seeLz) 
        self.Tool=Tool() 
    def getPage(self,pageNum): 
        try: 
            url=self.baseUrl+self.seeLz+'&pn='+str(pageNum) 
            request = urllib2.Request(url) 
            response = urllib2.urlopen(request) 
            return response.read() 
        except urllib2.URLError, e: 
            print "链接网络失败"+e.reason 
            return None 
    def getTitle(self): 
        html=self.getPage(1) 
        pattern = re.compile('core_title_txt pull-left text-overflow.*?>(.*?)</h3>',re.S) 
        result =re.search(pattern,html) 
        if result: 
            print result.group(1) 
        else: 
            return None 
    def getContent(self,page): 
        pattern  =re.compile('<div id="post_content_.*?>(.*?)</div>',re.S) 
        items = re.findall(pattern,page) 
        floor=1 
        for i in items: 
            print floor,u'楼--------------------------------------------\n' 
            print self.Tool.replace(i) 
            floor+=1 
 
 
 
 
class Tool:

#去除img标签,1-7位空格, 

removeImg = re.compile('<img.*?>| {1,7}| ')

#删除超链接标签

removeAddr = re.compile('<a.*?>|</a>')

#把换行的标签换为\n

replaceLine = re.compile('<tr>|<div>|</div>|</p>')

#将表格制表<td>替换为\t

replaceTD= re.compile('<td>')

#将换行符或双换行符替换为\n

replaceBR = re.compile('<br><br>|<br>')

#将其余标签剔除

removeExtraTag = re.compile('<.*?>')

#将多行空行删除

removeNoneLine = re.compile('\n+')

def replace(self,x):

x = re.sub(self.removeImg,"",x)

x = re.sub(self.removeAddr,"",x)

x = re.sub(self.replaceLine,"\n",x)

x = re.sub(self.replaceTD,"\t",x)

x = re.sub(self.replaceBR,"\n",x)

x = re.sub(self.removeExtraTag,"",x)

x = re.sub(self.removeNoneLine,"\n",x)

#strip()将前后多余内容删除

return x.strip()

 
 
 
 
baseURL = 'http://tieba.baidu.com/p/3138733512' 
bdtb = BDTB(baseURL,2) 
bdtb.getContent(bdtb.getPage(2))

IT源码网编程语言 Python

微信公众号号：IT虾米（左侧二维码扫一扫）欢迎添加！

Q : 823053937

热贴推荐

python百度贴吧爬虫