IT源码网

python百度贴吧爬虫

qq123 2021年02月16日 编程语言 467 0
# -*- coding: utf-8 -*- 
#coding=utf-8 
 
import urllib 
import urllib2 
import re 
import thread 
import time 
 
class BDTB: 
    def __init__(self,baseurl,seeLz): 
        self.baseUrl=baseurl 
        self.seeLz='?see_lz='+str(seeLz) 
        self.Tool=Tool() 
    def getPage(self,pageNum): 
        try: 
            url=self.baseUrl+self.seeLz+'&pn='+str(pageNum) 
            request = urllib2.Request(url) 
            response = urllib2.urlopen(request) 
            return response.read() 
        except urllib2.URLError, e: 
            print "链接网络失败"+e.reason 
            return None 
    def getTitle(self): 
        html=self.getPage(1) 
        pattern = re.compile('core_title_txt pull-left text-overflow.*?>(.*?)</h3>',re.S) 
        result =re.search(pattern,html) 
        if result: 
            print result.group(1) 
        else: 
            return None 
    def getContent(self,page): 
        pattern  =re.compile('<div id="post_content_.*?>(.*?)</div>',re.S) 
        items = re.findall(pattern,page) 
        floor=1 
        for i in items: 
            print floor,u'楼--------------------------------------------\n' 
            print self.Tool.replace(i) 
            floor+=1 
 
 
 
 
class Tool:
    #去除img标签,1-7位空格,&nbsp;
    removeImg = re.compile('<img.*?>| {1,7}|&nbsp;')
    #删除超链接标签
    removeAddr = re.compile('<a.*?>|</a>')
    #把换行的标签换为\n
    replaceLine = re.compile('<tr>|<div>|</div>|</p>')
    #将表格制表<td>替换为\t
    replaceTD= re.compile('<td>')
    #将换行符或双换行符替换为\n
    replaceBR = re.compile('<br><br>|<br>')
    #将其余标签剔除
    removeExtraTag = re.compile('<.*?>')
    #将多行空行删除
    removeNoneLine = re.compile('\n+')
 
  
    def replace(self,x):
 
  
        x = re.sub(self.removeImg,"",x)
 
  
        x = re.sub(self.removeAddr,"",x)
 
  
        x = re.sub(self.replaceLine,"\n",x)
 
  
        x = re.sub(self.replaceTD,"\t",x)
 
  
        x = re.sub(self.replaceBR,"\n",x)
 
  
        x = re.sub(self.removeExtraTag,"",x)
 
  
        x = re.sub(self.removeNoneLine,"\n",x)
 
  
        #strip()将前后多余内容删除
 
  
        return x.strip()
 
 
 
 
baseURL = 'http://tieba.baidu.com/p/3138733512' 
bdtb = BDTB(baseURL,2) 
bdtb.getContent(bdtb.getPage(2))

 

评论关闭
IT源码网

微信公众号号:IT虾米 (左侧二维码扫一扫)欢迎添加!