学了下beautifulsoup后,做个个网络爬虫,爬取读者杂志并用reportlab制作成pdf..

crawler.py

复制代码 代码如下:

!/usr/bin/env python

coding=utf-8

"""
Author: Anemone
Filename: getmain.py
Last modified: 2015-02-19 16:47
E-mail: anemone@82flex.com
"""
import urllib2
from bs4 import BeautifulSoup
import re
import sys
reload(sys)
sys.setdefaultencoding('utf-8')
def getEachArticle(url):

response = urllib2.urlopen('http://www.52duzhe.com/2015_01/duzh20150104.html')

response = urllib2.urlopen(url)  
html = response.read()  
soup = BeautifulSoup(html)#.decode("utf-8").encode("gbk"))  
#for i in soup.find_all('div'):  
#    print i,1  
title=soup.find("h1").string  
writer=soup.find(id="pub_date").string.strip()  
_from=soup.find(id="media_name").string.strip()  
text=soup.get_text()#.encode("utf-8")  
main=re.split("BAIDU_CLB.*;",text)  
result={"title":title,"writer":writer,"from":_from,"context":main[1]}  
return result  
#new=open("new.txt","w")  
#new.write(result["title"]+"\n\n")  
#new.write(result["writer"]+"  "+result["from"])  
#new.write(result["context"])  
#new.close()  

def getCatalog(issue):
url="http://www.52duzhe.com/"+issue[:4]+"_"+issue[-2:]+"/"
firstUrl=url+"duzh"+issue+"01.html"
firstUrl=url+"index.html"
duzhe=dict()
response = urllib2.urlopen(firstUrl)
html = response.read()
soup=BeautifulSoup(html)
firstUrl=url+soup.table.a.get("href")
response = urllib2.urlopen(firstUrl)
html = response.read()
soup = BeautifulSoup(html)
all=soup.find_all("h2")
for i in all:
print i.string
duzhe[i.string]=list()
for link in i.parent.find_all("a"):
href=url+link.get("href")
print href
while 1:
try:
article=getEachArticle(href)
break
except:
continue
duzhe[i.string].append(article)
return duzhe
def readDuZhe(duzhe):
for eachColumn in duzhe:
for eachArticle in duzhe[eachColumn]:
print eachArticle["title"]
if name == 'main':

issue=raw_input("issue(201501):")

readDuZhe(getCatalog("201424"))  

getpdf.py

复制代码 代码如下:

!/usr/bin/env python

coding=utf-8

"""
Author: Anemone
Filename: writetopdf.py
Last modified: 2015-02-20 19:19
E-mail: anemone@82flex.com
"""

coding=utf-8

import reportlab.rl_config
from reportlab.pdfbase import pdfmetrics
from reportlab.pdfbase.ttfonts import TTFont
from reportlab.lib import fonts
import copy
from reportlab.platypus import Paragraph, SimpleDocTemplate,flowables
from reportlab.lib.styles import getSampleStyleSheet
import crawler
def writePDF(issue,duzhe):
reportlab.rl_config.warnOnMissingFontGlyphs = 0
pdfmetrics.registerFont(TTFont('song',"simsun.ttc"))
pdfmetrics.registerFont(TTFont('hei',"msyh.ttc"))
fonts.addMapping('song', 0, 0, 'song')
fonts.addMapping('song', 0, 1, 'song')
fonts.addMapping('song', 1, 0, 'hei')
fonts.addMapping('song', 1, 1, 'hei')
stylesheet=getSampleStyleSheet()
normalStyle = copy.deepcopy(stylesheet['Normal'])
normalStyle.fontName ='song'
normalStyle.fontSize = 11
normalStyle.leading = 11
normalStyle.firstLineIndent = 20
titleStyle = copy.deepcopy(stylesheet['Normal'])
titleStyle.fontName ='song'
titleStyle.fontSize = 15
titleStyle.leading = 20
firstTitleStyle = copy.deepcopy(stylesheet['Normal'])
firstTitleStyle.fontName ='song'
firstTitleStyle.fontSize = 20
firstTitleStyle.leading = 20
firstTitleStyle.firstLineIndent = 50
smallStyle = copy.deepcopy(stylesheet['Normal'])
smallStyle.fontName ='song'
smallStyle.fontSize = 8
smallStyle.leading = 8
story = []
story.append(Paragraph("读者{0}期".format(issue), firstTitleStyle))
for eachColumn in duzhe:
story.append(Paragraph('__'*28, titleStyle))
story.append(Paragraph('{0}'.format(eachColumn), titleStyle))
for eachArticle in duzhe[eachColumn]:
story.append(Paragraph(eachArticle["title"],normalStyle))
story.append(flowables.PageBreak())
for eachColumn in duzhe:
for eachArticle in duzhe[eachColumn]:
story.append(Paragraph("{0}".format(eachArticle["title"]),titleStyle))
story.append(Paragraph(" {0} {1}".format(eachArticle["writer"],eachArticle["from"]),smallStyle))
para=eachArticle["context"].split("　　")
for eachPara in para:
story.append(Paragraph(eachPara,normalStyle))
story.append(flowables.PageBreak())

story.append(Paragraph("context",normalStyle))

doc = SimpleDocTemplate("duzhe"+issue+".pdf")  
print "Writing PDF..."  
doc.build(story)  

def main(issue):
duzhe=crawler.getCatalog(issue)
writePDF(issue,duzhe)
if name == 'main':
issue=raw_input("Enter issue(201501):")
main(issue)

以上就是本文的全部内容了，希望大家能够喜欢。