python实现批量下载新浪博客的方法

本文实例讲述了python实现批量下载新浪博客的方法。分享给大家供大家参考。具体实现方法如下：

    # coding=utf-8 
    import urllib2
    import sys, os
    import re
    import string
    from BeautifulSoup import BeautifulSoup
    def encode(s):
      return s.decode('utf-8').encode(sys.stdout.encoding, 'ignore')
    def getHTML(url):
      #proxy_handler = urllib2.ProxyHandler({'http':'http://211.138.124.211:80'})
      #opener = urllib2.build_opener(proxy_handler)
      #urllib2.install_opener(opener)
      req = urllib2.Request(url)
      response = urllib2.urlopen(req, timeout=15)
      return BeautifulSoup(response, convertEntities=BeautifulSoup.HTML_ENTITIES)
    def visible(element):
      '''抓取可见的文本元素'''
      if element.parent.name in ['style', 'script', '[document]', 'head', 'title']:
        return False
      elif re.match('<!--.*-->', str(element)):
        return False
      elif element == u'\xa0':
        return False
      return True
    def delReturn(element):
      '''删除元素内的换行'''
      return re.sub('(?<!^)\n+(?!$)', ' ', str(element)).decode('utf-8')
    def validFilename(filename):
      # windows
      return re.sub('[\/:*?<>"|\xa0]', '', filename)
    def writeToFile(text, filename, dirname):
      if not os.path.exists(dirname):
        os.makedirs(dirname)
        print encode('保存到目录'), dirname
      filename = validFilename(filename)
      print encode('保存文章'), filename
      path = os.path.join(dirname, filename)
      if not os.path.exists(path):
        f = open(path, 'w')
        f.write(text)
        f.close()
      else:
        print filename, encode('已经存在')
    def formatContent(url, title=''):
      '''格式化文章内容'''
      page = getHTML(url)
      content = page.find('div', {'class':'articalContent'})
      art_id = re.search('blog_(\w+)\.html', url).group(1)
      blog_name = page.find('span', id='blognamespan').string
      if title == '':
        title = page.find('h2', id=re.compile('^t_')).string
      temp_data = filter(visible, content.findAll(text=True)) # 去掉不可见元素
      temp_data = ''.join(map(delReturn, temp_data)) # 删除元素内的换行符
      temp_data = temp_data.strip() # 删除文章首尾的空行
      temp_data = re.sub('\n{2,}', '\n\n', temp_data) # 删除文章内过多的空行
      # 输出到文件
      # 编码问题
      temp_data = '本文地址:'.decode('utf-8') + url + '\n\n' + temp_data
      op_text = temp_data.encode('utf-8')
      op_file = title + '_' + art_id +'.txt'
      writeToFile(op_text, op_file, blog_name)
    def articlelist(url):
      articles = {}
      page = getHTML(url)
      pages = page.find('ul', {'class':'SG_pages'}).span.string
      page_num = int(re.search('(\d+)', pages).group(1))
      for i in range(1, page_num+1):
        print encode('生成第%d页文章索引'%i)
        if i != 1:
          url = re.sub('(_)\d+(\.html)$', '\g<1>'+str(i)+'\g<2>', url)
          page = getHTML(url)
        article = page.findAll('span', {'class':'atc_title'})
        for art in article:
          art_title = art.a['title']
          art_href = art.a['href']
          articles[art_title] = art_href
      return articles
    def blog_dld(articles):
      if not isinstance(articles, dict):
        return False
      print encode('开始下载文章')
      for art_title, art_href in articles.items():
        formatContent(art_href, art_title)
    if __name__ == '__main__':
      sel = raw_input(encode('你要下载的是(1)全部文章还是(2)单篇文章，输入1或者2: '))
      if sel == '1':
        #articlelist_url = 'http://blog.sina.com.cn/s/articlelist_1303481411_0_1.html'
        articlelist_url = raw_input(encode('请输入博客文章目录链接: '))
        articles = articlelist(articlelist_url)
        blog_dld(articles)
      else:
        #article_url = 'http://blog.sina.com.cn/s/blog_4db18c430100gxc5.html'
        article_url = raw_input(encode('请输入博客文章链接: '))
        formatContent(article_url)

希望本文所述对大家的Python程序设计有所帮助。