python采集jb51电子书资源并自动下载到本地实例教程

# -*- coding:utf-8 -*-
import re
import urllib2
import urllib
import sys
import os
reload(sys)
sys.setdefaultencoding('utf-8')
def getHtml(url):
 request = urllib2.Request(url)
 page = urllib2.urlopen(request)
 htmlcontent = page.read()
 #解决中文乱码问题
 htmlcontent = htmlcontent.decode('gbk', 'ignore').encode("utf8",'ignore')
 return htmlcontent
def report(count, blockSize, totalSize):
 percent = int(count*blockSize*100/totalSize)
 sys.stdout.write("r%d%%" % percent + ' complete')
 sys.stdout.flush()
def getBookInfo(url):
 htmlcontent = getHtml(url);
 #print "htmlcontent=",htmlcontent; # you should see the ouput html
 #<h1 class="h1user">crifan</h1>
 regex_  = '<h1s+?itemprop="name">(?P< >.+?)</h1>';
   = re.search(regex_ , htmlcontent);
 if( ):
    =  .group(" ");
  print "书籍名字:", ;
  file_ .write('书籍名字:'+ +'r');
 #<li>书籍大小：<span itemprop="fileSize">27.2MB</span></li>
 filesize = re.search('<spans+?itemprop="fileSize">(?P<filesize>.+?)</span>', htmlcontent);
 if(filesize):
  filesize = filesize.group("filesize");
  print "文件大小:",filesize;
  file_ .write('文件大小:'+filesize+'r');
 #<div class="picthumb"><a href="//files.jb51.net/do/uploads/litimg/151210/1A9262GO2.jpg" target="_blank"
 bookimg = re.search('<divs+?class="picthumb"><a href="(?P<bookimg>.+?)" rel="external nofollow" target="_blank"', htmlcontent);
 if(bookimg):
  bookimg = bookimg.group("bookimg");
  print "封面图片：",bookimg;
  file_ .write('封面图片:'+bookimg+'r');
 #<li><a href="http://xz6.jb51.net:81/201512/books/JavaWeb100(jb51.net).rar" target="_blank">酷云中国电信下载</a></li>
 downurl1 = re.search('<li><a href="(?P<downurl1>.+?)" rel="external nofollow" target="_blank">酷云中国电信下载</a></li>', htmlcontent);
 if(downurl1):
  downurl1 = downurl1.group("downurl1");
  print "下载地址1:",downurl1; 
  file_ .write('下载地址1:'+downurl1+'r');
  sys.stdout.write('rFetching ' +   + '...n')
    =  .replace(' ', '');
    =  .replace('/', '');
  saveFile = '/Users/superl/Desktop/pythonbook/'+ +'.rar';
  if os.path.exists(saveFile):
   print "该文件已经下载了！";
  else:
   urllib.urlretrieve(downurl1, saveFile, reporthook=report);
   sys.stdout.write("rDownload complete, saved as %s" % (saveFile) + 'nn')
   sys.stdout.flush()
   file_ .write('文件下载成功!r');
 else:
  print "下载地址1不存在";
  file_error.write(url+'r');
  file_error.write( +"下载地址1不存在!文件没有自动下载！r");
  file_error.write('r');
 #<li><a href="http://pan.baidu.com/s/1pKfVNwJ" rel="external nofollow" target="_blank">百度网盘下载2</a></li>
 downurl2 = re.search('</a></li><li><a href="(?P<downurl2>.+?)" rel="external nofollow" target="_blank">百度网盘下载2</a></li>', htmlcontent);
 if(downurl2):
  downurl2 = downurl2.group("downurl2");
  print "下载地址2:",downurl2; 
  file_ .write('下载地址2:'+downurl2+'r');
 else:
  #file_error.write(url+'r');
  print "下载地址2不存在"; 
  file_error.write( +"下载地址2不存在r");
  file_error.write('r');
 file_ .write('r');
 print "n";
def getBooksUrl(url):
 htmlcontent = getHtml(url);
 #<ul class="cur-cat-list"><a href="/books/438381.html" rel="external nofollow" class="tit"</ul></div><!--end #content -->
 urls = re.findall('<a href="(?P<urls>.+?)" rel="external nofollow" class="tit"', htmlcontent);
 for url in urls:
  url = "http://www.topjishu.net"+url;
  print url+"n";
  file_ .write(url+'r');
  getBookInfo(url)
  #print "url->", url
if __name__=="__main__":
 file_  = open('/Users/superl/Desktop/python.txt','w+');
 file_error = open('/Users/superl/Desktop/pyth .txt','w+');
 pagenum = 3;
 for pagevalue in range(1,pagenum+1):
  listurl = "http://www.topjishu.net/ books/list476_%d.html"%pagevalue;
  print listurl;
  file_ .write(listurl+'r');
  getBooksUrl(listurl);
 file_ .close();
 file_error.close();