python爬虫爬取后下载小说实现思路及源码分享

#coding:utf-8
import urllib
import urllib.request
import multiprocessing
from bs4 import BeautifulSoup
import re
import os
import time

def get_pages(url):
    soup=""
    try:
        # 创建请求日志文件夹
        if 'Log' not in os.listdir('.'):
            os.mkdir(r".\\Log")

        # 请求当前章节页面  params为请求参数
        request = urllib.request.Request(url)
        response = urllib.request.urlopen(request)
        content = response.read()
        data = content.decode('gbk')
        # soup转换
        soup = BeautifulSoup(data, "html.parser")

    except Exception as e:
        print(url+" 请求错误\\n")
        with open(r".\\Log\\req_error.txt",'a',encoding='utf-8') as f:
            f.write(url+" 请求错误\\n")
        f.close()
    return soup

# 通过章节的url下载内容，并返回下一页的url
def get_ChartTxt(url, ):
    soup=get_pages(url)

    # 获取章节名称
    sub  = soup.select('#htmltimu')[0].text
    # 获取章节文本
    content = soup.select('#htmlContent')[0].text
    # 按照指定格式替换章节内容，运用正则表达式
    content = re.sub(r'\\(.*?\\)', '', content)
    content = re.sub(r'\\r\\n', '', content)
    content = re.sub(r'\\n+', '\\n', content)
    content = re.sub(r'<.*?>+', '', content)
    # # 下一页
    # pageDown = soup.select('#htmlxiazhang')[0].attrs['href']
    # pageDown =   + pageDown

    # 写入这一章
    try:
        # 判断是否有感言
        if re.search(r'.*?章',sub ) is not None:
            with open(r'.\\%s\\%s.txt' % ( , sub ), 'w', encoding='utf-8') as f:
                f.write(sub  + content)
            f.close()
            print(sub , '下载成功')

    except Exception as e:
        print(sub , '下载失败',url)
        with open(".\\%s\\error_url.txt",'a',encoding='utf-8') as f:
            f.write(sub +"下载失败 "+url+'\\n')
    # return pageDown
    return

# 通过首页获得该小说的所有章节链接后下载这本书
def thread_getOneBook(indexUrl):
    soup = get_pages(indexUrl)
    # 获取书名
      = soup.select('#htmldhshuming')[0].text
    # 根据书名创建文件夹
    if   not in os.listdir('.'):
        os.mkdir(r".\\%s" % ( ))
        print( , "文件夹创建成功———————————————————")

    # 加载此进程开始的时间
    print('下载 %s 的PID：%s...' % ( , os.getpid()))
    start = time.time()

    # 获取这本书的所有章节
    charts_url = []
    # 提取出书的每章节不变的url
    indexUrl = re.sub(r'index.html', '', indexUrl)
    charts = soup.select(".zjlist4 li a")
    for i in charts:
        # print(j+i.attrs['href'])
        charts_url.append(indexUrl + i.attrs['href'])

    # 创建下载这本书的进程
    p = multiprocessing.Pool()
    for i in charts_url:
        p.apply_async(get_ChartTxt, args=(i, ))
    print('等待 %s 所有的章节被加载......' % ( ))
    p.close()
    p.join()
    end = time.time()
    print('下载 %s  完成，运行时间  %0.2f s.' % ( , (end - start)))
    return

# 创建下载多本书书的进程
def process_getAllBook( ):
    # 输入你要下载的书的首页地址
    print('主程序的PID：%s' % os.getpid())
    book_indexUrl=[
        'http://www.yznnw.com/files/article/html/1/1129/index.html',
        'http://www.yznnw.com/files/article/html/17/17455/index.html'
    ]
    print("开始下载..............")
    p=[]
    for i in book_indexUrl:
        p.append(multiprocessing.Process(target=thread_getOneBook,args=(i,)))
    print("等待所有的主进程加载完成........")
    for i in p:
        i.start()
    for i in p:
        i.join()
    print("全部下载完成")
    return

if __name__=="__main__":
    # # 主页
      = 'http://www.yznnw.com'
    # 下载指定的书
    process_getAllBook( )