Python3爬虫爬取百姓网列表并保存为json功能示例【基于request、lxml和json模块】

import requests
from l  import etree
import json
#构造头文件，模拟浏览器访问
url=\"http://xian.baixing.com/meirongfuwu/\"
headers = {\'User-Agent\':\'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36\',\'referer\':url}
response=requests.get(url,headers=headers)
body=response.text #获取网页内容
html=etree.HTML(body,etree.HTMLParser())
gethtml=html.xpath(\'//div[contains(@class,\"media-body- \")]\')
# 存储为数组list
jsondata = []
for item in gethtml:
  jsonone={}
  jsonone[\' \']=item.xpath(\'.//a[contains(@class,\"ad- \")]/text()\')[0]
  jsonone[\'url\']=item.xpath(\'.//a[contains(@class,\"ad- \")]/attribute::href\')[0]
  jsonone[\'phone\']=item.xpath(\'.//button[contains(@class,\"contact-button\")]/attribute::data-contact\')[0]
  jsondata.append(jsonone)
# 保存为json
with open(\"./d.json\",\'w\',encoding=\'utf-8\') as json_file:
  json.dump(jsondata,json_file,ensure_ascii=False)