.前言
说实话这次的爬虫可能是目前我遇到的最难的一个爬虫，主要之前爬取的都是一些静态资源的网站，这次的网站虽然 反爬机制 虽然也只是低层次的，但是对于新手的我来说也算是比较难的了。
2.反爬过程
这样我们便能通过xpath直接定位得到了
def getLinks(html):
chapter_link=[]
chapter_title=[]
parse=parsel.Selector(html)
links=parse.xpath('//div[@class="tab-content tab-content-selected zj_list_con autoHeight"]/ul[@class="list_con_li autoHeight"]/li/a/@href').getall()
titles=parse.xpath('//div[@class="tab-content tab-content-selected zj_list_con autoHeight"]/ul[@class="list_con_li autoHeight"]/li/a/span[@class="list_con_zj"]/text()').getall()
for link in links:
chapter_link.insert(0,link)
for title in titles:
chapter_title.insert(0, title)
return chapter_link,chapter_title
1234567891011
只要注意他这里章节是降序排列的，所以我们爬取的过程中需要将他翻转过来，所以不能只用 append 方法，应用 insert 方法。
源码
我的代码:
import requests
import parsel
import pypinyin
from bs4 import BeautifulSoup
import re
import os
import time
# 伪装浏览器。设置请求头
headers={"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36",}
# 返回网页的请求信息
def askUrl(url):
response=requests.get(url,headers=headers)
html=response.content.decode('utf-8')
return html
# 获取所有的章节链接以及章节名称
def getLinks(html):
chapter_link=[]
chapter_title=[]
parse=parsel.Selector(html)
links=parse.xpath('//div[@class="tab-content tab-content-selected zj_list_con autoHeight"]/ul[@class="list_con_li autoHeight"]/li/a/@href').getall()
titles=parse.xpath('//div[@class="tab-content tab-content-selected zj_list_con autoHeight"]/ul[@class="list_con_li autoHeight"]/li/a/span[@class="list_con_zj"]/text()').getall()
for link in links:
chapter_link.insert(0,link)
for title in titles:
chapter_title.insert(0, title)
return chapter_link,chapter_title
# 获取所有漫画的链接
def getImgs(link):
pic_url=[]
response=requests.get(link,headers=headers)
html=BeautifulSoup(response.text,'lxml')
script_info=html.script
one = re.findall("\|(\d{4})\|", str(script_info))[0]
two = re.findall("\|(\d{5})\|", str(script_info))[0]
threes=re.findall('\d{13,14}',str(script_info))
for i, three in enumerate(threes):
if len(three) == 13:
threes[i] = three + '0'
threes = sorted(threes, key=lambda x: int(x))
for three in threes:
if three[-1]=='0':
pic_url.append("https://images.dmzj.com/img/chapterpic/"+one+"/"+two+"/"+three[:-1]+".jpg")
else:
pic_url.append("https://images.dmzj.com/img/chapterpic/" + one + "/" + two + "/" + three + ".jpg")
return pic_url
# 下载漫画
def download(url,links,dir_name):
headers1={
'Referer': url,
}
i=1;
for link in links:
pic_name = '%03d.jpg' % (i)
new_dir_name = os.path.join(dir_name, pic_name)
response=requests.get(link,headers=headers1)
with open(new_dir_name, 'wb')as f:
f.write(response.content)
print(pic_name+"下载完成")
i+=1
# main方法
def main():
manhuas=input("请输入你要下载的漫画名：")
dir_name = r'D:\漫画'
if not os.path.exists(dir_name + './' + manhuas):
os.makedirs(dir_name + './' + manhuas)
dir_name=dir_name + './' + manhuas
manhuas=pypinyin.pinyin(manhuas,style=pypinyin.NORMAL)
name=''
for manhua in manhuas:
name=name+''.join(manhua)
url="https://www.dmzj.com/info/"+name+".html"
html=askUrl(url)
links=getLinks(html)[0]
names = getLinks(html)[1]
for i,link in enumerate(links):
if not os.path.exists(dir_name + './' + str(names[i])):
os.makedirs(dir_name + './' + str(names[i]))
print("开始下载："+names[i])
imglinks=getImgs(link)
download(link,imglinks,dir_name + './' + str(names[i]))
print(names[i]+"下载完毕")
print("休息一会儿，稍微继续下载下一章")
time.sleep(10)
print("————————————————————————————————————————————————————————————————————————————————")
外汇常见问题https://www.kaifx.cn/lists/question/
print(manhuas+"已经完全下载完毕")
#主函数入口
if __name__ == '__main__':
main()
12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394
老哥的代码：
import requests
import os
import re
from bs4 import BeautifulSoup
from contextlib import closing
from tqdm import tqdm
import time
# 创建保存目录
save_dir = '妖神记'
if save_dir not in os.listdir('./'):
os.mkdir(save_dir)
target_url = "https://www.dmzj.com/info/yaoshenji.html"
# 获取动漫章节链接和章节名
r = requests.get(url = target_url)
bs = BeautifulSoup(r.text, 'lxml')
list_con_li = bs.find('ul', class_="list_con_li")
cartoon_list = list_con_li.find_all('a')
chapter_names = []
chapter_urls = []
for cartoon in cartoon_list:
href = cartoon.get('href')
name = cartoon.text
chapter_names.insert(0, name)
chapter_urls.insert(0, href)
# 下载漫画
for i, url in enumerate(tqdm(chapter_urls)):
download_header = {
'Referer': url
}
name = chapter_names[i]
# 去掉.
while '.' in name:
name = name.replace('.', '')
chapter_save_dir = os.path.join(save_dir, name)
if name not in os.listdir(save_dir):
os.mkdir(chapter_save_dir)
r = requests.get(url = url)
html = BeautifulSoup(r.text, 'lxml')
script_info = html.script
pics = re.findall('\d{13,14}', str(script_info))
for j, pic in enumerate(pics):
if len(pic) == 13:
pics[j] = pic + '0'
pics = sorted(pics, key=lambda x:int(x))
chapterpic_hou = re.findall('\|(\d{5})\|', str(script_info))[0]
chapterpic_qian = re.findall('\|(\d{4})\|', str(script_info))[0]
for idx, pic in enumerate(pics):
if pic[-1] == '0':
url = 'https://images.dmzj.com/img/chapterpic/' + chapterpic_qian + '/' + chapterpic_hou + '/' + pic[:-1] + '.jpg'
else:
url = 'https://images.dmzj.com/img/chapterpic/' + chapterpic_qian + '/' + chapterpic_hou + '/' + pic + '.jpg'
pic_name = '%03d.jpg' % (idx + 1)
pic_save_path = os.path.join(chapter_save_dir, pic_name)
with closing(requests.get(url, headers = download_header, stream = True)) as response:
chunk_size = 1024
content_size = int(response.headers['content-length'])
if response.status_code == 200:
with open(pic_save_path, "wb") as file:
for data in response.iter_content(chunk_size=chunk_size):
file.write(data)
else:
print('链接异常')
time.sleep(10)
|
|
收藏
淘帖
踩
