糗事百科爬虫

糗事百科爬虫

不怎么看糗事百科,用来试一下PyQuery选择器还是好的

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
import requests
from lxml import etree
import time
from pyquery import PyQuery
import pandas as pd

def get_data(page):
text_data = []
for i in range(1, page + 1):
url = 'https://www.qiushibaike.com/text/page/{}/'.format(i)
print("正在爬去第{}页\n".format(i))
print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()))
r = requests.get(url)
s = etree.HTML(r.text)
jpy = PyQuery(s)
data = jpy('#content-left > div > a >div > span').text()
text_data.append(data)
time.sleep(1)
df = pd.DataFrame(text_data)
df.to_csv('糗事百科.csv', encoding='utf-8-sig')


if __name__ == '__main__':
get_data(2)
print("爬去完成")
print(time.strftime("%Y-%m-%d %H:%M:%S",time.localtime()))

这个样子的结果

-------------End Of This ArticleThank You For Reading-------------