04爬取拉勾网Python岗位分析报告

# 导入需要的包
import requests
import time,random
from openpyxl import Workbook
import pymysql.cursors
#@ 连接数据库；
# 这个是我本地上边运行的程序，用来获取代理服务器。
def get_proxy():
 try:
 PROXY_POOL_URL = 'http://localhost:5555/random'
 response = requests.get(PROXY_POOL_URL)
 print(response.text)
 if response.status_code == 200:
 return response.text
 except ConnectionError:
 return None
# 用来连接本地mysql，可以不连接，直接写入Excel中
def get_conn():
 """连接本地数据库"""
 # 定义要连接的主机IP，账号名称和密码，连接的数据库，编码等等
 conn = pymysql.connect(host = 'localhost',
 user = 'root',
 password = '123456',
 db = 'python',
 charset = 'utf8mb4',
 cursorclass = pymysql.cursors.DictCursor)
 return conn
# 将数据写入到数据库中
def insert(conn,info):
 """数据写入数据库"""
 with conn.cursor() as cursor:
 sql = "INSERT INTO `python` (`companyShortName`, `companyFullName`, `industryField`, `companySize`, `salary`, `city`, `education`) VALUES (%s, %s, %s, %s, %s, %s, %s)"
 cursor.execute(sql, info)
 conn.commit()
# 获取当前网址的信息
def get_json(url,page,lang_name):
 """返回当前页面的信息列表"""
 data = {'first':'false','pn':page,'kd':lang_name}
 proxies = get_proxy()
 proxies = {
 "http": "http://" + proxies
 }
 json = ses.post(url,data,proxies = proxies).json()
 list_con = json['content']['positionResult']['result']
 info_list = []
 for i in list_con:
 info = []
 info.append(i.get('companyShortName','无')) # 公司名称
 info.append(i.get('companyFullName','无'))
 info.append(i.get('industryField','无'))
 info.append(i.get('companySize','无'))
 info.append(i.get('salary','无'))
 info.append(i.get('city','无'))
 info.append(i.get('education','无'))
 info_list.append(info)
 return info_list

def main():
 lang_name = 'python'
 wb = Workbook() # 打开Excel工作薄
 conn = get_conn() # 建立数据库连接 不存放数据，注释此行
 for i in ['北京','上海','广州','深圳','杭州']: #五个城市
 page = 1
 wsl = wb.active
 wsl.title = lang_name
 url = 'https://www.lagou.com/jobs/positionAjax.json?city={}&needAddtionalResult=false'.format(i)
 while page < 2: # 每个城市30页信息
 info = get_json(url,page,lang_name)
 page += 1
 # time.sleep(random.randint(10,20))
 for row in info:
 # 插入数据库，若不想存入 注释此行
 insert(conn,tuple(row))
 wsl.append(row)
 # 关闭数据库连接，不存放数据，注释此行
 conn.close()
 wb.save('{}职位信息.xlsx'.format(lang_name))

if __name__ == "__main__":
 my_headers = {
 "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36",
 "Referer": "https://www.lagou.com/jobs/list_Python?city=%E5%85%A8%E5%9B%BD&cl=false&fromSearch=true&labelWords=&suginput=",
 "Content-Type": "application/x-www-form-urlencoded;charset = UTF-8"
 }
 # time.sleep(5)
 ses = requests.session() # 获取 session
 ses.headers.update(my_headers) # 更新
 ses.get(
 "https://www.lagou.com/jobs/list_python?city=%E5%85%A8%E5%9B%BD&cl=false&fromSearch=true&labelWords=&suginput=")
 main()

来源：oschina

链接：https://my.oschina.net/u/4343506/blog/4437052