问题
I need to scrape a website with a "load more button". This is my spider code written in Python:
import scrapy
import json
import requests
import re
from parsel import Selector
from scrapy.selector import Selector
from scrapy.http import HtmlResponse
headers = {
'origin': 'https://www.tayara.tn',
'accept-encoding': 'gzip, deflate, br',
'accept-language': 'en-US,en;q=0.9',
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36',
'content-type': 'application/json',
'accept': '*/*',
'referer': 'https://www.tayara.tn/sc/immobilier/bureaux-et-plateaux',
'authority': 'www.tayara.tn',
'dnt': '1',
}
data = '{"query":"query ListingsPage($page: Page, $filter: SearchFilter, $sortBy: SortOrder) {\\n listings: searchAds(page: $page, filter: $filter, sortBy: $sortBy) {\\n items {\\n uuid\\n title\\n price\\n currency\\n thumbnail\\n createdAt\\n category {\\n id\\n name\\n engName\\n __typename\\n }\\n user {\\n uuid\\n displayName\\n avatar(width: 96, height: 96) {\\n url\\n __typename\\n }\\n __typename\\n }\\n __typename\\n }\\n trackingInfo {\\n transactionId\\n listName\\n recommenderId\\n experimentId\\n variantId\\n __typename\\n }\\n totalCount\\n pageInfo {\\n startCursor\\n hasPreviousPage\\n endCursor\\n hasNextPage\\n __typename\\n }\\n __typename\\n }\\n}\\n","variables":{"page":{"count":36,"offset":"cDEwbg==.MjAxOC0xMi0wMlQxMzo1MDoxMlo=.MzY="},"filter":{"queryString":null,"category":"140","regionId":null,"attributeFilters":[]},"sortBy":"CREATED_DESC"},"operationName":"ListingsPage"}'
class Tun(scrapy.Spider):
name="tayaracommercial"
start_urls = [
'https://www.tayara.tn/sc/immobilier/bureaux-et-plateaux'
]
def parse(self, response):
yield Request('https://www.tayara.tn/graphql', method='post', headers=headers, body=data, self.parse_item)
def parse_item(self, response):
source = 'Tayara'
reference = response.url.split('//')[1].split('/')[3]
titre = response.xpath('//h1[@data-name="adview_title"]/text()').extract()
yield{'Source':source, 'Reference':reference, 'Titre':titre}
This is my modest trial. I know that is false. can you correct me please ?
来源:https://stackoverflow.com/questions/54171736/needing-help-to-simulate-an-xhr-request