问题
I've passed the last week trying to scrap information from Epic Games Store webpage (https://www.epicgames.com/store/en-US/), I first tried using the Requests module, but I soon realized I needed a module which supports javascript webs. And that's what I'm trying now, but there is a problem... When I use "inspect element" on the page, everything's fine, but when I execute this:
from requests_html import HTMLSession
session = HTMLSession()
r = session.get("https://www.epicgames.com/store/en-US/")
r.html.render()
print(r.html.html)
The result is an unreadable html file without most of the elements loaded. Result: https://pastebin.com/zQ9m1gr2
You can test this, pick a game from the web and then ctrl + f It's name in the result file. You will realize there are no matches. What can I do?
Thank you in advance! :)
Edit: It occurs exactly the same when I download the HTML from the browser manually.
回答1:
So the main page not containing the data you look for means that, store data is received after. So we can use requests to get the data by simulating what browser did.
If you look network tab in developer tools, you will see that when the page loads, it receives store data from graphql endpoint. That means if you simulate the request, you can get the store data:
import requests
endpoint = "https://graphql.epicgames.com/graphql"
# This query thing is what was sent to the server
# when loading the page, I couldn't figure out how
# to write it ourselves so I basically copy pasted
# the binary data in the payload.
query = b'{"query":"\\n query storefrontDiscoverQuery(\\n $locale:String,\\n $country:String\u0021\\n ) {\\n Storefront {\\n storefrontModules(locale: $locale) {\\n ... on StorefrontBreaker {\\n type\\n title\\n titleGroup\\n description\\n backgroundColors\\n layout\\n link {\\n src\\n linkText\\n }\\n image {\\n src\\n alt\\n }\\n }\\n ... on StorefrontFreeGames {\\n type\\n title\\n }\\n ... on StorefrontCardGroup {\\n type\\n title\\n link {\\n src\\n linkText\\n }\\n offers {\\n namespace\\n id\\n offer {\\n \\n title\\n id\\n namespace\\n description\\n keyImages {\\n type\\n url\\n }\\n seller {\\n id\\n name\\n }\\n urlSlug\\n items {\\n id\\n namespace\\n }\\n customAttributes {\\n key\\n value\\n }\\n categories {\\n path\\n }\\n price(country: $country) {\\n totalPrice {\\n discountPrice\\n originalPrice\\n voucherDiscount\\n discount\\n fmtPrice(locale: $locale) {\\n originalPrice\\n discountPrice\\n intermediatePrice\\n }\\n }\\n lineOffers {\\n appliedRules {\\n id\\n endDate\\n }\\n }\\n }\\n linkedOfferId\\n linkedOffer {\\n effectiveDate\\n customAttributes {\\n key\\n value\\n }\\n }\\n \\n }\\n }\\n }\\n ... on StorefrontFeaturedCarousel {\\n type\\n title\\n slides {\\n title\\n eyebrow\\n description\\n backgroundColor\\n image {\\n src\\n alt\\n }\\n mobileImage {\\n src\\n alt\\n }\\n link {\\n src\\n linkText\\n }\\n }\\n }\\n ... on StorefrontTiles {\\n type\\n title\\n tiles {\\n label\\n genre\\n link {\\n src\\n linkText\\n }\\n }\\n }\\n }\\n }\\n }\\n ","variables":{"locale":"en-US","country":"US"}}'
data = requests.post(endpoint, headers={"Content-type": "application/json;charset=UTF-8"
}, data=query)
print(data.json())
And it gives us this data. (Be careful, its pretty big.)
And also you can get per product information by using this:
import requests, json
endpoint = "https://graphql.epicgames.com/graphql"
query = {
"query": "\n query catalogQuery(\n $productNamespace:String!,\n $offerId:String!,\n $locale:String,\n $country:String!,\n $lineOffers: [LineOfferReq]!) {\n Catalog {\n catalogOffer(namespace: $productNamespace,\n id: $offerId,\n locale: $locale) {\n namespace\n effectiveDate\n id\n customAttributes {\n key\n value\n }\n items {\n id\n status\n customAttributes {\n key\n value\n }\n }\n }\n }\n PriceEngine {\n price(country: $country, lineOffers: $lineOffers) {\n totalPrice {\n discountPrice\n originalPrice\n voucherDiscount\n discount\n currencyCode\n currencyInfo {\n decimals\n }\n fmtPrice(locale: $locale) {\n originalPrice\n discountPrice\n intermediatePrice\n }\n }\n lineOffers {\n appliedRules {\n endDate\n discountSetting {\n discountType\n }\n }\n }\n }\n }\n }\n ",
"variables": {
"productNamespace": "cosmos",
"offerId": "1c55202badfc4212b4f82553d5d22c3e", # This is found in the first request we made,
"locale": "en-US", # data.Storefront.storefrontModules[1].offers[""0""].id to be more precise.
"country": "US",
"lineOffers": [{
"offerId": "1c55202badfc4212b4f82553d5d22c3e", # The same id goes here too.
"quantity": 1
}],
"calculateTax": False}
}
data = requests.post(endpoint, headers={"Content-type": "application/json;charset=UTF-8"
}, data=json.dumps(query)) # We added json.dumps because it basically turns dictionary
# into JSON string.
print(data.json())
That gives us:
{
"data": {
"Catalog": {
"catalogOffer": {
"namespace": "cosmos",
"effectiveDate": "2019-07-12T00:00:00.000Z",
"id": "1c55202badfc4212b4f82553d5d22c3e",
"customAttributes": [
{
"key": "com.epicgames.app.blacklist",
"value": "KR"
},
{
"key": "isPrepurchase",
"value": "true"
},
{
"key": "availableDate",
"value": "1573570800"
},
{
"key": "developerName",
"value": "Human Head Studios, Inc."
}
],
"items": [
{
"id": "70c30983cf0948e4bffc23505f232b11",
"status": "ACTIVE",
"customAttributes": [
{
"key": "SupportedPlatforms",
"value": "Windows"
}
]
},
{
"id": "974e25b4bce6425d9af79cd5ffd64152",
"status": "ACTIVE",
"customAttributes": [
{
"key": "SupportedPlatforms",
"value": "Windows"
}
]
},
{
"id": "159d92ebec254ecf8373709a99388a62",
"status": "ACTIVE",
"customAttributes": [
{
"key": "SupportedPlatforms",
"value": "Windows"
}
]
},
{
"id": "cc67628ab455419cb3d4ecc907febbb7",
"status": "ACTIVE",
"customAttributes": [
{
"key": "SupportedPlatforms",
"value": "Windows"
}
]
},
{
"id": "2f742aa604a441d1a145f70411e9d8d2",
"status": "ACTIVE",
"customAttributes": [
{
"key": "SupportedPlatforms",
"value": "Windows"
}
]
}
]
}
},
"PriceEngine": {
"price": {
"totalPrice": {
"discountPrice": 2999,
"originalPrice": 2999,
"voucherDiscount": 0,
"discount": 0,
"currencyCode": "USD",
"currencyInfo": {
"decimals": 2
},
"fmtPrice": {
"originalPrice": "$29.99",
"discountPrice": "$29.99",
"intermediatePrice": "$29.99"
}
},
"lineOffers": [
{
"appliedRules": []
}
]
}
}
},
"extensions": {
"cacheControl": {
"version": 1,
"hints": [
{
"path": [
"Catalog"
],
"maxAge": 0
},
{
"path": [
"Catalog",
"catalogOffer"
],
"maxAge": 0
},
{
"path": [
"PriceEngine"
],
"maxAge": 0
},
{
"path": [
"PriceEngine",
"price"
],
"maxAge": 0
},
{
"path": [
"Catalog",
"catalogOffer",
"customAttributes"
],
"maxAge": 0
},
{
"path": [
"Catalog",
"catalogOffer",
"items"
],
"maxAge": 0
},
{
"path": [
"Catalog",
"catalogOffer",
"items",
0,
"customAttributes"
],
"maxAge": 0
},
{
"path": [
"Catalog",
"catalogOffer",
"items",
1,
"customAttributes"
],
"maxAge": 0
},
{
"path": [
"Catalog",
"catalogOffer",
"items",
2,
"customAttributes"
],
"maxAge": 0
},
{
"path": [
"Catalog",
"catalogOffer",
"items",
3,
"customAttributes"
],
"maxAge": 0
},
{
"path": [
"Catalog",
"catalogOffer",
"items",
4,
"customAttributes"
],
"maxAge": 0
},
{
"path": [
"PriceEngine",
"price",
"totalPrice"
],
"maxAge": 0
},
{
"path": [
"PriceEngine",
"price",
"totalPrice",
"currencyInfo"
],
"maxAge": 0
},
{
"path": [
"PriceEngine",
"price",
"totalPrice",
"fmtPrice"
],
"maxAge": 0
},
{
"path": [
"PriceEngine",
"price",
"lineOffers"
],
"maxAge": 0
},
{
"path": [
"PriceEngine",
"price",
"lineOffers",
0,
"appliedRules"
],
"maxAge": 0
}
]
}
}
}
Apparently, you can get free-games-collection ids from this url. Then you can query with this id to get the game list:
import requests, json
endpoint = "https://graphql.epicgames.com/graphql"
gamesCollectionQuery = {
"query":"\n query catalogQuery($productNamespace:String!, $offerId:String!, $locale:String, $country:String!) {\n Catalog {\n catalogOffer(namespace: $productNamespace, id: $offerId, locale: $locale) {\n title\n collectionOffers {\n \n title\n id\n namespace\n description\n keyImages {\n type\n url\n }\n seller {\n id\n name\n }\n urlSlug\n items {\n id\n namespace\n }\n customAttributes {\n key\n value\n }\n categories {\n path\n }\n price(country: $country) {\n totalPrice {\n discountPrice\n originalPrice\n voucherDiscount\n discount\n fmtPrice(locale: $locale) {\n originalPrice\n discountPrice\n intermediatePrice\n }\n }\n lineOffers {\n appliedRules {\n id\n endDate\n }\n }\n }\n linkedOfferId\n linkedOffer {\n effectiveDate\n customAttributes {\n key\n value\n }\n }\n \n }\n customAttributes {\n key\n value\n }\n }\n }\n }\n ",
"variables":{
"productNamespace":"epic",
"offerId":"7f22b3b15abc4821bba634340e2dd1ef",
"locale":"es-ES",
"country":"EN"
}
}
data = requests.post(endpoint, headers={"Content-type": "application/json;charset=UTF-8"
}, data=json.dumps(gamesCollectionQuery))
print(data.content)
来源：https://stackoverflow.com/questions/59010090/problem-trying-to-scrap-a-js-web-with-requests-html-python-3-6