问题
In scrapy how would I go about having scrapy crawl only 1 level deep for all links outside the allowed domains. Within the crawl, I want to be able to make sure all outbound links within the site are working and not 404'd. I do not want it to crawl the whole site of the non-allowed domain. I am currently processing allowed domain 404s. I know that I can set a DEPTH_LIMIT of 1, but that will affect the allowed domain as well.
my code:
from scrapy.selector import Selector
from scrapy.spiders import CrawlSpider, Rule
from scrapy.linkextractors import LinkExtractor
from smcrawl.items import Website
import smcrawl.util
def iterate(lists):
for a in lists:
return a
class WalmartSpider(CrawlSpider):
handle_httpstatus_list = [200, 302, 404, 500, 502]
name = "surveymonkeycouk"
allowed_domains = ["surveymonkey.co.uk", "surveymonkey.com"]
start_urls = ['https://www.surveymonkey.co.uk/']
rules = (
Rule(
LinkExtractor(
allow=(),
deny=(),
process_value=smcrawl.util.trim),
callback="parse_items",
follow=True,),
)
#process_links=lambda links: [link for link in links if not link.nofollow] = filter nofollow links
#parses start urls
def parse_start_url(self, response):
list(self.parse_items(response))
def parse_items(self, response):
hxs = Selector(response)
sites = response.selector.xpath('//html')
items = []
for site in sites:
if response.status == 404:
item = Website()
item['url'] = response.url
item['referer'] = response.request.headers.get('Referer')
item['canonical'] = site.xpath('//head/link[@rel="canonical"]/@href').extract()
item['robots'] = site.xpath('//meta[@name="robots"]/@content').extract()
item['original_url'] = response.meta.get('redirect_urls', [response.url])[0]
item['description'] = site.xpath('//meta[@name="description"]/@content').extract()
item['redirect'] = response.status
elif response.status == 200:
item = Website()
item['url'] = response.url
item['referer'] = response.request.headers.get('Referer')
item['canonical'] = site.xpath('//head/link[@rel="canonical"]/@href').extract()
item['robots'] = site.xpath('//meta[@name="robots"]/@content').extract()
item['original_url'] = response.meta.get('redirect_urls', [response.url])[0]
item['description'] = site.xpath('//meta[@name="description"]/@content').extract()
item['redirect'] = response.status
titles = site.xpath('/html/head/title/text()').extract()
try:
titles = iterate(titles)
titles = titles.strip()
except:
pass
item['title'] = titles
h1 = site.xpath('//h1/text()').extract()
try:
h1 = iterate(h1)
h1 = h1.strip()
except:
pass
item['h1'] = h1
elif response.status == 302:
item = Website()
item['url'] = response.url
item['referer'] = response.request.headers.get('Referer')
item['canonical'] = site.xpath('//head/link[@rel="canonical"]/@href').extract()
item['robots'] = site.xpath('//meta[@name="robots"]/@content').extract()
item['original_url'] = response.meta.get('redirect_urls', [response.url])[0]
item['description'] = site.xpath('//meta[@name="description"]/@content').extract()
item['redirect'] = response.status
titles = site.xpath('/html/head/title/text()').extract()
try:
titles = iterate(titles)
titles = titles.strip()
except:
pass
item['title'] = titles
h1 = site.xpath('//h1/text()').extract()
try:
h1 = iterate(h1)
h1 = h1.strip()
except:
pass
item['h1'] = h1
elif response.status == 404:
item = Website()
item['url'] = response.url
item['referer'] = response.request.headers.get('Referer')
item['canonical'] = site.xpath('//head/link[@rel="canonical"]/@href').extract()
item['robots'] = site.xpath('//meta[@name="robots"]/@content').extract()
item['original_url'] = response.meta.get('redirect_urls', [response.url])[0]
item['description'] = site.xpath('//meta[@name="description"]/@content').extract()
item['redirect'] = response.status
titles = site.xpath('/html/head/title/text()').extract()
try:
titles = iterate(titles)
titles = titles.strip()
except:
pass
item['title'] = titles
h1 = site.xpath('//h1/text()').extract()
try:
h1 = iterate(h1)
h1 = h1.strip()
except:
pass
item['h1'] = h1
elif response.status == 500:
item = Website()
item['url'] = response.url
item['referer'] = response.request.headers.get('Referer')
item['canonical'] = site.xpath('//head/link[@rel="canonical"]/@href').extract()
item['robots'] = site.xpath('//meta[@name="robots"]/@content').extract()
item['original_url'] = response.meta.get('redirect_urls', [response.url])[0]
item['description'] = site.xpath('//meta[@name="description"]/@content').extract()
item['redirect'] = response.status
titles = site.xpath('/html/head/title/text()').extract()
try:
titles = iterate(titles)
titles = titles.strip()
except:
pass
item['title'] = titles
h1 = site.xpath('//h1/text()').extract()
try:
h1 = iterate(h1)
h1 = h1.strip()
except:
pass
item['h1'] = h1
elif response.status == 502:
item = Website()
item['url'] = response.url
item['referer'] = response.request.headers.get('Referer')
item['canonical'] = site.xpath('//head/link[@rel="canonical"]/@href').extract()
item['robots'] = site.xpath('//meta[@name="robots"]/@content').extract()
item['original_url'] = response.meta.get('redirect_urls', [response.url])[0]
item['description'] = site.xpath('//meta[@name="description"]/@content').extract()
item['redirect'] = response.status
titles = site.xpath('/html/head/title/text()').extract()
try:
titles = iterate(titles)
titles = titles.strip()
except:
pass
item['title'] = titles
h1 = site.xpath('//h1/text()').extract()
try:
h1 = iterate(h1)
h1 = h1.strip()
except:
pass
item['h1'] = h1
else:
item = Website()
item['url'] = response.url
item['referer'] = response.request.headers.get('Referer')
item['canonical'] = site.xpath('//head/link[@rel="canonical"]/@href').extract()
item['robots'] = site.xpath('//meta[@name="robots"]/@content').extract()
item['original_url'] = response.meta.get('redirect_urls', [response.url])[0]
item['description'] = site.xpath('//meta[@name="description"]/@content').extract()
item['redirect'] = response.status
titles = site.xpath('/html/head/title/text()').extract()
try:
titles = iterate(titles)
titles = titles.strip()
except:
pass
item['title'] = titles
h1 = site.xpath('//h1/text()').extract()
try:
h1 = iterate(h1)
h1 = h1.strip()
except:
pass
item['h1'] = h1
items.append(item)
return items
回答1:
Ok, one thing you could do would be to avoid using allowed_domains, so you don't filter any offsite request.
But to make it interesting you could create your own OffsiteMiddleware, with something like this:
from scrapy.spidermiddlewares.offsite import OffsiteMiddleware
class MyOffsiteMiddleware(OffsiteMiddleware):
offsite_domains = set()
def should_follow(self, request, spider):
regex = self.host_regex
host = urlparse_cached(request).hostname or ''
if host in offsite_domains:
return False
if not bool(regex.search(host)):
self.offsite_domains.add(host)
return True
I haven't test it, but it should work, remember that you should disable the default middleware and enable yours on settings:
SPIDER_MIDDLEWARES = {
'myproject.middlewares.MyOffsiteMiddleware': 543,
'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': None,
}
回答2:
I've referred to Scrapy set depth limit per allowed_domains as the answer. It's a little different than the solution I was looking for, but with a whitelist of URLs that I am willing to crawl, then end result is the same. Thank you!
来源：https://stackoverflow.com/questions/34189979/scrapy-crawl-1-level-deep-on-offsite-links