scrapy-spider

I am using the following to check for (internet) connection errors in my spider.py : def start_requests(self): for url in self.start_urls: yield scrapy.Request(url, callback=self.parse, errback=self.handle_error) def handle_error(self, failure): if failure.check(DNSLookupError): # or failure.check(UnknownHostError): request = failure.request self.logger.error('DNSLookupError on: %s', request.url) print("\nDNS Error! Please check your internet connection!\n") elif failure.check(HttpError): response = failure.value.response self.logger.error('HttpError on: %s', response.url) print('\nSpider

I'm using scrapy and i'm trying to use celery to manage multiple spiders on one machine. The problem i have is (bit difficult to explain), that the spiders gets multiplied -> meaning if my first spider starts and i start a second spider the first spider executes twice. See my code here: ProcessJob.py class ProcessJob(): def processJob(self, job): #update job mysql = MysqlConnector.Mysql() db = mysql.getConnection(); cur = db.cursor(); job.status = 1 update = "UPDATE job SET status=1 WHERE id=" + str(job.id) cur.execute(update) db.commit() db.close() #Start new crawler configure_logging()

I am quite new to Scrapy and want to try the following: Extract some values from a webpage, store it in a variable and use it in my main script. Therefore I followed their tutorial and changed code for my purposes: import scrapy from scrapy.crawler import CrawlerProcess class QuotesSpider(scrapy.Spider): name = "quotes" start_urls = [ 'http://quotes.toscrape.com/page/1/' ] custom_settings = { 'LOG_ENABLED': 'False', } def parse(self, response): global title # This would work, but there should be a better way title = response.css('title::text').extract_first() process = CrawlerProcess({ 'USER

import Scrapy class NgaSpider(Scrapy.Spider): name = "NgaSpider" host = "http://bbs.ngacn.cc/" start_urls = [ "http://bbs.ngacn.cc/thread.php?fid=406", ] def parse(self, response): print ("response.body") Error: ModuleNotFoundError: No module named 'Scrapy' What is going on to fix this issue？ You are incorrectly importing the scrapy module. Find a simple tutorial and references from here. You have to do the following changes: import scrapy # Change here class NgaSpider(scrapy.Spider): # Change here too name = "NgaSpider" host = "http://bbs.ngacn.cc/" start_urls = [ "http://bbs.ngacn.cc/thread

I am using sitemap spider in scrapy, python. The sitemap seems to have unusual format with '//' in front of urls: <url> <loc>//www.example.com/10/20-baby-names</loc> </url> <url> <loc>//www.example.com/elizabeth/christmas</loc> </url> myspider.py from scrapy.contrib.spiders import SitemapSpider from myspider.items import * class MySpider(SitemapSpider): name = "myspider" sitemap_urls = ["http://www.example.com/robots.txt"] def parse(self, response): item = PostItem() item['url'] = response.url item['title'] = response.xpath('//title/text()').extract() return item I am getting this error: raise