i am using scrapy to collect some data and i need to delay the code when extracting email. So not the entire code but only when it reaches the email extraction part. Any help is very appreciated.
import re
import scrapy
from scrapy.http import Request
# item class included here
class DmozItem(scrapy.Item):
# define the fields for your item here like:
link = scrapy.Field()
attr = scrapy.Field()
title = scrapy.Field()
tag = scrapy.Field()
class DmozSpider(scrapy.Spider):
name = "dmoz"
allowed_domains = ["craigslist.org"]
start_urls = [
"http://asheville.craigslist.org/bab/5078377070.html"
]
BASE_URL = 'http://asheville.craigslist.org/'
def parse_contact(self, response):
item = response.meta['item']
item["attr"] = "".join(response.xpath("//div[@class='anonemail']//text()").extract())
return item
Use
time.sleep()
for this. It "suspend[s] execution of the current thread for the given number of seconds."