from celery_app import app

from scrapy.crawler import CrawlerProcess

from scrapy.utils.project import get_project_settings

from tutorial.spiders.quotes import QuotesSpider

def crawl_run():

    scope = 'all'

    process = CrawlerProcess(settings=get_project_settings())

    process.crawl(QuotesSpider, scope)

    process.start()

    process.join()

@app.task(queue='default')

def execute_task():

    return crawl_run()

from crawler.tutorial.crawler.tutorial.spiders.quotes import QuotesSpider

from scrapy.utils.project import get_project_settings

import scrapy.crawler as crawler

from crochet import setup

setup()

import os

class Scraper():

    def crawl_run(self):

        spider = QuotesSpider()

        settings = get_project_settings()

        runner = crawler.CrawlerRunner(settings)

        runner.crawl(spider, 'all')

        runner.join()

if __name__ == '__main__':

    scraper = Scraper()

    scraper.crawl_run()