import Crawler.Spider as Spider
import time
from DataBase import mongoDB as db


def scrape_api(url_api, book_num_api, author_num_api):
    """ Function called by other module to run the scraper """
    print('Request received, start scraping')
    book_dict_api = {}
    author_dict_api = {}
    similar_book_dict_api = {}
    count_api = 1

    print("Crawling book #" + str(count_api))
    start_page_api = Spider.Spider(url_api)
    start_page_api.crawl(book_dict_api, author_dict_api, similar_book_dict_api)

    while len(book_dict_api) < int(book_num_api) or len(author_dict_api) < int(author_num_api):
        all_related_books_api = list(similar_book_dict_api.items())
        for book_url_pair_api in all_related_books_api:
            if len(book_dict_api) >= int(book_num_api) and len(author_dict_api) >= int(author_num_api):
                break
            book_name_api = book_url_pair_api[0]
            if book_name_api in book_dict_api:
                pass
            else:
                count_api += 1
                print("Crawling book #" + str(count_api))
                book_url_api = book_url_pair_api[1]
                book_spider_api = Spider.Spider(book_url_api)
                book_spider_api.crawl(book_dict_api, author_dict_api, similar_book_dict_api)

                # since the network delay from asia to goodread.com is already very high,
                # a small amount of time of sleep is used here.
                time.sleep(0.2)

    db.insert_dicts(book_dict_api, 0)
    db.insert_dicts(author_dict_api, 1)
    print("Scraping completed")