Newer
Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
import Crawler.Spider as Spider
import time
from DataBase import mongoDB as db
def scrape_api(url_api, book_num_api, author_num_api):
print('Request received, start scraping')
book_dict_api = {}
author_dict_api = {}
similar_book_dict_api = {}
count_api = 1
print("Crawling book #" + str(count_api))
start_page_api = Spider.Spider(url_api)
start_page_api.crawl(book_dict_api, author_dict_api, similar_book_dict_api)
while len(book_dict_api) < int(book_num_api) or len(author_dict_api) < int(author_num_api):
all_related_books_api = list(similar_book_dict_api.items())
for book_url_pair_api in all_related_books_api:
if len(book_dict_api) >= int(book_num_api) and len(author_dict_api) >= int(author_num_api):
break
book_name_api = book_url_pair_api[0]
if book_name_api in book_dict_api:
pass
else:
count_api += 1
print("Crawling book #" + str(count_api))
book_url_api = book_url_pair_api[1]
book_spider_api = Spider.Spider(book_url_api)
book_spider_api.crawl(book_dict_api, author_dict_api, similar_book_dict_api)
# since the network delay from asia to goodread.com is already very high,
# a small amount of time of sleep is used here.
time.sleep(0.2)
db.insert_dicts(book_dict_api, 0)
db.insert_dicts(author_dict_api, 1)
print("Scraping completed")