iNocturnis/webcrawler
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

(andy) first move recent discussed issue

Browse Source
This commit is contained in:
unknown
2022-04-21 20:31:38 -07:00
parent 320fe26c23
commit 754d3b4af6
3 changed files with 88 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
spacetime-crawler4py-master/crawler/worker.py
View File

@@ -29,6 +29,17 @@ class Worker(Thread):
resp = download(tbd_url, self.config, self.logger)
toc = time.perf_counter()
print(f"Took {toc - tic:0.4f} seconds to do download url")
tic = time.perf_counter()
self.frontier.q1(tbd_url)
toc = time.perf_counter()
print(f"Took {toc - tic:0.4f} seconds to do download url")
tic = time.perf_counter()
self.frontier.q234(tbd_url, resp)
toc = time.perf_counter()
print(f"Took {toc - tic:0.4f} seconds to do download url")
self.logger.info(
f"Downloaded {tbd_url}, status <{resp.status}>, "
f"using cache {self.config.cache_server}.")