From 74063e5d00d0915d17a8a347735a8e12d69d6395 Mon Sep 17 00:00:00 2001 From: Hieuhuy Pham Date: Sat, 23 Apr 2022 02:13:12 -0700 Subject: [PATCH] Fixed a lot of racing issues, there potentially could be a writer reader confusion type of thing, but it should not matter that much, as long as server is healthy we can let this bad boi lose --- spacetime-crawler4py-master/Logs/FRONTIER.log | 39 + spacetime-crawler4py-master/Logs/Worker.log | 967 ++++++++++++++++++ .../crawler/frontier.py | 60 +- spacetime-crawler4py-master/crawler/worker.py | 42 +- spacetime-crawler4py-master/scraper.py | 22 +- 5 files changed, 1090 insertions(+), 40 deletions(-) diff --git a/spacetime-crawler4py-master/Logs/FRONTIER.log b/spacetime-crawler4py-master/Logs/FRONTIER.log index 8f96db5..877123a 100644 --- a/spacetime-crawler4py-master/Logs/FRONTIER.log +++ b/spacetime-crawler4py-master/Logs/FRONTIER.log @@ -52,3 +52,42 @@ 2022-04-22 12:57:19,541 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. 2022-04-22 13:02:40,174 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. 2022-04-22 13:07:26,611 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-22 17:25:16,739 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-22 17:27:01,372 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-22 17:28:24,395 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-22 17:33:03,228 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-22 17:33:14,391 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-22 17:34:11,862 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-22 17:35:05,121 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-22 17:36:23,994 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-22 17:36:31,564 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-22 17:38:41,035 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-22 17:39:43,493 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-22 17:43:12,698 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-22 17:46:27,304 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-22 17:52:23,826 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-22 17:52:38,658 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-22 17:59:19,523 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-22 18:00:51,039 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-22 18:01:45,112 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-23 00:46:46,850 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-23 00:49:09,876 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-23 00:53:26,894 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-23 00:53:54,532 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-23 01:40:16,310 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-23 01:41:34,284 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-23 01:43:18,453 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-23 01:46:32,822 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-23 01:47:34,475 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-23 01:48:29,467 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-23 01:48:56,671 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-23 01:50:51,864 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-23 01:53:07,556 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-23 01:53:56,693 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-23 01:54:34,028 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-23 01:55:03,124 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-23 01:56:20,721 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-23 01:59:29,951 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-23 01:59:57,446 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-23 02:02:46,431 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-23 02:05:59,557 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. diff --git a/spacetime-crawler4py-master/Logs/Worker.log b/spacetime-crawler4py-master/Logs/Worker.log index 902c217..2ee0815 100644 --- a/spacetime-crawler4py-master/Logs/Worker.log +++ b/spacetime-crawler4py-master/Logs/Worker.log @@ -301,3 +301,970 @@ 2022-04-22 13:11:42,004 - Worker-2 - INFO - Downloaded https://www.ics.uci.edu/community/news/view_news.php?id=2039, status <200>, using cache ('styx.ics.uci.edu', 9004). 2022-04-22 13:11:45,064 - Worker-3 - INFO - Downloaded https://mswe.ics.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9004). 2022-04-22 13:11:47,846 - Worker-1 - INFO - Downloaded https://mt-live.ics.uci.edu/alumni/corporate-engagement, status <200>, using cache ('styx.ics.uci.edu', 9004). +2022-04-22 17:25:16,753 - Worker-2 - INFO - Frontier is empty. Stopping Crawler. +2022-04-22 17:25:16,756 - Worker-4 - INFO - Frontier is empty. Stopping Crawler. +2022-04-22 17:25:16,756 - Worker-3 - INFO - Frontier is empty. Stopping Crawler. +2022-04-22 17:25:16,881 - Worker-1 - INFO - Downloaded https://www.cs.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:25:16,907 - Worker-0 - INFO - Downloaded https://www.informatics.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:25:22,016 - Worker-1 - INFO - Frontier is empty. Stopping Crawler. +2022-04-22 17:25:28,582 - Worker-0 - INFO - Frontier is empty. Stopping Crawler. +2022-04-22 17:27:01,385 - Worker-3 - INFO - Frontier is empty. Stopping Crawler. +2022-04-22 17:27:01,385 - Worker-2 - INFO - Frontier is empty. Stopping Crawler. +2022-04-22 17:27:01,388 - Worker-4 - INFO - Frontier is empty. Stopping Crawler. +2022-04-22 17:27:01,509 - Worker-1 - INFO - Downloaded https://www.cs.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:27:01,532 - Worker-0 - INFO - Downloaded https://www.informatics.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:27:05,460 - Worker-1 - INFO - Frontier is empty. Stopping Crawler. +2022-04-22 17:27:07,041 - Worker-0 - INFO - Frontier is empty. Stopping Crawler. +2022-04-22 17:28:24,407 - Worker-3 - INFO - Frontier is empty. Stopping Crawler. +2022-04-22 17:28:24,407 - Worker-4 - INFO - Frontier is empty. Stopping Crawler. +2022-04-22 17:28:24,525 - Worker-0 - INFO - Downloaded https://www.cs.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:28:24,534 - Worker-1 - INFO - Downloaded https://www.stat.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:28:24,580 - Worker-2 - INFO - Downloaded https://www.informatics.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:28:27,976 - Worker-0 - INFO - Frontier is empty. Stopping Crawler. +2022-04-22 17:28:29,909 - Worker-2 - INFO - Downloaded https://www.informatics.uci.edu/support/set-future-agenda, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:28:30,325 - Worker-2 - INFO - Frontier is empty. Stopping Crawler. +2022-04-22 17:28:30,700 - Worker-1 - INFO - Downloaded https://www.stat.uci.edu/seminar-series, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:28:31,624 - Worker-1 - INFO - Downloaded http://www.ics.uci.edu/ugrad/courses/listing.php?year=2016&level=Lower-Division&department=STATS&program=ALL, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:28:35,376 - Worker-1 - INFO - Frontier is empty. Stopping Crawler. +2022-04-22 17:33:03,239 - Worker-0 - INFO - Frontier is empty. Stopping Crawler. +2022-04-22 17:33:03,240 - Worker-1 - INFO - Frontier is empty. Stopping Crawler. +2022-04-22 17:33:03,241 - Worker-2 - INFO - Frontier is empty. Stopping Crawler. +2022-04-22 17:33:03,242 - Worker-4 - INFO - Frontier is empty. Stopping Crawler. +2022-04-22 17:33:03,242 - Worker-3 - INFO - Frontier is empty. Stopping Crawler. +2022-04-22 17:35:05,135 - Worker-4 - INFO - Frontier is empty. Stopping Crawler. +2022-04-22 17:35:05,263 - Worker-3 - INFO - Downloaded https://www.stat.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:35:05,278 - Worker-2 - INFO - Downloaded https://www.cs.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:35:05,315 - Worker-0 - INFO - Downloaded https://www.informatics.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:35:09,264 - Worker-2 - INFO - Frontier is empty. Stopping Crawler. +2022-04-22 17:35:10,072 - Worker-0 - INFO - Frontier is empty. Stopping Crawler. +2022-04-22 17:35:10,172 - Worker-1 - INFO - Downloaded https://www.ics.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:35:11,932 - Worker-3 - INFO - Frontier is empty. Stopping Crawler. +2022-04-22 17:35:14,389 - Worker-1 - INFO - Frontier is empty. Stopping Crawler. +2022-04-22 17:36:31,583 - Worker-4 - INFO - Frontier is empty. Stopping Crawler. +2022-04-22 17:36:31,695 - Worker-2 - INFO - Downloaded https://www.cs.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:36:31,706 - Worker-3 - INFO - Downloaded https://www.stat.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:36:31,746 - Worker-0 - INFO - Downloaded https://www.informatics.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:37:07,956 - Worker-2 - INFO - Frontier is empty. Stopping Crawler. +2022-04-22 17:37:07,956 - Worker-0 - INFO - Frontier is empty. Stopping Crawler. +2022-04-22 17:37:07,958 - Worker-3 - INFO - Frontier is empty. Stopping Crawler. +2022-04-22 17:37:08,069 - Worker-1 - INFO - Downloaded https://www.ics.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:37:14,243 - Worker-1 - INFO - Frontier is empty. Stopping Crawler. +2022-04-22 17:38:47,929 - Worker-3 - INFO - Downloaded https://www.stat.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:38:53,459 - Worker-2 - INFO - Downloaded https://www.cs.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:39:05,610 - Worker-0 - INFO - Downloaded https://www.informatics.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:39:54,314 - Worker-0 - INFO - Downloaded https://www.informatics.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:40:00,680 - Worker-3 - INFO - Downloaded https://www.stat.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:40:06,770 - Worker-2 - INFO - Downloaded https://www.cs.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:43:26,669 - Worker-0 - INFO - Downloaded https://www.informatics.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:43:32,118 - Worker-3 - INFO - Downloaded https://www.stat.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:43:38,320 - Worker-2 - INFO - Downloaded https://www.cs.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:46:27,553 - Worker-0 - INFO - Downloaded https://www.informatics.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:46:34,384 - Worker-0 - INFO - Downloaded https://www.informatics.uci.edu/undergrad/special-opportunities, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:46:38,955 - Worker-0 - INFO - Downloaded http://www.ics.uci.edu/community/scholarships/index.php, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:46:45,414 - Worker-0 - INFO - Downloaded http://www.ics.uci.edu/grad/admissions/index, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:46:56,207 - Worker-0 - INFO - Downloaded https://www.ics.uci.edu/grad, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:46:56,755 - Worker-0 - INFO - Downloaded https://mswe.ics.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:47:01,406 - Worker-0 - INFO - Downloaded https://mswe.ics.uci.edu/faq, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:52:38,798 - Worker-0 - INFO - Downloaded https://www.informatics.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:52:44,650 - Worker-0 - INFO - Downloaded https://www.informatics.uci.edu/research/undergraduate-research, status <608>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:52:45,278 - Worker-0 - INFO - Downloaded https://www.informatics.uci.edu/very-top-footer-menu-items/site-map, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:52:45,818 - Worker-0 - INFO - Downloaded http://www.informatics.uci.edu/menu-very-top/site-map, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:52:46,371 - Worker-0 - INFO - Downloaded http://www.informatics.uci.edu/menu-very-top/news, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:52:58,523 - Worker-0 - INFO - Downloaded https://www.informatics.uci.edu/2017/12, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:53:03,800 - Worker-0 - INFO - Downloaded https://www.informatics.uci.edu/michigan-virtual-learning-research-institute-mvlri-interview-with-mimi-ito, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:53:04,571 - Worker-0 - INFO - Downloaded https://www.informatics.uci.edu/from-homebound-to-school-bound-with-telepresence-robots, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:53:09,529 - Worker-0 - INFO - Downloaded http://www.informatics.uci.edu/explore/faculty-profiles/judy-olson, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:53:10,065 - Worker-0 - INFO - Downloaded https://www.informatics.uci.edu/explore/faculty-profiles/kylie-peppler, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:53:10,693 - Worker-0 - INFO - Downloaded https://www.informatics.uci.edu/explore/faculty-profiles/anne-marie-piper, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:53:11,232 - Worker-0 - INFO - Downloaded https://www.informatics.uci.edu/explore/faculty-profiles/madhu-reddy, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:53:11,771 - Worker-0 - INFO - Downloaded https://www.informatics.uci.edu/explore/faculty-profiles/david-redmiles, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:53:12,305 - Worker-0 - INFO - Downloaded https://www.informatics.uci.edu/explore/faculty-profiles/debra-richardson, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:53:12,801 - Worker-0 - INFO - Downloaded http://www.ics.uci.edu/~djr, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:53:13,444 - Worker-0 - INFO - Downloaded https://www.informatics.uci.edu/explore/faculty-profiles/katie-salen-tekinbas, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:53:13,981 - Worker-0 - INFO - Downloaded https://www.informatics.uci.edu/explore/faculty-profiles/kurt-squire, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:53:14,519 - Worker-0 - INFO - Downloaded https://www.informatics.uci.edu/explore/faculty-profiles/constance-steinkuehler, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:53:15,053 - Worker-0 - INFO - Downloaded https://www.informatics.uci.edu/explore/faculty-profiles/tess-tanenbaum, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:53:16,168 - Worker-0 - INFO - Downloaded https://www.informatics.uci.edu/explore/faculty-profiles/richard-taylor, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:53:16,704 - Worker-0 - INFO - Downloaded https://www.informatics.uci.edu/explore/faculty-profiles/bill-tomlinson, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:53:17,218 - Worker-0 - INFO - Downloaded http://www.ics.uci.edu/~wmt, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:53:25,649 - Worker-0 - INFO - Downloaded https://www.informatics.uci.edu/explore/faculty-profiles/aaron-trammell, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:53:26,220 - Worker-0 - INFO - Downloaded https://www.informatics.uci.edu/explore/faculty-profiles/andre-van-der-hoek, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:53:26,768 - Worker-0 - INFO - Downloaded https://www.informatics.uci.edu/explore/faculty-profiles/sean-young, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:53:27,363 - Worker-0 - INFO - Downloaded https://www.informatics.uci.edu/explore/faculty-profiles/kai-zheng, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:53:27,901 - Worker-0 - INFO - Downloaded https://www.informatics.uci.edu/explore/faculty-profiles/hadar-ziv, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:53:28,491 - Worker-0 - INFO - Downloaded https://www.informatics.uci.edu/explore/faculty-profiles/mark-baldwin, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:53:29,031 - Worker-0 - INFO - Downloaded https://www.informatics.uci.edu/explore/faculty-profiles/matthew-bietz, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:53:31,395 - Worker-0 - INFO - Downloaded https://www.informatics.uci.edu/explore/faculty-profiles/darren-denenberg, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:53:32,026 - Worker-0 - INFO - Downloaded https://www.informatics.uci.edu/explore/faculty-profiles/emily-navarro, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:53:32,520 - Worker-0 - INFO - Downloaded https://www.ics.uci.edu/~emilyo/SimSE, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:53:33,158 - Worker-0 - INFO - Downloaded https://www.ics.uci.edu/~emilyo, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:53:49,788 - Worker-0 - INFO - Downloaded https://www.ics.uci.edu/~emilyo/alien-mastermind, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:53:50,288 - Worker-0 - INFO - Downloaded http://www.ics.uci.edu/~zhengkai, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:53:50,883 - Worker-0 - INFO - Downloaded http://www.ics.uci.edu/~andre, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:53:55,520 - Worker-0 - INFO - Downloaded http://sdcl.ics.uci.edu/research/knocap-2, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:54:03,128 - Worker-0 - INFO - Downloaded http://sdcl.ics.uci.edu/contact, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:54:03,670 - Worker-0 - INFO - Downloaded http://sdcl.ics.uci.edu/papers, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:54:10,429 - Worker-0 - INFO - Downloaded http://sdcl.ics.uci.edu/research/past-projects/calico, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:54:11,316 - Worker-0 - INFO - Downloaded http://sdcl.ics.uci.edu/research/past-projects/crowddesign, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:54:11,824 - Worker-0 - INFO - Downloaded http://sdcl.ics.uci.edu/people, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:54:12,332 - Worker-0 - INFO - Downloaded http://sdcl.ics.uci.edu/research/past-projects/crowd-fault-localization, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:54:12,836 - Worker-0 - INFO - Downloaded http://www.ics.uci.edu/~adrianoc, status <404>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:54:13,440 - Worker-0 - INFO - Downloaded http://sdcl.ics.uci.edu/papers/dissertations, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:54:14,257 - Worker-0 - INFO - Downloaded http://sdcl.ics.uci.edu/research/past-projects, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:54:14,760 - Worker-0 - INFO - Downloaded http://sdcl.ics.uci.edu/research/past-projects/code-topics, status <404>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:54:15,367 - Worker-0 - INFO - Downloaded http://sdcl.ics.uci.edu/papers/technical-reports, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:54:15,872 - Worker-0 - INFO - Downloaded http://sdcl.ics.uci.edu/sponsors, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:57:09,336 - Worker-0 - INFO - Downloaded http://sdcl.ics.uci.edu/research/past-projects/crowd-development, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:57:10,623 - Worker-0 - INFO - Downloaded http://www.ics.uci.edu/~tlatoza, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:57:11,227 - Worker-0 - INFO - Downloaded http://sdcl.ics.uci.edu/contact/directions, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:57:11,718 - Worker-0 - INFO - Downloaded http://sdcl.ics.uci.edu/wp-login.php, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:57:12,342 - Worker-0 - INFO - Downloaded https://sdcl.ics.uci.edu/wp-login.php?action=lostpassword, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:57:12,956 - Worker-0 - INFO - Downloaded https://sdcl.ics.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:59:19,669 - Worker-0 - INFO - Downloaded https://www.informatics.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:59:27,266 - Worker-0 - INFO - Downloaded https://www.informatics.uci.edu/grad/phd-software-engineering, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:59:28,145 - Worker-0 - INFO - Downloaded http://www.informatics.uci.edu/grad/student-profiles/925-2, status <404>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:59:28,672 - Worker-0 - INFO - Downloaded https://www.informatics.uci.edu/explore/history-of-the-department, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:59:32,112 - Worker-0 - INFO - Downloaded https://www.informatics.uci.edu/grad/student-profiles, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:59:32,658 - Worker-0 - INFO - Downloaded https://www.informatics.uci.edu/grad/student-profiles/emory-edwards, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:59:33,188 - Worker-0 - INFO - Downloaded http://www.informatics.uci.edu/explore/faculty-profiles/bonnie-ruberg, status <404>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:59:33,714 - Worker-0 - INFO - Downloaded https://www.informatics.uci.edu/grad/student-profiles/graduate-student-profile-namrata-puri, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:59:34,254 - Worker-0 - INFO - Downloaded https://www.informatics.uci.edu/grad/student-profiles/kaj-dreef, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:59:35,030 - Worker-0 - INFO - Downloaded http://www.informatics.uci.edu/explore/faculty-profiles/james-a-jones, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 17:59:35,348 - Worker-0 - INFO - Downloaded http://www.ics.uci.edu/~jajones, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 18:00:51,191 - Worker-0 - INFO - Downloaded https://www.informatics.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 18:00:57,716 - Worker-0 - INFO - Downloaded https://www.informatics.uci.edu/research/labs-centers, status <608>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 18:01:45,260 - Worker-0 - INFO - Downloaded https://www.informatics.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 18:01:50,787 - Worker-0 - INFO - Downloaded https://www.informatics.uci.edu/undergrad/bs-software-engineering, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-22 18:01:52,519 - Worker-0 - INFO - Downloaded https://www.informatics.uci.edu/grad/ms-informatics, status <200>, using cache ('styx.ics.uci.edu', 9007). +2022-04-23 00:46:47,000 - Worker-0 - INFO - Downloaded https://www.informatics.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 00:46:47,541 - Worker-0 - INFO - Downloaded https://www.informatics.uci.edu/admissions/graduate-application-process, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 00:46:48,034 - Worker-0 - INFO - Downloaded http://www.ics.uci.edu/grad/qa, status <403>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 00:46:48,655 - Worker-0 - INFO - Downloaded http://www.informatics.uci.edu/menu-very-top/contact, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 00:46:49,165 - Worker-0 - INFO - Downloaded https://mhcid.ics.uci.edu/admissions/costs-and-financial-aid, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 00:46:49,675 - Worker-0 - INFO - Downloaded https://mhcid.ics.uci.edu/people/faculty-and-staff, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 00:46:50,181 - Worker-0 - INFO - Downloaded https://www.ics.uci.edu/faculty/profiles/view_faculty.php?ucinetid=ddenenbe, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 00:46:50,733 - Worker-0 - INFO - Downloaded https://www.ics.uci.edu/grad/funding/index, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 00:46:51,291 - Worker-0 - INFO - Downloaded https://www.ics.uci.edu/grad/funding, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 00:46:51,833 - Worker-0 - INFO - Downloaded https://www.ics.uci.edu/community/scholarships, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 00:46:52,367 - Worker-0 - INFO - Downloaded https://www.ics.uci.edu/community, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 00:46:52,875 - Worker-0 - INFO - Downloaded https://www.ics.uci.edu/ics/community/friends/leadershipcouncil/index, status <404>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 00:46:53,460 - Worker-0 - INFO - Downloaded https://tad.ics.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 00:46:54,044 - Worker-0 - INFO - Downloaded https://tad.ics.uci.edu/login, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 00:46:54,627 - Worker-0 - INFO - Downloaded https://tad.ics.uci.edu/site/index, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 00:46:55,236 - Worker-0 - INFO - Downloaded https://www.ics.uci.edu/grad, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 00:46:55,744 - Worker-0 - INFO - Downloaded https://www.ics.uci.edu/community/scholarships/ , status <404>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 00:46:56,353 - Worker-0 - INFO - Downloaded https://www.ics.uci.edu/ugrad/QA_Petitions, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 00:46:56,898 - Worker-0 - INFO - Downloaded https://www.ics.uci.edu/ugrad/policies, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 00:46:57,392 - Worker-0 - INFO - Downloaded https://ugradforms.ics.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 00:50:01,625 - Worker-0 - INFO - Downloaded https://www.informatics.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 00:50:01,655 - Worker-2 - INFO - Downloaded https://www.cs.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 00:50:01,717 - Worker-3 - INFO - Downloaded https://www.stat.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 00:53:27,577 - Worker-3 - INFO - Downloaded https://www.stat.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 00:53:27,608 - Worker-0 - INFO - Downloaded https://www.informatics.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 00:53:27,624 - Worker-2 - INFO - Downloaded https://www.cs.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 00:53:55,284 - Worker-3 - INFO - Downloaded https://www.stat.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 00:53:55,314 - Worker-2 - INFO - Downloaded https://www.cs.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 00:53:55,329 - Worker-0 - INFO - Downloaded https://www.informatics.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 01:40:17,022 - Worker-2 - INFO - Downloaded https://www.cs.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 01:40:17,131 - Worker-3 - INFO - Downloaded https://www.stat.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 01:40:17,161 - Worker-0 - INFO - Downloaded https://www.informatics.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 01:41:35,077 - Worker-0 - INFO - Downloaded https://www.informatics.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 01:41:35,123 - Worker-2 - INFO - Downloaded https://www.cs.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 01:41:35,138 - Worker-3 - INFO - Downloaded https://www.stat.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 01:43:19,173 - Worker-3 - INFO - Downloaded https://www.stat.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 01:43:19,232 - Worker-2 - INFO - Downloaded https://www.cs.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 01:43:19,309 - Worker-0 - INFO - Downloaded https://www.informatics.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 01:46:33,488 - Worker-3 - INFO - Downloaded https://www.stat.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 01:46:33,520 - Worker-2 - INFO - Downloaded https://www.cs.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 01:46:33,611 - Worker-0 - INFO - Downloaded https://www.informatics.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 01:47:35,193 - Worker-0 - INFO - Downloaded https://www.informatics.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 01:47:35,304 - Worker-3 - INFO - Downloaded https://www.stat.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 01:47:35,334 - Worker-2 - INFO - Downloaded https://www.cs.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 01:48:30,147 - Worker-2 - INFO - Downloaded https://www.cs.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 01:48:30,193 - Worker-0 - INFO - Downloaded https://www.informatics.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 01:48:30,332 - Worker-3 - INFO - Downloaded https://www.stat.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 01:48:57,402 - Worker-3 - INFO - Downloaded https://www.stat.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 01:48:57,433 - Worker-2 - INFO - Downloaded https://www.cs.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 01:48:57,448 - Worker-0 - INFO - Downloaded https://www.informatics.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 01:50:52,606 - Worker-0 - INFO - Downloaded https://www.informatics.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 01:50:52,621 - Worker-3 - INFO - Downloaded https://www.stat.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 01:50:52,667 - Worker-2 - INFO - Downloaded https://www.cs.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 01:53:07,568 - Worker-4 - INFO - Frontier is empty. Stopping Crawler. +2022-04-23 01:53:07,672 - Worker-3 - INFO - Downloaded https://www.stat.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 01:53:07,674 - Worker-2 - INFO - Downloaded https://www.cs.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 01:53:07,676 - Worker-0 - INFO - Downloaded https://www.informatics.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 01:53:08,167 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/mds-student-ty-shao-aims-to-make-an-impact-in-healthcare, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 01:53:56,708 - Worker-4 - INFO - Frontier is empty. Stopping Crawler. +2022-04-23 01:53:56,839 - Worker-2 - INFO - Downloaded https://www.cs.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 01:53:56,840 - Worker-3 - INFO - Downloaded https://www.stat.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 01:53:56,854 - Worker-0 - INFO - Downloaded https://www.informatics.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 01:53:57,334 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/mds-student-ty-shao-aims-to-make-an-impact-in-healthcare, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 01:54:34,042 - Worker-4 - INFO - Frontier is empty. Stopping Crawler. +2022-04-23 01:54:34,136 - Worker-3 - INFO - Downloaded https://www.stat.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 01:54:34,138 - Worker-2 - INFO - Downloaded https://www.cs.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 01:54:34,152 - Worker-0 - INFO - Downloaded https://www.informatics.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 01:54:34,630 - Worker-3 - INFO - Downloaded https://www.stat.uci.edu/contact-the-department, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 01:55:03,138 - Worker-4 - INFO - Frontier is empty. Stopping Crawler. +2022-04-23 01:55:03,235 - Worker-3 - INFO - Downloaded https://www.stat.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 01:55:03,235 - Worker-2 - INFO - Downloaded https://www.cs.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 01:55:03,249 - Worker-0 - INFO - Downloaded https://www.informatics.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 01:55:03,824 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/culture-of-collaboration-leads-to-cutting-edge-work-in-ai, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 01:56:20,738 - Worker-4 - INFO - Frontier is empty. Stopping Crawler. +2022-04-23 01:56:20,831 - Worker-3 - INFO - Downloaded https://www.stat.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 01:56:20,832 - Worker-2 - INFO - Downloaded https://www.cs.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 01:56:20,846 - Worker-0 - INFO - Downloaded https://www.informatics.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 01:56:21,327 - Worker-1 - INFO - Downloaded https://www.ics.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 01:56:21,415 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/slider/b-s-in-data-science, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 01:59:29,966 - Worker-4 - INFO - Frontier is empty. Stopping Crawler. +2022-04-23 01:59:30,072 - Worker-2 - INFO - Downloaded https://www.cs.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 01:59:30,073 - Worker-3 - INFO - Downloaded https://www.stat.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 01:59:30,086 - Worker-0 - INFO - Downloaded https://www.informatics.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 01:59:30,574 - Worker-1 - INFO - Downloaded https://www.ics.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 01:59:30,654 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/m-s-ph-d-in-statistics, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 01:59:57,462 - Worker-4 - INFO - Frontier is empty. Stopping Crawler. +2022-04-23 01:59:57,593 - Worker-2 - INFO - Downloaded https://www.cs.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 01:59:57,594 - Worker-3 - INFO - Downloaded https://www.stat.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 01:59:57,607 - Worker-0 - INFO - Downloaded https://www.informatics.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 01:59:58,084 - Worker-3 - INFO - Downloaded https://www.stat.uci.edu/mds-student-ty-shao-aims-to-make-an-impact-in-healthcare, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 01:59:58,097 - Worker-1 - INFO - Downloaded https://www.ics.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 01:59:58,605 - Worker-2 - INFO - Downloaded http://www.stat.uci.edu/news, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:46,447 - Worker-4 - INFO - Frontier is empty. Stopping Crawler. +2022-04-23 02:02:46,547 - Worker-2 - INFO - Downloaded https://www.cs.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:46,549 - Worker-3 - INFO - Downloaded https://www.stat.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:46,564 - Worker-0 - INFO - Downloaded https://www.informatics.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:47,048 - Worker-3 - INFO - Downloaded https://www.stat.uci.edu/slider/uci-launches-new-professional-program-master-of-data-science, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:47,063 - Worker-1 - INFO - Downloaded https://www.ics.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:47,557 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/culture-of-collaboration-leads-to-cutting-edge-work-in-ai, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:47,666 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/minor-in-statistics, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:47,780 - Worker-1 - INFO - Downloaded https://www.stat.uci.edu/a-look-at-health-and-technology-trends-for-2021-and-beyond, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:47,905 - Worker-3 - INFO - Downloaded https://www.stat.uci.edu/a-campus-gem-ucis-statistical-consulting-services, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:48,108 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/uci-news-ucis-graduate-programs-shine-in-u-s-news-world-report-rankings, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:48,210 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/research, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:48,314 - Worker-1 - INFO - Downloaded https://www.stat.uci.edu/amstat-news-celebrating-women-in-statistics-2021-annie-qu-mentioned, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:48,415 - Worker-3 - INFO - Downloaded https://www.stat.uci.edu/orange-county-business-journal-uci-forecasts-covid-19-trends-in-oc, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:48,621 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/uci-deep-learning-researchers-advance-theory-and-practice-of-data-compression, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:48,729 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/uci-launches-new-professional-program-master-of-data-science, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:48,833 - Worker-1 - INFO - Downloaded https://www.stat.uci.edu/three-ics-students-receive-2021-nsf-graduate-research-fellowships, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:48,936 - Worker-3 - INFO - Downloaded https://www.stat.uci.edu/oc_covid_model, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:49,154 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/uci-news-uci-researchers-receive-state-funding-to-study-pandemic-forecasting, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:49,257 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/kuci-office-hours-podcast-keeping-up-with-coronavirus-statistics-vladimir-minin-interviewed, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:49,357 - Worker-1 - INFO - Downloaded https://www.stat.uci.edu/uci-news-hal-s-stern-is-named-uci-provost-and-executive-vice-chancellor, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:49,553 - Worker-3 - INFO - Downloaded https://www.stat.uci.edu/covid19/index.html, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:50,488 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/university-of-california-health-and-cdph-covid-modeling-consortium-awards-grant-funding-to-further-covid-19-data-research, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:50,587 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/oc_covid_model/index.html, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:50,763 - Worker-1 - INFO - Downloaded https://www.stat.uci.edu/exploring-and-strengthening-sports-stats-at-uci, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:50,871 - Worker-3 - INFO - Downloaded https://www.stat.uci.edu/uci-statisticians-release-new-online-orange-county-covid-19-information-resource, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:51,089 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/womens-history-month-resources-for-empowering-women-in-tech, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:51,208 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/faculty/bin-nan, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:51,310 - Worker-1 - INFO - Downloaded https://www.stat.uci.edu/uci-news-efforts-to-treat-covid-19-patients-chronicled-in-uc-health-medications-data-bin-nan-mentioned, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:51,413 - Worker-3 - INFO - Downloaded https://www.stat.uci.edu/ics-statistics-researchers-release-new-online-oc-covid-situation-report, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:51,616 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/statistics-ph-d-student-isaac-goldstein-receives-la-verne-noyes-fellowship, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:51,719 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/contact-the-department, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:51,821 - Worker-1 - INFO - Downloaded https://www.stat.uci.edu/teach-access-recognizes-mine-dogucu-as-curriculum-development-awardee, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:51,923 - Worker-3 - INFO - Downloaded https://www.stat.uci.edu/orange-county-business-journal-uci-releases-covid-19-website, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:52,124 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/uci-announces-launch-of-institute-for-precision-health, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:52,234 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/statistics-internships-employment-opportunities, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:52,334 - Worker-1 - INFO - Downloaded https://www.stat.uci.edu/a-bold-decision-to-study-abroad-in-high-school-led-sirui-hu-to-uci-and-beyond-data-science-major, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:52,437 - Worker-3 - INFO - Downloaded https://www.stat.uci.edu/yahoo-news-data-glitch-confounds-coronavirus-totals-in-orange-county-state-vladimir-minin-quoted, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:52,655 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/black-history-month-sharing-resources-to-expand-diversity-in-tech, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:52,775 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/ics-graduate-programs-in-statistics-and-computer-science-among-top-20-for-public-universities, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:52,878 - Worker-1 - INFO - Downloaded https://www.stat.uci.edu/capstone-program-showcases-growing-talent-of-ics-students, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:52,987 - Worker-3 - INFO - Downloaded https://www.stat.uci.edu/uci-news-uci-researchers-launch-first-of-its-kind-coronavirus-statistics-portal, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:53,206 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/4-ics-professors-among-7-uci-researchers-named-aaas-fellows, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:53,352 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/covid19/incidence-maps.html, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:53,618 - Worker-1 - INFO - Downloaded https://www.stat.uci.edu/the-resilience-of-the-class-of-2021, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:53,726 - Worker-3 - INFO - Downloaded https://www.stat.uci.edu/professor-annie-qu-works-to-enhance-the-detection-of-invasive-cancers-using-medical-imaging-data, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:53,830 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/xu-receives-slds-student-paper-award, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:53,930 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/faculty/annie-qu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:54,137 - Worker-1 - INFO - Downloaded https://www.stat.uci.edu/qu-appointed-ims-program-secretary, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:54,239 - Worker-3 - INFO - Downloaded https://www.stat.uci.edu/hal-stern-selected-as-a-fellow-of-the-international-society-of-bayesian-analysis, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:54,379 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/tech-trends-for-2022, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:54,495 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/dr-joni-ricks-oddie-applies-data-driven-lens-in-run-for-long-beach-city-council, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:54,719 - Worker-1 - INFO - Downloaded https://www.stat.uci.edu/mine-dogucu-receives-young-investigator-award, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:54,821 - Worker-3 - INFO - Downloaded https://www.stat.uci.edu/uci-news-national-institute-awards-20-million-in-renewed-funding-to-forensic-science-center, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:54,924 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/graduate-student-spotlight-mds-ambassador-adelynn-paik-shares-her-academic-professional-goals, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:55,029 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/m-s-ph-d-in-statistics, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:55,231 - Worker-1 - INFO - Downloaded https://www.stat.uci.edu/choc-teams-up-with-uci-to-offer-pediatric-capstone-project-in-data-science, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:55,335 - Worker-3 - INFO - Downloaded https://www.stat.uci.edu/michelle-nuno-selected-to-participate-in-the-70th-lindau-nobel-laureate-meeting, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:55,436 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/disney-research-supports-stephan-mandts-work-on-generative-modeling-with-50000-gift, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:55,537 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/what-is-statistics, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:55,757 - Worker-1 - INFO - Downloaded https://www.stat.uci.edu/from-theory-to-practice-ucis-machine-learning-hackathon-delivers, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:55,863 - Worker-3 - INFO - Downloaded https://www.stat.uci.edu/uci-mind-alzheimers-disease-clinical-trials-and-covid-19-with-dr-daniel-gillen, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:55,966 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/beyond-the-t-test-uci-professors-publish-new-statistical-guidance-for-neuroscience-research, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:56,072 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/seminar-series, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:56,293 - Worker-1 - INFO - Downloaded https://www.stat.uci.edu/uci-news-uci-led-study-finds-disparities-in-o-c-rates-of-covid-19-infection-mortality-vladimir-minin-mentioned, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:56,398 - Worker-3 - INFO - Downloaded https://www.stat.uci.edu/uci-brain-researcher-spotlight-dr-babak-shahbaba, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:56,501 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/statistics-ph-d-student-tianyu-pan-wins-two-competitive-awards, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:56,635 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/wp-content/uploads/Shujie-Ma-Abstract-5-5-22, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:57,587 - Worker-1 - INFO - Downloaded https://www.stat.uci.edu/lifetime-achievement-and-dinsdale-awards-double-the-accolades-for-professor-emerita-jessica-utts, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:57,696 - Worker-3 - INFO - Downloaded https://www.stat.uci.edu/nsf-announces-2020-graduate-research-fellows, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:57,799 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/elizabeth-lee-wins-data-science-for-social-good-award, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:57,907 - Worker-0 - INFO - Downloaded http://www.stat.uci.edu/seminar-series/seminar-series-2019-2020, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:58,116 - Worker-1 - INFO - Downloaded https://www.stat.uci.edu/harnessing-the-data-revolution-ics-researchers-and-collaborators-receive-1-5m-data-science-grant, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:58,226 - Worker-3 - INFO - Downloaded https://www.stat.uci.edu/amstat-news-celebrating-rising-undergraduate-women-in-statistics-and-data-science, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:58,327 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/stephan-mandt-and-collaborators-receive-3-5-million-to-study-machine-learning-for-climate-science, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:58,437 - Worker-0 - INFO - Downloaded http://www.stat.uci.edu/seminar-series/seminar-series-2017-2018, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:58,649 - Worker-1 - INFO - Downloaded http://www.stat.uci.edu/seminar-series/seminar-series-2018-2019, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:58,760 - Worker-3 - INFO - Downloaded https://www.stat.uci.edu/statistics-ph-d-student-mary-ryan-receives-public-impact-fellowship, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:58,863 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/seminar-series-2020-2021, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:58,999 - Worker-0 - INFO - Downloaded http://www.stat.uci.edu/seminar-series/seminar-series-2015-2016, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:02:59,240 - Worker-1 - INFO - Downloaded http://www.stat.uci.edu/wp-content/uploads/TianZhengAbstract5-30-19, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:00,229 - Worker-3 - INFO - Downloaded https://www.stat.uci.edu/professor-berrocal-elected-chair-of-the-section-in-environmental-sciences-of-the-isba, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:00,367 - Worker-2 - INFO - Downloaded http://www.stat.uci.edu/wp-content/uploads/StephanMandtAbstract5-16-19, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:01,397 - Worker-0 - INFO - Downloaded http://www.stat.uci.edu/ICS/statistics/research/seminarseries/2010-2011/index, status <404>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:01,502 - Worker-1 - INFO - Downloaded http://www.stat.uci.edu/ICS/statistics/research/seminarseries/2011-2012/index, status <404>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:01,600 - Worker-3 - INFO - Downloaded https://www.stat.uci.edu/staff-spotlight-rosemary-bustas-journey-from-chemistry-to-statistics-to-scotland-and-beyond, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:01,724 - Worker-2 - INFO - Downloaded http://www.stat.uci.edu/ICS/statistics/research/seminarseries/2012-2013/index, status <404>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:01,932 - Worker-0 - INFO - Downloaded http://www.stat.uci.edu/ICS/statistics/research/seminarseries/2013-2014/index, status <404>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:02,054 - Worker-1 - INFO - Downloaded http://www.stat.uci.edu/wp-content/uploads/JuliaPalaciosAbstract6-6-19, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:02,994 - Worker-3 - INFO - Downloaded https://www.stat.uci.edu/kpcc-vexed-by-college-statistics-courses-new-approaches-emphasize-practical-learning-jessica-utts-interviewed, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:03,139 - Worker-2 - INFO - Downloaded http://www.stat.uci.edu/wp-content/uploads/XinTongAbstract4-25-19, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:04,182 - Worker-0 - INFO - Downloaded http://www.stat.uci.edu/wp-content/uploads/LorinCrawfordAbstract5-2-19, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:05,331 - Worker-1 - INFO - Downloaded http://www.stat.uci.edu/seminar-series/seminar-series-2014-2015, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:05,442 - Worker-3 - INFO - Downloaded https://www.stat.uci.edu/laist-socal-professors-push-to-make-college-level-statistics-less-painful-jessica-utts-quoted, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:05,543 - Worker-2 - INFO - Downloaded http://www.stat.uci.edu/seminar-series/seminar-series-2016-2017, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:05,652 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/slider/b-s-in-data-science, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:05,861 - Worker-1 - INFO - Downloaded https://www.stat.uci.edu/chairs-message, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:05,964 - Worker-3 - INFO - Downloaded https://www.stat.uci.edu/ics-welcomes-8-new-faculty-for-2019, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:06,124 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/faculty, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:06,281 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/grad-student-directory, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:06,394 - Worker-1 - INFO - Downloaded https://www.stat.uci.edu/tutoring-resources, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:06,499 - Worker-3 - INFO - Downloaded https://www.stat.uci.edu/senior-spotlight-taneisha-arora-pursues-her-passions-from-working-in-industry-to-running-a-bakery, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:06,615 - Worker-2 - INFO - Downloaded http://www.stat.uci.edu/faculty-directory/veronica-berrocal, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:06,838 - Worker-0 - INFO - Downloaded http://www.stat.uci.edu/faculty/vladimir-minin, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:06,942 - Worker-1 - INFO - Downloaded http://www.stat.uci.edu/faculty-directory/dan-gillen, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:07,045 - Worker-3 - INFO - Downloaded https://www.stat.uci.edu/statistics-professors-shahbaba-and-minin-help-develop-framework-to-investigate-complex-biological-systems, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:07,150 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/faculty-directory/annie-qu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:07,356 - Worker-0 - INFO - Downloaded http://www.stat.uci.edu/faculty-directory/wesley-johnson, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:07,457 - Worker-1 - INFO - Downloaded http://www.stat.uci.edu/faculty-directory/mine-dogucu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:07,580 - Worker-3 - INFO - Downloaded https://www.stat.uci.edu/hal-stern-appointed-vice-provost-for-academic-planning, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:07,681 - Worker-2 - INFO - Downloaded http://www.stat.uci.edu/faculty-directory/tianchen-qian, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:07,892 - Worker-0 - INFO - Downloaded http://www.stat.uci.edu/faculty-directory/weining-shen, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:07,996 - Worker-1 - INFO - Downloaded http://www.stat.uci.edu/faculty-directory/hal-stern, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:08,101 - Worker-3 - INFO - Downloaded https://www.stat.uci.edu/professor-nan-awarded-nsf-grant-to-improve-statistical-inference, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:08,207 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/faculty-directory/babak-shahbab, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:08,431 - Worker-0 - INFO - Downloaded http://www.stat.uci.edu/faculty-directory/koko-gulesserian, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:08,533 - Worker-1 - INFO - Downloaded http://www.stat.uci.edu/faculty/brigitte-baldi, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:08,635 - Worker-3 - INFO - Downloaded https://www.stat.uci.edu/renewed-funding-lets-hal-stern-continue-research-of-early-life-adversity-brain-development-with-the-uci-conte-center, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:08,742 - Worker-2 - INFO - Downloaded http://www.stat.uci.edu/faculty-directory/michele-guindani, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:08,960 - Worker-0 - INFO - Downloaded http://www.stat.uci.edu/faculty-directory/lee-kucera, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:09,064 - Worker-1 - INFO - Downloaded http://www.stat.uci.edu/faculty-directory/jessica-utts, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:09,166 - Worker-3 - INFO - Downloaded https://www.stat.uci.edu/conference-honors-statistics-professor-emeritus-wesley-johnson, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:09,272 - Worker-2 - INFO - Downloaded http://www.stat.uci.edu/faculty-directory/zhaoxia-yu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:09,479 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/employers-of-statistics-grad-students, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:09,583 - Worker-1 - INFO - Downloaded https://www.stat.uci.edu/applications-open-for-new-uci-summer-biostatistics-institute-for-undergraduate-students, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:09,693 - Worker-3 - INFO - Downloaded https://www.stat.uci.edu/ics-students-win-best-web-app-at-hacksc, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:09,800 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/mds-student-ty-shao-aims-to-make-an-impact-in-healthcare, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:10,030 - Worker-0 - INFO - Downloaded http://www.stat.uci.edu/news, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:10,045 - Worker-2 - INFO - Downloaded https://www.ics.uci.edu/~mguindan, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:10,182 - Worker-1 - INFO - Downloaded https://www.stat.uci.edu/isi-buds/apply.html, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:10,282 - Worker-3 - INFO - Downloaded https://www.stat.uci.edu/professor-guindani-named-fellow-of-the-american-statistical-association, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:10,623 - Worker-0 - INFO - Downloaded https://www.ics.uci.edu/~wjohnson, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:10,724 - Worker-1 - INFO - Downloaded https://www.ics.uci.edu/ugrad/degrees/degree_se.php, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:10,838 - Worker-3 - INFO - Downloaded https://www.ics.uci.edu/~pattis, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:10,947 - Worker-2 - INFO - Downloaded https://www.ics.uci.edu/~aburtsev, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:11,177 - Worker-0 - INFO - Downloaded http://www.ics.uci.edu/~wjohnson/BIDA/BIDABook.html, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:11,293 - Worker-1 - INFO - Downloaded https://www.ics.uci.edu/ugrad/policies, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:11,408 - Worker-3 - INFO - Downloaded http://tutors.ics.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:11,520 - Worker-2 - INFO - Downloaded http://www.ics.uci.edu/informatics, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:11,720 - Worker-0 - INFO - Downloaded http://www.ics.uci.edu/~wjohnson/BIDA/Ch13/OvarianCancerData.txt, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:11,798 - Worker-1 - INFO - Downloaded http://www.ics.uci.edu/~wjohnson/BIDA/Ch9/Model2.txt, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:11,990 - Worker-3 - INFO - Downloaded http://tutors.ics.uci.edu/index.php/becoming-a-tutor, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:12,077 - Worker-2 - INFO - Downloaded http://tutors.ics.uci.edu/index.php/tutor-resources, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:12,275 - Worker-0 - INFO - Downloaded http://tutors.ics.uci.edu/index.php/tutor-coordinators, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:12,368 - Worker-1 - INFO - Downloaded http://tutors.ics.uci.edu/index.php/contact, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:12,550 - Worker-3 - INFO - Downloaded http://www.ics.uci.edu/~wjohnson/BIDA/Ch5/DiasorinModel.txt, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:12,629 - Worker-2 - INFO - Downloaded http://tutors.ics.uci.edu/index.php/tutor-resources/81-cpp-resources/100-cpp-resources, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:12,812 - Worker-0 - INFO - Downloaded http://tutors.ics.uci.edu/index.php/tutor-resources/80-java-resources/96-java-resources, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:12,894 - Worker-1 - INFO - Downloaded http://tutors.ics.uci.edu/index.php/0-python-resources/85-python-resources, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:13,079 - Worker-3 - INFO - Downloaded http://tutors.ics.uci.edu/index.php/tutor-resources/82-assembly-resources/110-assembly-resources, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:13,175 - Worker-2 - INFO - Downloaded http://tutors.ics.uci.edu/index.php/tutor-resources/81-cpp-resources/119-cpp-big-three, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:13,389 - Worker-0 - INFO - Downloaded http://tutors.ics.uci.edu/index.php/tutor-resources/80-java-resources/97-java-basics, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:13,499 - Worker-1 - INFO - Downloaded http://tutors.ics.uci.edu/index.php/79-python-resources/93-string-list-basics, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:13,600 - Worker-3 - INFO - Downloaded http://tutors.ics.uci.edu/index.php/tutor-resources/82-assembly-resources/111-assembly-syntax-register, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:13,712 - Worker-2 - INFO - Downloaded http://tutors.ics.uci.edu/index.php/0-python-resources/89-for-loop-basics, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:13,922 - Worker-0 - INFO - Downloaded http://tutors.ics.uci.edu/index.php/79-python-resources/108-python-list-exercises, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:14,009 - Worker-1 - INFO - Downloaded http://tutors.ics.uci.edu/index.php/79-python-resources/103-for-while-basics, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:14,108 - Worker-3 - INFO - Downloaded http://tutors.ics.uci.edu/index.php/79-python-resources/123-recursion-examples, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:14,322 - Worker-2 - INFO - Downloaded http://tutors.ics.uci.edu/index.php/79-python-resources/89-for-loop-basics, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:14,516 - Worker-0 - INFO - Downloaded http://tutors.ics.uci.edu/index.php/79-python-resources/116-python-exercises-2, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:14,599 - Worker-1 - INFO - Downloaded http://tutors.ics.uci.edu/index.php/79-python-resources/106-python-function-exercise, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:14,697 - Worker-3 - INFO - Downloaded http://tutors.ics.uci.edu/index.php/79-python-resources/121-printing-text-shapes, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:14,899 - Worker-2 - INFO - Downloaded http://tutors.ics.uci.edu/index.php/79-python-resources/104-try-except, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:15,106 - Worker-0 - INFO - Downloaded http://tutors.ics.uci.edu/index.php/0-python-resources/88-traversing-lists-with-for-loops, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:15,210 - Worker-1 - INFO - Downloaded http://tutors.ics.uci.edu/index.php/79-python-resources/105-python-try-exercises, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:15,296 - Worker-3 - INFO - Downloaded http://tutors.ics.uci.edu/index.php/79-python-resources/113-python-output-exercises, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:15,479 - Worker-2 - INFO - Downloaded http://tutors.ics.uci.edu/index.php/79-python-resources/85-python-resources, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:15,679 - Worker-0 - INFO - Downloaded http://tutors.ics.uci.edu/index.php/79-python-resources/88-traversing-lists-with-for-loops, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:15,779 - Worker-1 - INFO - Downloaded http://tutors.ics.uci.edu/index.php/79-python-resources/109-while-true, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:15,881 - Worker-3 - INFO - Downloaded http://tutors.ics.uci.edu/index.php/0-python-resources/87-comprehensions, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:16,089 - Worker-2 - INFO - Downloaded http://tutors.ics.uci.edu/index.php/79-python-resources/102-burrito-class, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:16,197 - Worker-0 - INFO - Downloaded http://tutors.ics.uci.edu/index.php/79-python-resources/118-python-for-exercises, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:16,298 - Worker-1 - INFO - Downloaded http://tutors.ics.uci.edu/index.php/79-python-resources/117-nested-lists, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:16,399 - Worker-3 - INFO - Downloaded http://tutors.ics.uci.edu/index.php/79-python-resources/87-comprehensions, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:16,608 - Worker-2 - INFO - Downloaded http://tutors.ics.uci.edu/index.php/79-python-resources/99-python-exercises, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:16,715 - Worker-0 - INFO - Downloaded http://tutors.ics.uci.edu/index.php/0-python-resources/86-breaking-down-a-problem, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:16,800 - Worker-1 - INFO - Downloaded http://tutors.ics.uci.edu/index.php/79-python-resources/114-python-function-loop-exercises, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:17,013 - Worker-3 - INFO - Downloaded http://tutors.ics.uci.edu/index.php/79-python-resources/115-abstraction-modules, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:17,113 - Worker-2 - INFO - Downloaded http://tutors.ics.uci.edu/index.php/79-python-resources/98-python-quiz, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:17,218 - Worker-0 - INFO - Downloaded http://tutors.ics.uci.edu/index.php/79-python-resources/86-breaking-down-a-problem, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:17,318 - Worker-1 - INFO - Downloaded http://tutors.ics.uci.edu/index.php/79-python-resources/107-while-basics, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:17,528 - Worker-3 - INFO - Downloaded http://tutors.ics.uci.edu/index.php/tutor-resources/81-cpp-resources/112-c-pointers, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:17,637 - Worker-2 - INFO - Downloaded http://tutors.ics.uci.edu/index.php/tutor-resources/81-cpp-resources/120-cpp-google-tests, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:17,745 - Worker-0 - INFO - Downloaded http://tutors.ics.uci.edu/index.php/tutor-resources/81-cpp-resources/101-cpp-exercises, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:17,851 - Worker-1 - INFO - Downloaded http://tutors.ics.uci.edu/index.php/tutor-resources/81-cpp-resources/122-cpp-ref-pointer-operators, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:18,052 - Worker-3 - INFO - Downloaded http://www.ics.uci.edu/~wjohnson/BIDA/Ch8/Ch8Rcode.txt, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:18,135 - Worker-2 - INFO - Downloaded http://www.ics.uci.edu/~wjohnson/BIDA/Ch15/DPMdensity.odc, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:18,324 - Worker-0 - INFO - Downloaded http://www.ics.uci.edu/~wjohnson/BIDA/Ch9/Model6.txt, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:18,416 - Worker-1 - INFO - Downloaded http://www.ics.uci.edu/~wjohnson/BIDA/DiagnosticTestsPart2/CodeandDataMedicalTestsPart2.txt, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:18,635 - Worker-3 - INFO - Downloaded http://www.ics.uci.edu/~wjohnson/BIDA/Ch9/Ch9WinBUGScodeANOVA.odc, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:18,777 - Worker-2 - INFO - Downloaded http://www.ics.uci.edu/~wjohnson/BIDA/AppendixC/RcodeAppendix.txt, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:18,861 - Worker-0 - INFO - Downloaded http://www.ics.uci.edu/~wjohnson/BIDA/Ch6/Ch6WinBUGScode.odc, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:18,949 - Worker-1 - INFO - Downloaded http://www.ics.uci.edu/~wjohnson/BIDA/AppendixC/leukemia.txt, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:19,127 - Worker-3 - INFO - Downloaded http://www.ics.uci.edu/~wjohnson/BIDA/Ch3/RcodeCh3.txt, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:19,322 - Worker-2 - INFO - Downloaded http://www.ics.uci.edu/~wjohnson/BIDA/Ch1/SurvivalDataArmadilloHunting.txt, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:19,406 - Worker-0 - INFO - Downloaded http://www.ics.uci.edu/~wjohnson/BIDA/Ch7/Ch7Rcode.txt, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:19,578 - Worker-1 - INFO - Downloaded http://www.ics.uci.edu/~wjohnson/BIDA/Ch8/posterioriterates.txt, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:19,887 - Worker-3 - INFO - Downloaded http://www.ics.uci.edu/~wjohnson/BIDA/Ch8/ToenailCodebook.txt, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:19,973 - Worker-2 - INFO - Downloaded http://www.ics.uci.edu/~wjohnson/BIDA/Ch8/trauma300.txt, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:20,061 - Worker-0 - INFO - Downloaded http://www.ics.uci.edu/~wjohnson/BIDA/Ch5/RcodeDiasorinExample.txt, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:20,177 - Worker-1 - INFO - Downloaded http://www.ics.uci.edu/~wjohnson/BIDA/Ch5/Ch5WinBUGScode.odc, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:20,432 - Worker-3 - INFO - Downloaded http://www.ics.uci.edu/~wjohnson/BIDA/Ch1/ArmadilloHuntingRepeatedMeasuresData.txt, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:20,514 - Worker-2 - INFO - Downloaded http://www.ics.uci.edu/~wjohnson/BIDA/Ch10/CSmodel.txt, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:20,596 - Worker-0 - INFO - Downloaded http://www.ics.uci.edu/~wjohnson/BIDA/Ch14/Ch14WinBUGScode.odc, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:20,704 - Worker-1 - INFO - Downloaded http://www.ics.uci.edu/~wjohnson/BIDA/Ch10/DogCode.txt, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:20,990 - Worker-3 - INFO - Downloaded http://www.ics.uci.edu/~wjohnson/BIDA/Ch4/Ch4WinBUGScode.odc, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:21,088 - Worker-2 - INFO - Downloaded http://www.ics.uci.edu/~wjohnson/BIDA/Ch12/LeukemiaData.txt, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:21,170 - Worker-0 - INFO - Downloaded http://www.ics.uci.edu/~wjohnson/BIDA/Ch15/Chap15DPpackage.txt, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:21,273 - Worker-1 - INFO - Downloaded http://www.ics.uci.edu/~wjohnson/BIDA/Ch9/Ch9Rcode.txt, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:21,568 - Worker-3 - INFO - Downloaded http://www.ics.uci.edu/~wjohnson/BIDA/Ch11/FMDData.txt, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:21,653 - Worker-2 - INFO - Downloaded http://www.ics.uci.edu/~wjohnson/BIDA/Ch13/Larynx-Cancer-Data.txt, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:21,740 - Worker-0 - INFO - Downloaded http://www.ics.uci.edu/~wjohnson/BIDA/Ch9/Ch9RcodeANOVA.txt, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:21,827 - Worker-1 - INFO - Downloaded http://www.ics.uci.edu/~wjohnson/BIDA/Ch13/leukemia.txt, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:22,111 - Worker-3 - INFO - Downloaded http://www.ics.uci.edu/~wjohnson/BIDA/Ch7/FEVdataAge10to19.txt, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:22,214 - Worker-2 - INFO - Downloaded http://www.ics.uci.edu/~wjohnson/BIDA/Ch9/FullFEVdataExercise9-21.txt, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:22,309 - Worker-0 - INFO - Downloaded http://www.ics.uci.edu/~wjohnson/BIDA/Ch5/SampleSizeProportions.txt, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:22,412 - Worker-1 - INFO - Downloaded http://www.ics.uci.edu/~wjohnson/BIDA/Ch10/Ch10Rcode.txt, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:22,706 - Worker-3 - INFO - Downloaded http://www.ics.uci.edu/~wjohnson/BIDA/Ch13/Ch13RcodeLeukemia.txt, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:22,810 - Worker-2 - INFO - Downloaded http://www.ics.uci.edu/~wjohnson/BIDA/Ch12/Ch12Rcode.txt, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:22,929 - Worker-0 - INFO - Downloaded http://www.ics.uci.edu/~wjohnson/BIDA/Ch10/Ch10WinBUGScode.odc, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:23,457 - Worker-1 - INFO - Downloaded http://www.ics.uci.edu/~wjohnson/BIDA/Ch1/BrassAlloyZincData.txt, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:23,544 - Worker-3 - INFO - Downloaded http://www.ics.uci.edu/~wjohnson/BIDA/Ch8/OringData.txt, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:23,627 - Worker-2 - INFO - Downloaded http://www.ics.uci.edu/~wjohnson/BIDA/Ch9/DiasorinANOVAModel.txt, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:23,716 - Worker-0 - INFO - Downloaded http://www.ics.uci.edu/~wjohnson/BIDA/Ch9/Model3.txt, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:03:23,822 - Worker-1 - INFO - Downloaded http://www.ics.uci.edu/~wjohnson/BIDA/Ch13/AcheDataExercise13-20.txt, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:05:59,568 - Worker-3 - INFO - Frontier is empty. Stopping Crawler. +2022-04-23 02:05:59,568 - Worker-4 - INFO - Frontier is empty. Stopping Crawler. +2022-04-23 02:05:59,664 - Worker-0 - INFO - Downloaded https://www.cs.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:05:59,665 - Worker-1 - INFO - Downloaded https://www.stat.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:05:59,684 - Worker-2 - INFO - Downloaded https://www.informatics.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:00,248 - Worker-2 - INFO - Downloaded https://www.cs.uci.edu/ics-graduate-programs-in-statistics-and-computer-science-among-top-20-for-public-universities, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:00,256 - Worker-1 - INFO - Downloaded https://www.stat.uci.edu/grad-student-directory, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:00,264 - Worker-0 - INFO - Downloaded https://www.informatics.uci.edu/undergrad/bs-business-information-management, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:00,684 - Worker-2 - INFO - Downloaded https://www.cs.uci.edu/culture-of-collaboration-leads-to-cutting-edge-work-in-ai, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:00,804 - Worker-1 - INFO - Downloaded https://www.stat.uci.edu/applications-open-for-new-uci-summer-biostatistics-institute-for-undergraduate-students, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:00,819 - Worker-0 - INFO - Downloaded https://www.ics.uci.edu/~smyth, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:01,192 - Worker-2 - INFO - Downloaded https://www.cs.uci.edu/uci-news-ucis-graduate-programs-shine-in-u-s-news-world-report-rankings, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:01,306 - Worker-0 - INFO - Downloaded http://www.ics.uci.edu/~smyth/courses/stats5, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:01,408 - Worker-1 - INFO - Downloaded https://www.stat.uci.edu/covid19/index.html, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:01,765 - Worker-2 - INFO - Downloaded https://www.cs.uci.edu/uci-deep-learning-researchers-advance-theory-and-practice-of-data-compression, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:02,328 - Worker-2 - INFO - Downloaded http://www.ics.uci.edu/~smyth/courses/stats5/Forms, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:02,401 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/isi-buds/apply.html, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:02,401 - Worker-1 - INFO - Downloaded https://www.cs.uci.edu/ukr-ai-develops-ai-model-to-detect-misinformation-on-russia-ukraine-war, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:02,515 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/oc_covid_model/index.html, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:02,847 - Worker-2 - INFO - Downloaded https://www.ics.uci.edu/community/news/view_news?id=2121, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:02,949 - Worker-1 - INFO - Downloaded https://www.cs.uci.edu/uci-researchers-draft-legislation-to-incentivize-better-cybersecurity, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:03,029 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/contact-the-department, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:03,377 - Worker-2 - INFO - Downloaded https://www.ics.uci.edu/~sjordan, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:03,480 - Worker-1 - INFO - Downloaded https://www.cs.uci.edu/uci-news-uci-researchers-develop-hybrid-human-machine-framework-for-building-smarter-ai, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:03,582 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/ics-graduate-programs-in-statistics-and-computer-science-among-top-20-for-public-universities, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:03,886 - Worker-2 - INFO - Downloaded http://www.ics.uci.edu/computerscience, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:04,005 - Worker-1 - INFO - Downloaded https://www.cs.uci.edu/hack-at-uci-hosts-hybrid-hackuci-2022, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:04,088 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/seminar-series, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:04,398 - Worker-2 - INFO - Downloaded http://www.ics.uci.edu/news, status <404>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:04,523 - Worker-1 - INFO - Downloaded https://www.cs.uci.edu/ph-d-student-ping-xiang-chen-awarded-j-yang-family-foundation-fellowship, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:04,603 - Worker-0 - INFO - Downloaded http://www.stat.uci.edu/seminar-series/seminar-series-2018-2019, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:05,023 - Worker-2 - INFO - Downloaded https://www.ics.uci.edu/~dutt, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:05,043 - Worker-1 - INFO - Downloaded https://www.cs.uci.edu/womens-history-month-resources-for-empowering-women-in-tech, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:05,153 - Worker-0 - INFO - Downloaded http://www.stat.uci.edu/wp-content/uploads/JuliaPalaciosAbstract6-6-19, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:05,945 - Worker-1 - INFO - Downloaded https://www.cs.uci.edu/student-spotlight-computer-science-and-music-double-major-joseph-wong-finds-harmony-at-uci, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:05,976 - Worker-2 - INFO - Downloaded https://www.ics.uci.edu/~djr/DebraJRichardson/Home.html, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:06,146 - Worker-2 - INFO - Downloaded https://transformativeplay.ics.uci.edu/Tess-Tanenbaum, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:06,150 - Worker-0 - INFO - Downloaded http://www.stat.uci.edu/wp-content/uploads/StephanMandtAbstract5-16-19, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:06,618 - Worker-1 - INFO - Downloaded https://www.cs.uci.edu/alumni-spotlight-anthony-d-mays-is-opening-doors-for-the-next-generation-of-tech-talent, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:07,229 - Worker-0 - INFO - Downloaded http://www.stat.uci.edu/wp-content/uploads/TianZhengAbstract5-30-19, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:07,485 - Worker-2 - INFO - Downloaded https://www.cs.uci.edu/student-spotlight-cs-major-alex-meng-makes-an-impact-as-a-resident-advisor, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:07,581 - Worker-1 - INFO - Downloaded https://transformativeplay.ics.uci.edu/magia-transformo, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:08,264 - Worker-0 - INFO - Downloaded http://www.stat.uci.edu/wp-content/uploads/LorinCrawfordAbstract5-2-19, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:08,354 - Worker-2 - INFO - Downloaded https://transformativeplay.ics.uci.edu/shadowcast, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:08,447 - Worker-1 - INFO - Downloaded https://www.cs.uci.edu/black-history-month-sharing-resources-to-expand-diversity-in-tech, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:09,475 - Worker-0 - INFO - Downloaded http://www.stat.uci.edu/wp-content/uploads/XinTongAbstract4-25-19, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:09,475 - Worker-2 - INFO - Downloaded http://www.cecs.uci.edu, status <604>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:09,643 - Worker-1 - INFO - Downloaded https://www.ics.uci.edu/community/news/view_news?id=1928, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:10,522 - Worker-2 - INFO - Downloaded http://www.stat.uci.edu/seminar-series/seminar-series-2014-2015, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:10,548 - Worker-1 - INFO - Downloaded https://www.cs.uci.edu/events/distinguished-lecture-series, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:10,548 - Worker-0 - INFO - Downloaded https://www.ics.uci.edu/community/news/view_news?id=2092, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:10,665 - Worker-2 - INFO - Downloaded http://www.stat.uci.edu/ICS/statistics/research/seminarseries/2013-2014/index, status <404>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:11,115 - Worker-1 - INFO - Downloaded https://www.ics.uci.edu/community/news/view_news.php?id=2093, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:11,120 - Worker-0 - INFO - Downloaded http://www.cs.uci.edu/events/distinguished-lecture-series/distinguished-lecture-series-2019, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:11,174 - Worker-2 - INFO - Downloaded http://www.stat.uci.edu/ICS/statistics/research/seminarseries/2012-2013/index, status <404>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:11,658 - Worker-1 - INFO - Downloaded https://www.cs.uci.edu/events/seminar-series/?seminar_id=1029, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:11,658 - Worker-0 - INFO - Downloaded https://www.ics.uci.edu/community/news/view_news.php?id=2092, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:11,680 - Worker-2 - INFO - Downloaded http://www.stat.uci.edu/ICS/statistics/research/seminarseries/2010-2011/index, status <404>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:12,186 - Worker-1 - INFO - Downloaded http://www.cs.uci.edu/seminar-series-archive, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:12,210 - Worker-2 - INFO - Downloaded https://www.ics.uci.edu/community/news/view_news.php?id=2094, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:12,225 - Worker-0 - INFO - Downloaded http://www.stat.uci.edu/ICS/statistics/research/seminarseries/2011-2012/index, status <404>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:12,719 - Worker-0 - INFO - Downloaded http://www.cs.uci.edu/seminar-series-archive/?seminar_id=1091, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:12,736 - Worker-1 - INFO - Downloaded https://www.ics.uci.edu/community/news/view_news.php?id=2095, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:12,737 - Worker-2 - INFO - Downloaded http://www.stat.uci.edu/seminar-series/seminar-series-2015-2016, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:13,277 - Worker-1 - INFO - Downloaded http://www.cs.uci.edu/seminar-series-archive/?seminar_id=1094, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:13,284 - Worker-0 - INFO - Downloaded https://www.ics.uci.edu/community/news/view_news?id=1883, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:13,284 - Worker-2 - INFO - Downloaded http://www.stat.uci.edu/seminar-series/seminar-series-2017-2018, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:13,778 - Worker-0 - INFO - Downloaded https://www.ics.uci.edu/community/news/view_news.php?id=1884, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:13,823 - Worker-1 - INFO - Downloaded http://www.cs.uci.edu/seminar-series-archive/?seminar_id=1093, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:13,893 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/wp-content/uploads/Shujie-Ma-Abstract-5-5-22, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:14,724 - Worker-0 - INFO - Downloaded https://www.ics.uci.edu/community/news/view_news.php?id=1883, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:14,773 - Worker-1 - INFO - Downloaded http://www.cs.uci.edu/seminar-series-archive/?seminar_id=1092, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:14,869 - Worker-2 - INFO - Downloaded http://www.stat.uci.edu/seminar-series/seminar-series-2019-2020, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:15,015 - Worker-0 - INFO - Downloaded http://www.cs.uci.edu/seminar-series-archive/?seminar_id=1103, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:15,016 - Worker-1 - INFO - Downloaded https://www.ics.uci.edu/community/news/view_news.php?id=1885, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:15,381 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/seminar-series-2020-2021, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:15,578 - Worker-1 - INFO - Downloaded http://www.cs.uci.edu/seminar-series-archive/?seminar_id=1112, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:15,579 - Worker-0 - INFO - Downloaded https://www.ics.uci.edu/community/news/view_news.php?id=1886, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:15,905 - Worker-2 - INFO - Downloaded http://www.stat.uci.edu/seminar-series/seminar-series-2016-2017, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:16,123 - Worker-0 - INFO - Downloaded http://www.cs.uci.edu/seminar-series-archive/?seminar_id=1088, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:16,124 - Worker-1 - INFO - Downloaded https://www.ics.uci.edu/community/news/view_news.php?id=1887, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:16,450 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/faculty, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:16,650 - Worker-0 - INFO - Downloaded http://www.economics.uci.edu/~ivan, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:16,678 - Worker-1 - INFO - Downloaded http://www.cs.uci.edu/seminar-series-archive/?seminar_id=1078, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:16,989 - Worker-2 - INFO - Downloaded http://www.stat.uci.edu/faculty/vladimir-minin, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:17,164 - Worker-0 - INFO - Downloaded http://www.economics.uci.edu/~ivan/classes.html, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:17,192 - Worker-1 - INFO - Downloaded http://www.cs.uci.edu/seminar-series-archive/?seminar_id=1110, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:17,498 - Worker-2 - INFO - Downloaded http://www.stat.uci.edu/faculty-directory/wesley-johnson, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:17,773 - Worker-0 - INFO - Downloaded http://www.cs.uci.edu/seminar-series-archive/?seminar_id=1080, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:17,789 - Worker-1 - INFO - Downloaded https://www.ics.uci.edu/~wjohnson, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:18,015 - Worker-2 - INFO - Downloaded http://www.stat.uci.edu/faculty-directory/dan-gillen, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:18,297 - Worker-1 - INFO - Downloaded http://www.cs.uci.edu/seminar-series-archive/?seminar_id=1111, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:18,313 - Worker-0 - INFO - Downloaded https://www.ics.uci.edu/~dgillen, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:18,540 - Worker-2 - INFO - Downloaded http://www.stat.uci.edu/faculty-directory/jessica-utts, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:18,823 - Worker-0 - INFO - Downloaded http://www.cs.uci.edu/seminar-series-archive/?seminar_id=1077, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:18,824 - Worker-1 - INFO - Downloaded https://www.ics.uci.edu/~jutts, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:19,211 - Worker-2 - INFO - Downloaded http://www.stat.uci.edu/faculty-directory/michele-guindani, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:19,355 - Worker-0 - INFO - Downloaded http://www.cs.uci.edu/seminar-series-archive/?seminar_id=1076, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:19,386 - Worker-1 - INFO - Downloaded https://www.ics.uci.edu/~mguindan, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:19,641 - Worker-2 - INFO - Downloaded http://www.stat.uci.edu/faculty-directory/tianchen-qian, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:19,902 - Worker-0 - INFO - Downloaded http://www.ics.uci.edu/~jutts/response.html, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:19,918 - Worker-1 - INFO - Downloaded http://www.cs.uci.edu/seminar-series-archive/?seminar_id=1089, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:20,155 - Worker-2 - INFO - Downloaded http://www.stat.uci.edu/faculty-directory/mine-dogucu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:20,410 - Worker-0 - INFO - Downloaded http://www.ics.uci.edu/~jutts/110, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:20,424 - Worker-1 - INFO - Downloaded http://www.cs.uci.edu/seminar-series-archive/?seminar_id=1095, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:20,669 - Worker-2 - INFO - Downloaded http://www.stat.uci.edu/faculty-directory/hal-stern, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:20,942 - Worker-0 - INFO - Downloaded https://www.ics.uci.edu/~sternh, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:20,962 - Worker-1 - INFO - Downloaded http://www.cs.uci.edu/seminar-series-archive/?seminar_id=1075, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:21,186 - Worker-2 - INFO - Downloaded http://www.stat.uci.edu/faculty-directory/lee-kucera, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:21,549 - Worker-1 - INFO - Downloaded http://www.ics.uci.edu/~lab/lab_schedule/index.php, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:21,555 - Worker-0 - INFO - Downloaded http://www.cs.uci.edu/seminar-series-archive/?seminar_id=1079, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:21,695 - Worker-2 - INFO - Downloaded http://www.stat.uci.edu/faculty/brigitte-baldi, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:21,971 - Worker-1 - INFO - Downloaded http://www.ics.uci.edu/ugrad/courses/index, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:22,096 - Worker-0 - INFO - Downloaded http://www.cs.uci.edu/events/distinguished-lecture-series/distinguished-lecture-series-2017, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:22,209 - Worker-2 - INFO - Downloaded http://www.stat.uci.edu/faculty-directory/weining-shen, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:22,486 - Worker-1 - INFO - Downloaded https://www.ics.uci.edu/ugrad/courses, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:22,603 - Worker-0 - INFO - Downloaded http://www.cs.uci.edu/events/distinguished-lecture-series/distinguished-lecture-series-2018, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:22,717 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/faculty-directory/annie-qu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:23,021 - Worker-1 - INFO - Downloaded https://www.ics.uci.edu/~lab/lab_schedule, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:23,109 - Worker-0 - INFO - Downloaded https://www.cs.uci.edu/graduate-computer-science-programs, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:23,226 - Worker-2 - INFO - Downloaded http://www.stat.uci.edu/faculty-directory/koko-gulesserian, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:23,529 - Worker-1 - INFO - Downloaded http://www.ics.uci.edu/grad/degrees/degree_cs.php, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:23,627 - Worker-0 - INFO - Downloaded https://www.cs.uci.edu/computer-science-ph-d-student-vikram-narayanan-receives-ibm-fellowship, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:23,732 - Worker-2 - INFO - Downloaded http://www.stat.uci.edu/faculty-directory/zhaoxia-yu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:24,080 - Worker-1 - INFO - Downloaded https://www.ics.uci.edu/~zhaoxia, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:24,160 - Worker-0 - INFO - Downloaded https://www.cs.uci.edu/senior-spotlight-kazeem-salaam-promotes-stem-learning-combining-programming-skills-with-a-passion-for-helping-others, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:24,260 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/faculty-directory/babak-shahbab, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:24,594 - Worker-1 - INFO - Downloaded https://www.ics.uci.edu/~zhaoxia/Data/BeyondTandANOVA, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:24,677 - Worker-0 - INFO - Downloaded https://www.cs.uci.edu/sang-woo-jun-and-collaborators-win-best-paper-award-for-predictive-maintenance-at-the-edge, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:24,772 - Worker-2 - INFO - Downloaded http://www.stat.uci.edu/faculty/bin-nan, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:25,183 - Worker-0 - INFO - Downloaded https://www.ics.uci.edu/~swjun, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:25,199 - Worker-1 - INFO - Downloaded https://www.cs.uci.edu/2020-acm-thacker-breakthrough-in-computing-award-awarded-to-professor-michael-franz, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:25,281 - Worker-2 - INFO - Downloaded http://www.stat.uci.edu/faculty-directory/veronica-berrocal, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:25,688 - Worker-0 - INFO - Downloaded https://www.ics.uci.edu/community/news/view_news?id=2063", status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:25,705 - Worker-1 - INFO - Downloaded https://www.cs.uci.edu/david-eppstein-receives-2022-siam-best-paper-award, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:25,801 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/minor-in-statistics, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:26,196 - Worker-0 - INFO - Downloaded https://www.ics.uci.edu/~eppstein, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:26,244 - Worker-1 - INFO - Downloaded https://www.cs.uci.edu/icssc-hosts-annual-webjam-competition-to-teach-web-development, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:26,310 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/research, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:26,804 - Worker-1 - INFO - Downloaded https://studentcouncil.ics.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:26,809 - Worker-0 - INFO - Downloaded https://www.cs.uci.edu/stephan-mandt-and-collaborators-receive-3-5-million-to-study-machine-learning-for-climate-science, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:26,831 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/slider/uci-launches-new-professional-program-master-of-data-science, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:27,319 - Worker-1 - INFO - Downloaded https://mds.ics.uci.edu/prospective-students/cost-and-financial-aid, status <404>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:27,334 - Worker-0 - INFO - Downloaded https://www.cs.uci.edu/ics-welcomes-3-new-faculty-for-2021, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:27,351 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/a-campus-gem-ucis-statistical-consulting-services, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:27,859 - Worker-2 - INFO - Downloaded https://www.cs.uci.edu/faisal-nawab-receives-facebook-award-for-next-generation-data-infrastructure, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:27,917 - Worker-1 - INFO - Downloaded https://www.stat.uci.edu/orange-county-business-journal-uci-forecasts-covid-19-trends-in-oc, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:27,918 - Worker-0 - INFO - Downloaded http://www.ics.uci.edu/~babaks/index.html, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:28,441 - Worker-2 - INFO - Downloaded https://www.ics.uci.edu/community/news/view_news?id=2036, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:28,445 - Worker-1 - INFO - Downloaded https://www.stat.uci.edu/kuci-office-hours-podcast-keeping-up-with-coronavirus-statistics-vladimir-minin-interviewed, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:28,455 - Worker-0 - INFO - Downloaded https://www.cs.uci.edu/nsf-civic-innovation-challenge-awards-1-million-to-uci-for-disaster-resilience-in-aging-communities, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:28,927 - Worker-2 - INFO - Downloaded https://www.ics.uci.edu/community/news/view_news.php?id=2035, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:28,940 - Worker-0 - INFO - Downloaded https://www.cs.uci.edu/chen-li-receives-nsf-grant-for-collaborative-data-analytics, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:29,019 - Worker-1 - INFO - Downloaded https://www.stat.uci.edu/uci-statisticians-release-new-online-orange-county-covid-19-information-resource, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:29,458 - Worker-2 - INFO - Downloaded https://www.ics.uci.edu/community/news/view_news.php?id=2036, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:29,465 - Worker-0 - INFO - Downloaded https://www.cs.uci.edu/sangeetha-abdu-jyothis-work-on-solar-superstorms-raises-concerns-of-internet-apocalypse, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:29,531 - Worker-1 - INFO - Downloaded https://www.stat.uci.edu/ics-statistics-researchers-release-new-online-oc-covid-situation-report, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:30,004 - Worker-2 - INFO - Downloaded https://www.ics.uci.edu/~sabdujyo, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:30,005 - Worker-0 - INFO - Downloaded https://www.cs.uci.edu/harnessing-the-data-revolution-ics-researchers-and-collaborators-receive-1-5m-data-science-grant, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:30,056 - Worker-1 - INFO - Downloaded https://www.stat.uci.edu/orange-county-business-journal-uci-releases-covid-19-website, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:30,590 - Worker-0 - INFO - Downloaded https://cml.ics.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:30,592 - Worker-2 - INFO - Downloaded https://www.cs.uci.edu/wired-a-bad-solar-storm-could-cause-an-internet-apocalypse-sangeetha-abdu-jyothi-quoted, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:30,593 - Worker-1 - INFO - Downloaded https://www.stat.uci.edu/yahoo-news-data-glitch-confounds-coronavirus-totals-in-orange-county-state-vladimir-minin-quoted, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:31,154 - Worker-2 - INFO - Downloaded https://cml.ics.uci.edu/aiml, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:31,155 - Worker-0 - INFO - Downloaded https://www.cs.uci.edu/marco-levorato-receives-nsf-grant-for-mobile-autonomous-systems-mas-research, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:31,156 - Worker-1 - INFO - Downloaded https://www.stat.uci.edu/uci-news-uci-researchers-launch-first-of-its-kind-coronavirus-statistics-portal, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:31,654 - Worker-2 - INFO - Downloaded https://cml.ics.uci.edu/aiml/page/2, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:31,689 - Worker-0 - INFO - Downloaded https://web.archive.org/web/20220101151252/https://www.cs.uci.edu/david-eppstein-receives-2022-siam-best-paper-award, status <604>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:31,741 - Worker-1 - INFO - Downloaded https://www.stat.uci.edu/professor-annie-qu-works-to-enhance-the-detection-of-invasive-cancers-using-medical-imaging-data, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:32,197 - Worker-2 - INFO - Downloaded https://www.ics.uci.edu/~jingz31, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:32,269 - Worker-1 - INFO - Downloaded https://web.archive.org/web/20220101151252/http://www.cs.uci.edu/eppstein-shares-research-at-conferences-in-australia-canada-and-tokyo, status <604>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:32,286 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/hal-stern-selected-as-a-fellow-of-the-international-society-of-bayesian-analysis, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:32,724 - Worker-2 - INFO - Downloaded https://www.ics.uci.edu/~jingz31/paper, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:32,747 - Worker-1 - INFO - Downloaded https://web.archive.org/web/20220101151252/http://www.cs.uci.edu/eppstein-named-aaas-fellow, status <604>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:32,795 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/uci-news-national-institute-awards-20-million-in-renewed-funding-to-forensic-science-center, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:33,241 - Worker-2 - INFO - Downloaded https://www.ics.uci.edu/~jingz31/contact, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:33,348 - Worker-1 - INFO - Downloaded https://www.stat.uci.edu/michelle-nuno-selected-to-participate-in-the-70th-lindau-nobel-laureate-meeting, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:33,364 - Worker-0 - INFO - Downloaded https://www.cs.uci.edu/alumni-spotlight-devin-held-15-shares-her-journey-from-uci-to-arteria-ai, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:33,769 - Worker-2 - INFO - Downloaded https://www.ics.uci.edu/~jingz31/people, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:33,879 - Worker-1 - INFO - Downloaded https://www.cs.uci.edu/four-winning-teams-recognized-at-zothacks-2021, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:33,880 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/uci-mind-alzheimers-disease-clinical-trials-and-covid-19-with-dr-daniel-gillen, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:34,285 - Worker-2 - INFO - Downloaded https://www.ics.uci.edu/community/news/view_news?id=2072, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:34,434 - Worker-0 - INFO - Downloaded https://www.cs.uci.edu/disney-research-supports-stephan-mandts-work-on-generative-modeling-with-50000-gift, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:34,435 - Worker-1 - INFO - Downloaded https://www.stat.uci.edu/uci-brain-researcher-spotlight-dr-babak-shahbaba, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:34,815 - Worker-2 - INFO - Downloaded https://www.ics.uci.edu/community/news/view_news?id=2005, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:34,979 - Worker-0 - INFO - Downloaded https://www.cs.uci.edu/ics-students-train-for-embedded-capture-the-flag-competition, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:34,998 - Worker-1 - INFO - Downloaded https://www.stat.uci.edu/nsf-announces-2020-graduate-research-fellows, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:35,325 - Worker-2 - INFO - Downloaded https://www.ics.uci.edu/~marios, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:35,490 - Worker-0 - INFO - Downloaded https://www.cs.uci.edu/nalini-venkatasubramanian-named-acm-2021-distinguished-member, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:35,509 - Worker-1 - INFO - Downloaded https://www.stat.uci.edu/amstat-news-celebrating-rising-undergraduate-women-in-statistics-and-data-science, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:35,929 - Worker-2 - INFO - Downloaded http://ics.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:36,013 - Worker-0 - INFO - Downloaded https://www.cs.uci.edu/tech-startup-emerge-is-transforming-virtual-communication, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:36,016 - Worker-1 - INFO - Downloaded https://www.stat.uci.edu/statistics-ph-d-student-mary-ryan-receives-public-impact-fellowship, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:36,441 - Worker-2 - INFO - Downloaded https://www.ics.uci.edu/~marymr, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:36,568 - Worker-1 - INFO - Downloaded https://www.stat.uci.edu/professor-berrocal-elected-chair-of-the-section-in-environmental-sciences-of-the-isba, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:36,584 - Worker-0 - INFO - Downloaded https://www.cs.uci.edu/tech-trends-for-2022, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:36,982 - Worker-2 - INFO - Downloaded https://www.ics.uci.edu/community/news/view_news?id=1978, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:37,107 - Worker-1 - INFO - Downloaded https://www.cs.uci.edu/ucis-new-university-uci-students-launch-the-zotmeal-app, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:37,109 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/staff-spotlight-rosemary-bustas-journey-from-chemistry-to-statistics-to-scotland-and-beyond, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:37,499 - Worker-2 - INFO - Downloaded https://www.ics.uci.edu/community/news/view_news.php?id=1977, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:37,628 - Worker-1 - INFO - Downloaded https://www.cs.uci.edu/multidisciplinary-collaborators-set-their-sights-on-color-vision-in-the-dark, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:37,644 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/kpcc-vexed-by-college-statistics-courses-new-approaches-emphasize-practical-learning-jessica-utts-interviewed, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:38,023 - Worker-2 - INFO - Downloaded https://www.ics.uci.edu/community/news/view_news.php?id=1978, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:38,156 - Worker-1 - INFO - Downloaded https://www.cs.uci.edu/nlp-benchmarks-ml-devops-interoperability-with-yasaman-razeghi-and-prof-sameer-singh-machine-learning-street-talk-podcast, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:38,157 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/laist-socal-professors-push-to-make-college-level-statistics-less-painful-jessica-utts-quoted, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:38,544 - Worker-2 - INFO - Downloaded https://www.ics.uci.edu/~gts, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:38,685 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/ics-welcomes-8-new-faculty-for-2019, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:38,705 - Worker-1 - INFO - Downloaded https://www.cs.uci.edu/cs-research-areas, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:39,154 - Worker-2 - INFO - Downloaded https://www.ics.uci.edu/faculty/profiles/view_faculty.php?ucinetid=smehrotr, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:39,232 - Worker-0 - INFO - Downloaded http://plrg.eecs.uci.edu, status <604>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:39,249 - Worker-1 - INFO - Downloaded https://www.stat.uci.edu/senior-spotlight-taneisha-arora-pursues-her-passions-from-working-in-industry-to-running-a-bakery, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:39,673 - Worker-2 - INFO - Downloaded https://www.ics.uci.edu/ugrad/degrees/degree_se.php, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:39,730 - Worker-0 - INFO - Downloaded https://www.cs.uci.edu/research-centers, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:39,766 - Worker-1 - INFO - Downloaded https://www.stat.uci.edu/statistics-professors-shahbaba-and-minin-help-develop-framework-to-investigate-complex-biological-systems, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:40,182 - Worker-2 - INFO - Downloaded http://www.ics.uci.edu/~theory, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:40,242 - Worker-0 - INFO - Downloaded https://www.cs.uci.edu/machine-learning-street-talk-podcast-features-reality-check-from-sameer-singh-and-yasaman-razeghi, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:40,278 - Worker-1 - INFO - Downloaded https://www.stat.uci.edu/hal-stern-appointed-vice-provost-for-academic-planning, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:40,791 - Worker-0 - INFO - Downloaded https://www.ics.uci.edu/~fukuzaws, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:40,806 - Worker-1 - INFO - Downloaded https://www.stat.uci.edu/professor-nan-awarded-nsf-grant-to-improve-statistical-inference, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:40,807 - Worker-2 - INFO - Downloaded https://www.cs.uci.edu/current-graduate-students, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:41,289 - Worker-0 - INFO - Downloaded http://www.ics.uci.edu/grad/policies/GradPolicies_CopyrightInfringement.php, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:41,339 - Worker-1 - INFO - Downloaded https://www.cs.uci.edu/ics-researchers-win-best-paper-award-at-eurosys-2022, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:41,340 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/renewed-funding-lets-hal-stern-continue-research-of-early-life-adversity-brain-development-with-the-uci-conte-center, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:41,808 - Worker-0 - INFO - Downloaded https://www.ics.uci.edu/~dabrowsa, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:41,871 - Worker-1 - INFO - Downloaded http://www.cs.uci.edu/research-areas, status <404>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:41,873 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/conference-honors-statistics-professor-emeritus-wesley-johnson, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:42,323 - Worker-0 - INFO - Downloaded https://www.ics.uci.edu/~dgens, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:42,382 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/ics-students-win-best-web-app-at-hacksc, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:42,401 - Worker-1 - INFO - Downloaded https://www.cs.uci.edu/faculty, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:42,901 - Worker-2 - INFO - Downloaded https://www.cs.uci.edu/levorato-receives-nsf-grant-for-automated-design-solutions-for-real-time-computing, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:42,921 - Worker-1 - INFO - Downloaded https://www.ics.uci.edu/~mustafai, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:42,937 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/professor-guindani-named-fellow-of-the-american-statistical-association, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:43,424 - Worker-2 - INFO - Downloaded http://www.ics.uci.edu/faculty/profiles/view_faculty.php?ucinetid=jwongma, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:43,426 - Worker-1 - INFO - Downloaded http://www.cs.uci.edu/distinguished-lecture-series, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:43,447 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/center-for-statistical-consulting-a-one-stop-shop-for-data-analysis, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:43,997 - Worker-2 - INFO - Downloaded https://www.cs.uci.edu/qi-alfred-chen-receives-nsf-career-award-for-ai-security-research, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:43,998 - Worker-1 - INFO - Downloaded https://statconsulting.ics.uci.edu/research-highlights/client-testimonials, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:44,002 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/statistics-ph-d-students-awarded-nsf-graduate-research-fellowships, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:44,543 - Worker-2 - INFO - Downloaded https://www.cs.uci.edu/undergraduate-programs, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:44,550 - Worker-1 - INFO - Downloaded https://www.ics.uci.edu/community/news/view_news?id=1306, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:44,553 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/new-data-science-scholarship-to-promote-social-good, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:45,113 - Worker-0 - INFO - Downloaded http://www.cs.uci.edu/professor-of-teaching-in-computer-science, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:45,132 - Worker-2 - INFO - Downloaded https://www.ics.uci.edu/community/news/view_news.php?id=1307, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:45,132 - Worker-1 - INFO - Downloaded https://www.stat.uci.edu/professor-shens-collaborations-exemplify-the-significance-of-statistics, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:45,592 - Worker-2 - INFO - Downloaded https://www.ics.uci.edu/community/news/view_news.php?id=1308, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:45,625 - Worker-0 - INFO - Downloaded https://www.cs.uci.edu/job-opportunities, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:45,697 - Worker-1 - INFO - Downloaded https://www.stat.uci.edu/trio-of-ics-professors-preview-tech-trends-for-2019, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:46,114 - Worker-2 - INFO - Downloaded https://www.ics.uci.edu/community/news/view_news.php?id=1310, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:46,151 - Worker-0 - INFO - Downloaded https://www.cs.uci.edu/news, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:46,225 - Worker-1 - INFO - Downloaded https://www.stat.uci.edu/statistics-ph-d-alumnus-andrew-holbrook-18-named-a-finalist-for-the-savage-award, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:46,651 - Worker-2 - INFO - Downloaded https://www.ics.uci.edu/community/news/view_news?id=1312, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:46,681 - Worker-0 - INFO - Downloaded https://www.cs.uci.edu/news/page/2, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:46,757 - Worker-1 - INFO - Downloaded https://www.stat.uci.edu/multidepartmental-collaboration-on-detecting-code-clones-leads-to-distinguished-paper-award, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:47,209 - Worker-2 - INFO - Downloaded https://www.informatics.uci.edu/explore/faculty-profiles/cristina-lopes, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:47,228 - Worker-0 - INFO - Downloaded https://www.cs.uci.edu/news/page/4, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:47,273 - Worker-1 - INFO - Downloaded https://www.stat.uci.edu/senior-spotlight-james-purpura-goes-from-watching-moneyball-to-earning-data-science-degree, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:47,729 - Worker-2 - INFO - Downloaded http://www.ics.uci.edu/~lopes, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:47,785 - Worker-0 - INFO - Downloaded https://www.cs.uci.edu/news/page/5, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:47,796 - Worker-1 - INFO - Downloaded https://www.stat.uci.edu/knowable-magazine-when-courtroom-science-goes-wrong-and-how-stats-can-fix-it-hal-stern-featured, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:48,338 - Worker-0 - INFO - Downloaded https://www.ics.uci.edu/community/news/view_news?id=1643, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:48,338 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/olivia-bernstein-named-outstanding-ta-in-statistics, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:48,341 - Worker-1 - INFO - Downloaded https://www.cs.uci.edu/graduate-student-spotlight-daniel-chengs-computer-science-research-has-real-world-impact, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:48,852 - Worker-0 - INFO - Downloaded https://www.cs.uci.edu/facebook-awards-research-gift-to-dutt-and-maity-for-studying-hyperscale-data-center-platform-power-management, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:48,852 - Worker-1 - INFO - Downloaded https://www.ics.uci.edu/community/news/view_news.php?id=1642, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:48,932 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/hina-arora-tong-zou-share-2018-newcomb-graduate-award-in-statistics, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:49,411 - Worker-1 - INFO - Downloaded https://www.cs.uci.edu/wired-no-facebook-and-google-are-not-public-utilities-scott-jordan-quoted, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:49,419 - Worker-0 - INFO - Downloaded https://www.ics.uci.edu/community/news/spotlight/spotlight_bob_newcomb, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:49,471 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/professor-guindani-named-incoming-editor-in-chief-of-bayesian-analysis, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:49,961 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/professor-gillen-receives-1-2m-grant-to-study-alzheimers-disease-clinical-trial-study-partners, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:49,972 - Worker-1 - INFO - Downloaded https://www.ics.uci.edu/community/news/spotlight, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:49,973 - Worker-0 - INFO - Downloaded https://www.cs.uci.edu/3514-2, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:50,457 - Worker-1 - INFO - Downloaded http://alumni.ics.uci.edu/2008/07/10/answering-the-call, status <601>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:50,531 - Worker-2 - INFO - Downloaded https://www.cs.uci.edu/computer-science-researchers-win-article-of-the-year-award, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:50,531 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/data-science-student-raj-parekh-receives-distinguished-anteater-award, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:51,039 - Worker-1 - INFO - Downloaded http://alumni.ics.uci.edu/2009/03/05/blended-passion, status <601>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:51,066 - Worker-0 - INFO - Downloaded https://www.cs.uci.edu/reappointment-of-dean-marios-papaefthymiou, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:51,067 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/stern-co-directs-award-winning-csafe-team, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:51,614 - Worker-0 - INFO - Downloaded https://www.cs.uci.edu/four-ics-professors-receive-nsf-career-awards, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:51,624 - Worker-2 - INFO - Downloaded https://www.ics.uci.edu/~galbraic, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:51,662 - Worker-1 - INFO - Downloaded https://www.stat.uci.edu/ics-staff-faculty-honored-at-inaugural-faculty-staff-awards-celebration, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:52,174 - Worker-0 - INFO - Downloaded https://www.ics.uci.edu/community/news/view_news?id=1353, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:52,187 - Worker-1 - INFO - Downloaded https://www.stat.uci.edu/maricela-cruz-receives-latino-excellence-award-for-ics, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:52,188 - Worker-2 - INFO - Downloaded https://www.cs.uci.edu/the-resilience-of-the-class-of-2021, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:52,750 - Worker-0 - INFO - Downloaded https://www.ics.uci.edu/community/news/view_news.php?id=1352, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:52,776 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/professor-nan-receives-1-2-million-grant-to-develop-new-statistical-methods, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:52,787 - Worker-1 - INFO - Downloaded https://www.cs.uci.edu/capstone-program-showcases-growing-talent-of-ics-students, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:53,276 - Worker-0 - INFO - Downloaded https://www.ics.uci.edu/community/news/view_news?id=1827, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:53,333 - Worker-1 - INFO - Downloaded https://www.stat.uci.edu/2018-carl-cotman-young-investigator-award-recipient-is-a-rising-star, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:53,338 - Worker-2 - INFO - Downloaded https://www.cs.uci.edu/pierre-baldis-new-book-explores-deep-learning-in-science, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:53,827 - Worker-0 - INFO - Downloaded https://www.ics.uci.edu/community/news/view_news?id=1983, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:53,875 - Worker-1 - INFO - Downloaded https://www.cs.uci.edu/lawfare-fighting-insider-abuse-after-van-buren-by-bryan-cunningham, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:53,880 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/ucis-graduate-programs-shine-in-u-s-news-world-report-rankings, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:54,348 - Worker-0 - INFO - Downloaded https://www.ics.uci.edu/community/news/view_news.php?id=1984, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:54,388 - Worker-1 - INFO - Downloaded https://www.cs.uci.edu/irvine-standard-ucis-cybersecurity-club-makes-history, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:54,402 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/the-center-for-statistical-consultings-new-director-envisions-a-one-stop-shop-for-data-analysis-needs, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:54,866 - Worker-0 - INFO - Downloaded https://www.ics.uci.edu/community/news/view_news.php?id=1985, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:54,913 - Worker-1 - INFO - Downloaded https://www.cs.uci.edu/maestro-a-new-platform-for-building-reliable-and-robust-ai-systems, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:54,916 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/shahbaba-receives-1-7-million-neural-data-analysis-grant, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:55,392 - Worker-0 - INFO - Downloaded https://hombao.ics.uci.edu/hernando.html, status <601>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:55,448 - Worker-2 - INFO - Downloaded https://www.cs.uci.edu/a-bold-decision-to-study-abroad-in-high-school-led-sirui-hu-to-uci-and-beyond-data-science-major, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:55,449 - Worker-1 - INFO - Downloaded https://www.stat.uci.edu/shahbaba-receives-1-7-million-grant-to-develop-novel-models-for-neural-data-analysis, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:55,996 - Worker-2 - INFO - Downloaded https://www.ics.uci.edu/community/news/view_news?id=1523, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:55,996 - Worker-1 - INFO - Downloaded https://www.cs.uci.edu/kabc-biden-weighs-direct-action-against-russian-targets-following-cyberattacks-bryan-cunningham-quoted, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:55,996 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/minin-co-edits-special-section-on-infectious-diseases-in-statistical-science, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:56,571 - Worker-1 - INFO - Downloaded https://www.cs.uci.edu/professor-mohsen-imani-awarded-two-grants-to-study-brain-inspired-hyperdimensional-computing, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:56,572 - Worker-0 - INFO - Downloaded https://www.ics.uci.edu/community/news/view_news.php?id=1524, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:56,573 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/lakeland-times-autism-numbers-spike-the-latest-call-to-action-stern-quoted, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:57,080 - Worker-0 - INFO - Downloaded https://www.cs.uci.edu/nalini-venkatasubramanian-named-one-of-10-stars-in-computer-networking-and-communications, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:57,082 - Worker-1 - INFO - Downloaded https://www.ics.uci.edu/community/news/view_news.php?id=1523, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:57,150 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/utts-discusses-statistics-for-good-governance-at-international-conference-in-sri-lanka, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:57,625 - Worker-0 - INFO - Downloaded https://www.ics.uci.edu/community/news/view_news?id=1538, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:57,625 - Worker-1 - INFO - Downloaded https://www.cs.uci.edu/medium-epic-women-in-cyber-saltanat-mashirova-ics-alumna, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:57,683 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/2018-ics-projects-and-predictions, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:58,179 - Worker-1 - INFO - Downloaded http://www.ics.uci.edu/community/news/view_news?id=1267, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:58,193 - Worker-0 - INFO - Downloaded https://www.cs.uci.edu/new-security-threat-for-self-driving-cars-invisible-road-obstacles, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:58,198 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/new-faculty-spotlight-statistics-professor-vladimir-minin-is-pleased-to-join-growing-community, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:58,715 - Worker-1 - INFO - Downloaded http://www.ics.uci.edu/community/news/view_news.php?id=1266, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:58,766 - Worker-0 - INFO - Downloaded https://www.cs.uci.edu/11-ics-professors-included-on-guide2researchs-2021-ranking-of-top-scientists-in-computer-science, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:58,767 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/uci-well-represented-at-women-in-statistics-data-science-conference, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:59,243 - Worker-1 - INFO - Downloaded http://www.ics.uci.edu/~amirr1, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:59,301 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/zhu-wins-2017-newcomb-graduate-award-in-statistics, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:59,319 - Worker-0 - INFO - Downloaded https://www.cs.uci.edu/venushacks-fosters-gender-inclusivity-in-stem, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:59,780 - Worker-1 - INFO - Downloaded http://www.informatics.uci.edu/explore/faculty-profiles/kai-zheng, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:59,849 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/alexandra-peterson-named-outstanding-ta-in-statistics, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:06:59,875 - Worker-2 - INFO - Downloaded https://www.cs.uci.edu/graduate-student-spotlight-persistence-pays-off-for-ph-d-candidate-ivan-de-oliveira-nunes, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:00,290 - Worker-1 - INFO - Downloaded http://www.ics.uci.edu/~zhengkai, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:00,403 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/new-faculty-spotlight-statistics-professor-bin-nan-welcomes-collaboration, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:00,404 - Worker-0 - INFO - Downloaded https://www.cs.uci.edu/phys-org-an-artificial-neural-network-joins-the-fight-against-receding-glaciers-ph-d-student-daniel-cheng-mentioned, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:00,906 - Worker-1 - INFO - Downloaded https://www.informatics.uci.edu/explore/faculty-profiles/sean-young, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:00,939 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/uci-news-uci-neurobiologists-aim-to-identify-biomarkers-for-alzheimers-disease-dan-gillen-mentioned, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:00,951 - Worker-0 - INFO - Downloaded https://www.cs.uci.edu/computer-science-undergraduate-pratyush-muthukumar-named-2021-goldwater-scholar, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:01,451 - Worker-1 - INFO - Downloaded https://www.informatics.uci.edu/explore/faculty-profiles/andre-van-der-hoek, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:01,474 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/statistics-department-welcomes-two-new-faculty-for-2017, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:01,475 - Worker-2 - INFO - Downloaded https://www.cs.uci.edu/uci-finishes-in-top-5-at-national-collegiate-cyber-defense-competition, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:01,976 - Worker-1 - INFO - Downloaded https://cyberclub.ics.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:02,047 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/ucis-data-science-program-ranked-18th-in-nation-by-bcss, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:02,048 - Worker-0 - INFO - Downloaded https://www.cs.uci.edu/uci-news-michael-franz-wins-acm-chuck-thacker-breakthrough-in-computing-award, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:02,576 - Worker-2 - INFO - Downloaded https://cyberclub.ics.uci.edu/blog/TSEC-talk, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:02,592 - Worker-0 - INFO - Downloaded https://www.cs.uci.edu/uci-news-risk-to-reward-one-alums-journey, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:02,592 - Worker-1 - INFO - Downloaded https://www.stat.uci.edu/gillen-furthers-healthcare-research-with-two-nih-grants, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:03,064 - Worker-2 - INFO - Downloaded https://cyberclub.ics.uci.edu/board, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:03,127 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/mazmanian-regan-and-shahbaba-appointed-decade-graduate-faculty-mentors, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:03,128 - Worker-1 - INFO - Downloaded https://www.cs.uci.edu/3416-2, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:03,652 - Worker-1 - INFO - Downloaded https://www.cs.uci.edu/michael-franz-wins-acm-chuck-thacker-breakthrough-in-computing-award, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:03,655 - Worker-0 - INFO - Downloaded https://cyberclub.ics.uci.edu/sponsor, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:03,690 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/2017-ics-deans-award-winners, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:04,170 - Worker-0 - INFO - Downloaded https://cyberclub.ics.uci.edu/blog, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:04,204 - Worker-1 - INFO - Downloaded https://www.cs.uci.edu/computer-science-student-cheng-zhang-receives-facebook-ph-d-fellowship, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:04,222 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/data-scientist-ranked-top-u-s-job-by-glassdoor, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:04,733 - Worker-2 - INFO - Downloaded https://www.cs.uci.edu/uci-news-uci-advances-to-national-cyber-defense-competition-following-historic-regionals-win, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:04,779 - Worker-1 - INFO - Downloaded https://www.ics.uci.edu/~chengz20, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:04,782 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/ph-d-students-gao-and-cruz-receive-asa-paper-awards, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:05,293 - Worker-2 - INFO - Downloaded http://www.ics.uci.edu/community/news/view_news?id=1115, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:05,303 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/ombao-uci-space-time-modeling-group-contribute-to-new-handbook-of-neuroimaging-data-analysis, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:05,309 - Worker-1 - INFO - Downloaded https://www.cs.uci.edu/alumni-spotlight-archana-senthilkumars-real-world-education-helps-bring-fictional-worlds-to-life, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:05,771 - Worker-2 - INFO - Downloaded https://www.informatics.uci.edu/students-present-blueprints-for-new-ar-vr-theater-experiences, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:05,892 - Worker-0 - INFO - Downloaded https://www.cs.uci.edu/new-wics-mentorship-program-helps-high-school-girls-explore-computer-science, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:05,893 - Worker-1 - INFO - Downloaded https://www.stat.uci.edu/statistics-ph-d-students-wang-gao-win-enar-distinguished-paper-awards, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:06,350 - Worker-2 - INFO - Downloaded https://wics.ics.uci.edu/about-community-outreach, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:06,428 - Worker-0 - INFO - Downloaded https://www.cs.uci.edu/uci-health-a-digital-prosthetic-eye-for-the-21st-century-ian-harris-mentioned, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:06,428 - Worker-1 - INFO - Downloaded https://www.stat.uci.edu/3-ics-researchers-named-aaas-fellows, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:06,884 - Worker-2 - INFO - Downloaded https://wics.ics.uci.edu/apply-to-wics-committee, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:06,959 - Worker-0 - INFO - Downloaded https://www.cs.uci.edu/ucis-computer-science-program-ranked-4th-by-best-value-schools, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:06,961 - Worker-1 - INFO - Downloaded https://www.stat.uci.edu/los-angeles-times-why-giving-people-5-to-take-a-government-survey-is-money-well-spent-by-jessica-utts, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:07,440 - Worker-2 - INFO - Downloaded https://wics.ics.uci.edu/exploreics, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:07,502 - Worker-0 - INFO - Downloaded https://www.cs.uci.edu/tippers-wins-naval-information-warfare-systems-command-navwar-innovation-award, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:07,505 - Worker-1 - INFO - Downloaded https://www.stat.uci.edu/nuno-pluta-receive-graduate-statistics-award, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:07,944 - Worker-2 - INFO - Downloaded https://tippersweb.ics.uci.edu/team, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:08,029 - Worker-0 - INFO - Downloaded https://www.cs.uci.edu/ics-researchers-work-to-find-clarity-for-covid-long-haulers, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:08,030 - Worker-1 - INFO - Downloaded https://www.stat.uci.edu/ics-welcomes-four-new-faculty-members-for-fall-2016-quarter, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:08,561 - Worker-1 - INFO - Downloaded https://www.stat.uci.edu/new-dean-named-for-ics, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:08,562 - Worker-0 - INFO - Downloaded https://www.cs.uci.edu/the-new-york-times-many-long-covid-patients-had-no-symptoms-from-their-initial-infection-nikil-dutt-and-yong-huang-referenced, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:08,578 - Worker-2 - INFO - Downloaded https://wics.ics.uci.edu/category/news, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:09,076 - Worker-2 - INFO - Downloaded https://wics.ics.uci.edu/category/news/wics-meeting, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:09,127 - Worker-1 - INFO - Downloaded https://www.stat.uci.edu/nsf-awards-professor-shahbaba-uci-team-250k-for-big-data-analysis-research, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:09,147 - Worker-0 - INFO - Downloaded https://www.cs.uci.edu/allyship-webinar-provides-practical-advice-on-advocating-for-inclusivity, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:09,591 - Worker-2 - INFO - Downloaded http://www.ics.uci.edu/~babaks/Site/Home.html, status <404>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:09,660 - Worker-1 - INFO - Downloaded https://www.cs.uci.edu/hackuci-2021-award-winning-hacks-from-home, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:09,661 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/stern-receives-degroot-prize-for-bayesian-data-analysis, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:10,170 - Worker-2 - INFO - Downloaded https://wics.ics.uci.edu/category/news/wics-meeting/project-meeting, status <601>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:10,200 - Worker-1 - INFO - Downloaded https://www.stat.uci.edu/dean-sterns-co-led-csafe-cited-in-white-house-impact-report, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:10,216 - Worker-0 - INFO - Downloaded https://www.cs.uci.edu/uci-students-cultivate-culture-of-innovation-with-new-vc-fund, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:10,748 - Worker-0 - INFO - Downloaded https://www.cs.uci.edu/3338-2, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:10,754 - Worker-1 - INFO - Downloaded https://wics.ics.uci.edu/event/spring-2022-week-4-projects-with-chipotle, status <601>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:10,773 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/gillen-ombao-named-asa-fellows, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:11,249 - Worker-1 - INFO - Downloaded https://www.ics.uci.edu/community/news/view_news?id=1813, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:11,273 - Worker-0 - INFO - Downloaded https://www.cs.uci.edu/ics-researchers-publish-novel-paper-on-system-design-for-virtual-beings, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:11,299 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/wsj-foreign-students-seen-cheating-more-than-domestic-ones-hancock-quoted, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:11,796 - Worker-1 - INFO - Downloaded https://www.ics.uci.edu/community/news/view_news?id=1707, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:11,833 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/two-statistics-ph-d-students-receive-honorable-mentions-in-nsf-grfp, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:11,834 - Worker-2 - INFO - Downloaded https://www.cs.uci.edu/professor-amiri-sanis-research-group-wins-2020-android-security-and-privacy-research-aspire-award, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:12,310 - Worker-1 - INFO - Downloaded https://www.ics.uci.edu/~yingtong/index.html, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:12,364 - Worker-2 - INFO - Downloaded https://www.cs.uci.edu/professor-sameer-singh-awarded-fairness-in-ai-grant-from-nsf-and-amazon, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:12,368 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/jessica-utts-an-ambassador-for-statistics, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:12,881 - Worker-0 - INFO - Downloaded https://www.cs.uci.edu/the-zotbins-team-continues-to-grow-supporting-ucis-move-to-zero-waste, status <601>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:12,919 - Worker-2 - INFO - Downloaded https://www.ics.uci.edu/community/news/view_news.php?id=1709, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:12,920 - Worker-1 - INFO - Downloaded https://www.stat.uci.edu/statistics-ph-d-student-receives-enar-distinguished-student-paper-award, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:13,470 - Worker-1 - INFO - Downloaded https://www.ics.uci.edu/community/news/view_news.php?id=1710, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:13,489 - Worker-2 - INFO - Downloaded https://www.cs.uci.edu/uci-news-uci-institute-for-future-health-to-harness-technology-to-build-personalized-health-model, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:13,489 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/faculty/annie-qu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:14,003 - Worker-2 - INFO - Downloaded https://www.ics.uci.edu/community/news/view_news.php?id=1712, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:14,027 - Worker-1 - INFO - Downloaded https://www.cs.uci.edu/a-look-at-health-and-technology-trends-for-2021-and-beyond, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:14,052 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/covid19/incidence-maps.html, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:14,552 - Worker-2 - INFO - Downloaded https://www.ics.uci.edu/~yunanc, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:14,582 - Worker-1 - INFO - Downloaded https://www.stat.uci.edu/oc_covid_model, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:14,583 - Worker-0 - INFO - Downloaded https://www.cs.uci.edu/professor-sharad-mehrotra-named-ieee-fellow, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:15,150 - Worker-1 - INFO - Downloaded https://www.ics.uci.edu/~claram, status <403>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:15,166 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/uci-launches-new-professional-program-master-of-data-science, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:15,183 - Worker-0 - INFO - Downloaded https://www.cs.uci.edu/computer-science-and-software-engineering-professional-programs-showcase-student-success, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:15,644 - Worker-1 - INFO - Downloaded https://www.ics.uci.edu/community/news/view_news?id=1623, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:15,715 - Worker-2 - INFO - Downloaded https://www.cs.uci.edu/uci-news-uci-researchers-use-deep-learning-to-identify-gene-regulation-at-single-cell-level-xiaohui-xie-quoted, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:15,732 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/a-look-at-health-and-technology-trends-for-2021-and-beyond, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:16,170 - Worker-1 - INFO - Downloaded https://www.ics.uci.edu/community/news/view_news.php?id=1624, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:16,246 - Worker-2 - INFO - Downloaded https://www.cs.uci.edu/ieee-computer-society-announces-2021-fellows-sharad-mehrotra-named, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:16,267 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/amstat-news-celebrating-women-in-statistics-2021-annie-qu-mentioned, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:16,694 - Worker-1 - INFO - Downloaded https://www.ics.uci.edu/community/news/view_news.php?id=1623, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:16,780 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/three-ics-students-receive-2021-nsf-graduate-research-fellowships, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:16,783 - Worker-2 - INFO - Downloaded https://www.cs.uci.edu/senior-spotlight-daniel-afework-finds-opportunities-to-learn-everywhere, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:17,244 - Worker-1 - INFO - Downloaded https://www.ics.uci.edu/community/news/view_news.php?id=1625, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:17,349 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/uci-news-hal-s-stern-is-named-uci-provost-and-executive-vice-chancellor, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:17,350 - Worker-0 - INFO - Downloaded https://www.cs.uci.edu/professors-gillian-hayes-and-sharad-mehrotra-named-distinguished-members-of-the-acm, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:17,780 - Worker-1 - INFO - Downloaded https://www.ics.uci.edu/community/news/view_news.php?id=1627, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:17,886 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/exploring-and-strengthening-sports-stats-at-uci, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:17,904 - Worker-2 - INFO - Downloaded https://www.cs.uci.edu/ics-researchers-win-best-paper-award-for-detecting-covid-19-misinformation, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:18,302 - Worker-1 - INFO - Downloaded https://iasl.ics.uci.edu/people/yoshitom, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:18,428 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/uci-news-efforts-to-treat-covid-19-patients-chronicled-in-uc-health-medications-data-bin-nan-mentioned, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:18,428 - Worker-0 - INFO - Downloaded https://www.cs.uci.edu/uci-launches-new-professional-program-master-of-data-science, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:18,837 - Worker-1 - INFO - Downloaded https://iasl.ics.uci.edu/publications, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:18,987 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/teach-access-recognizes-mine-dogucu-as-curriculum-development-awardee, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:19,007 - Worker-2 - INFO - Downloaded https://www.cs.uci.edu/faculty-spotlight-professor-jing-zhang-works-to-advance-genomics, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:19,415 - Worker-1 - INFO - Downloaded https://iasl.ics.uci.edu/research, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:19,539 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/a-bold-decision-to-study-abroad-in-high-school-led-sirui-hu-to-uci-and-beyond-data-science-major, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:19,558 - Worker-0 - INFO - Downloaded https://www.cs.uci.edu/ics-researchers-receive-1-5-million-nsf-grant-for-smart-water-infrastructures, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:19,962 - Worker-1 - INFO - Downloaded https://iasl.ics.uci.edu/people, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:20,091 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/capstone-program-showcases-growing-talent-of-ics-students, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:20,092 - Worker-2 - INFO - Downloaded https://www.cs.uci.edu/computer-science-researchers-win-best-paper-award-for-work-on-smart-community-monitoring, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:20,482 - Worker-1 - INFO - Downloaded https://iasl.ics.uci.edu/people/ali-tazarv, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:20,614 - Worker-2 - INFO - Downloaded https://www.cs.uci.edu/computer-science-researchers-contribute-to-editors-choice-paper-on-deep-learning-and-hair-loss, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:20,633 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/the-resilience-of-the-class-of-2021, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:21,011 - Worker-1 - INFO - Downloaded https://iasl.ics.uci.edu/people/forough-shirin-abkenar, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:21,132 - Worker-2 - INFO - Downloaded https://www.cs.uci.edu/virtual-workshop-on-privacy-aspects-of-contact-tracing-oct-2, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:21,150 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/qu-appointed-ims-program-secretary, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:21,541 - Worker-1 - INFO - Downloaded https://iasl.ics.uci.edu/people/sharon-l-g-contreras, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:21,654 - Worker-2 - INFO - Downloaded https://www.cs.uci.edu/ics-team-takes-1st-place-at-baidus-autodriving-capture-the-flag-competition, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:21,674 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/mine-dogucu-receives-young-investigator-award, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:22,054 - Worker-1 - INFO - Downloaded https://iasl.ics.uci.edu/people/marco-levorato, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:22,187 - Worker-2 - INFO - Downloaded https://www.cs.uci.edu/alumni-spotlight-dan-woolleys-success-in-building-from-scratch, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:22,191 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/choc-teams-up-with-uci-to-offer-pediatric-capstone-project-in-data-science, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:22,580 - Worker-1 - INFO - Downloaded https://iasl.ics.uci.edu/people/armin-karamzade, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:22,715 - Worker-2 - INFO - Downloaded https://www.cs.uci.edu/professor-rina-dechter-receives-2020-classic-paper-award, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:22,749 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/from-theory-to-practice-ucis-machine-learning-hackathon-delivers, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:23,088 - Worker-1 - INFO - Downloaded https://iasl.ics.uci.edu/people/anas-alsoliman, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:23,237 - Worker-2 - INFO - Downloaded https://www.cs.uci.edu/women-in-technology-at-uci-first-event-offers-encouragement-and-empowerment, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:23,266 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/uci-news-uci-led-study-finds-disparities-in-o-c-rates-of-covid-19-infection-mortality-vladimir-minin-mentioned, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:23,603 - Worker-1 - INFO - Downloaded https://iasl.ics.uci.edu/people/peyman-tehrani, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:23,763 - Worker-2 - INFO - Downloaded https://www.cs.uci.edu/faculty-spotlight-rina-dechter-is-pushing-the-frontiers-of-research, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:23,773 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/lifetime-achievement-and-dinsdale-awards-double-the-accolades-for-professor-emerita-jessica-utts, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:24,121 - Worker-1 - INFO - Downloaded https://iasl.ics.uci.edu/people/tim-johnsen, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:24,282 - Worker-2 - INFO - Downloaded https://www.cs.uci.edu/uci-ranked-25th-in-undergraduate-computer-science-programs-by-u-s-news-world-report, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:24,302 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/harnessing-the-data-revolution-ics-researchers-and-collaborators-receive-1-5m-data-science-grant, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:24,630 - Worker-1 - INFO - Downloaded https://studentcouncil.ics.uci.edu/index.html, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:24,794 - Worker-2 - INFO - Downloaded https://www.cs.uci.edu/professor-mandt-awarded-nsf-grant-on-deep-variational-data-compression, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:24,815 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/stephan-mandt-and-collaborators-receive-3-5-million-to-study-machine-learning-for-climate-science, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:25,148 - Worker-1 - INFO - Downloaded https://iasl.ics.uci.edu/people/yoshitomo-matsubara, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:25,304 - Worker-2 - INFO - Downloaded https://www.cs.uci.edu/professors-amiri-sani-and-tsudik-investigate-techniques-to-authenticate-photos-and-videos, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:25,324 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/elizabeth-lee-wins-data-science-for-social-good-award, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:25,688 - Worker-1 - INFO - Downloaded https://www.ics.uci.edu/community/news/view_news?id=2031, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:25,816 - Worker-2 - INFO - Downloaded https://www.cs.uci.edu/professor-sameer-singh-receives-nsf-grant-to-explore-and-address-vulnerabilities-in-machine-learning-explanations, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:25,837 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/statistics-ph-d-student-tianyu-pan-wins-two-competitive-awards, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:26,211 - Worker-1 - INFO - Downloaded https://www.ics.uci.edu/community/news/view_news.php?id=2030, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:26,331 - Worker-2 - INFO - Downloaded https://www.cs.uci.edu/workflow-colleges-reopen-with-an-assist-from-tech-sharad-mehrotra-quoted, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:26,348 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/beyond-the-t-test-uci-professors-publish-new-statistical-guidance-for-neuroscience-research, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:26,728 - Worker-1 - INFO - Downloaded https://www.ics.uci.edu/community/news/view_news.php?id=2031, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:26,846 - Worker-2 - INFO - Downloaded https://www.cs.uci.edu/twiml-ai-podcast-beyond-accuracy-behavioral-testing-of-nlp-models-with-sameer-singh, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:26,866 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/disney-research-supports-stephan-mandts-work-on-generative-modeling-with-50000-gift, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:27,256 - Worker-1 - INFO - Downloaded https://www.ics.uci.edu/community/news/view_news.php?id=2029, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:27,375 - Worker-2 - INFO - Downloaded https://www.cs.uci.edu/ics-undergrad-adarsh-pachori-co-founds-ai-driven-recruiting-app, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:27,376 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/graduate-student-spotlight-mds-ambassador-adelynn-paik-shares-her-academic-professional-goals, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:27,791 - Worker-1 - INFO - Downloaded https://www.ics.uci.edu/community/news/view_news.php?id=2028, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:27,951 - Worker-0 - INFO - Downloaded https://www.cs.uci.edu/nature-millions-of-students-are-returning-to-us-universities-in-a-vast-unplanned-pandemic-experiment-sharad-mehrotra-quoted, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:27,968 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/tech-trends-for-2022, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:28,316 - Worker-1 - INFO - Downloaded https://www.ics.uci.edu/community/news/view_news.php?id=2027, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:28,463 - Worker-0 - INFO - Downloaded https://www.cs.uci.edu/professor-amiri-sani-ucr-professor-receive-500000-nsf-award-to-improve-security-of-os-kernels, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:28,497 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/xu-receives-slds-student-paper-award, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:28,849 - Worker-1 - INFO - Downloaded https://www.ics.uci.edu/community/news/view_news.php?id=2026, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:28,990 - Worker-0 - INFO - Downloaded https://www.cs.uci.edu/security-vulnerability-in-self-driving-cars-unveils-achilles-heel-of-sensor-fusion, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:29,016 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/4-ics-professors-among-7-uci-researchers-named-aaas-fellows, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:29,372 - Worker-1 - INFO - Downloaded https://www.ics.uci.edu/community/news/view_news.php?id=2025, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:29,523 - Worker-0 - INFO - Downloaded https://www.cs.uci.edu/sameer-singh-wins-best-paper-award-at-acl-2020-for-tool-that-test-nlp-models, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:29,551 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/black-history-month-sharing-resources-to-expand-diversity-in-tech, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:29,907 - Worker-1 - INFO - Downloaded https://www.ics.uci.edu/community/news/view_news?id=1532, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:30,045 - Worker-0 - INFO - Downloaded https://www.cs.uci.edu/ics-team-explores-distance-based-mental-health-services-for-minority-students, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:30,122 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/uci-announces-launch-of-institute-for-precision-health, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:30,443 - Worker-1 - INFO - Downloaded https://www.ics.uci.edu/community/news/view_news.php?id=1531, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:30,593 - Worker-0 - INFO - Downloaded https://www.cs.uci.edu/ics-alumnus-mamadou-diallo-wins-galileo-award-for-cyber-science-technology, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:30,644 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/statistics-ph-d-student-isaac-goldstein-receives-la-verne-noyes-fellowship, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:30,967 - Worker-1 - INFO - Downloaded https://www.ics.uci.edu/community/news/view_news.php?id=1530, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:31,106 - Worker-0 - INFO - Downloaded https://www.cs.uci.edu/health-it-analytics-uci-uses-campus-wi-fi-to-test-covid-19-contact-tracing-app, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:31,184 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/womens-history-month-resources-for-empowering-women-in-tech, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:31,486 - Worker-1 - INFO - Downloaded https://www.ics.uci.edu/community/news/view_news.php?id=1529, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:31,618 - Worker-0 - INFO - Downloaded https://www.cs.uci.edu/venturebeat-ai-researchers-create-testing-tool-to-find-bugs-in-nlp-from-amazon-google-and-microsoft, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:31,713 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/university-of-california-health-and-cdph-covid-modeling-consortium-awards-grant-funding-to-further-covid-19-data-research, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:32,009 - Worker-1 - INFO - Downloaded https://www.ics.uci.edu/community/news/view_news.php?id=1528, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:32,142 - Worker-0 - INFO - Downloaded https://www.cs.uci.edu/uci-news-uci-researchers-use-campus-as-test-bed-for-coronavirus-contact-tracing-system, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:32,244 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/uci-news-uci-researchers-receive-state-funding-to-study-pandemic-forecasting, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:32,532 - Worker-1 - INFO - Downloaded https://www.ics.uci.edu/community/news/view_news.php?id=1527, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:32,673 - Worker-0 - INFO - Downloaded https://www.cs.uci.edu/computer-science-professor-pierre-baldi-involved-in-multiple-multidisciplinary-collaborations, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:32,757 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/uci-deep-learning-researchers-advance-theory-and-practice-of-data-compression, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:33,052 - Worker-1 - INFO - Downloaded https://www.physics.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:33,191 - Worker-0 - INFO - Downloaded https://www.cs.uci.edu/uci-news-five-uci-junior-faculty-named-hellman-fellows-for-2020-21-sameer-singh-named, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:33,278 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/uci-news-ucis-graduate-programs-shine-in-u-s-news-world-report-rankings, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:33,716 - Worker-0 - INFO - Downloaded https://www.cs.uci.edu/tabellion-provides-secure-legal-contracts-on-mobile-devices, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:33,791 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/mds-student-ty-shao-aims-to-make-an-impact-in-healthcare, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:34,319 - Worker-2 - INFO - Downloaded https://www.cs.uci.edu/cs-researchers-vikram-narayanan-and-anton-burtsev-win-best-paper-award, status <200>, using cache ('styx.ics.uci.edu', 9006). +2022-04-23 02:07:34,832 - Worker-2 - INFO - Downloaded https://www.stat.uci.edu/dr-joni-ricks-oddie-applies-data-driven-lens-in-run-for-long-beach-city-council, status <200>, using cache ('styx.ics.uci.edu', 9006). diff --git a/spacetime-crawler4py-master/crawler/frontier.py b/spacetime-crawler4py-master/crawler/frontier.py index dd41788..a4ac470 100644 --- a/spacetime-crawler4py-master/crawler/frontier.py +++ b/spacetime-crawler4py-master/crawler/frontier.py @@ -4,6 +4,8 @@ import shelve from threading import Thread, Lock,Semaphore from queue import Queue, Empty +import time + from utils import get_logger, get_urlhash, normalize from scraper import is_valid from datacollection import * @@ -27,6 +29,7 @@ class Frontier(object): self.balance_index = 0 + #Semaphore for each domain to keep each domain noice and tidy with politeness self.domain_semaphores = [Lock(),Lock(),Lock(),Lock(),Lock()] #Local data lock @@ -37,6 +40,7 @@ class Frontier(object): self.file_2_mutex = Lock() self.file_3_mutex = Lock() self.file_4_mutex = Lock() + # data collection is going to happen in the frontier # uniques encompass overall unique links @@ -88,8 +92,19 @@ class Frontier(object): ###CRITICAL SECTION self.data_mutex.acquire() try: - hold = self.to_be_downloaded[balance_index].pop() - balance_index = balance_index + 1 + initial = self.balance_index + print("Initial " + str(initial)) + self.balance_index = self.balance_index + 1 + if self.balance_index > 4: + self.balance_index = 0 + while not self.to_be_downloaded[self.balance_index]: + self.balance_index = self.balance_index + 1 + if self.balance_index > 4: + self.balance_index = 0 + if self.balance_index == initial: + self.data_mutex.release() + return None + hold = self.to_be_downloaded[self.balance_index].pop() self.data_mutex.release() return hold except IndexError: @@ -111,14 +126,12 @@ class Frontier(object): urlhash = get_urlhash(url) ##CRITICAL SECTION - self.data_mutex.acquire() if urlhash not in self.save: # This should not happen. self.logger.error( f"Completed url {url}, but have not seen it before.") self.save[urlhash] = (url, True) self.save.sync() - self.data_mutex.release() ##CRITICAL SECTION @@ -134,7 +147,8 @@ class Frontier(object): elif "today.uci.edu/department/information_computer_sciences/" in url: return 4 else: - println("ERROR") + print(url) + print("ERROR") @@ -149,6 +163,12 @@ class Frontier(object): def release_data_mutex(self): return self.data_mutex.release() + + def acquire_234_mutex(self): + return self.file_2_3_4_mutex.acquire() + + def release_234_mutex(self): + return self.file_2_3_4_mutex.release() def q1(self, url): @@ -162,11 +182,11 @@ class Frontier(object): self.file_1_mutex.acquire() if (os.path.exists(my_filename)): f = open(my_filename, 'a') - f.write(str(removeFragment(url))) + f.write(str(removeFragment(url)) + "\n") f.close() else: f = open(my_filename, 'w') - f.write(str(removeFragment(url))) + f.write(str(removeFragment(url)) + "\n") f.close() self.file_1_mutex.release() @@ -177,12 +197,13 @@ class Frontier(object): if resp.status != 200: return - self.file_2_mutex.acquire() + tic = time.perf_counter() path_to_script = os.path.dirname(os.path.abspath(__file__)) my_filename = os.path.join(path_to_script, "q2.txt") try: tempTok = tokenize(resp) + self.file_2_mutex.acquire() if len(tempTok) > self.max: self.max = len(tempTok) self.longest = url @@ -191,17 +212,21 @@ class Frontier(object): f.close() except: print("resp dying for some reason ?") + self.file_2_mutex.release() - - tempTok = removeStopWords(tempTok) - computeFrequencies(tempTok, self.grand_dict) + toc = time.perf_counter() + print(f"Took {toc - tic:0.4f} seconds to save file 2 !") + + tic = time.perf_counter() + tempTok = removeStopWords(tempTok) + self.file_3_mutex.acquire() + computeFrequencies(tempTok, self.grand_dict) # rakslice (8 May 2013) Stackoverflow. https://stackoverflow.com/questions/16430258/creating-a-python-file-in-a-local-directory # this saves to the local directory, so I can constantly access the right file and check if it exists or not path_to_script = os.path.dirname(os.path.abspath(__file__)) my_filename = os.path.join(path_to_script, "q3.txt") - self.file_3_mutex.acquire() f = open(my_filename, "w") sortedGrandDict = {k: v for k, v in sorted(self.grand_dict.items(), key=lambda item: item[1], reverse = True)} @@ -213,11 +238,16 @@ class Frontier(object): f.write("{}: {}\n".format(k, v)) i += 1 f.close() - self.file_3_mutex.release() + toc = time.perf_counter() + print(f"Took {toc - tic:0.4f} seconds to save file 3 !") + + tic = time.perf_counter() + fragless = removeFragment(url) domain = findDomains(fragless.netloc) + self.file_4_mutex.acquire() if domain[1] == 'ics': if domain[0] not in self.ics: self.ics[domain[0]] = urlData(url, domain[0], domain[1]) @@ -231,11 +261,13 @@ class Frontier(object): my_filename = os.path.join(path_to_script, "q4.txt") # creating text file for question 4 - self.file_4_mutex.acquire() sortedDictKeys = sorted(self.ics.keys()) f = open(my_filename, "w") for i in sortedDictKeys: f.write("{url}, {num}".format(url = self.ics[i].getNiceLink(), num = len(self.ics[i].getUniques()))) f.close() self.file_4_mutex.release() + + toc = time.perf_counter() + print(f"Took {toc - tic:0.4f} seconds to save file 4 !") \ No newline at end of file diff --git a/spacetime-crawler4py-master/crawler/worker.py b/spacetime-crawler4py-master/crawler/worker.py index 4ccca16..f141884 100644 --- a/spacetime-crawler4py-master/crawler/worker.py +++ b/spacetime-crawler4py-master/crawler/worker.py @@ -32,6 +32,24 @@ class Worker(Thread): toc = time.perf_counter() print(f"Took {toc - tic:0.4f} seconds to do download url") + self.logger.info( + f"Downloaded {tbd_url}, status <{resp.status}>, " + f"using cache {self.config.cache_server}.") + + tic = time.perf_counter() + scraped_urls = scraper.scraper(tbd_url, resp) + toc = time.perf_counter() + print(f"Took {toc - tic:0.4f} seconds to do scrape url") + + tic = time.perf_counter() + print(self.frontier.acquire_data_mutex()) + for scraped_url in scraped_urls: + self.frontier.add_url(scraped_url) + self.frontier.mark_url_complete(tbd_url) + self.frontier.release_data_mutex() + toc = time.perf_counter() + print(f"Took {toc - tic:0.4f} seconds to do add_url stuffs") + tic = time.perf_counter() self.frontier.q1(tbd_url) toc = time.perf_counter() @@ -41,30 +59,12 @@ class Worker(Thread): self.frontier.q234(tbd_url, resp) toc = time.perf_counter() print(f"Took {toc - tic:0.4f} seconds to do log q234 url") + - self.logger.info( - f"Downloaded {tbd_url}, status <{resp.status}>, " - f"using cache {self.config.cache_server}.") - - tic = time.perf_counter() - scraped_urls = scraper.scraper(tbd_url, resp) - toc = time.perf_counter() - print(f"Took {toc - tic:0.4f} seconds to do scrape url") - - - tic = time.perf_counter() - self.frontier.acquire_data_mutex() - for scraped_url in scraped_urls: - self.frontier.add_url(scraped_url) - self.frontier.release_data_mutex() - toc = time.perf_counter() - print(f"Took {toc - tic:0.4f} seconds to do add_url stuffs") + - tic = time.perf_counter() - self.frontier.mark_url_complete(tbd_url) - toc = time.perf_counter() - print(f"Took {toc - tic:0.4f} seconds to do store stuffs") while start + self.config.time_delay > time.perf_counter(): + print("Sleeping") time.sleep(self.config.time_delay/5) self.frontier.release_polite(tbd_url) diff --git a/spacetime-crawler4py-master/scraper.py b/spacetime-crawler4py-master/scraper.py index 62758b2..a08bd7e 100644 --- a/spacetime-crawler4py-master/scraper.py +++ b/spacetime-crawler4py-master/scraper.py @@ -10,13 +10,16 @@ from bs4 import BeautifulSoup from robotsokay import * def scraper(url, resp): + links = extract_next_links(url, resp) + links_valid = set() #valid_links = open("valid_links.txt",'a') #invalid_links = open("invalid_links.txt",'a') - + tic = time.perf_counter() + for link in links: if is_valid(link): links_valid.add(link) @@ -26,6 +29,8 @@ def scraper(url, resp): #invalid_links.write(link + "\n") pass + toc = time.perf_counter() + print(f"Took {toc - tic:0.4f} seconds to validate !!!") return links_valid @@ -42,7 +47,7 @@ def extract_next_links(url, resp): pages = set() if resp.status == 200: #do stuff - soup = BeautifulSoup(resp.raw_response.content) + soup = BeautifulSoup(resp.raw_response.content,'lxml') #tempFile = open("test6.txt", 'a') #Getting all the links, href = true means at least theres a href value, dont know what it is yet for link in soup.find_all('a', href=True): @@ -65,11 +70,18 @@ def extract_next_links(url, resp): # don't know if this is too expensive, otherwise idk # takes parsed url and if not ok on robots goes next, else we can write file - parsed = urlparse(href_link) + + """ + #For now robot checking too time expensive and incorrectly implemented + parsed = urlparse(href_link) + tic = time.perf_counter() + print(parsed) if not robots_are_ok(parsed): continue - - + toc = time.perf_counter() + print(f"Took {toc - tic:0.4f} seconds to robots_are_ok !!!") + """ + #tempFile.write(href_link + "\n") #Adding to the boi wonder pages pages.add(href_link)