From d0dde4a4dbaaa8a8a2bcaf1c2bd3657bac16eaae Mon Sep 17 00:00:00 2001 From: Hieuhuy Pham Date: Wed, 20 Apr 2022 03:52:14 -0700 Subject: [PATCH 1/2] Fixes error in syntax for new merged code from data collection branch, fixed 'infinite loop', added timers to measure performance of functions. --- spacetime-crawler4py-master/Logs/CRAWLER.log | 0 spacetime-crawler4py-master/Logs/FRONTIER.log | 28 +++++++++++ spacetime-crawler4py-master/Logs/Worker.log | 30 +++++++++++ .../crawler/frontier.py | 49 +++++++++++++++--- spacetime-crawler4py-master/crawler/worker.py | 13 +++++ spacetime-crawler4py-master/datacollection.py | 25 +++++++--- spacetime-crawler4py-master/scraper.py | 50 +++++-------------- 7 files changed, 144 insertions(+), 51 deletions(-) create mode 100644 spacetime-crawler4py-master/Logs/CRAWLER.log create mode 100644 spacetime-crawler4py-master/Logs/FRONTIER.log create mode 100644 spacetime-crawler4py-master/Logs/Worker.log diff --git a/spacetime-crawler4py-master/Logs/CRAWLER.log b/spacetime-crawler4py-master/Logs/CRAWLER.log new file mode 100644 index 0000000..e69de29 diff --git a/spacetime-crawler4py-master/Logs/FRONTIER.log b/spacetime-crawler4py-master/Logs/FRONTIER.log new file mode 100644 index 0000000..7a0fa0e --- /dev/null +++ b/spacetime-crawler4py-master/Logs/FRONTIER.log @@ -0,0 +1,28 @@ +2022-04-20 02:33:03,517 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-20 02:34:22,952 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-20 02:34:41,682 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-20 02:37:03,708 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-20 02:38:32,080 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-20 02:47:05,419 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-20 02:47:20,616 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-20 02:50:20,425 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-20 02:58:18,494 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-20 03:06:07,345 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-20 03:07:45,102 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-20 03:08:46,162 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-20 03:11:05,040 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-20 03:11:21,977 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-20 03:12:18,315 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-20 03:15:53,501 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-20 03:18:48,477 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-20 03:20:00,913 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-20 03:23:14,356 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-20 03:36:00,708 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-20 03:37:24,758 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-20 03:37:24,758 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-20 03:37:42,260 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-20 03:42:16,880 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-20 03:42:23,013 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-20 03:45:40,958 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-20 03:49:54,245 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-20 03:50:02,043 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. diff --git a/spacetime-crawler4py-master/Logs/Worker.log b/spacetime-crawler4py-master/Logs/Worker.log new file mode 100644 index 0000000..f028367 --- /dev/null +++ b/spacetime-crawler4py-master/Logs/Worker.log @@ -0,0 +1,30 @@ +2022-04-20 03:11:10,483 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9009). +2022-04-20 03:11:24,819 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9009). +2022-04-20 03:12:22,276 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9009). +2022-04-20 03:15:59,251 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9009). +2022-04-20 03:18:53,674 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9009). +2022-04-20 03:20:01,017 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9009). +2022-04-20 03:23:14,468 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9009). +2022-04-20 03:26:22,309 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9009). +2022-04-20 03:28:00,535 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9009). +2022-04-20 03:30:54,421 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9009). +2022-04-20 03:36:00,819 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9009). +2022-04-20 03:36:23,270 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/from-theory-to-practice-ucis-machine-learning-hackathon-delivers, status <200>, using cache ('styx.ics.uci.edu', 9009). +2022-04-20 03:36:39,470 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/choc-teams-up-with-uci-to-offer-pediatric-capstone-project-in-data-science, status <200>, using cache ('styx.ics.uci.edu', 9009). +2022-04-20 03:37:07,340 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/mine-dogucu-receives-young-investigator-award, status <200>, using cache ('styx.ics.uci.edu', 9009). +2022-04-20 03:37:24,747 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/qu-appointed-ims-program-secretary, status <200>, using cache ('styx.ics.uci.edu', 9009). +2022-04-20 03:37:24,856 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9009). +2022-04-20 03:37:24,856 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9009). +2022-04-20 03:37:42,359 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9009). +2022-04-20 03:38:03,484 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/from-theory-to-practice-ucis-machine-learning-hackathon-delivers, status <200>, using cache ('styx.ics.uci.edu', 9009). +2022-04-20 03:38:19,620 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/choc-teams-up-with-uci-to-offer-pediatric-capstone-project-in-data-science, status <200>, using cache ('styx.ics.uci.edu', 9009). +2022-04-20 03:38:36,526 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/mine-dogucu-receives-young-investigator-award, status <200>, using cache ('styx.ics.uci.edu', 9009). +2022-04-20 03:38:55,383 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/qu-appointed-ims-program-secretary, status <200>, using cache ('styx.ics.uci.edu', 9009). +2022-04-20 03:39:12,794 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/the-resilience-of-the-class-of-2021, status <200>, using cache ('styx.ics.uci.edu', 9009). +2022-04-20 03:39:29,963 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/capstone-program-showcases-growing-talent-of-ics-students, status <200>, using cache ('styx.ics.uci.edu', 9009). +2022-04-20 03:42:16,982 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9009). +2022-04-20 03:42:23,150 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9009). +2022-04-20 03:45:41,061 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9009). +2022-04-20 03:46:32,554 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/from-theory-to-practice-ucis-machine-learning-hackathon-delivers, status <200>, using cache ('styx.ics.uci.edu', 9009). +2022-04-20 03:49:54,351 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9009). +2022-04-20 03:50:02,145 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9009). diff --git a/spacetime-crawler4py-master/crawler/frontier.py b/spacetime-crawler4py-master/crawler/frontier.py index 1c8d28f..2a95f03 100644 --- a/spacetime-crawler4py-master/crawler/frontier.py +++ b/spacetime-crawler4py-master/crawler/frontier.py @@ -73,12 +73,24 @@ class Frontier(object): self.save.sync() self.to_be_downloaded.append(url) + + + def mark_url_complete(self, url): + urlhash = get_urlhash(url) + if urlhash not in self.save: + # This should not happen. + self.logger.error( + f"Completed url {url}, but have not seen it before.") + + + + # Q1 self.uniques.add(removeFragment(url)) # Q2 tempTok = tokenize(url) - if len(tempTok) > max: + if len(tempTok) > self.max: self.max = len(tempTok) self.longest = url @@ -97,12 +109,35 @@ class Frontier(object): self.ics[domain[0]].appendUnique(fragless) - def mark_url_complete(self, url): - urlhash = get_urlhash(url) - if urlhash not in self.save: - # This should not happen. - self.logger.error( - f"Completed url {url}, but have not seen it before.") + + f = open("q1.txt", "w") + f.write("Number of unique pages: {length}\n".format(length = len(uniques))) + f.close() + + # creating text file for question 2 + f = open("q2.txt", "w") + f.write("Largest page url: {url} \nLength of page: {length}".format(url = longest, length = max)) + f.close() + + # creating text file for question 3 + f = open("q3.txt", "w") + sortedGrandDict = {k: v for k, v in sorted(grand_dict.items(), key=lambda item: item[1], reverse = True)} + i = 0 + for k, v in sortedGrandDict.items(): + if i == 50: + break + else: + f.write("{}: {}\n".format(k, v)) + i += 1 + f.close() + + # creating text file for question 4 + sortedDictKeys = sorted(ics.keys()) + f = open("q4.txt", "w") + for i in sortedDictKeys: + f.write("{url}, {num}".format(url = ics[i].getNiceLink(), num = len(ics[i].getUniques()))) + f.close() + self.save[urlhash] = (url, True) self.save.sync() diff --git a/spacetime-crawler4py-master/crawler/worker.py b/spacetime-crawler4py-master/crawler/worker.py index c92736d..8a514f1 100644 --- a/spacetime-crawler4py-master/crawler/worker.py +++ b/spacetime-crawler4py-master/crawler/worker.py @@ -18,16 +18,29 @@ class Worker(Thread): def run(self): while True: + tic = time.perf_counter() tbd_url = self.frontier.get_tbd_url() + toc = time.perf_counter() + print(f"Took {toc - tic:0.4f} seconds to get_tbd_url") if not tbd_url: self.logger.info("Frontier is empty. Stopping Crawler.") break + tic = time.perf_counter() resp = download(tbd_url, self.config, self.logger) + toc = time.perf_counter() + print(f"Took {toc - tic:0.4f} seconds to do download url") self.logger.info( f"Downloaded {tbd_url}, status <{resp.status}>, " f"using cache {self.config.cache_server}.") + tic = time.perf_counter() scraped_urls = scraper.scraper(tbd_url, resp) + toc = time.perf_counter() + print(f"Took {toc - tic:0.4f} seconds to do scrape url") + tic = time.perf_counter() + for scraped_url in scraped_urls: self.frontier.add_url(scraped_url) self.frontier.mark_url_complete(tbd_url) + toc = time.perf_counter() + print(f"Took {toc - tic:0.4f} seconds to do store stuffs") time.sleep(self.config.time_delay) diff --git a/spacetime-crawler4py-master/datacollection.py b/spacetime-crawler4py-master/datacollection.py index 2aabb80..9a10aa7 100644 --- a/spacetime-crawler4py-master/datacollection.py +++ b/spacetime-crawler4py-master/datacollection.py @@ -1,4 +1,5 @@ import re + import urllib.request from urllib.parse import urlparse from urllib.parse import urljoin @@ -6,10 +7,16 @@ from bs4 import BeautifulSoup from nltk.tokenize import word_tokenize from nltk.corpus import stopwords from nltk.corpus import words +import re import html2text import nltk -# nltk.download('stopwords') -# nltk.download('words') +#nltk.download('stopwords') +#nltk.download('words') +#nltk.download('punkt') + +english_words = words.words() +english_stop_words = stopwords.words('english') + # there is another nltk.download() requirement but I removed it so i forgot what it was # it'll show in the console/terminal if you run the code i believe # it showed in mine @@ -77,7 +84,7 @@ def tokenize(url): # getting connection from url page = urllib.request.urlopen(url) data = page.read() - + valid = re.compile(r'[^a-zA-Z0-9]+') # named it tSoup for merge convience # need the 'lxml' parser for this. # When extract_next_links is called it returns a list full of links with no resp, and I had to find a way to get text from just link. @@ -89,10 +96,15 @@ def tokenize(url): clean_text = ' '.join(tSoup.stripped_strings) token = word_tokenize(clean_text) + clean_token = list() # This used the nltk.corpus and just removes the tokens that aren't words - token = [i for i in token if i.lower() in words.words()] + #token = [i for i in token if i.lower() in english_words] - return token + for word in token: + if not valid.match(word): + clean_token.append(word.lower()) + + return clean_token #added this so the scraper code is not too redundant def computeFrequencies(tokens, d): @@ -103,8 +115,7 @@ def computeFrequencies(tokens, d): d[t] += 1 def removeStopWords(toks): - stopWords = set(stopwords.words('english')) - return [t for t in toks if t.lower() if not t.lower() in stopWords] + return [t for t in toks if t.lower() if not t.lower() in english_stop_words] def removeFragment(u): # turn into a urlparse object diff --git a/spacetime-crawler4py-master/scraper.py b/spacetime-crawler4py-master/scraper.py index 1dbabd0..a490616 100644 --- a/spacetime-crawler4py-master/scraper.py +++ b/spacetime-crawler4py-master/scraper.py @@ -1,7 +1,7 @@ from distutils.filelist import findall from operator import truediv import re - +import time import urllib.request from urllib import robotparser from urllib.parse import urlparse @@ -23,48 +23,24 @@ from datacollection import * def scraper(url, resp): links = extract_next_links(url, resp) + links_valid = list() - valid_links = open("valid_links.txt",'a') - invalid_links = open("invalid_links.txt",'a') + #valid_links = open("valid_links.txt",'a') + #invalid_links = open("invalid_links.txt",'a') for link in links: + tic = time.perf_counter() if is_valid(link): links_valid.append(link) - valid_links.write(link + "\n") + toc = time.perf_counter() + print(f"Took {toc - tic:0.4f} seconds to do validate url") + #valid_links.write(link + "\n") else: - invalid_links.write("From: " + url + "\n") - invalid_links.write(link + "\n") + # invalid_links.write("From: " + url + "\n") + #invalid_links.write(link + "\n") + pass - # Needs to be moved - # creating text file that includes the number of unique links - f = open("q1.txt", "w") - f.write("Number of unique pages: {length}\n".format(length = len(uniques))) - f.close() - - # creating text file for question 2 - f = open("q2.txt", "w") - f.write("Largest page url: {url} \nLength of page: {length}".format(url = longest, length = max)) - f.close() - - # creating text file for question 3 - f = open("q3.txt", "w") - sortedGrandDict = {k: v for k, v in sorted(grand_dict.items(), key=lambda item: item[1], reverse = True)} - i = 0 - for k, v in sortedGrandDict.items(): - if i == 50: - break - else: - f.write("{}: {}\n".format(k, v)) - i += 1 - f.close() - - # creating text file for question 4 - sortedDictKeys = sorted(ics.keys()) - f = open("q4.txt", "w") - for i in sortedDictKeys: - f.write("{url}, {num}".format(url = ics[i].getNiceLink(), num = len(ics[i].getUniques()))) - f.close() return links_valid @@ -111,7 +87,7 @@ def extract_next_links(url, resp): if resp.status == 200: #do stuff soup = BeautifulSoup(resp.raw_response.content) - tempFile = open("test6.txt", 'a') + #tempFile = open("test6.txt", 'a') #Getting all the links, href = true means at least theres a href value, dont know what it is yet for link in soup.find_all('a', href=True): #There is a lot of relative paths stuff here gotta add them @@ -137,7 +113,7 @@ def extract_next_links(url, resp): continue ''' - tempFile.write(href_link + "\n") + #tempFile.write(href_link + "\n") #Adding to the boi wonder pages pages.append(href_link) else: From 58d15918d5753caee1439b14166dda8fa16bd80d Mon Sep 17 00:00:00 2001 From: Hieuhuy Pham Date: Wed, 20 Apr 2022 04:03:58 -0700 Subject: [PATCH 2/2] Change more syntax to get data collection working, check extracturl and sorted links into sets instead of lists to signifcantly reduce url extractions --- spacetime-crawler4py-master/Logs/FRONTIER.log | 4 ++++ spacetime-crawler4py-master/Logs/Worker.log | 6 ++++++ spacetime-crawler4py-master/crawler/frontier.py | 10 +++++----- spacetime-crawler4py-master/scraper.py | 8 ++++---- 4 files changed, 19 insertions(+), 9 deletions(-) diff --git a/spacetime-crawler4py-master/Logs/FRONTIER.log b/spacetime-crawler4py-master/Logs/FRONTIER.log index 7a0fa0e..48ab184 100644 --- a/spacetime-crawler4py-master/Logs/FRONTIER.log +++ b/spacetime-crawler4py-master/Logs/FRONTIER.log @@ -26,3 +26,7 @@ 2022-04-20 03:45:40,958 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. 2022-04-20 03:49:54,245 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. 2022-04-20 03:50:02,043 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-20 03:59:18,104 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-20 04:00:14,031 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-20 04:01:31,386 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. +2022-04-20 04:02:16,043 - FRONTIER - INFO - Found save file frontier.shelve, deleting it. diff --git a/spacetime-crawler4py-master/Logs/Worker.log b/spacetime-crawler4py-master/Logs/Worker.log index f028367..c022b90 100644 --- a/spacetime-crawler4py-master/Logs/Worker.log +++ b/spacetime-crawler4py-master/Logs/Worker.log @@ -28,3 +28,9 @@ 2022-04-20 03:46:32,554 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/from-theory-to-practice-ucis-machine-learning-hackathon-delivers, status <200>, using cache ('styx.ics.uci.edu', 9009). 2022-04-20 03:49:54,351 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9009). 2022-04-20 03:50:02,145 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9009). +2022-04-20 03:52:31,224 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9009). +2022-04-20 03:59:18,220 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9009). +2022-04-20 04:00:14,134 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9009). +2022-04-20 04:01:31,499 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9009). +2022-04-20 04:02:16,153 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu, status <200>, using cache ('styx.ics.uci.edu', 9009). +2022-04-20 04:02:35,264 - Worker-0 - INFO - Downloaded https://www.stat.uci.edu/stephan-mandt-and-collaborators-receive-3-5-million-to-study-machine-learning-for-climate-science, status <200>, using cache ('styx.ics.uci.edu', 9009). diff --git a/spacetime-crawler4py-master/crawler/frontier.py b/spacetime-crawler4py-master/crawler/frontier.py index 2a95f03..162a732 100644 --- a/spacetime-crawler4py-master/crawler/frontier.py +++ b/spacetime-crawler4py-master/crawler/frontier.py @@ -111,17 +111,17 @@ class Frontier(object): f = open("q1.txt", "w") - f.write("Number of unique pages: {length}\n".format(length = len(uniques))) + f.write("Number of unique pages: {length}\n".format(length = len(self.uniques))) f.close() # creating text file for question 2 f = open("q2.txt", "w") - f.write("Largest page url: {url} \nLength of page: {length}".format(url = longest, length = max)) + f.write("Largest page url: {url} \nLength of page: {length}".format(url = self.longest, length = self.max)) f.close() # creating text file for question 3 f = open("q3.txt", "w") - sortedGrandDict = {k: v for k, v in sorted(grand_dict.items(), key=lambda item: item[1], reverse = True)} + sortedGrandDict = {k: v for k, v in sorted(self.grand_dict.items(), key=lambda item: item[1], reverse = True)} i = 0 for k, v in sortedGrandDict.items(): if i == 50: @@ -132,10 +132,10 @@ class Frontier(object): f.close() # creating text file for question 4 - sortedDictKeys = sorted(ics.keys()) + sortedDictKeys = sorted(self.ics.keys()) f = open("q4.txt", "w") for i in sortedDictKeys: - f.write("{url}, {num}".format(url = ics[i].getNiceLink(), num = len(ics[i].getUniques()))) + f.write("{url}, {num}".format(url = self.ics[i].getNiceLink(), num = len(self.ics[i].getUniques()))) f.close() diff --git a/spacetime-crawler4py-master/scraper.py b/spacetime-crawler4py-master/scraper.py index a490616..d903864 100644 --- a/spacetime-crawler4py-master/scraper.py +++ b/spacetime-crawler4py-master/scraper.py @@ -24,7 +24,7 @@ from datacollection import * def scraper(url, resp): links = extract_next_links(url, resp) - links_valid = list() + links_valid = set() #valid_links = open("valid_links.txt",'a') #invalid_links = open("invalid_links.txt",'a') @@ -32,7 +32,7 @@ def scraper(url, resp): for link in links: tic = time.perf_counter() if is_valid(link): - links_valid.append(link) + links_valid.add(link) toc = time.perf_counter() print(f"Took {toc - tic:0.4f} seconds to do validate url") #valid_links.write(link + "\n") @@ -83,7 +83,7 @@ def extract_next_links(url, resp): # resp.raw_response.url: the url, again # resp.raw_response.content: the content of the page! # Return a list with the hyperlinks (as strings) scrapped from resp.raw_response.content - pages = list() + pages = set() if resp.status == 200: #do stuff soup = BeautifulSoup(resp.raw_response.content) @@ -115,7 +115,7 @@ def extract_next_links(url, resp): #tempFile.write(href_link + "\n") #Adding to the boi wonder pages - pages.append(href_link) + pages.add(href_link) else: print("Page error !") return pages