201 lines
6.9 KiB
Python
201 lines
6.9 KiB
Python
import re
|
|
import urllib.request
|
|
from urllib.parse import urlparse
|
|
from urllib.parse import urljoin
|
|
from bs4 import BeautifulSoup
|
|
from nltk.tokenize import word_tokenize
|
|
from nltk.corpus import stopwords
|
|
from nltk.corpus import words
|
|
import html2text
|
|
import nltk
|
|
#moved all my code to a separted py file and imported it here
|
|
from datacollection import *
|
|
|
|
# nltk.download('stopwords')
|
|
# nltk.download('words')
|
|
# there is another nltk.download() requirement but I removed it so i forgot what it was
|
|
# it'll show in the console/terminal if you run the code i believe. it appeared in mine
|
|
|
|
def scraper(url, resp):
|
|
# initialize set for unique links
|
|
# used a set for elimatining duplicates
|
|
uniques = set()
|
|
# have to add the original url to the unique set
|
|
copyoriginal = url
|
|
uniques.add(removeFragment(copyoriginal))
|
|
|
|
# initializing longest for finding the longest page
|
|
max = -9999
|
|
longest = None
|
|
|
|
# have to do this for the original url
|
|
tok = tokenize(url)
|
|
if len(tok) > max:
|
|
max = len(tok)
|
|
longest = url
|
|
|
|
# grand_dict is a dictionary that contains every word over the entire set of pages (excluding stop words)
|
|
# key: word , value: frequencies
|
|
grand_dict = dict()
|
|
tok = removeStopWords(tok)
|
|
computeFrequencies(tok, grand_dict)
|
|
|
|
# ics is a dict with subdomains
|
|
ics = dict()
|
|
|
|
|
|
links = extract_next_links(url, resp)
|
|
links_valid = list()
|
|
valid_links = open("valid_links.txt",'a')
|
|
invalid_links = open("invalid_links.txt",'a')
|
|
|
|
|
|
for link in links:
|
|
if is_valid(link):
|
|
links_valid.append(link)
|
|
valid_links.write(link + "\n")
|
|
|
|
# Answering q1 for report
|
|
uniques.add(removeFragment(link))
|
|
|
|
# Answering q2
|
|
tempTok = tokenize(link)
|
|
if len(tempTok) > max:
|
|
max = len(tempTok)
|
|
longest = link
|
|
|
|
|
|
# Answering q3
|
|
tempTok = removeStopWords(tempTok)
|
|
computeFrequencies(tempTok, grand_dict)
|
|
|
|
# Answering q4
|
|
fragless = removeFragment(link)
|
|
domain = findDomains(fragless.netloc)
|
|
if domain[1] == 'ics':
|
|
if domain[0] not in ics:
|
|
ics[domain[0]] = urlData(link, domain[0], domain[1])
|
|
else:
|
|
if fragless not in ics[domain[0]].getUniques():
|
|
ics[domain[0]].appendUnique(fragless)
|
|
|
|
|
|
else:
|
|
invalid_links.write("From: " + url + "\n")
|
|
invalid_links.write(link + "\n")
|
|
|
|
# creating text file that includes the number of unique links
|
|
f = open("q1.txt", "w")
|
|
f.write("Number of unique pages: {length}".format(length = len(uniques)))
|
|
f.close()
|
|
|
|
# creating text file for question 2
|
|
f = open("q2.txt", "w")
|
|
f.write("Largest page url: {url} \nLength of page: {length}".format(url = longest, length = max))
|
|
f.close()
|
|
|
|
# creating text file for question 3
|
|
f = open("q3.txt", "w")
|
|
sortedGrandDict = {k: v for k, v in sorted(grand_dict.items(), key=lambda item: item[1], reverse = True)}
|
|
i = 0
|
|
for k, v in sortedGrandDict.items():
|
|
if i == 50:
|
|
break
|
|
else:
|
|
f.write(k, ':', v)
|
|
f.close()
|
|
|
|
# creating text file for question 4
|
|
sortedDictKeys = sorted(ics.keys())
|
|
f = open("q4.txt", "w")
|
|
for i in sortedDictKeys:
|
|
f.write("{url}, {num}".format(url = ics[i].getNiceLink(), num = len(ics[i].getUniques())))
|
|
f.close()
|
|
|
|
return links_valid
|
|
|
|
def extract_next_links(url, resp):
|
|
# Implementation required.
|
|
# url: the URL that was used to get the page
|
|
# resp.url: the actual url of the page
|
|
# resp.status: the status code returned by the server. 200 is OK, you got the page. Other numbers mean that there was some kind of problem.
|
|
# resp.error: when status is not 200, you can check the error here, if needed.
|
|
# resp.raw_response: this is where the page actually is. More specifically, the raw_response has two parts:
|
|
# resp.raw_response.url: the url, again
|
|
# resp.raw_response.content: the content of the page!
|
|
# Return a list with the hyperlinks (as strings) scrapped from resp.raw_response.content
|
|
pages = list()
|
|
if resp.status == 200:
|
|
#do stuff
|
|
soup = BeautifulSoup(resp.raw_response.content)
|
|
tempFile = open("test6.txt", 'a')
|
|
#Getting all the links, href = true means at least theres a href value, dont know what it is yet
|
|
for link in soup.find_all('a', href=True):
|
|
#There is a lot of relative paths stuff here gotta add them
|
|
href_link = link.get('href')
|
|
|
|
#Relative path
|
|
##Suprisingly index fail safe
|
|
#some <a href> give //thenlink.com...
|
|
if href_link.startswith("//"):
|
|
href_link = href_link[2:]
|
|
|
|
#Relative path fixing
|
|
if(href_link.startswith("/")):
|
|
href_link = urljoin(url,href_link)
|
|
|
|
#skipping query with specific actions which mutate the websites and cause a trap
|
|
if "do=" in href_link:
|
|
continue
|
|
|
|
tempFile.write(href_link + "\n")
|
|
#Adding to the boi wonder pages
|
|
pages.append(href_link)
|
|
else:
|
|
print("Page error !")
|
|
return pages
|
|
|
|
#*.ics.uci.edu/*
|
|
#*.cs.uci.edu/*
|
|
#*.informatics.uci.edu/*
|
|
#*.stat.uci.edu/*
|
|
#today.uci.edu/department/information_computer_sciences/*
|
|
|
|
def is_valid(url):
|
|
|
|
|
|
# Decide whether to crawl this url or not.
|
|
# If you decide to crawl it, return True; otherwise return False.
|
|
# There are already some conditions that return False.
|
|
|
|
try:
|
|
#Gotta check if they are in the domain
|
|
parsed = urlparse(url)
|
|
if parsed.scheme not in set(["http", "https"]):
|
|
return False
|
|
elif re.match(
|
|
r".*\.(css|js|bmp|gif|jpe?g|ico"
|
|
+ r"|png|tiff?|mid|mp2|mp3|mp4"
|
|
+ r"|wav|avi|mov|mpeg|ram|m4v|mkv|ogg|ogv|pdf"
|
|
+ r"|ps|eps|tex|ppt|pptx|doc|docx|xls|xlsx|names"
|
|
+ r"|data|dat|exe|bz2|tar|msi|bin|7z|psd|dmg|iso"
|
|
+ r"|epub|dll|cnf|tgz|sha1"
|
|
+ r"|thmx|mso|arff|rtf|jar|csv"
|
|
+ r"|rm|smil|wmv|swf|wma|zip|rar|gz)$",parsed.path.lower()):
|
|
return False
|
|
elif not re.match(
|
|
r".*ics.uci.edu/.*"
|
|
+ r"|.*cs.uci.edu/.*"
|
|
+ r"|.*informatics.uci.edu/.*"
|
|
+ r"|.*stat.uci.edu/.*"
|
|
+ r"|today.uci.edu/department/information_computer_sciences/.*",url):
|
|
return False
|
|
elif parsed.fragment:
|
|
return False
|
|
else:
|
|
return True
|
|
|
|
except TypeError:
|
|
print ("TypeError for ", parsed)
|
|
raise
|