66 lines
2.2 KiB
Python
66 lines
2.2 KiB
Python
from threading import Thread
|
|
import json
|
|
import os
|
|
import shelve
|
|
from bs4 import BeautifulSoup
|
|
from time import perf_counter
|
|
import time
|
|
import pickle
|
|
|
|
import re
|
|
|
|
|
|
#Data process
|
|
from nltk.tokenize import word_tokenize
|
|
from nltk.stem import PorterStemmer
|
|
from sklearn.feature_extraction.text import TfidfVectorizer
|
|
import pandas as pd
|
|
import numpy as np
|
|
from collections import Counter
|
|
|
|
from posting import Posting
|
|
|
|
|
|
import sys
|
|
|
|
class Worker(Thread):
|
|
def __init__(self,indexer,target):
|
|
self.file = target
|
|
self.indexer = indexer
|
|
super().__init__(daemon=True)
|
|
|
|
def run(self):
|
|
print("Target: " + str(self.file))
|
|
ticker = perf_counter()
|
|
file_load = open(self.file)
|
|
data = json.load(file_load)
|
|
soup = BeautifulSoup(data["content"],features="lxml")
|
|
# Gets a cleaner version text comparative to soup.get_text()
|
|
tic = perf_counter()
|
|
clean_text = ' '.join(soup.stripped_strings)
|
|
# Looks for large white space, tabbed space, and other forms of spacing and removes it
|
|
# Regex expression matches for space characters excluding a single space or words
|
|
clean_text = re.sub(r'\s[^ \w]', '', clean_text)
|
|
# Tokenizes text and joins it back into an entire string. Make sure it is an entire string is essential for get_tf_idf to work as intended
|
|
clean_text = " ".join([i for i in clean_text.split() if i != "" and re.fullmatch('[A-Za-z0-9]+', i)])
|
|
# Stems tokenized text
|
|
clean_text = " ".join([self.indexer.stemmer.stem(i) for i in clean_text.split()])
|
|
# Put clean_text as an element in a list because get_tf_idf workers properly with single element lists
|
|
x = [clean_text]
|
|
toc = perf_counter()
|
|
print("Took " + str(toc - tic) + " seconds to create clean text")
|
|
# ngrams is a dict
|
|
# structure looks like {ngram : {0: tf-idf score}}
|
|
ngrams = self.indexer.get_tf_idf(x)
|
|
if ngrams != -1:
|
|
tic = perf_counter()
|
|
for ngram, tfidf in ngrams.items():
|
|
posting = Posting(self.indexer.get_url_id(data["url"]), tfidf[0])
|
|
self.indexer.save_index(ngram,posting)
|
|
toc = perf_counter()
|
|
print("Took " + str(toc - tic) + " seconds to save ngram")
|
|
|
|
tocker = perf_counter()
|
|
print("Took " + str(tocker - ticker) + " seconds to work")
|
|
|