Changed tf_idf model into the new one, try it on the current dataset

indexer.py

@@ -116,6 +116,7 @@ class Indexer():
 			print(word)
 			print("You have somehow went beyond the magic")
 			return self.save_5
+
 	def get_save_lock(self,word):
 		word_lower = word.lower()
 		if re.match(r"^[a-d0-1].*",word_lower):
@@ -130,6 +131,7 @@ class Indexer():
 			print(word)
 			print("You have somehow went beyond the magic")
 			return self.save_5_lock.acquire()
+
 	# I have a test file (mytest.py) with pandas but couldn't figure out how to grab just a single cell.
 	# so I came up with this, if anyone knows how to get a single cell and can explain it to
 	# me I would love to know, as I think that method might be quicker, maybe, idk it like
@@ -178,7 +180,7 @@ class Indexer():
 
 	def get_data(self):
 
-		num_threads = 8
+		num_threads = 1
 		threads = list()
 
 		for directory in os.listdir(self.path):

worker.py

@@ -52,49 +52,54 @@ class Worker(Thread):
 		tic = perf_counter()
 		for word in words:
 			if word != "" and re.fullmatch('[A-Za-z0-9]+',word):
-				#So all the tokenized words are here,
 				tokenized_words.append(word)
 		toc = perf_counter()
 		if toc - tic > 1 :
 			print("Took " + str(toc - tic) + "seconds to isalnum text !")
-		#YOUR CODE HERE
 
 		tic = perf_counter()
 		for word in tokenized_words:
 			stemmed_words.append(self.indexer.stemmer.stem(word))
-			#stemming,
+
-			#tf_idf
-			#get_tf_idf(stemmed_words,word)
-			#post = Posting()
 		toc = perf_counter()
 		if toc - tic > 1 :
 			print("Took " + str(toc - tic) + "seconds to stemmed text !")
 
-		counts = Counter(stemmed_words)
-		size = len(stemmed_words)
-		for word in counts:
-			#posting = Posting(data["url"],self.get_tf_idf(list(' '.join(stemmed_words)),word))
-			tic = perf_counter()
-			weight = 1.0
-			index = 0
 			"""
-			for group in important:
+		tfidf = TfidfVectorizer(ngram_range=(1,3)) # ngram_range is range of n-values for different n-grams to be extracted (1,3) gets unigrams, bigrams, trigrams
-				for word_important in group:
+		tfidf_matrix = tfidf.fit_transform(stemmed_words)  # fit trains the model, transform creates matrix
-					if word_important.lower() == word.lower():
+		#df = pd.DataFrame(tfidf_matrix.toarray(), columns = tfidf.get_feature_names_out()) # store value of matrix to associated word/n-gram
-						if index == 0:
+		tfidf.sget_feature_names_out()
-							weight = 1.2
+		#tf_idf_dict = df.to_dict() # transform dataframe to dict *could be expensive the larger the data gets, tested on ~1000 word doc and took 0.002 secs to run
-						elif index == 1:
+		
-							weight = 1.8
+		print(tfidf_matrix)
-						elif index == 2:
-							weight = 1.5
-						elif index == 3:
-							weight = 1.3
-						elif index == 4:
-							weight = 2.0
-				index = index + 1
 		"""
 
-			posting = Posting(data["url"],counts[word]/size*weight)
+		tfIdfVectorizer=TfidfVectorizer(use_idf=True)
+		tfIdf = tfIdfVectorizer.fit_transform(stemmed_words)
+		df = pd.DataFrame(tfIdf[0].T.todense(), index=tfIdfVectorizer.get_feature_names_out(), columns=["TF-IDF"])
+		df = df.sort_values('TF-IDF', ascending=False)
+
+		print(df.head(25))
+
+		for word in tf_idf_dict.keys():
+			tic = perf_counter()
+			print(tf_idf_dict)
+			weight = 1.0
+			for k,v in important.items():
+				if k == 'b' and word in v:
+					weight = 1.2
+				elif k == 'h1' and word in v:
+					weight = 1.75
+				elif k == 'h2' and word in v:
+					weight = 1.5
+				elif k == 'h3' and word in v:
+					weight = 1.2
+				elif k == 'title' and word in v:
+					weight = 2
+			
+			posting = Posting(data["url"],tf_idf_dict[word]*weight)
+			
 			toc = perf_counter()
 			if toc - tic > 1 :
 				print("Took " + str(toc - tic) + "seconds to tf_idf text !")