iNocturnis/Search_Engine
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update indexer.py

Browse Source 
had incorrect implementation
This commit is contained in:
Aaron 2022-05-12 17:58:31 -07:00 committed by GitHub
parent c4b3512df7
commit e7c4170cc2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 6 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
indexer.py
View File

@ -147,6 +147,7 @@ class Indexer():
#word the word we finding the score for #word the word we finding the score for
#return the score #return the score
try: try:
'''
tfidf = TfidfVectorizer() tfidf = TfidfVectorizer()
tfidf_matrix = tfidf.fit_transform(words) tfidf_matrix = tfidf.fit_transform(words)
df = pd.DataFrame(tfidf_matrix.toarray(), columns = tfidf.get_feature_names_out()) df = pd.DataFrame(tfidf_matrix.toarray(), columns = tfidf.get_feature_names_out())
@ -166,12 +167,14 @@ class Indexer():
#print(df) #print(df)
except KeyError: except KeyError:
return -1 return -1
'''
try:
tfidf = TfidfVectorizer(ngram_range=(1,3)) # ngram_range is range of n-values for different n-grams to be extracted (1,3) gets unigrams, bigrams, trigrams tfidf = TfidfVectorizer(ngram_range=(1,3)) # ngram_range is range of n-values for different n-grams to be extracted (1,3) gets unigrams, bigrams, trigrams
tfidf_matrix = tfidf.fit_transform(words) # fit trains the model, transform creates matrix tfidf_matrix = tfidf.fit_transform(words) # fit trains the model, transform creates matrix
df = pd.DataFrame(tfidf_matrix.toarray(), columns = tfidf.get_feature_names_out()) # store value of matrix to associated word/n-gram df = pd.DataFrame(tfidf_matrix.toarray(), columns = tfidf.get_feature_names_out()) # store value of matrix to associated word/n-gram
#return(df.iloc[0][''.join(word)]) #used for finding single word in dataset #return(df.iloc[0][''.join(word)]) #used for finding single word in dataset
data = df.to_dict() # transform dataframe to dict *could be expensive the larger the data gets, tested on ~1000 word doc and took 0.002 secs to run tfidf_dict = df.to_dict() # transform dataframe to dict *could be expensive the larger the data gets, tested on ~1000 word doc and took 0.002 secs to run
return data # returns the dict of words/n-grams with tf-idf return tfidf_dict # returns the dict of words/n-grams with tf-idf as value
#print(df) # debugging #print(df) # debugging
except: except:
print("Error in tf_idf!") print("Error in tf_idf!")
@ -229,4 +232,4 @@ def main():
indexer.get_data() indexer.get_data()
if __name__ == "__main__": if __name__ == "__main__":
main() main()