Upload nltk_utils.py

nltk_utils.py

@@ -0,0 +1,24 @@
+# Import Libraries
+import nltk
+from nltk.tokenize import word_tokenize
+nltk.download('punkt')
+from sklearn.feature_extraction.text import TfidfVectorizer
+from nltk.stem import SnowballStemmer
+stemmer= SnowballStemmer(language= 'english')
+from nltk.corpus import stopwords
+nltk.download('stopwords')
+
+# Tokenize text i.e make all text be in a list format e.g "I am sick" = ['i', 'am', 'sick']
+def tokenize(text):
+  return [stemmer.stem(token) for token in word_tokenize(text)]
+
+# Create stopwords to reduce noise in data
+english_stopwords= stopwords.words('english')
+
+# Create a vectosizer to learn all words in order to convert them into numbers
+def vectorizer():
+    vectorizer= TfidfVectorizer(tokenizer=tokenize,
+                                stop_words=english_stopwords,
+                                )
+    return vectorizer
+