# import libraries used:
import nltk
import pandas as pd
import numpy as np
from nltk.tokenize import word_tokenize, sent_tokenize
from nltk import pos_tag, ne_chunk
import gensim
import re
import spacy
from time import time
from gensim.models import Word2Vec

from nltk.stem import WordNetLemmatizer

from gensim.models.phrases import Phrases, Phraser
import multiprocessing
from tqdm import tqdm
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from tqdm import tqdm
from keras.models import Sequential
from keras.layers import Embedding,LSTM,Dense,SpatialDropout1D
from keras.initializers import Constant
from sklearn.model_selection import train_test_split
from keras.optimizers import Adam
from nltk.corpus import stopwords
from nltk.stem import PorterStemmer
import string
import re

df = pd.read_csv("/content/simpsons_dataset.csv")

df.head()

df.info()

df.describe()

df.dtypes

df.isnull().sum()

df = df.dropna().reset_index(drop=True)
df.isnull().sum()

nlp = spacy.load("en_core_web_sm", disable=['ner', 'parser'])
# nlp = spacy.load('en', disable=['ner', 'parser']) # disabling Named Entity Recognition for speed

def cleaning(doc):
    # Lemmatizes and removes stopwords
    # doc needs to be a spacy Doc object
    txt = [token.lemma_ for token in doc if not token.is_stop]
    # Word2Vec uses context words to learn the vector representation of a target word,
    # if a sentence is only one or two words long,
    # the benefit for the training is very small
    if len(txt) > 2:
        return ' '.join(txt)

brief_cleaning = (re.sub("[^A-Za-z']+", ' ', str(row)).lower() for row in df['spoken_words'])

t = time()
txt = [cleaning(doc) for doc in nlp.pipe(brief_cleaning, batch_size=5000, n_process=-1)]

print('Time to clean up everything: {} mins'.format(round((time() - t) / 60, 2)))

df_clean = pd.DataFrame({'clean': txt})
df_clean = df_clean.dropna().drop_duplicates()
df_clean.shape

sent = [row.split() for row in df_clean['clean']]

phrases = Phrases(sent, min_count=30, progress_per=10000)

bigram = Phraser(phrases)

sentences = bigram[sent]

cores = multiprocessing.cpu_count()

w2v_model = Word2Vec(min_count=20,
                     window=2,
                     vector_size=300,
                     sample=6e-5,
                     alpha=0.03,
                     min_alpha=0.0007,
                     negative=20,
                     workers=cores-1)

t = time()

w2v_model.build_vocab(sentences, progress_per=10000)

print('Time to build vocab: {} mins'.format(round((time() - t) / 60, 2)))

t = time()

w2v_model.train(sentences, total_examples=w2v_model.corpus_count, epochs=30, report_delay=1)

print('Time to train the model: {} mins'.format(round((time() - t) / 60, 2)))

w2v_model.init_sims(replace=True)

w2v_model.wv.most_similar(positive=["homer"])

w2v_model.wv.most_similar(positive=["homer_simpson"])

df = pd.read_table("/content/Restaurant_Reviews.tsv")

df.head()

df.info()

nlp = spacy.load('en_core_web_sm')

array = df.to_numpy()#convert to numpy

#stop words list
stop_words = set(stopwords.words('english'))

#add other things to stop_words
stop_words.add(',')
stop_words.add('.')
stop_words.add('“')
stop_words.add('’')
stemmer= PorterStemmer()
lemmatizer=WordNetLemmatizer()
l = len(array)

for i in range(l):
    array[i][0] = array[i][0].lower() #Upper to lower
    array[i][0] = ''.join((z for z in array[i][0] if not z.isdigit())) #removing numbers
    array[i][0] = array[i][0].strip() #Removing white space
    array[i][0] = array[i][0].translate(str.maketrans('', '', string.punctuation))#remove Punctuation
    array[i][0] = re.sub('http://\S+|https://\S+', '', array[i][0])#remove http adress
    word_tokens = word_tokenize(array[i][0]) #Tokenize
    array[i][0] = word_tokens

    array[i][0] = [word for word in array[i][0] if not word in stopwords.words()]
    a = []
    le = len(array[i][0])

    for word in array[i][0]:

            w1 = stemmer.stem(word)
            w2 = "".join(word)
            #w2 = lemmatizer.lemmatize(word)#lemmatize
            a.append(w2)
            array[i][0] = a

a_list = array[:,0]
Total_keywords = []

l = len(a_list)
for i in range(l):
    l2 = len(a_list[i])
    for j in range(l2):
        Total_keywords.append(a_list[i][j])

#make a unique vectore
Total_keywords = set(Total_keywords)
print('Total number of keywords is : ',len(Total_keywords))

n = len(Total_keywords)#lenght of the dimention of the quadratic matrix
matrix = np.zeros(shape=(n,n))# n*n zero matrix

Total_keywords = list(Total_keywords)

l = len(array)
for i in range(n):
    for j in range(l):
        l2 = len(array[j,0])
        for k in range(l2):
            if Total_keywords[i] == array[j,0][k]:
                if k>0 and k<l2-1:
                    for s in range(n):
                        if array[j,0][k-1] == Total_keywords[s]:
                            matrix[s][i] = matrix[s][i]+1
                        elif array[j,0][k+1] == Total_keywords[s]:
                            matrix[s][i] = matrix[s][i]+1
                elif k==0 and l2>1:
                    for s in range(n):
                        if array[j,0][k+1] == Total_keywords[s]:
                            matrix[s][i] = matrix[s][i]+1
                elif k==l2-1 and l2!=0:
                    for s in range(n):
                        if array[j,0][k-1] == Total_keywords[s]:
                            matrix[s][i] = matrix[s][i]+1

Glove = pd.DataFrame(matrix, columns = Total_keywords )
Glove.set_index([pd.Index(Total_keywords)])

#Comparing two vectors by Cosin distance
from scipy.spatial import distance
a = Glove['unsatisfying']#word vector a
b = Glove['refused']#word vector b
a = a.to_numpy()
b = b.to_numpy()

print('cosin similarity is :', distance.cosine(a, b))

a = Glove['menu']#word vector a
b = Glove['great']#word vector a
a = a.to_numpy()
b = b.to_numpy()
l1 = len(array)
counter = 0
for i in range(l1):
    l2 = len(array[i,0])
    for j in range(l2):
        if 'menu' == array[i,0][j]:
            counter = counter + 1
d = len(Total_keywords)
for i in range(d):
    if 'menu' == Total_keywords[i]:
        f = i
s = Glove.loc[f, 'great']
print('Total number of word a :',counter)
print('Total number of word a after or befor b:',Glove.loc[f, 'great'])
print('probability of event word a to word b is:',s/counter*100,'%')
print('cosin similarity is :', round(distance.cosine(a, b),3))