Introduction To Word Embeddings - How To Train Your Own Word Vectors Using The Simpsons Dialogues?

# Install/upgrade Gensim if needed:
# !pip install gensim --upgrade

# Import Gensim and data science standard libraries:
import gensim # to get access to word2vec tool
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline

# Import Simpsons dataset, and read it in a pandas dataframe:
df = pd.read_csv('simpsons_dataset.csv')

# let's explore quickly the data:

# Data is showing as text strings, contains 2 columns and 158314 rows, and has some null values in each of the 2 columns:
print('Dataframe information: ')
print()
df.info()

Dataframe information:

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 158314 entries, 0 to 158313
Data columns (total 2 columns):
raw_character_text    140500 non-null object
spoken_words          131855 non-null object
dtypes: object(2)
memory usage: 2.4+ MB

# Data looks great!
print('Dataframe top rows:')
df.head()

Dataframe top rows:

# Here is the total of null values per columns:
print('Number of null values in dataframe: ')
df.isnull().sum()

Number of null values in dataframe:





raw_character_text    17814
spoken_words          26459
dtype: int64

# Dataset is large enough to run our demo, so let's get rid of all null values:
df = df.dropna()

# Just making sure that our dataset is free of nulls now:
print('Number of null values remaining: ')
df.isnull().sum()

Number of null values remaining:





raw_character_text    0
spoken_words          0
dtype: int64

# After dropping the nulls, 131853 rows remain in the dataset:
print('Dataframe shape:', df.shape)

Dataframe shape: (131853, 2)

# Let's rename the 2 columns:
df.rename(columns={'raw_character_text': 'character', 'spoken_words': 'sequence'}, inplace=True)

# Import text nltk pre-processing tools:
from nltk.stem import WordNetLemmatizer # to lemmatize words
from nltk.tokenize import RegexpTokenizer # to'tokenize' words
from nltk.corpus import stopwords # to remove stopwords.
import re # to access to regular expression matching operations

def preprocess (x): # create a pre-processing function:
    string_text = str(x) # convert text to strings
    lower_case = string_text.lower() # lowercase text
    lower_case = re.sub(r'[^a-zA-Z]', ' ', lower_case) # remove non-alphabetic characters
    retokenizer = RegexpTokenizer(r'\w+')
    words = retokenizer.tokenize(lower_case) # split strings into substrings using a regular expression
    lemmer = WordNetLemmatizer() # take words and attempt to return their base/dictionary form.
    stops = set(stopwords.words('english')) # remove english stopwords, some example of stopwords are as 'what', 'as', 'of', 'into'
    meaningful_words = [w for w in words if not w in stops]
    if len(meaningful_words) >2: # Let's remove sentences that are less than two words long as Word2Vec uses context words to learn the vector representation of a target word
        return " ".join([lemmer.lemmatize(word) for word in meaningful_words if not word in stops])

# Apply our function to the Simpsons dialogues:
df.sequence = df.sequence.apply(preprocess)

# Pre-processing is now done, but sentences that were 1 or 2 words long have been removed from our dataset, see 'sequence' at row [1]:
print('Pre-processed dataframe top rows:')
df.head()

Pre-processed dataframe top rows:

# 38061 new nulls appear throughout our preprocessed dataset:
print('Number of null values in pre-processed dataframe: ', df.sequence.isnull().sum())

Number of null values in pre-processed dataframe:  38061

# ..so let's remove them as well as all duplicates:
df_clean = df.dropna().drop_duplicates()

# Our 'clean' dataset includes now 93317 rows of pre-processed text:
print('Shape of the clean version of our dataframe: ', df_clean.shape)

Shape of the clean version of our dataframe:  (93317, 2)

# Let's use Gensim Phrases package to automatically detect bigrams (common phrases) from a list of sentences.
# This will connect "bart" and "simpson" when they appear side by side, so the model will treat 'homer_simpson' as one word
from gensim.models.phrases import Phrases, Phraser # Phrases() takes a list of list of words as input:

sent = [row.split() for row in df_clean['sequence']]
phrases = Phrases(sent, min_count=20) # we will ignore all bigrams with total collected count lower than 20.
bigram = Phraser(phrases)
sentences = bigram[sent]

print('First 15 bigrams in alpha order:' )
pd.DataFrame(sorted(bigram.phrasegrams)[:15], columns=['first_word', 'second_word'])

# We can see at row 12 that when 'bart' and 'simpson' show up side by side in a sentence,
# they will be now considered as one word, the bigram 'bart_simpson'

First 15 bigrams in alpha order:

from collections import defaultdict # defaultdict will allow us to count the number of occurrences for each word.

word_count = defaultdict(int)
for sent in sentences:
    for i in sent:
        word_count[i] += 1

print("Number of words in our dataframe: ", len(word_count))

Number of words in our dataframe:  32943

# What are the most frequent words appearing in our data?
print('Most frequent words:')
pd.DataFrame(sorted(word_count.items(), key=lambda x: x[1], reverse=True)[:15], columns=['word', 'frequency'])

# Homer and Bart are respectively the 10th and 12th most frequently used word in the show,
# which makes sense as they are the 2 main characters of the series

Most frequent words:

from gensim.models import Word2Vec # import word2vec from Gensim library

w2v_model = Word2Vec(
                     size=300, # number of dimensions of our word vectors
                     window=10, # set the number of "context words" at 10
                     min_count=20, # ignore all words with total frequency lower than 20
                     sample=0.0001, # set the threshold for configuring which higher-frequency words are randomly downsampled
                     negative=20, # set the number of noise words to be drawn at 20
                     workers=4) # number of threads used in parallel to train the machine

# This step will build the vocabulary table:
w2v_model.build_vocab(sentences)

# Model can now be trained:
w2v_model.train(
    sentences,  
    total_examples=w2v_model.corpus_count, # number of sentences in the corpus
    epochs=30 # number of iterations in the corpus
)

(8742042, 18370410)

# init_sims will precompute L2-normalized vectors and ,if true, will forget the original vectors and only keep the normalized ones, this can save lots of memory
w2v_model.init_sims(replace=False)

print('Top 10 most similar words to Homer:')
pd.DataFrame(w2v_model.wv.most_similar(positive=["homer"]), columns=['word', 'frequency'])

Top 10 most similar words to Homer:

print('Top 10 most similar words to Bart:')
pd.DataFrame(w2v_model.wv.most_similar(positive=["bart"]), columns=['word', 'frequency'])

Top 10 most similar words to Bart:

print('Top 10 most similar words to Lisa:')
pd.DataFrame(w2v_model.wv.most_similar(positive=["lisa"]), columns=['word', 'frequency'])

Top 10 most similar words to Lisa:

print('Top 10 most similar words to Marge:')
pd.DataFrame(w2v_model.wv.most_similar(positive=["marge"]), columns=['word', 'frequency'])

Top 10 most similar words to Marge:

print('Word to exclude from the list:', w2v_model.wv.doesnt_match(['bart', 'lisa', 'maggie', 'milhouse']))

Word to exclude from the list: milhouse


C:\Users\Administrator\Anaconda3\lib\site-packages\gensim\models\keyedvectors.py:877: FutureWarning: arrays to stack must be passed as a "sequence" type such as list or tuple. Support for non-sequence iterables such as generators is deprecated as of NumPy 1.16 and will raise an error in the future.
  vectors = vstack(self.word_vec(word, use_norm=True) for word in used_words).astype(REAL)

print('Word to exclude from the list:', w2v_model.wv.doesnt_match(['moe', 'lenny', 'carl', 'homer', 'marge']))

Word to exclude from the list: marge

w2v_model.wv.most_similar(positive=["woman", "homer"], negative=["man"], topn=10)

[('marge', 0.5261433124542236),
 ('husband', 0.4614260196685791),
 ('homie', 0.449959933757782),
 ('brunch', 0.4463199973106384),
 ('wife', 0.44602805376052856),
 ('marriage', 0.44539350271224976),
 ('grownup', 0.44457897543907166),
 ('wasted', 0.4415558874607086),
 ('affair', 0.4279390275478363),
 ('luann', 0.4259093403816223)]