# Notebook from sgmark3/Topic-modelling-of-podcast-episodes
Path: model/.ipynb_checkpoints/LDA_model_all_genres-checkpoint.ipynb

#### Import all the required packages_____no_output_____

## basic packages
import numpy as np
import re
import csv
import pandas as pd

##gensim
import gensim
import gensim.corpora as corpora
from gensim.utils import simple_preprocess
from gensim.models import CoherenceModel

##spacy and nltk
import spacy
from nltk.corpus import stopwords
from spacy.lang.en.stop_words import STOP_WORDS

##vis
# import pyLDAvis
# import pyLDAvis.gensim_models as gensimvis
# pyLDAvis.enable_notebook()

import matplotlib.pyplot as plt
import seaborn as sns
sns.set_style('darkgrid')

import warnings
warnings.filterwarnings("ignore",category=DeprecationWarning)_____no_output_____

#### load the metadata of podcast transcripts_____no_output_____

global df, show_descriptions

meta_data = []
with open("../data/metadata.tsv") as csvfile:
    csvreader = csv.reader(csvfile,delimiter="\t")
    for row in csvreader:
        meta_data.append(row)

df = pd.DataFrame(meta_data[1:],columns=meta_data[0])
show_filename_prefixes = df.show_filename_prefix
episode_filename_prefixes = df.episode_filename_prefix
shows = df.groupby(by=['show_name'])
show_names = shows.apply(lambda x: x.show_name.unique()[0])_____no_output_____# show_descriptions_aggregated = shows.apply(lambda x: x.show_description.unique()[0])
# episode_descriptions_aggregated = shows.apply(lambda x: list(x.episode_description))
# descriptions_aggregated = {}
# for k,text in enumerate(episode_descriptions_aggregated):
#     descriptions_aggregated[show_names[k]] = [show_descriptions_aggregated[k]]+text_____no_output_____genres_topics = ["comedy","news","crime","science","economics","politics","education",\
                  "sports","lifestyle","health","wellbeing","religion","faith","music",\
                  "art","fashion","literature","humanities","drama","fitness","drama",\
                  "fantasy","scifi","gameshow","news quiz","games","mental",\
                  "humor","research","technology","society","social","culture","lifestyle",\
                  "songs","cooking","culinary","food","travel","films","film","movies","tv",\
                  "climate","space","planet","digital","artificial intelligence", "ai",\
                  "cars","car","nutrition","wellness","family","history","geography","physics",\
                  "mathematics","math","chemistry","biology","documentary","commentary","nfl",\
                  "mls","nba","mlb","stocks","stock","market","wall","street","business",\
                  "reality","shows","investing","social media","biography","biographies",\
                  "data science","medicine","media","books","book","europe","asia","canada",\
                  "south america","north america","america","usa","netflix","adventure","pets","dogs",\
                  "cats","dog","cat","nintendo","xbox","playstation","ps4","ps5","theatre","mars"\
                  "tennis","australia","conspiracy","war","epidemic","pandemic","climate","change"\
                  "astrology","novel","church","christ","romance","english","kids","astronomy"\
                  "design","entrepreneurship","marketing","digital"]_____no_output_____podcasts_genres_topics = {}
for k,show in enumerate(show_names):
    keywords = show.lower().split(" ")
    for word in keywords:
        if word in genres_topics:
            if (k,show) in podcasts_genres_topics:
                if word not in podcasts_genres_topics[(k,show)]:
                    podcasts_genres_topics[(k,show)].append(word)
            else:
                podcasts_genres_topics[(k,show)] = [word]
podcasts = [item[1] for item in podcasts_genres_topics.keys()]_____no_output_____nlp = spacy.load("en_core_web_sm")
stops_nltk = set(stopwords.words("english"))
stops_spacy = STOP_WORDS.union({'ll', 've', 'pron','okay','oh','like','know','yeah','yea','yep',\
                                "like like","oh like","yeah like","yeah yeah","oh okay","wow","podcast",\
                               "support","anchor","podcasts","episode","http","https","5star","reviews",\
                               "review","instagram","tiktok","amazon","apple","twitter","goole",\
                               "facebook","send","voice message","message","voice","subscribe","follow",\
                               "sponsor","links","easiest","way","fuck","fucking","talk","discuss",\
                               "world","time","want","join","learn","week","things","stuff","find",\
                               "enjoy","welcome","share","talk","talking","people","gmail","help","today",\
                               "listen","best","stories","story","hope","tips","great","journey",\
                               "topics","email","questions","question","going","life","good","friends",\
                               "friend","guys","discussing","live","work","student","students","need",\
                               "hear","think","change","free","better","little","fucking","fuck","shit",\
                                "bitch","sex","easiest","way","currently","follow","follows","needs",\
                               "grow","stay","tuned","walk","understand","tell","tells","ask","helps",\
                                "feel","feels","look","looks","meet","relate","soon","quick","dude","girl",\
                                "girls","guy","literally","spotify","google","totally","played","young",\
                               "begin","began","create","month","year","date","day","terms","lose","list",\
                               "bought","brings","bring","buy","percent","rate","increase","words","value",\
                               "search","awesome","followers","finn","jake","mark","america","american",\
                               "speak","funny","hours","hour","honestly","states","united","franklin",\
                               "patrick","john","build","dave","excited","process","processes","based",\
                                "focus","star","mary","chris","taylor","gotta","liked","hair","adam","chat",\
                               "named","died","born","country","mother","father","children","tools",\
                               "countries","jordan","tommy","listeners","water","jason","lauren","alex",\
                               "laguna","jessica","kristen","examples","example","heidi","stephen","utiful",\
                               "everybody","sorry","came","come","meet","whoa","whoaa","yay","whoaw",\
                               "anybody","somebody","cool","watch","nice","shall"})
stops = stops_nltk.union(stops_spacy)_____no_output_____

#### Filtering the shows to be analyzed based on the total number of episodes in the dataset_____no_output_____

# ten_to_fifty_episode_podcasts = [item1 for (item1,item2) in descriptions_aggregated.items()\
#                                 if (len(item2) >= 10) and (len(item2) < 50)]
# ten_to_fifty_episodes = [(df[df.show_name == name].show_filename_prefix.iloc[0],\
#                           df[df.show_name == name].episode_filename_prefix)\
#                          for name in ten_to_fifty_episode_podcasts]_____no_output_____

#### The individual transcript location_____no_output_____

# def file_location(show,episode):
#     search_string = local_path + "/spotify-podcasts-2020" + "/podcasts-transcripts" \
#                                + "/" + show[0] \
#                                + "/" + show[1] \
#                                + "/" + "show_" + show \
#                                + "/"
#     return search_string_____no_output_____

#### load the transcripts_____no_output_____

transcripts = {}
for podcast in podcasts:
    for i in shows.get_group(podcast)[['show_filename_prefix','episode_filename_prefix']].index:
        show,episode = shows.get_group(podcast)[['show_filename_prefix','episode_filename_prefix']].loc[i]
        s = show.split("_")[1]
        try:
            with open('../podcast_transcripts/'+s[0]+'/'+s[1]+'/'+show+'/'+episode+'.txt','r') as f:
                transcripts[(show,episode)] = f.readlines()
            f.close()
        except Exception as e:
            pass

keys = list(transcripts.keys())_____no_output_____# transcripts = {}
# for k,showname in enumerate(show_filename_prefixes):
#     s = showname.split("_")[1]
#     try:
#         with open('../podcast_transcripts/'+s[0]+'/'+s[1]+'/'+showname+'/'+episode_filename_prefixes[k]+'.txt','r') as f:
#             transcripts[(showname,episode_filename_prefixes[k])] = f.readlines()
#         f.close()
#     except Exception as e:
#         pass

# keys = list(transcripts.keys())

# transcripts = {}
# for name,list_of_episodes in ten_to_fifty_episodes:
#     s = name.split("_")[1]
#     for episode in list_of_episodes:
#         try:
#             with open('../data/shows_ten_to_fifty_episodes/'+name+'/'+episode+'.txt','r') as f:
#                 transcripts[(name,episode)] = f.readlines()
#             f.close()
#         except Exception as e:
#             pass

# keys = list(transcripts.keys())_____no_output_____# Cleaning & remove urls and links
def remove_stops(text,stops):
    final = []
    for word in text:
        if (word not in stops) and (len(word)>3) and (not word.endswith('ing')) and (not word.endswith('ly')):
            final.append(word)
    return final

def clean_text(docs):
    final = []
    for doc in docs:
        clean_doc = remove_stops(doc, stops)
        final.extend(clean_doc)
    return final

def lemmatization(text_data):
    nlp = spacy.load("en_core_web_sm")
    texts = []
    for text in text_data:
        doc = nlp(text)
        lem_text = []
        for token in doc:
            if (token.pos_=="VERB") or (token.pos_=="ADV"):
                pass
            else:
                lem_text.append(token.lemma_)
        texts.append(lem_text)
    return texts_____no_output_____

#### tokenize/convert text into words_____no_output_____

def normalize_docs(text_data):
    final_texts = []
    for text in text_data:
        new_text = gensim.utils.simple_preprocess(text,deacc=True)
        final_texts.append(new_text)
    return final_texts_____no_output_____docs = []
for text in transcripts.values():
    docs.append(' '.join(clean_text(normalize_docs(text))))

texts = lemmatization(docs)
texts = [remove_stops(text,stops) for text in texts]_____no_output_____# with open('lemmatized_docs.txt','w') as f:
#     for doc in lemmatized_docs:
#         f.write(doc)
#         f.write('\n')
# with open('/home1/sgmark/capstone-project/model/lemmatized_docs.txt','r') as f:
#     read_lemmatized_docs = f.readlines()_____no_output_____

### Using bigrams_____no_output_____

from gensim.models.phrases import Phrases

bigram = Phrases(texts, min_count=5)

for i in range(len(texts)):
    for token in bigram[texts[i]]:
        if '_' in token:
            texts[i].append(token)_____no_output_____

#### Construct a corpus of words as a bag of words _____no_output_____

dictionary = corpora.Dictionary(texts)

dictionary.filter_extremes(no_below=9,no_above=0.3)_____no_output_____corpus = [dictionary.doc2bow(text) for text in texts]_____no_output_____from itertools import product

number_of_topics = [15,20,25,30]
df_parameters = list(product([2,3,4,5,6,7,8,9,10],[0.3,0.4,0.5,0.6,0.7,0.8,0.9]))
coherence_scores_umass = np.zeros((len(number_of_topics),len(df_parameters)))
coherence_scores_uci = np.zeros((len(number_of_topics),len(df_parameters)))
coherence_scores_npmi = np.zeros((len(number_of_topics),len(df_parameters)))

j = 0
for num in number_of_topics[3:]:
    i = 0
    for n,m in df_parameters[52:]:
        dictionary = corpora.Dictionary(texts)
        dictionary.filter_extremes(no_below=n,no_above=m)
        corpus = [dictionary.doc2bow(text) for text in texts]
        num_topics = num
        chunksize = 10000
        passes = 20
        iterations = 500
        eval_every = None

        lda_model = gensim.models.ldamodel.LdaModel(corpus,
                                                   id2word=dictionary,
                                                   num_topics=num_topics,
                                                   chunksize=chunksize,
                                                   passes=passes,
                                                   iterations=iterations,
                                                   alpha='auto',
                                                   eta='auto',
                                                   random_state = 123,
                                                   eval_every=eval_every)
        cm = CoherenceModel(lda_model, texts=texts,corpus=corpus, coherence= 'c_uci')
        coherence_scores_uci[j,i] = cm.get_coherence()
        cm = CoherenceModel(lda_model, texts=texts,corpus=corpus, coherence= 'c_npmi')
        coherence_scores_npmi[j,i] = cm.get_coherence()
        cm = CoherenceModel(lda_model, corpus=corpus, coherence= 'u_mass')
        coherence_scores_umass[j,i] = cm.get_coherence()
        with open("coherence_scores.csv",'a') as f:
            writer = csv.writer(f)
            writer.writerow([num,n,m,coherence_scores_uci[j,i],coherence_scores_npmi[j,i],\
                             coherence_scores_umass[j,i]])
        i += 1
        print(i)
    j += 1
    print(j)1
2
%%time
import logging
logging.basicConfig(filename='example.log', encoding='utf-8',format='%(asctime)s : %(levelname)s : %(message)s', level=logging.DEBUG)
# logging.basicConfig(filename='example.log', filemode = 'w+', encoding='utf-8', level=logging.DEBUG)
# logging.debug('This message should go to the log file')
# logging.info('So should this')
# logging.warning('And this, too')
# logging.error('And non-ASCII stuff, too, like Øresund and Malmö')

num_topics = 30
chunksize = 10000
passes = 50
iterations = 1000
eval_every = None

lda_model = gensim.models.ldamodel.LdaModel(corpus,
                                           id2word=dictionary,
                                           num_topics=num_topics,
                                           chunksize=chunksize,
                                           passes=passes,
                                           iterations=iterations,
                                           alpha='auto',
                                           eta='auto',
                                           random_state=123,
                                           eval_every=eval_every)CPU times: user 1h 45min 35s, sys: 3h 46min 25s, total: 5h 32min
Wall time: 29min 52s
num_topics = 30
top_topics = lda_model.top_topics(corpus) #, num_words=20)

# Average topic coherence is the sum of topic coherences of all topics, divided by the number of topics.
avg_topic_coherence = sum([t[1] for t in top_topics])/num_topics
print('Average topic coherence: %.4f.' % avg_topic_coherence)Average topic coherence: -1.4218.
cm = CoherenceModel(lda_model, texts = texts, corpus=corpus, coherence='c_npmi')
coherence = cm.get_coherence()
print(coherence)
for x in cm.get_coherence_per_topic(): print(x)0.06839929939377389
-0.04139600062276654
0.12084543421352392
0.07445232825467579
0.06323677321070126
0.017157704066082295
0.10796859510784866
-0.03349006835597917
0.05726519314590546
0.0460092216318655
0.07954979360679083
0.10899650654193957
0.03804871677703496
0.0540325978319245
0.05657790460355731
0.02156085028278504
0.14556261661111605
0.11066366105467208
0.08442901466880408
0.09380129988728274
0.03064434468230435
0.11213925735825896
0.05380938433680618
0.04083377065973443
0.08079994798265114
0.07991047460669295
0.015224654821260421
0.09843815760960242
0.096284017588724
0.17500498180655857
0.06361784784285882

### Visualizing data_____no_output_____

# vis = pyLDAvis.gensim_models.prepare(lda_model,corpus,dictionary,mds="mmds",R=10)
# vis_____no_output_____# from pprint import pprint
# pprint(top_topics)_____no_output_____import pickle

pickle.dump(lda_model,open('../model/genre_assigned_shows_lda_model.pkl','wb'))
pickle.dump(dictionary,open('../model/genre_assigned_shows_dictionary.pkl','wb'))
pickle.dump(corpus,open('../model/genre_assigned_shows_corpus.pkl','wb'))
pickle.dump(lda_model,open('most_recent_lda_model.pkl','wb'))_____no_output_____import pickle

file = open('../model/most_recent_lda_model.pkl','rb')
lda_model = pickle.load(file)
file.close()

file = open('../model/genre_assigned_shows_corpus.pkl','rb')
corpus = pickle.load(file)
file.close()

file = open('../model/genre_assigned_shows_dictionary.pkl','rb')
dictionary = pickle.load(file)
file.close()_____no_output_____# import csv

# with open("genre_assigned_shows_corpus.csv","w+") as f:
#     writer = csv.writer(f)
#     for lis in corpus:
#         b = []
#         for tup in lis:
#             b.append(tup[0])
#             b.append(tup[1])

#         writer.writerow(b)_____no_output_____# with open("top_topics_50_to_100.csv","w+") as f:
#     writer = csv.writer(f)
#     for item1,item2 in top_topics:
#         b = [item2]
#         for tup in item1:
#             b.append(tup[0])
#             b.append(tup[1])

#         writer.writerow(b)_____no_output_____def get_main_topic_df(model, bow, texts):
    topic_list = []
    percent_list = []
    keyword_list = []
    podcast_list = []
    episode_list = []
    duration_list = []
    publisher_list = []

    for key,wc in zip(keys,bow):
        podcast_list.append(df[(df['show_filename_prefix'] == key[0])&(df['episode_filename_prefix'] == key[1])].show_name.iloc[0])
        episode_list.append(df[(df['show_filename_prefix'] == key[0])&(df['episode_filename_prefix'] == key[1])].episode_name.iloc[0])
        duration_list.append(df[(df['show_filename_prefix'] == key[0])&(df['episode_filename_prefix'] == key[1])].duration.iloc[0])
        publisher_list.append(df[(df['show_filename_prefix'] == key[0])&(df['episode_filename_prefix'] == key[1])].publisher.iloc[0])

        topic, percent = sorted(model.get_document_topics(wc), key=lambda x: x[1], reverse=True)[0]
        topic_list.append(topic)
        percent_list.append(round(percent, 3))
        keyword_list.append(' '.join(sorted([x[0] for x in model.show_topic(topic)])))

    result_df = pd.concat([pd.Series(podcast_list, name='Podcast_name'),
                           pd.Series(episode_list, name='Episode_name'),
                           pd.Series(topic_list, name='Dominant_topic'),
                           pd.Series(percent_list, name='Percent'),
                           pd.Series(texts, name='Processed_text'),
                           pd.Series(keyword_list, name='Keywords'),
                           pd.Series(duration_list, name='Duration of the episode'),
                           pd.Series(publisher_list, name='Publisher of the show')], axis=1)

    return result_df_____no_output_____import pandas as pd

# main_topic_df = pd.read_csv('lda_model_summary1_df.csv')
main_topic_df = get_main_topic_df(lda_model,corpus,texts)_____no_output_____main_topic_df.head(5)_____no_output_____# import matplotlib.pyplot as plt
# import seaborn as sns
# sns.set_style("darkgrid")
# plt.figure(figsize=(10,8))
# grouped_topics = main_topic_df.groupby('Dominant_topic')
# grouped_topics.count()['Processed_text'].\
#     plot.bar(rot=0).\
#     set(title='Dominant Topic Frequency in the {} podcast episodes'.format(len(texts)),
#         ylabel='Topic frequency'); _____no_output_____plt.figure(figsize=(10,8))
grouped_topics = main_topic_df.groupby('Dominant_topic')
grouped_topics.count()['Podcast_name'].\
    plot.bar(rot=0).\
    set(title='Dominant Topic Frequency in the {} podcast episodes'.format(len(texts)),
        ylabel='Topic frequency'); _____no_output_____representatives = pd.DataFrame()

for k in grouped_topics.groups.keys():
    representatives = pd.concat([representatives,
                                 grouped_topics.get_group(k).sort_values(['Percent'], ascending=False).head(1)])

representatives.head(5)_____no_output_____print('Document: {}  Dominant topic: {}\n'.format(representatives.index[0],
                                       representatives.loc[representatives.index[0]]['Dominant_topic']))
print([sentence.strip() for sentence in transcripts[keys[representatives.index[0]]]])Document: 5775  Dominant topic: 0

["Hey afterbuzzers, before we move on to your next topic. We just want to say thanks to our sponsor. Anchor. If you haven't heard about anchor it's the easiest way to make a podcast. Let me explain it's free. Plus there are creation tools that allow you to record and edit your podcast right from your phone or computer. Also anchor will distribute your podcast for you. So it could be heard on Spotify Apple podcasts and many more plus you can make money from your podcast with no minimum listenership, and it's everything you need to make a podcast in one place download the free anchor app or go to Anchor.", "Um to get started we've got a vaginal rejuvenation, but it sounds like we need a friendship Rejuvenation stay tuned is the Married to Medicine after show you're tuned in to AfterBuzz TV the ESPN of tv top now let the fun begin. Hi guys. Welcome. Welcome. Welcome. Welcome into the season 7 premiere of Married to Medicine. We are so excited the first", "So the season and when I tell you they came in with a bang they came in with a bang. This is the first episode is called open and honest and I don't know if people weren't necessarily being open and honest or maybe a little bit too open and honest but overall it was a great show, but I can't do this show without the help of my co-host. She's not married, but she does use medicine its Pietra. That was a good one at least you doing tonight. Yes. We're holding it down today. We've got a lot.", "To talk about we've got an open and honest podcast. Everybody got new houses vaginal rejuvenations and a surgery center party as well as some of our favorite tweets of the night our top three and some juicy news and gossip. But before we do all that, what did you think of tonight's show? Um, honestly, it kind of started off slow for me. I was like board up into the party at the end. So I was not expecting a premier to be like that using the premiere. They hit us with a bang and we're looking", "To the next show, but I don't know if they tried to do that. So at the end it's like okay like now is starts like it's gearing us up to look for the next show, but it kind of started off slow and I was bored and resting on I can see that it was really just more of an introduction. I feel like we're kind of like catching up with all of the girls. So just hearing about how like their sons left school and people learning to drive. We don't want to hear about that. We want to get to all the drama but I can say My overall thought is I can't believe everybody is", "into it. Yeah, like usually, you know, you have the normals like you have quad usually into it with somebody Heavenly usually one or two girls are into it, but it seems like there's like a true divided between the group what yeah that I was surprised to see that because normally it's only like you said one or two people with a little beef here in their shade, but I do think it's going to make the show more show more interesting. I feel like because everybody's into it. It's going to give us more and more thing to talk about each week. So I'm looking forward to see how things going to unfold. Yes. Yes. Yes.", "All right. Well, let's get right onto it. We open up with the open and honest podcast, which is a podcast that Jackie and Heavenly are doing together, which I was a little surprised by because you know, usually Jackie and Simone are like the two that do everything together. Usually Jackie's having to tell Heavenly about herself or they're into it about something. So I was surprised to see them be the ones who decide to do a podcast together team like a weird match to me. What did you think? It is a weird match, but I do feel like Jackie's kind of coming out of her shell.", "Show I don't know what it is about this season. I don't know if she's trying to prove herself. She's trying to not be The Peacemaker all of a sudden but she's really showing us some attitude and I seen a preview she's going to give us a little bit more. So I'm looking forward to see how she's going to unravel like. Yeah, that's so true. Usually, she's just like the neutral one was kind of, you know, just straddling the fence, but she was really, you know coming for everybody's business this time. She was asking about Mariah's accusations of her doing drugs.", "Quieter to mention that she kind of bruise some subtle shade with people not talking about people's husbands. Like it felt like she she thought she was being the mediator by not necessarily name-dropping the people who were doing those things, but it was kind of more in a passive aggressive way like our friends are doing this thing. Let me mention it but not mentioning but it's because she's always trying to like I said Play The Peacemaker, but it's almost like she wanna get her feet wet as far as really showing the girls like a look. I have another side to me. But at the same time she's trying to", "Play it safe and not say the wrong things, but I definitely do see what you mean as far as like she doesn't say too much, but she says just enough to crush a you know, yep. Definitely she's playing her part to stay on the show. Yeah. So you said that was thinking that I was like, what was it kind of a scare for Jackie that she wasn't going to be on the show anymore like this. She felt like she had to turn up the drama more to have a storyline because honestly, I was really shocked to see her kind of stirring the pot the way she was it seemed", "Some of the girls got upset with not only Heavenly but Jackie because of the podcast you think that was warranted like do you understand or do you think they're reacting? Honestly, I think they are overreacting because in the podcast I didn't hear anything that was super just bad and just like Heavenly said the things that she mentioned was things that was public knowledge everybody already knew, I do feel like they were stretching it a bit and maybe that was because they needed something for the premiere. I don't know but it was really nothing that was said they're just doing them.", "I was trying to make something out of nothing. Yeah, I agree with you. I don't think everything that they said we already knew from yeah seasons are things that came out on social media, but I think it was just more of a shock because it was coming from Jackie who's not usually the person to do that. Nobody was shocked by Heavenly because that's what she does but right Jackie she's not that person. So well before we go on Pietra you have a message for our viewers. I do I just want to thank you all for supporting us and making us the ESPN of TB talk. I wanted to say", "Thank you so much. I'm not sure where you all are listening. But if you're on YouTube right now hit the thumbs up button for sure and subscribe to our Channel. And if you're on iTunes, please give us a 5-star rating. We really really appreciate you guys. And if you're in the live chat, please talk. Let us know you're there get involved in the conversation and being a part of AfterBuzz TV has meant so much to us. So thank you for supporting us and keep enjoying our show. Yes. Yes. Yes. Thank you so much. Make sure to like comment and subscribe and if you're watching on the", "Live-chat make sure to write and let us know what you think because we are reading them and we love y'all talking back to us. Yeah, let's go on. So after the opening on his podcast we catch up with quad and heavily they went and got a vaginal rejuvenation. Oh my God, that seems to be the thing on all these Broadway shows. Okay is everybody doing that? Nowadays? Like okay. Yeah. That's a thing. All right, I guess some have to try it one of these days because everybody's doing it it's okay. We're gonna go together. Okay. Yeah, we're going to do it after the show. So while", "Are we doing there that in our Rejuvenation? We had Mariah Toya and Simone out at dinner and it was kind of like we were watching both groups because they were talking about each other of course and that's when we kind of saw with the Divide was what were you shocked by singing Mariah Toya and Simone together out like that.", "No kind of a little bit in between because I know now they're all trying to click up. So it's just like, you know, you don't like her. I don't like her either. Let's all hang together type thing. It's kind of like high school. Right? But I understand we are watching a reality shows that they have to keep it interesting and I do think when they click up it's more to talk about right. So yeah, exactly. I think what's really been a shock for me is that it's just so weird like the Dynamics are all", "Like usually you have like Simone and Jackie usually have quad and Heavenly Toya's kind of usually more with Simone and Jackie. She's been closer to Mariah, but you never really saw some own with Mariah. You know, Jackie's just always just kind of been in the middle, but it's weird seeing her kind of more on the quad and heavenly side. It's just the Dynamics are just so off to me. But what I was surprised by was that there was this beef with Heavenly and Simone and it came out it was because Cecil and", "Heavenly exchanged some words on Twitter, which I was shocked by because it seems like the husbands are always saying stay at Little Miss. Donna says other women's drama but it seemed like seafood was the one who started it with Heavenly. Yeah, you think of that I'm not surprised you're not I'm not going to because I don't know it kind of reminds me of what's his name from the other show Peter. Mmm. It kind of reminds me of Peter. Like there's always a guy that's going to start something with the woman and I kind of think that spices it up to because it", "You know, we know that the guys are there but realistically we're looking at it for the women, but you know, it kind of keeps the guys like, you know involved in some type of way. I do feel like at some point production maybe probably talks to the guys and have to say something but I'm not surprised at all interesting. I'm not were you surprised that one that Heavenly brought up Cecil being jobless on Twitter or to that Simone was mad about Heavenly bringing it up because while it was kind of a touchy subject", "And we did all know that he didn't have a job. Right? So whose side are you kind of on in that accent? I feel like as a friend is certain things you just don't do and like they mention they all have broken the Friendship code in some type of way. I just yeah What's Understood doesn't have to be explained type thing. So it's just like yeah, I get it again. It's a show but it's certain things. You just don't say, I mean you did it for a reason. So that's my take on it. Yeah, and why?", "We were at the vaginal rejuvenation with Simone and Heavenly not Simone in Heavenly with Heavenly and quad Heavenly told quad about the party that was coming up which happened at the end of the episode but that she was going to invite all of the girls. Were you surprised that she invited all the girls something so good idea. I think it was a great idea because there there's no show like we need it. I need it that because personally, I don't know about you guys, but I was bored I'm really was because I'm like, okay I get it like you said,", "And we seeing everybody okay cool, whatever but where's the drama? Right because let's be real about it. That's what we're watching it for. So I think it was a great idea and if we wouldn't have gotten the girls together, I don't know who have so many people tuning in the next episode true. I mean it's good for us. But at the same time if Heavenly really claims she wants to be drama-free and she didn't want to have any issues at a really big event for her husband. You would think that she wouldn't have invited them. Yeah. It's like do you love the drama more than you care about your husband having a", "It will be even better you have to take into account to these are very how do I put it kind of southern belles in a way so they want to show off. So at the end of the day, yeah, it's going to bring drama. But I also want you guys to see what my husband is doing and I want you to see our sinner. So that's what it was about as well because that's very it couldn't be a party without having to let them know like I'm going to invite you so you can see what we're doing the drama is", "Going to be there, but I'm going to actually put it outside in the parking lot. So it doesn't tear up the place. Yeah, everything was well thought out and even though they kept talking about the parking lot. I didn't see anything wrong with that. Yeah, because if I if me and my husband opened up a new Surgery Center, I don't want you guys to start anything we have business to run right? So yeah, bring this show there, you know, bring the show there, you know, I'll have a little section outside for everyone to be at but where are you going to actually set up anything?", "Surgery center right away from me partying with in my way to bed. Like hey, yeah, they kept throwing shade about that, but I didn't see anything wrong with that. It's like where else are you going to put it right totally agree with you on the phone? Yeah, we'll leave it up to Heavenly to plan out a perfectly dramatic way to have drama at a party. I don't know poor Damon is always so embarrassed. Okay, well because it was the premiere episode we made sure that we caught up with each.", "Each person individually and we saw that two people got new houses Jackie got a new house and Toya got a new house. What do you think about that? Because honestly, I would feel like out of the two Jackie would be the one who has the most money who's able to get the big house. But Toya and her husband with all of their tax issues got a humongous house. What did you think? Well, well, well I was thinking about that because in 2016 when they were talking about", "A hundred and seventy thousand dollars owed in taxes. I was looking at that like you you got out of the house that you were in and you cut back on expenses, but then you turn around and get a six bedroom house 9,000 square feet 2-story closet. No. No, I'm trying to not be judgmental. But at the same time it could be a different area.", "Be aware is cheaper. Yeah area. So you just never know. Yeah, and then Atlanta and again, we're dealing with the who's who, you know of they're like click so they want to show off they're trying to they're in directly competing with each other all the time. Yes, because what are the odds this season first episode everybody buying a house? Yeah. Hmm, maybe Bravo done cut some bonuses. Everybody got up started to season because everybody got it.", "I was but I was excited about you know, Jackie and Twain these new houses. Mainly for Jackie is that it looks like Jackie and her new husband. Mike has completely just done one. What is a 180? I don't know math. It's okay 60. It's all the same. Either way. They're in a really happy place and they are getting their house redone. Like I don't know. I was just really happy to see Jackie and her husband happy. Yeah, me too because they've come such a long way and let's be real 16 years, honestly.", "Ali I get it we're all black strong women but let's be real 16 years is investment you have kids involved like it's just a lot of stuff that's going on with that situation where I was rooting for them to stay together because you don't want to start over and start over at that age. It's kind of more difficult than starting all over at a younger age. So I wanted to see them work it out and I'm happy that they're back together. So yes definitely so another copy couple that we saw.", "Together but not together physically was Scott and Contessa. It came out that she's living in Tennessee not because she went back to school. I don't how did you feel about that before I go into how I felt? Yeah. Um, okay as far as what part as far as okay, so I guess I can just start with me I guess so then we can talk about it. So basically, you know last season contestant has had all this guilt about being away from her kids, and she walked away from medicine because she wanted to be a full-time.", "Time Mom and really focus and then the next season comes and she's moved to a completely different city. It's like do you not have that care anymore about being there for your kids full-time? And you know, why did you have to go all the way to Tennessee to go back to school when there's plenty of schools in Georgia? My mom just got her doctor and public health to shout out to my mama. She mom and she did it in Georgia and Georgia State so it's like why did you have to go all the way to Tennessee especially with you trying to be a better mom to your kids?", "I think is one of those things where because of the group that we're dealing with. It's always about the best. It has to be the best this or the best that and I'm sure that she definitely went through options locally. I would assume so being that you know, she wanted to be there for her kids, but it's one of those things where if you want better you have to do better type things and sometimes the best of schools isn't where you're at like for example for me. I'm trying to", "Turn back for my masters and I want to go to an HBCU. There's no HBCU here. So in order for me to God hbcus, I know right for me to go back to where I want. It has to be the South or DC or something like that. So sometimes certain circumstances you have to like leave in order to do what you want and being that I don't think it's going to be a long like period of time. I like the fact that she's a because I think he said only a few times out the week, so", "It seems like she's going to be doing a lot of traveling. Yeah, and I don't think she thinks yeah, I don't think she's staying like months at a time or something like that. So I think they've worked out something where it makes sense, right? You have the kids money with him Friday I go to school Thursday Tuesday weekends or something like that. So I understand like yeah, I can understand I get it. I mean, I understand as a single person like why not just go go to the best school, but it's just it was just ironic to me how you know you wanted to be around me.", "And that's the reason why you walked away from medicine and now because you walked away from medicine, you're starting a whole new career Venture that still took you away from your kids. So that's like was that more important than having, you know that relationship with your kids and not even just her kids with her husband because yeah the strain on a marriage specially him like he wasn't complaining but he did make a few comments as far as being a single dad is hard and I mean, let's face it ladies like we do it the best. Okay, and when it comes down to", "Children, especially girls we know how to handle them a little bit better than men. Right. So for him, he might, you know get over it quick and it's like look we need to figure this out. I don't know if you can finish school. So you have to think about stuff like that, especially him being a doctor to like, yeah. It happened for you you walked away from medicine because it took you away from your kids, but then you leave your kids with someone who's also in medicine who may not have the time either and you know, I was trying to think about but I don't know which what doctors they are, but I know most doctors some", "Some of them are on call so you just never know right may come up. So if it does come up, I'm wondering how that situation will go. She's not there. What does he do? Right especially last season they had that whole drama with the nanny and then firing the nanny so you don't have a nanny and now you're gone and you're just leaving him with Scott. So I guess we'll just have to wait and see what happens and I'm also curious to see how this is going to affect her relationship with the other girls by her being gone. Yeah. We will see we will see another", "Update we will catch up we had with whipped quad who says she's in her final steps of her divorce how we food about that. I don't like divorces. Yeah, I don't like to see people go through divorces because I don't care how strong you appear to be and how happy you seem to be single again, like a piece of you is Left Behind. He's been all that time with somebody and like let them into your heart and stuff like that. Like you have to give yourself time to", "Heal and get back out there, right? And I don't know if it's just for the cameras and stuff. But we seek we've seen quite go through it. Yeah, like so I don't like it. I don't like it at all. I don't like divorces. I agree. I don't like divorce is like I never want to see a divorce but I'm actually kind of happy to see quad be single. You know, I know we watched Jackie forgive her husband for cheating, but I don't know if I could ever stay in and you know cheat", "You just never cuz I've always been the type to say that to you cheat. I'm done. You just never know right because you just never know and like I don't even like to judge situations. I give my opinion on situations unless I've been through it because until you've experienced it yourself. You don't know what you would do, right? You know, that's so true. But it is going to be fun to see her like get back in the dating scene and because we've never seen a single quad we've only seen her with dr. Greg. So now we'll get to see quad date and go out and do what she wants.", "To do but at the same time she's the only single person in the group now and that may be a thing because I know a lot of these shows there's always a couples trip this or that but then again it may be interesting because she might bring somebody new right? So we'll see how that goes. Yeah, we'll have to do is wait and see. All right the moment we've all been waiting for let's talk about this party for Damon had his surgery center party and Heavenly invited all of the girls, even though every nobody likes each other right now. And of course there was drama.", "What did you think about the drama that went down? I loved it. Yes, again, the only reason why I loved it and I'm saying that I loved it. It's not because I love drama but because without that whole getup the show, I don't even know what have been a show because it was nothing going on prior to that. I do think I was able to see a lot of hookah Tessa is because this is my first time really like seeing her because I've watched, you know Married to Medicine every season.", "Season off and on but I'm really able to see like a little bit of her so I liked it. I mean, yeah, I also was surprised to see Tanya kind of back down, you know, but we'll get to that. Yeah, what what automatically got me out of the gate is literally as soon as like Toya and Heavenly not heavily Toya and Mariah keep missing mixing up my Ryan heavily, but Toya and Mariah they came in like just all shade. It's like if you", "In if you don't like her if you don't want to be there if you just gonna be there to talk Miss why come you know, my why that's that was really my big question. Like if you just are are coming and want to mess things up or you don't want to be happy for them. You could have just stayed home it goes back to one of those things where you want to be on the scene. You want to see you don't want to miss out on nothing type thing. You know, I couldn't imagine if I didn't like someone I wouldn't go to their stuff, but it", "Again, it's one of those things where a girl. Let's just go see what's going on. Like let me see who's gonna be there. Let me see like you don't want to miss out on nothing. Yeah, you know, and I'm sure they wanted to be there for Damon because Damon don't bother nobody he stay at all the mess so they probably wanted to support him to but why come in there with guns blazing they was just talking about her. We're how it was in the parking lot about how her carpet wasn't read. It was maroon. It's just like all the shade it's unnecessary and to add insult to injury Mariah.", "Her sister Lake, who as we know from last year's reunions. She accused quad of cheating on her husband with her sister's husband. So do you think she was being messy by inviting Lake to come? Absolutely absolutely. Like there's no doubt about it. That was the whole point and I don't see how we could have, you know went through that whole scene without addressing that that would have been really weird. Yeah. I think it should have been addressed to and hopefully they do in the next episode. But how would you feel if you were quad bike?", "What do you think? She's thinking?", "I don't know. What quad I just don't know. I don't know but I will say Joseph is in a live chat right now. Shout out to you Joseph and he said I think her husband is partly jealous because if Contessa does go to school and events in her career, she will be home more than he is. Hmm. Good point. Yeah. Hmm. Hmm interesting. Well what also happened at the party was speaking of that we saw", "I am Contessa get into it really wasn't their fight to begin with but of course they find any way to get into it and Contessa called Toya's husband the b-word and Toya said absolutely nothing back. And the thing is she's always the one that's in little shady stuff and starting stuff. So for me, I was able to see ketosis is a whole Thug Oh contestants for sure about that life. She's like not playing like she got up in her face like and what?", "What? Yeah, and and and the fact that tall you didn't say anything and now she's like, oh, I just love her show her her true colors in the but you kind of started it. Yeah, like you kind of brought it up. And then now that she says something now is oh she's showing her true colors. It's one of those things where will people poke poke poke you just to get you to retaliate and once you do is like see told you yeah, I don't like that. I think she was scared because we've seen Toya for all these seasons and", "You've never back down from my argument, please if anybody ever comes to her husband like when Contessa said I was like, oh it's about to go down and Toya was silent mute. I was shocked but then I also know that contest is for sure about that life. He's a she's in the military, and she's also bigger than everybody like tall, so I'm not surprised. That's all she is. I don't know, but she be towering over everybody and I guess Toya knew today was not the day to mess with her and we'll just have to wait and see if something comes up later.", "All right, so that is a wrap up of our show. Let's get you to some tweets that we saw from viewers. Yes.", "All right. This one is from add. I think that says Charisse don't have my glasses on dad. What happened to all that fake crying over missing Milestone with her kids. I think they're talking about Contessa. Yeah, because she was crying last season about not being there and now she's in Tennessee. So I guess that doesn't matter anymore Charisse we have another one from at like Bentley. It says IDK why Simone and Cecil even went to the event? I agree true. I understand like okay Simone.", "It's Simone is messy she's gonna go regardless, but I think it's Cecil really had an issue with Heavenly. Why go to her husband's he been absolutely I totally agree with that. Yeah. All right another one from at TJ's business. Now that Simone is on the outs with the other ladies her and Mariah are besties. I was a combat true and I can't help but Wonder because we didn't really see a lot of interaction with Jackie and Simone if like they're into it and like or they're just trying to stay on there.", "They're like them their sides and their still fine, but they didn't they weren't really around each other much. We usually see them Kiki a lot with the more with each other. So I'm wondering what that means. Is there an issue with Jackie and Simona. Hmm. That would be interesting. Yeah, it would be interesting. We very interesting and they are kind of avoiding each other and it's like they're both it seems like Jackie's more. So on the Contessa Heavenly quad side and Simone's mom the Toya and Mariah side, so it's like where do they live?", "Mm-hmm. I will be looking out for that next. Yeah, we'll have to wait and see what happens. Okay back to our sweets. It says at Afrocentric Gypsy Mariah's in her feelings, but she's live stream and talk mountains of nasty stuff about the other women. So what's the issue if that's the case, none of them should ever be mad about anything because I agree. Let's talk about each other. That's all they do. They tweet about each other they talk about each other all the time. Yep. Yep. Yep. Yep, and then we have another one from Atlas shell H. It says I really", "If this is the last movement will have to see with Toya and Eugene and honestly, I'm not understanding. How do you did they ever explain that a hundred and seventy thousand dollars, they downsized and paid it off, but it's like if you're going to continue what you already did you? Yeah, you paid it off, but you can go right back in that same place. They have a better plan this time where it's like they downsize to pay it off and now they're moving back into the home that they always want to because now they can do it the right way. Yeah and maybe also making more money.", "Now because you know this show has been on for a long time. You can start negotiating a salary. Yeah, absolutely. Yep. Alright, let's get into some news and gossip news and gossip. All right guys, so this news and gossip this week is not about any of the ladies Married to Medicine, but it is about another strong beautiful African-American woman that we all remember some of us remember her from The Jamie Foxx Show. She was one of the leading ladies and some of us may remember her from", "Hosting and being the face of extra but we are talking about miss garcelle boo. They now most of us know her from acting but looks like she will be joining the ladies of Housewives of Beverly Hills and I'm really here for that because I think that would be great for her. She's a mom. She's successful. We know her in the industry and they need some color in Housewives of Beverly Hills. So I am here for that. What do you think about that? Yeah, she", "Um put on her Instagram how excited he was and that she's going to be the first black woman to be on Real Housewives of Beverly Hills. So that was exciting. I I've been watching Beverly Hills lately and they do need a shake-up in there. Yeah, and they're cast so it's not when we ourselves also sunstrike who's in the picture with her. So we're going to see a new Dynamic on Real Housewives of Beverly Hills. So when you're not watching Married to Medicine, make sure to take the time to watch another one of our favorite Bravo shows with Real Housewives of Beverly Hills and this time we'll see a black beautiful face. Yes.", "I'm here for that. Congratulations guys exciting exciting exciting. All right, since it is the first episode we're going to do a top three. Let's do it. I think because there was so much shade we should do our top three shadiest moments. Yes. Yes. Yes. Yeah. What are you thinking? I'm thinking when when Mariah told Jackie, I believe no. Yeah Mariah told Jackie you're showing your slip and Jackie's like, you know I have pants on.", "The end of the day that was really shady because I'm like now that's my new my little thing. Like you showed your slip tonight showing your slip is showing its lit. Yes real classy, but still letting you know you was wrong. Yeah. All right. So the number two when I have to go back to they didn't say it, but it was brought up because it was something that had been retweeted when she told Cecil but he's tweeting twerking and not work that way.", "That's the ultimate. Yeah disrespect me like really? Yes. I would been so mad and that's why I would have gone to the party policy side because how can you support somebody who says that exactly and then I think the top one has to be when Mariah showed up at heavenly's party and said she has a maroon carpet that she was parking lot people with the cab trucks and dingy decor and my thing is you were so right like if you're gonna come up to the", "It would negativity already why even be there. But again, I know she wanted to see stuff didn't even want to go there on the red carpet or anything but it's like okay you could have at least kept acute changes like, you know, okay, you could have thought it but why do you have to be like that? Right? You have to be like that? Exactly and what's funny is that there were saying all this stuff in there like interviews and when they were off together in their little cliques, but when they saw each other was not yeah, give me a hug. But that just goes to show how fake they are. Ya you're so fake. Yep.", "Yep. Yep, and it's it's in the South to Southern Belles. They were the Queen's of being nice nasty. So I am not at all. So very true clothes are our top three shadiest moments. Let's get into our prediction prediction. Okay. Let's see. What do I start? Um, I predict that.", "I think heavily and dr. Simone is going to have a showdown. I just see it coming because heavily doesn't back down from anything and when I'm starting to see with her is she just doesn't care in the more they drink the more like everything comes out. Yes, Contessa and Toya Toya scary. That's that I predict that I don't think that Toya would even try to put her hands on. Oh no sense of like, I don't even think it would even get to that point.", "With that situation, I just see Contessa like running everything. I hope that her schooling doesn't make her marriage go downhill because I'm predicting that that may happen if she continued to be away like that and everything is on him and it's kind of like you're popping in. Hey, I'm mom and then you're leaving. Yeah, so that's a prediction and lastly. Let's see. Who else", "I predict Miss Jackie is going to show her slit even in her pants. I think she's gonna show her slip this season and we're going to see another layout layer of her just with the girls. I don't think we'll see anything else for her marriage, especially being that they're just getting back on the Good Foot, but I definitely think she's going to show her sleep with these ladies. Yeah for sure. So I think my predictions are we're going to see a lot more quad dating, but I feel like", "What is going to be left out a lot more because she doesn't have a husband. This is Married to Medicine. She's no longer going to be Married to Medicine. So what is that going to mean with her Dynamic of the group? And is she going to be able to come around for certain things? So think that's going to become a thing. I do see some not hopefully not a lot that leads to a divorce but definitely some marital issues with Contessa and Scott. Yeah, and I just see a lot of drama this season. It doesn't seem like anybody's getting along even our old faithfuls.", "Jackie and Simone don't seem like they're getting along so they're not getting along then things are just all off. So I'm definitely seeing a lot of drama a lot of arguments and going back and forth. But all in all I think it's going to be a really great season. So I'm expecting tattoo. Yep. Yep. Yep. Well, thank you for tuning in with us for our first episode. We love talking about our stuff you and we're excited to continue off this season. So where can they find you future if you all are looking for any great things and I'm doing you can follow me on all social media.", "Platforms Pietra at least TV so it's PE tra e-l is ETV. Yes, and you can find me on Instagram and Twitter app. I am Lexi fears. I am L exif IE rce and until we meet again push you later. Peace our founder Keven undergaro and me Maria Menounos would like to thank you for tuning in to AfterBuzz TV. Remember, we're not just the first where the biggest in the world and were the", "the only destination for all your favorite TV shows whatever your grave we've got it. So go to AfterBuzz TV.com and check out our lineup buzz you later. You've expressed herein are those of the host only do not necessarily reflect the views of AfterBuzz TV or zoners our principal."]
num_topics = 30
def word_count_by_topic(topic=0):
    d_lens = [len(d) for d in grouped_topics.get_group(topic)['Processed_text']]
    plt.figure(figsize=(10,8))
    plt.hist(d_lens)
    large = plt.gca().get_ylim()[1]
    d_mean = round(np.mean(d_lens), 1)
    d_median = np.median(d_lens)
    plt.plot([d_mean, d_mean], [0,large], label='Mean = {}'.format(d_mean))
    plt.plot([d_median, d_median], [0,large], label='Median = {}'.format(d_median))
    plt.legend()
    plt.xlabel('Document word count',fontsize=16)
    plt.ylabel('Number of documents',fontsize=16)
    plt.xticks(fontsize=14)
    plt.yticks(fontsize=14)_____no_output_____from ipywidgets import interact, IntSlider

slider = IntSlider(min=0, max=num_topics-1, step=1, value=0, description='Topic')
interact(word_count_by_topic, topic=slider);_____no_output_____lda_top_words_index = set()
for i in range(lda_model.num_topics):
    lda_top_words_index = lda_top_words_index.union([k for (k,v) in lda_model.get_topic_terms(i)[:5]])

print('Indices of top words: \n{}\n'.format(lda_top_words_index))Indices of top words:
{1, 2819, 1283, 771, 519, 6668, 781, 13, 3853, 17, 275, 2069, 22, 3095, 3351, 1814, 2073, 1820, 798, 1058, 1585, 1331, 2356, 3379, 51, 2099, 1080, 572, 64, 1602, 66, 1090, 323, 5444, 1863, 581, 1098, 1872, 5201, 594, 1110, 4184, 88, 91, 1373, 350, 1630, 8289, 8036, 614, 872, 617, 2920, 2923, 2157, 11374, 111, 114, 116, 3704, 5497, 1148, 125, 1662, 638, 8575, 1151, 10369, 2429, 1156, 640, 2951, 9353, 7819, 397, 9614, 2446, 916, 661, 150, 2202, 3227, 3228, 2719, 3490, 3235, 2211, 1445, 3492, 169, 6570, 5035, 430, 9399, 6583, 10427, 700, 4798, 2752, 706, 4035, 8644, 8389, 10182, 199, 709, 450, 9675, 9421, 14542, 2512, 723, 1236, 467, 728, 2271, 9699, 6885, 21989, 230, 2280, 2025, 1260, 4846, 497, 760, 249, 3322, 1275, 1021}

words_we_care_about = [{dictionary[tup[0]]: tup[1] for tup in lst if tup[0] in list(lda_top_words_index)}
                       for lst in corpus]_____no_output_____lda_top_words_df = pd.DataFrame(words_we_care_about).fillna(0).astype(int).sort_index(axis=1)
lda_top_words_df['Cluster'] = main_topic_df['Dominant_topic']_____no_output_____clusterwise_words_dist = lda_top_words_df.groupby('Cluster').get_group(0)

clusterwise_words_dist.sum().transpose().\
         plot.bar(figsize=(30, 8), width=0.7).\
         set(ylabel='Word frequency',
         title='Word Frequencies by Topic, Combining the Top {} Words in Each Topic'.format(len(lda_top_words_index)));_____no_output_____