# Notebook from davidluther/Metis-Projects
Path: 4-say-it-loud/code/20-pipeline-dev.ipynb

# Pipeline Development_____no_output_____First draft of pipeline from loading to clusters. Final draft functions and classes were moved to the `fletchmod.py` module for use in the next phases, and updated as needed throughout the process. **Functions in this notebook are not necessarily indicitave of the final versions.**_____no_output_____

import pandas as pd
import numpy as np
import json
from itertools import chain
from pymongo import MongoClient
from bson.objectid import ObjectId
import re
import pprint
from IPython import display

from spacy.en import STOP_WORDS
import spacy
from nltk.stem import (PorterStemmer,
                       LancasterStemmer,
                       SnowballStemmer,
                       RegexpStemmer as REStemmer,
                       WordNetLemmatizer
                      )

from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.decomposition import NMF, TruncatedSVD
from sklearn.decomposition import LatentDirichletAllocation as LDA
from sklearn.cluster import KMeans
from sklearn.cluster import AgglomerativeClustering as Agg
from sklearn.preprocessing import StandardScaler

import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
import seaborn as sns
plt.style.use('seaborn-colorblind')
%matplotlib inline_____no_output_____# load collection
client = MongoClient()
p4_db = client.proj4
lyrics_coll = p4_db.lyrics_____no_output_____

## 1. Pulling lyrics from DB_____no_output_____

def lyrics_from_db(coll, sample_size='all', min_year=1965, max_year=1975):
    """
    Returns raw lyrics and corresponding BSON IDs for songs within the min
    year/max year range (inclusive). Returns all songs in range, or a sample of
    given size.
    ---
    IN
    coll: MongoDB lyrics collection
    sample_size: number of songs in sample (int), or 'all' if all in range (str)
    min_year: lower bound for samples (int)
    max_year: upper bound for samples (int)
    OUT
    lyrics: list of raw lyrics (list)
    song_ids: list of BSON IDs corresponding with lyrics list (list)
    """

    if sample_size == 'all':
        docs = coll.find({'Year': {"$gte": min_year, "$lte": max_year},
                          'Lyrics': {"$exists": True}})

    else:
        assert type(sample_size) == int, "size must be an integer if not 'all'"
        docs = coll.aggregate([{"$match":
                                {"$and": [{'Year': {"$gte": min_year,
                                                    "$lte": max_year}},
                                          {'Lyrics': {"$exists": True}}
                                         ]}},
                               {"$sample": {"size": sample_size}}
                              ])

    lyrics = []
    song_ids = []
    for doc in docs:
        lyrics.append(doc['Lyrics'])
        song_ids.append(doc['_id'])

    return lyrics, song_ids_____no_output_____

## 2. Cleaning Lyrics_____no_output_____To get an idea of what we might need to do in the cleaning process..._____no_output_____

# pulls a sample from a specific date range

sample = lyrics_coll.aggregate([{"$match": {'Year': {"$gte": 1965, "$lte": 1975}}},
                           {"$sample": {"size": 20}}
                           ])

for record in sample:
    pprint.pprint(record){'Artist': 'bobby hebb',
 'Instrumental': False,
 'Lyrics': ' sunny yesterday my life was filled with rain sunny you smiled at '
           'me and really eased the pain now the dark days are gone and the '
           'bright days are here my sunny one shines so sincere sunny one so '
           'true i love yousunny thank you for the sunshine bouquet sunny '
           'thank you for the love you brought my way you gave to me your all '
           'and all and now i feel ten feet tall sunny one so true i love '
           'yousunny thank you for the truth you let me see sunny thank you '
           'for the facts from a to z my life was torn like windblown sand and '
           'a rock was formed when you held my hand oh sunny sunny one so true '
           'i love yousunny thank you for the smile upon your face hmm sunny '
           'thank you thank you for the gleam that shows its grace youre my '
           'spark of natures fire youre my sweet complete desire sunny one so '
           'true yes i love you ',
 'Rank': 27,
 'Song': 'sunny',
 'Source': 1,
 'Year': 1966,
 '_id': ObjectId('59f903150f3e05a46c01d806')}
{'Artist': 'the partridge family',
 'Instrumental': False,
 'Lyrics': ' i go downtown and roam all around but every street i walk i find '
           'another dead end im on my own but im so all alone i need somebody '
           'so i wont have to pretendi know theres someone just waiting '
           'somewhere i look around for her but shes just not thereoh doesnt '
           'somebody want to be wanted like me where are you doesnt somebody '
           'want to be wanted like me just like meim running free but i dont '
           'want to be i couldnt take another day like yesterday im dead on my '
           'feet from walking the street i need somebody to help me find my '
           'wayive gotta get out of this town before i do ill take a last look '
           'aroundoh doesnt somebody want to be wanted like me where are you '
           'doesnt somebody want to be wanted like me just like meyou know im '
           'no different from anybody else i start each day and end each night '
           'but it gets real lonely when youre by yourself now where is love '
           'and who is love i gotta knowdoesnt somebody want to be wanted like '
           'me where are you doesnt somebody want to be wanted like me just '
           'like me doesnt somebody want to be wanted like me where are you '
           'doesnt somebody want to be wanted like me just like medoesnt '
           'somebody want to be wanted like me where are you ',
 'Rank': 53,
 'Song': 'doesnt somebody want to be wanted',
 'Source': 1,
 'Year': 1971,
 '_id': ObjectId('59f903150f3e05a46c01da14')}
{'Artist': 'paul  linda mccartney',
 'Instrumental': False,
 'Lyrics': 'were so sorry uncle albert were so sorry if we caused you any pain '
           'were so sorry uncle albert but theres no one left at home and i '
           'believe im gonna rain were so sorry but we havent heard a thing '
           'all day were so sorry uncle albert but if anything should happen '
           'well be sure to give a ring were so sorry uncle albert but we '
           'havent done a bloody thing all day were so sorry uncle albert but '
           'the kettles on the boil and were so easily called away hands '
           'across the water water heads across the sky hands across the water '
           'water heads across the sky admiral halsey notified me he had to '
           'have a berth or he couldnt get to sea i had another look and i had '
           'a cup of tea and butter pie butter piethe butter wouldnt melt so i '
           'put it in the pie hands across the water water heads across the '
           'sky hands across the water water heads across the sky live a '
           'little be a gypsy get aroundget around get your feet up off the '
           'ground live a little get around live a little be a gypsy get '
           'aroundget around get your feet up off the ground live a little get '
           'around hands across the water water heads across the sky hand '
           'across the water water heads across the sky',
 'Rank': 22,
 'Song': 'uncle albertadmiral halsey',
 'Source': 5,
 'Year': 1971,
 '_id': ObjectId('59f903150f3e05a46c01d9f5')}
{'Artist': 'the delfonics',
 'Instrumental': False,
 'Lyrics': ' i gave my heart and soul to you girl didnt i do it baby didnt i '
           'do it babygave you the love you never knew girl didnt i do it baby '
           'didnt i do it babyive tried so many times and thats no lie it '
           'seems to make you laugh each time i crydidnt i blow your mind this '
           'time didnt i didnt i blow your mind this time didnt i listeni '
           'thought that heart of yours was true girl now didnt i think it '
           'baby didnt i think it babybut this time im really leavin you girl '
           'hope you know it baby hope you know it babyten times or more yes '
           'ive walked out that door get this into your head therell be no '
           'moredidnt i blow your mind this time didnt i didnt i blow your '
           'mind this time didnt ididnt i do it baby didnt i do it baby didnt '
           'i do it baby didnt i do it babyten times or more yes ive walked '
           'out that door get this into your head therell be no moredidnt i '
           'blow your mind this time didnt i didnt i blow your mind this time '
           'didnt i i got to leave you babydidnt i blow your mind this time '
           'didnt i didnt i blow your mind this time didnt i ',
 'Rank': 71,
 'Song': 'didnt i blow your mind this time',
 'Source': 1,
 'Year': 1970,
 '_id': ObjectId('59f903150f3e05a46c01d9c2')}
{'Artist': 'the jackson 5',
 'Instrumental': False,
 'Lyrics': '  ',
 'Rank': 7,
 'Song': 'ill be there',
 'Source': 1,
 'Year': 1970,
 '_id': ObjectId('59f903150f3e05a46c01d982')}
{'Artist': 'the mindbenders',
 'Instrumental': False,
 'Lyrics': 'when im feeling blue all i have to do  is take a look at you then '
           'im not so blue  when youre close to me i can feel your heart beat  '
           'i can hear you breathing near my ear  wouldnt you agree baby you '
           'and me got a groovy kind of love  anytime you want to you can turn '
           'me onto  anything you want to anytime at all  when i kiss your '
           'lips ooh i start to shiver  cant control the quivering inside  '
           'wouldnt you agree baby you and me got a groovy kind of love oh  '
           'when im feeling blue all i have to do  is take a look at you then '
           'im not so blue  when im in your arms nothing seems to matter  my '
           'whole world could shatter i dont care  wouldnt you agree baby you '
           'and me got a groovy kind of love  we got a groovy kind of love  we '
           'got a groovy kind of love oh  we got a groovy kind of love',
 'Rank': 34,
 'Song': 'a groovy kind of love',
 'Source': 3,
 'Year': 1966,
 '_id': ObjectId('59f903150f3e05a46c01d80d')}
{'Artist': 'the temptations',
 'Instrumental': False,
 'Lyrics': ' i can turn the gray sky blue i can make it rain whenever i want '
           'it to oh i i can build a castle from a single grain of sand i can '
           'make a ship sail on dry land tell em yeah but my life is '
           'incomplete and im so blue cause i cant get next to youi cant get '
           'next to you babe next to you i cant get next to you i just cant '
           'get next you i cant get next to you babe i cant get next to youi '
           'can fly like a bird in the sky hey and i can buy anything that '
           'money cant buy oh i i can turn a river into a raging fire i can '
           'live forever if i so desiredunimportant are all the things i can '
           'do cause i cant get next to youi cant get next to you babe no '
           'matter what i do i cant get next to you uhyaoooooo oooooo chica '
           'boom chica boom chica boom boom boomi can turn back the hands of '
           'time you better believe i can i can make the seasons change just '
           'by waving my hand oh i i can change anything from old to new the '
           'things i want to do the most im unable to dounhappy am i with all '
           'the powers i possess cause girl youre the key to my happiness and '
           'icant get next to yougirl youre blowing my mind cause i cant get '
           'next to you cant you see these tears im crying i cant get next to '
           'you girl its you that i need i gotta get next to you cant you see '
           'these tears im crying i cant get next to you i i i i i cant get '
           'next to you i i i i i cant get next to you girl youre blowing my '
           'mind ',
 'Rank': 3,
 'Song': 'i cant get next to you',
 'Source': 1,
 'Year': 1969,
 '_id': ObjectId('59f903150f3e05a46c01d91a')}
{'Artist': 'simon  garfunkel',
 'Instrumental': False,
 'Lyrics': 'im sittin in the railway station got a ticket for my destination '
           'on a tour of one night stands my suitcase and guitar in hand and '
           'every stop is neatly planned for a poet and a one man band '
           'homeward bound i wish i was homeward bound home where my thoughts '
           'escapin home where my musics playin home where my love lies waitin '
           'silently for me everydays an endless stream of cigarettes and '
           'magazines and each town looks the same to me the movies and the '
           'factories and every strangers face i see reminds me that i long to '
           'be homeward bound i wish i was homeward bound home where my '
           'thoughts escapin home where my musics playin home where my love '
           'lies waitin silently for me tonight ill sing my songs again ill '
           'play the game and pretend but all my words come back to me in '
           'shades of mediocrity like emptiness in harmony i need someone to '
           'comfort me homeward bound i wish i was homeward bound home where '
           'my thoughts escapin home where my musics playin home where my love '
           'lies waitin silently for me silently for me',
 'Rank': 56,
 'Song': 'homeward bound',
 'Source': 3,
 'Year': 1966,
 '_id': ObjectId('59f903150f3e05a46c01d823')}
{'Artist': 'aretha franklin',
 'Instrumental': False,
 'Lyrics': ' the moment i wake up before i put on my makeup i say a little '
           'pray for you while combing my hair now and wondering what dress to '
           'wear now i say a little prayer for youforever and ever youll stay '
           'in my heart and i will love you forever and ever we never will '
           'part oh how i love you together forever thats how it must be to '
           'live without you would only mean heartbreak for mei run for the '
           'bus dear while riding i think of us dear i say a little prayer for '
           'you at work i just take time and all through my coffee break time '
           'i say a little prayer for youforever and ever youll stay in my '
           'heart and i will love you forever and ever we never will part oh '
           'how ill love you together forever thats how it must be to live '
           'without you would only mean heartbreak for mei say a little prayer '
           'for you i say a little prayer for youmy darling believe me believe '
           'me for me there is no one but you please love me too answer his '
           'pray and im in love with you answer his pray answer my prayer now '
           'babe answer his prayforever and ever youll stay in my heart and i '
           'will love you forever and ever we never will part oh how ill love '
           'you together forever thats how it must be to live without you '
           'would only mean heartbreak for me ',
 'Rank': 93,
 'Song': 'i say a little prayer',
 'Source': 1,
 'Year': 1968,
 '_id': ObjectId('59f903150f3e05a46c01d910')}
{'Artist': 'the flying machine',
 'Instrumental': False,
 'Lyrics': 'you really should accept this time hes gone for good hell never '
           'come back now even though he said he would so darling dry your '
           'eyes so many other guys would give the world im sure to wear the '
           'shoes he wore oh cmon smile a little smile for me rosemarie whats '
           'the use in cryin in a little while youll see rosemarie you must '
           'keep on tryin i know that he hurt you bad i know darling dont be '
           'sad and smile a little smile for me rosemarie rosemarie i guess '
           'youre lonely now loves comin to an end but darling only now are '
           'you free to start again lift up your pretty chin dont let those '
           'tears begin youre a big girl now and youll pull through somehow oh '
           'cmon smile a little smile for me rosemarie whats the use in cryin '
           'in a little while youll see rosemarie you must keep on tryin i '
           'know that he hurt you bad i know darling dont be sad and smile a '
           'little smile for me rosemarie rosemarie smile a little smile for '
           'me rosemarie rosemarie',
 'Rank': 76,
 'Song': 'smile a little smile for me',
 'Source': 3,
 'Year': 1969,
 '_id': ObjectId('59f903150f3e05a46c01d963')}
{'Artist': 'the bee gees',
 'Instrumental': False,
 'Lyrics': ' the preacher talked to me and he smiled said come and walk with '
           'me come and walk one more mile now for once in your life youre '
           'alone but you aint got a dime theres no time for the phoneive just '
           'got to get a message to you hold on hold on one more hour and my '
           'life will be through hold on hold oni told him im in no hurry but '
           'if i broke her heart wont you tell her im sorry and for once in my '
           'life im alone and ive got to let her know just in time before i '
           'goive just go to get a message to you hold on hold on one more '
           'hour and my life will be through hold on hold onwell i laughed but '
           'that didnt hurt and its only her love that keeps me wearing this '
           'dirt now im crying but deep down inside well i did it to him now '
           'its my turn to dieive just got to get a message to you hold on '
           'hold on one more hour and my life will be through hold on hold on '
           'hold on one more hour and my life will be through hold on hold on '
           'ive just got to get a message to her hold on hold on ',
 'Rank': 59,
 'Song': 'ive gotta get a message to you',
 'Source': 1,
 'Year': 1968,
 '_id': ObjectId('59f903150f3e05a46c01d8ee')}
{'Artist': 'jewel akens',
 'Instrumental': False,
 'Lyrics': 'let me tell ya bout the birds and the bees and the flowers and the '
           'trees and the moon up above and a thing called love let me tell ya '
           'bout the stars in the sky and a girl and a guy and the way they '
           'could kiss on a night like this when i look into your big brown '
           'eyes its so very plain to see that its time you learned about the '
           'facts of life starting from a to z let me tell ya bout the birds '
           'and the bees and the flowers and the trees and the moon up above '
           'and a thing called love yeah when i look into your big brown eyes '
           'its so very plain to see that its time you learned about the facts '
           'of life starting from a to z let me tell ya bout the birds and the '
           'bees and the flowers and the trees and the moon up above and a '
           'thing called love let me tell ya bout the birds and the bees and '
           'the flowers and the trees and the birds and the bees and the '
           'flowers and the trees bout the birds and the bees and the bees and '
           'the bees ',
 'Rank': 13,
 'Song': 'the birds and the bees',
 'Source': 3,
 'Year': 1965,
 '_id': ObjectId('59f903140f3e05a46c01d794')}
{'Artist': 'bobby sherman',
 'Instrumental': False,
 'Lyrics': ' takin the shade out of the sun whatever made me think that i was '
           'number one i oughta know easy come easy go sittin it out spinnin '
           'the dial thinkin about the chump ive been i have to smile didnt i '
           'know easy come easy go she wasnt kind i wasnt smart i lost my mind '
           'and fell apart i had to find myself in time now i can start all '
           'over again hangin around takin it slow happy i found i still can '
           'smile and dig the show lettin me know easy come easy go ',
 'Rank': 56,
 'Song': 'easy come easy go',
 'Source': 1,
 'Year': 1970,
 '_id': ObjectId('59f903150f3e05a46c01d9b3')}
{'Artist': 'roberta flack',
 'Instrumental': False,
 'Lyrics': 'strollin in the park watchin winter turn to spring walkin in the '
           'dark seein lovers do their thing ooh thats the time i feel like '
           'makin love to you thats the time i feel like makin dreams come '
           'true oh baby when you talk to me when youre moanin sweet and low '
           'when youre touchin me and my feelings start to show ooh thats the '
           'time i feel like makin love to you thats the time i feel like '
           'makin dreams come true oh baby in a restaurant holdin hands by '
           'candlelight while im touchin you wanting you with all my might ooh '
           'thats the time i feel like makin love to you thats the time i feel '
           'like makin dreams come true oh baby strollin in the park watchin '
           'winter turn to spring walkin in the dark seein lovers do their '
           'thing ooh thats the time i feel like makin love to you thats the '
           'time i feel like makin dreams come true',
 'Rank': 35,
 'Song': 'feel like makin love',
 'Source': 3,
 'Year': 1974,
 '_id': ObjectId('59f903150f3e05a46c01db2e')}
{'Artist': 'the carpenters',
 'Instrumental': False,
 'Lyrics': ' such a feelins comin over me there is wonder in most evry thing i '
           'see not a cloud in the sky got the sun in my eyes and i wont be '
           'surprised if its a dreameverything i want the world to be is now '
           'comin true especially for me and the reason is clear its because '
           'you are here youre the nearest thing to heaven that ive seenim on '
           'the top of the world lookin down on creation and the only '
           'explanation i can find is the love that ive found ever since youve '
           'been around your loves put me at the top of the worldsomethin in '
           'the wind has learned my name and its tellin me that things are not '
           'the same in the leaves on the trees and the touch of the breeze '
           'theres a pleasin sense of happiness for methere is only one wish '
           'on my mind when this day is through i hope that i will find that '
           'tomorrow will be just the same for you and me all i need will be '
           'mine if you are hereim on the top of the world lookin down on '
           'creation and the only explanation i can find is the love that ive '
           'found ever since youve been around your loves put me at the top of '
           'the worldim on the top of the world lookin down on creation and '
           'the only explanation i can find is the love that ive found ever '
           'since youve been around your loves put me at the top of the world ',
 'Rank': 39,
 'Song': 'top of the world',
 'Source': 1,
 'Year': 1974,
 '_id': ObjectId('59f903150f3e05a46c01db32')}
{'Artist': 'the pipkins',
 'Instrumental': False,
 'Lyrics': 'that̢s right that̢s right i̢m sad and blue ̢cause i can̢t do the '
           'boogaloo i̢m lost i̢m lost can̢t do my thing that̢s why i sing '
           'gimme gimme dat ding gimme dat gimme dat gimme gimme gimme dat '
           'gimme dat ding gimme dat gimme dat gimme gimme dat gimme gimme '
           'gimme dat ding oh sing it one more time mama oh gimme dat gimme '
           'dat gimme gimme gimme dat gimme dat ding gimme dat gimme gimme dat '
           'gimme dat ding gimme dat gimme gimme dat gimme gimme gimme dat '
           'ding ah you ain̢t doing̢ that late at night ah what good̢s a '
           'metronome without a bell for ringing how fast can anybody ever '
           'tell he swings how can you tell the rhythm written on a bar how '
           'can you ever hope to know just where you are gimme dat gimma dat '
           'gimme dat gimma dat gimme dat gimma dat gimme dat gimme gimme '
           'gimme gimme ah gotcha gimme dat gimme dat gimme gimme gimme dat '
           'gimme dat ding gimme dat gimme gimme dat gimme dat ding gimme dat '
           'gimme gimme dat gimme gimme gimme dat ding gimme dat gimme dat '
           'gimme gimme gimme dat gimme dat ding gimme dat gimme gimme dat '
           'gimme dat ding gimme dat gimme gimme dat gimme gimme gimme dat '
           'ding ah what good̢s a metronome without a bell for ringing how '
           'fast can anybody ever tell he swings how can you tell the rhythm '
           'written on a bar how can you ever hope to know just where you are '
           'gimme dat gimma dat gimme dat gimma dat gimme dat gimma dat gimme '
           'dat gimme gimme gimme gimme gimme dat gimme dat gimme gimme gimme '
           'dat gimme dat ding gimme dat gimme gimme dat gimme dat ding gimme '
           'dat gimme gimme dat gimme gimme gimme dat ding that̢s right',
 'Rank': 86,
 'Song': 'gimme dat ding',
 'Source': 3,
 'Year': 1970,
 '_id': ObjectId('59f903150f3e05a46c01d9d1')}
{'Artist': 'creedence clearwater revival',
 'Instrumental': False,
 'Lyrics': ' just got home from illinois lock the front door oh boy got to sit '
           'down take a rest on the porch imagination sets in pretty soon im '
           'singindoo doo doo lookin out my back doortheres a giant doing '
           'cartwheels a statue wearin high heels look at all the happy '
           'creatures dancing on the lawn a dinosaur victrola listning to buck '
           'owensdoo doo doo lookin out my back doortambourines and elephants '
           'are playing in the band wont you take a ride on the flyin spoon '
           'doo doo doo wondrous apparition provided by magiciandoo doo doo '
           'lookin out my back doortambourines and elephants are playing in '
           'the band wont you take a ride on the flyin spoon doo doo doo '
           'bother me tomorrow today ill buy no sorrowsdoo doo doo lookin out '
           'my back doorforward troubles illinois lock the front door oh boy '
           'look at all the happy creatures dancing on the lawn bother me '
           'tomorrow today ill buy no sorrowsdoo doo doo lookin out my back '
           'door ',
 'Rank': 36,
 'Song': 'lookin out my back door',
 'Source': 1,
 'Year': 1970,
 '_id': ObjectId('59f903150f3e05a46c01d99f')}
{'Artist': 'the beach boys',
 'Instrumental': False,
 'Lyrics': 'ii love the colorful clothes she wears and the way the sunlight '
           'plays upon her hair i hear the sound of a gentle word on the wind '
           'that lifts her perfume through the air im pickin up good '
           'vibrations shes giving me excitations oom bop bop im pickin up '
           'good vibrations good vibrations oom bop bop shes giving me '
           'excitations excitations oom bop bop good good good good vibrations '
           'oom bop bop shes giving me excitations excitations oom bop bop '
           'good good good good vibrations oom bop bop shes giving me '
           'excitations excitations close my eyes shes somehow closer now '
           'softly smile i know she must be kind when i look in her eyes she '
           'goes with me to a blossom world im pickin up good vibrations shes '
           'giving me excitations oom bop bop im pickin up good vibrations '
           'good vibrations oom bop bop shes giving me excitations excitations '
           'oom bop bop good good good good vibrations oom bop bop shes giving '
           'me excitations excitations oom bop bop good good good good '
           'vibrations oom bop bop shes giving me excitations excitations ahh '
           'ah my my what elation i dont know where but she sends me there oh '
           'my my what a sensation oh my my what elation oh my my what gotta '
           'keep those lovin good vibrations ahappenin with her gotta keep '
           'those lovin good vibrations ahappenin with her gotta keep those '
           'lovin good vibrations ahappenin ahh good good good good vibrations '
           'oom bop bop im pickin up good vibrations oom bop bop shes giving '
           'me excitations excitations oom bop bop good good good good '
           'vibrations na na na na na na na na na na na na na na na na bop '
           'bopbopbopbop bop do do do do do do do do bop bopbopbopbop bop do '
           'do do do do do do do bop bopbopbopbop bop',
 'Rank': 33,
 'Song': 'good vibrations',
 'Source': 3,
 'Year': 1966,
 '_id': ObjectId('59f903150f3e05a46c01d80c')}
{'Artist': 'the outsiders',
 'Instrumental': False,
 'Lyrics': ' i cant wait forever even though you want me to i cant wait '
           'forever to know if youll be true time wont let me time wont let me '
           'time wont let me wait too longcant you see ive waited too long to '
           'love you to hold you in my arms time wont let me time wont let me '
           'time wont let me aw ohi cant wait forever even though you want me '
           'to i cant wait forever to know if youll be true time wont let me '
           'time wont let me time wont let me wait that long it wont let me '
           'wait that long wait wait wait wait wait that long wait wait wait '
           'wait wait that long wait wait wait wait wait that long wait wait '
           'wait wait wait that long wait wait wait wait wait that long wait '
           'wait wait wait wait that long wait wait wait wait wait that long ',
 'Rank': 42,
 'Song': 'time wont let me',
 'Source': 1,
 'Year': 1966,
 '_id': ObjectId('59f903150f3e05a46c01d815')}
{'Artist': 'ronnie dyson',
 'Instrumental': False,
 'Lyrics': 'if you let me make love to you then why cant i touch you if you '
           'let me make love to you then why cant i touch you im not saying i '
           'want to change you i dont want to rearrange you if you let me make '
           'love to you then why cant i touch you from the very first moment i '
           'saw you its been a different world from the very first moment i '
           'saw you its been such a different world i seem to be a little bit '
           'stronger each moment seems just a little little bit longer the '
           'missing links and little kinks have never been found and uncurled '
           'but youre still the same youre like a frozen flame well it sure '
           'seems a shame that you let me make love to you but you wont let me '
           'touch you when i leave you im going someplace that ive never been '
           'before when i leave you im going someplace that ive never been '
           'before oh reach out and touch me why dont you reach out and touch '
           'me if you let me make love to you then why cant i touch you if you '
           'let me make love to you then why why cant i touch you',
 'Rank': 74,
 'Song': 'if you let me make love to you then why cant i touch you',
 'Source': 5,
 'Year': 1970,
 '_id': ObjectId('59f903150f3e05a46c01d9c5')}

**Lyric Issues**

1. Some songs include the header information, usually followed by several spaces
2. Some include pop-up text, other title info, sometimes separated by more than few spaces
3. Runnin v running (stemming should fix this)
4. Newlines were stripped and not replaced with a space, thus many words at the end of the line form compound words with those at the beginning of the next line
5. Potential misspellings
6. Excessive repetition of words

Removing header info and other forms of invasive text may be more difficult than actually re-scraping the data; same goes for the newline problem. Running everything through a spell-checker may transform these unintentionally compounded words into words that have nothing to do with the song whatsoever, same going for slang words. Binarizing the vectorizer would take care of excessive repetition, but may increase the strength of these compounded words as well.

I can do the following right off. This will need far more if I scrape my own..._____no_output_____

def clean_it_up(words):
    """
    Cleaning operations necessary to clean document and prepare for
    tokenization.
    ---
    IN
    words: string to be cleaned (str)
    OUT
    words: cleaned string (str)
    """

    # remove any 'word' beginning with a number
    words = re.sub(r'\b\d+\S*\b', '', words)

    # remove swaths of whitespace, strip at beginning and end
    words = re.sub(r'\s+', ' ', words).strip()

    return words_____no_output_____def pull_and_clean_lyrics(coll, size):
    """
    Aggregate the lyrics_from_db() and clean_it_up() functions.
    *** As of now, cannot specify year range in this one, defaults of
    lyrics_from_db() are used. ***
    ---
    IN
    coll: MongoDB collection of songs
    size: number of songs in sample, or 'all' for all
    OUT
    lyrics_clean: list of cleaned lyrics (list of strs)
    song_ids: corresponding list of BSON IDs for each song (list)
    """

    raw_lyrics, song_ids = lyrics_from_db(coll, sample_size=size)

    clean_lyrics = []
    for lyric in raw_lyrics:
        clean_lyrics.append(clean_it_up(lyric))

    return clean_lyrics, song_ids_____no_output_____test_lyrics, test_ids = pull_and_clean_lyrics(lyrics_coll, size=10)

for ID, lyric in zip(test_ids, test_lyrics):
    print("Song ID:", ID)
    print(lyric, '\n')Song ID: 59f903150f3e05a46c01d7ff
when a man loves a woman cant keep his mind on nothing else hell trade the world for the good thing hes found if shes bad he cant see it she can do no wrong turn his back on his best friend if he put her downwhen a man loves a woman spend his very last dime trying to hold on to what he needs hed give up all his comfort sleep out in the rain if she said thats the way it ought to bewell this man loves a woman i gave you everything i had trying to hold on to your precious love baby please dont treat me badwhen a man loves a woman down deep in his soul she can bring him such misery if she plays him for a fool hes the last one to know loving eyes cant ever seewhen a man loves a woman he can do no wrong he can never own some other girl yes when a man loves a woman i know exactly how he feels cause baby baby baby youre my worldwhen a man loves a woman

Song ID: 59f903150f3e05a46c01daa8
im comin home ive done my time now ive got to know what is and isnt mine if you received my letter telling you id soon be free then youll know just what to do if you still want me if you still want me whoa tie a yellow ribbon round the ole oak tree its been three long years do you still want me if i dont see a ribbon round the ole oak tree ill stay on the bus forget about us put the blame on me if i dont see a yellow ribbon round the ole oak tree bus driver please look for me cause i couldnt bear to see what i might see im really still in prison and my love she holds the key a simple yellow ribbons what i need to set me free and i wrote and told her please whoa tie a yellow ribbon round the ole oak tree its been three long years do you still want me if i dont see a ribbon round the ole oak tree ill stay on the bus forget about us put the blame on me if i dont see a yellow ribbon round the ole oak tree now the whole damned bus is cheerin and i cant believe i see a hundred yellow ribbons round the ole oak tree im comin home tie a ribbon round the ole oak tree tie a ribbon round the ole oak tree tie a ribbon round the ole oak tree tie a ribbon round the ole oak tree tie a ribbon round the ole oak tree tie a ribbon round the ole oak tree tie a ribbon round the ole oak tree tie a ribbon round the ole oak tree

Song ID: 59f903150f3e05a46c01d918
sugar honey honey you are my candy girl and you got me wanting you honey sugar sugar you are my candy girl and you got me wanting youi just cant believe the loveliness of loving you i just cant believe its true i just cant believe the wonder of this feeling too i just cant believe its truesugar honey honey you are my candy girl and you got me wanting you honey sugar sugar you are my candy girl and you got me wanting youwhen i kissed you girl i knew how sweet a kiss could be i know how sweet a kiss can be like the summer sunshine pour you sweetness over me pour your sweetness over meoh sugar pour your sugar on me honey pour your sugar on me baby im gonna make your life so sweet yeah yeah yeahpour your sugar on me oh yeah pour your sugar on me honey pour your sugar on me baby im gonna make your life so sweet yeah yeah yeah pour your sugar on me honeysugar honey honey you are my candy girl and you got me wanting you honey sugar sugar you are my candy girl

Song ID: 59f903150f3e05a46c01db7f
well i built me a raft and shes ready for floatin ol mississippi shes callin my name catfish are jumpin that paddle wheel thumpin black water keep rollin on past just the sameold black water keep on rollin mississippi moon wont you keep on shinin on me old black water keep on rollin mississippi moon wont you keep on shinin on me old black water keep on rollin mississippi moon wont you keep on shinin on me yeah keep on shinin your light gonna make everything pretty mama gonna make everything all right and i aint got no worries cause i aint in no hurry at allwell if it rains i dont care dont make no difference to me just take that street car thats goin up town yeah id like to hear some funky dixieland and dance a honky tonk and ill be buyin evrybody drinks all rounold black water keep on rollin mississippi moon wont you keep on shinin on me old black water keep on rollin mississippi moon wont you keep on shinin on me old black water keep on rollin mississippi moon wont you keep on shinin on me keep on shinin your light gonna make everything everything gonna make everything all right and i aint got no worries cause i aint in no hurry at allid like to hear some funky dixieland pretty mama come and take me by the hand by the hand take me by the hand pretty mama come and dance with your daddy all night long id like to hear some funky dixieland pretty mama come and take me by the hand i want to honky tonk honky tonk honky tonk with you all night longid like to hear some funky dixieland pretty mama come and take me by the hand by the hand take me by the hand pretty mama come and dance with your daddy all night long id like to hear some funky dixieland pretty mama come and take me by the hand i want to honky tonk honky tonk honky tonk with you all night longid like to hear some funky dixieland pretty mama come and take me by the hand by the hand take me by the hand pretty mama come and dance with your daddy all night long id like to hear some funky dixieland pretty mama come and take me by the hand i want to honky tonk honky tonk honky tonk with you all night long id like to hear some funky dixieland pretty mama come and take me by the hand by the hand take me by the hand pretty mama

Song ID: 59f903150f3e05a46c01d965
oh the games people play now every night and every day now never meaning what they say now never saying what they meanwhile they wile away the hours in their ivory towers till theyre covered up with flowers in the back of a black limousinechorus lada da da da da da da lada da da da da de talking bout you and me and the games people playoh we make one another cry break a heart then we say goodbye cross our hearts and we hope to die that the other was to blamebut neither one ever will give in so we gaze at an eight by ten thinking bout the things that might have been and its a dirty rotten shamerepeat choruspeople walking up to you singing glory hallelujah and they try to sock it to you in the name of the lordtheyre gonna teach you how to meditate read your horoscope cheat your fate and further more to hell with hate come on and get on boardrepeat choruslook around tell me what you see whats happening to you and me god grant me the serenity to remember who i amcause youve given up your sanity for your pride and your vanity turned your back on humanity and you dont give a da da da da darepeat chorus

Song ID: 59f903150f3e05a46c01db46
hey hey hey what ya got to say i said hey hey hey what ya got to say hollywood hollywood swinging hollywood hollywood swinging i remember not too long ago i went to a theater and i saw the kool the gang show i always wanted to fit into a band to sing my songs and then become a bad piano playing man so here i am in this hollywood city the city of the stars movies women and cars well i guess i i guess i star i said hey hey hey what ya got to say i said hey hey hey what ya got to say hollywood hollywood swinging hollywood hollywood swinging hollywood swinging hollywood swinging hollywood swinging

Song ID: 59f903150f3e05a46c01da67
what are ya doin tonight ya got no place to go gotta get out of the city why dontcha come along with me mighty fine out tonight getcha girl and hold her tight we can take it nice and easy you know it wont be long till well be well be down by the lazy river come as you please down by the lazy river one big family if youre all alone you wont be long just bring your guitar and sing your song down by the lazy river everybody follow me better make up your mind lets get the show on the road were gonna have a party come along or youll be sorry sorry you know theres plenty of room come on and sing a boom boom boom we can get nice and cozy why dontcha come along and mosey and down i said down come on down im goin down by the lazy river come as you please down by the lazy river one big family if youre all alone you wont be long just bring your guitar and sing your song down by the lazy river everybody follow me if youre all alone you wont be long just bring your guitar and sing your song down by the lazy river down im going down by the lazy river down im going down by the lazy river down im going down by the lazy river down

Song ID: 59f903150f3e05a46c01d832
somewhere my love there will be songs to sing although the snow covers the hope of spring somewhere a hill blossoms in green and gold and there are dreams all that your heart can hold someday well meet again my love someday whenever the spring breaks through youll come to me out of the long ago warm as the wind soft as the kiss of snow till then my sweet think of me now and then god speed my love till you are mine again lara my own think of me now and then god speed my love till you are mine again mine again

Song ID: 59f903150f3e05a46c01da33
well be fighting in the streets with our children at our feet and the morals that they worship will be gone and the men who spurred us on sit in judgement of all wrong they decide and the shotgun sings the song ill tip my hat to the new constitution take a bow for the new revolution smile and grin at the change all around pick up my guitar and play just like yesterday then ill get on my knees and pray we dont get fooled again the change it had to come we knew it all along we were liberated from the fold thats all and the world looks just the same and history aint changed cause the banners they are flown in the next war ill tip my hat to the new constitution take a bow for the new revolution smile and grin at the change all around pick up my guitar and play just like yesterday then ill get on my knees and pray we dont get fooled again no no ill move myself and my family aside if we happen to be left half alive ill get all my papers and smile at the sky though i know that the hypnotized never lie do ya theres nothing in the streets looks any different to me and the slogans are replaced bythebye and the parting on the left are now parting on the right and the beards have all grown longer overnight ill tip my hat to the new constitution take a bow for the new revolution smile and grin at the change all around pick up my guitar and play just like yesterday then ill get on my knees and pray we dont get fooled again dont get fooled again no no yeaaaaaaaaaaaaaaaaaaaaaaaaah meet the new boss same as the old boss

Song ID: 59f903150f3e05a46c01d872
hey where did we go days when the rains came down in the hollow playin a new gamelaughin and arunnin hey hey skippin and ajumpin in the misty mornin fog with our our hearts athumpinand you my brown eyed girl you my brown eyed girland whatever happened to tuesday and so slow going down the old man with a transistor radiostandin in the sunlight laughin hidin behind a rainbows wall slippin and aslidin all along the waterfallwith you my brown eyed girl you my brown eyed girldo you remember when we used to sing sha la la la la la la la la lla te da just like that sha la la la la la la la la lla te da la te daso hard to find my way now that im all on my own i saw you just the other day my how you have growncast my memory back there lord sometimes im overcome thinkin bout it makin love in the green grass behind the stadium with you my brown eyed girl you my brown eyed girldo you remember when we used to sing sha la la la la la la la la lla te da lyin in the green grass sha la la la la la la la la lla te da bit by bit by bit by bit by bit by bit sha la la la la la la la la lla te da sha la la la la la la la la la te da la te da la te da da da da sha la la la la la la la la lla te da

## 3. Tokenizing_____no_output_____

nlp = spacy.load('en')_____no_output_____unchained = lyrics.find_one({"Song": "unchained melody"})
um_lyrics = clean_it_up(unchained['Lyrics'])

um_doc = nlp(um_lyrics)_____no_output_____for word in um_doc:
    print((word,word.pos_, word.lemma_))(oh, 'INTJ', 'oh')
(my, 'ADJ', '-PRON-')
(love, 'NOUN', 'love')
(my, 'ADJ', '-PRON-')
(darling, 'NOUN', 'darling')
(i, 'PRON', '-PRON-')
(ve, 'VERB', 'have')
(hungered, 'NUM', 'hungered')
(for, 'ADP', 'for')
(your, 'ADJ', '-PRON-')
(touch, 'NOUN', 'touch')
(a, 'DET', 'a')
(long, 'ADJ', 'long')
(lonely, 'ADJ', 'lonely')
(time, 'NOUN', 'time')
(time, 'NOUN', 'time')
(goes, 'VERB', 'go')
(by, 'ADV', 'by')
(so, 'ADV', 'so')
(slowly, 'ADV', 'slowly')
(and, 'CCONJ', 'and')
(time, 'NOUN', 'time')
(can, 'VERB', 'can')
(do, 'VERB', 'do')
(so, 'ADV', 'so')
(much, 'ADV', 'much')
(are, 'VERB', 'be')
(you, 'PRON', '-PRON-')
(still, 'ADV', 'still')
(mine, 'PRON', '-PRON-')
(i, 'PRON', 'i')
(need, 'VERB', 'ne')
(your, 'ADJ', '-PRON-')
(love, 'NOUN', 'love')
(i, 'PRON', 'i')
(need, 'VERB', 'ne')
(your, 'ADJ', '-PRON-')
(love, 'NOUN', 'love')
(god, 'ADJ', 'god')
(speed, 'NOUN', 'speed')
(your, 'ADJ', '-PRON-')
(love, 'NOUN', 'love')
(to, 'ADP', 'to')
(me, 'PRON', '-PRON-')
(meaning, 'VERB', 'mean')
(byamandah, 'ADJ', 'byamandah')
(editor, 'NOUN', 'editor')
(written, 'VERB', 'write')
(in, 'ADP', 'in')
(unchained, 'ADJ', 'unchained')
(melody, 'NOUN', 'melody')
(has, 'VERB', 'have')
(become, 'VERB', 'become')
(one, 'NUM', 'one')
(of, 'ADP', 'of')
(the, 'DET', 'the')
(mostrecorded, 'ADJ', 'mostrecorded')
(songs, 'NOUN', 'song')
(of, 'ADP', 'of')
(the, 'DET', 'the')
(centurythe, 'ADJ', 'centurythe')
(righteous, 'ADJ', 'righteous')
(brothers, 'NOUN', 'brother')
(version, 'NOUN', 'version')
(remaining, 'VERB', 'remain')
(probably, 'ADV', 'probably')
(the, 'DET', 'the')
(most, 'ADV', 'most')
(see, 'VERB', 'see')
(all, 'DET', 'all')
(lonely, 'ADJ', 'lonely')
(rivers, 'NOUN', 'river')
(flow, 'VERB', 'flow')
(to, 'ADP', 'to')
(the, 'DET', 'the')
(sea, 'NOUN', 'sea')
(to, 'ADP', 'to')
(the, 'DET', 'the')
(sea, 'NOUN', 'sea')
(to, 'ADP', 'to')
(the, 'DET', 'the')
(open, 'ADJ', 'open')
(arms, 'NOUN', 'arm')
(of, 'ADP', 'of')
(the, 'DET', 'the')
(sea, 'NOUN', 'sea')
(lonely, 'ADJ', 'lonely')
(rivers, 'NOUN', 'river')
(sigh, 'ADJ', 'sigh')
(wait, 'VERB', 'wait')
(for, 'ADP', 'for')
(me, 'PRON', '-PRON-')
(wait, 'VERB', 'wait')
(for, 'ADP', 'for')
(me, 'PRON', '-PRON-')
(ill, 'VERB', 'ill')
(be, 'VERB', 'be')
(coming, 'VERB', 'come')
(home, 'ADV', 'home')
(wait, 'VERB', 'wait')
(for, 'ADP', 'for')
(meoh, 'NOUN', 'meoh')
(my, 'ADJ', '-PRON-')
(love, 'NOUN', 'love')
(my, 'ADJ', '-PRON-')
(darling, 'NOUN', 'darling')
(i, 'PRON', '-PRON-')
(ve, 'VERB', 'have')
(hungered, 'NUM', 'hungered')
(for, 'ADP', 'for')
(your, 'ADJ', '-PRON-')
(touch, 'NOUN', 'touch')
(a, 'DET', 'a')
(long, 'ADJ', 'long')
(lonely, 'ADJ', 'lonely')
(time, 'NOUN', 'time')
(time, 'NOUN', 'time')
(goes, 'VERB', 'go')
(by, 'ADV', 'by')
(so, 'ADV', 'so')
(slowly, 'ADV', 'slowly')
(and, 'CCONJ', 'and')
(time, 'NOUN', 'time')
(can, 'VERB', 'can')
(do, 'VERB', 'do')
(so, 'ADV', 'so')
(much, 'ADV', 'much')
(are, 'VERB', 'be')
(you, 'PRON', '-PRON-')
(still, 'ADV', 'still')
(mine, 'PRON', '-PRON-')
(i, 'PRON', 'i')
(need, 'VERB', 'ne')
(your, 'ADJ', '-PRON-')
(love, 'NOUN', 'love')
(i, 'PRON', 'i')
(need, 'VERB', 'ne')
(your, 'ADJ', '-PRON-')
(love, 'NOUN', 'love')
(god, 'ADJ', 'god')
(speed, 'NOUN', 'speed')
(your, 'ADJ', '-PRON-')
(love, 'NOUN', 'love')
(to, 'ADP', 'to')
(me, 'PRON', '-PRON-')
(lonely, 'ADJ', 'lonely')
(mountains, 'NOUN', 'mountain')
(gaze, 'VERB', 'gaze')
(at, 'ADP', 'at')
(the, 'DET', 'the')
(stars, 'NOUN', 'star')
(at, 'ADP', 'at')
(the, 'DET', 'the')
(stars, 'NOUN', 'star')
(waiting, 'VERB', 'wait')
(for, 'ADP', 'for')
(the, 'DET', 'the')
(dawn, 'NOUN', 'dawn')
(of, 'ADP', 'of')
(the, 'DET', 'the')
(dayall, 'NOUN', 'dayall')
(alone, 'ADV', 'alone')
(i, 'PRON', 'i')
(gaze, 'VERB', 'gaze')
(at, 'ADP', 'at')
(the, 'DET', 'the')
(stars, 'NOUN', 'star')
(at, 'ADP', 'at')
(the, 'DET', 'the')
(stars, 'NOUN', 'star')
(dreaming, 'VERB', 'dream')
(of, 'ADP', 'of')
(my, 'ADJ', '-PRON-')
(love, 'NOUN', 'love')
(far, 'ADV', 'far')
(away, 'ADV', 'away')
def look_at_words(mongo_sample):
    """
    Cycles through a sample of lyrics and shows word, word.pos_,
    and word.lemma_ for each song.
    """

    for song in mongo_sample:
        print(f"Song: {song['Song']}\n")
        print("WORD / POS / LEMMA\n")
        lyrics = clean_it_up(song['Lyrics'])
        lyrics_doc = nlp(lyrics)
        for word in lyrics_doc:
            print((word, word.pos_, word.lemma_))
        input("\n-- Enter to continue --\n")_____no_output_____look_at_words(sample_lyrics(20))Song: keep searchin well follow the sun

WORD / POS / LEMMA

(no, 'DET', 'no')
(one, 'NOUN', 'one')
(will, 'VERB', 'will')
(understand, 'VERB', 'understand')
(what, 'NOUN', 'what')
(i, 'PRON', '-PRON-')
(ve, 'VERB', 'have')
(got, 'VERB', 'get')
(ta, 'PART', 'ta')
(do, 'VERB', 'do')
(i, 'PRON', '-PRON-')
(ve, 'VERB', 'have')
(got, 'VERB', 'get')
(ta, 'PART', 'ta')
(find, 'VERB', 'find')
(a, 'DET', 'a')
(place, 'NOUN', 'place')
(to, 'PART', 'to')
(hide, 'VERB', 'hide')
(with, 'ADP', 'with')
(my, 'ADJ', '-PRON-')
(baby, 'NOUN', 'baby')
(by, 'ADP', 'by')
(my, 'ADJ', '-PRON-')
(side, 'NOUN', 'side')
(she, 'PRON', '-PRON-')
(s, 'VERB', 's')
(been, 'VERB', 'be')
(hurt, 'ADV', 'hurt')
(so, 'ADV', 'so')
(much, 'ADV', 'much')
(they, 'PRON', '-PRON-')
(treat, 'VERB', 'treat')
(her, 'PRON', '-PRON-')
(mean, 'ADJ', 'mean')
(and, 'CCONJ', 'and')
(cruel, 'ADJ', 'cruel')
(they, 'PRON', '-PRON-')
(try, 'VERB', 'try')
(to, 'PART', 'to')
(keep, 'VERB', 'keep')
(us, 'PRON', '-PRON-')
(far, 'ADV', 'far')
(apart, 'ADV', 'apart')
(there, 'ADV', 'there')
(s, 'VERB', 's')
(only, 'ADV', 'only')
(one, 'NUM', 'one')
(thing, 'NOUN', 'thing')
(left, 'VERB', 'leave')
(we, 'PRON', '-PRON-')
(can, 'VERB', 'can')
(dowe, 'VERB', 'dowe')
(got, 'VERB', 'get')
(ta, 'PART', 'ta')
(keep, 'VERB', 'keep')
(searchin, 'NOUN', 'searchin')
(searchin, 'NOUN', 'searchin')
(find, 'VERB', 'find')
(a, 'DET', 'a')
(place, 'NOUN', 'place')
(to, 'PART', 'to')
(hide, 'VERB', 'hide')
(searchin, 'NOUN', 'searchin')
(searchin, 'NOUN', 'searchin')
(shell, 'NOUN', 'shell')
(be, 'VERB', 'be')
(by, 'ADP', 'by')
(my, 'ADJ', '-PRON-')
(side, 'NOUN', 'side')
(if, 'ADP', 'if')
(we, 'PRON', '-PRON-')
(got, 'VERB', 'get')
(ta, 'PART', 'ta')
(keep, 'VERB', 'keep')
(on, 'ADP', 'on')
(the, 'DET', 'the')
(run, 'NOUN', 'run')
(well, 'ADV', 'well')
(follow, 'VERB', 'follow')
(the, 'DET', 'the')
(sunah, 'NOUN', 'sunah')
(weeooh, 'NOUN', 'weeooh')
(follow, 'VERB', 'follow')
(the, 'DET', 'the')
(sunah, 'NOUN', 'sunah')
(weeoohdoesnt, 'ADV', 'weeoohdoesnt')
(matter, 'ADV', 'matter')
(does, 'VERB', 'do')
(nt, 'ADV', 'not')
(matter, 'VERB', 'matter')
(what, 'NOUN', 'what')
(people, 'NOUN', 'people')
(might, 'VERB', 'may')
(say, 'VERB', 'say')
(she, 'PRON', '-PRON-')
(s, 'VERB', 's')
(mine, 'NOUN', 'mine')
(and, 'CCONJ', 'and')
(i, 'PRON', '-PRON-')
(m, 'VERB', 'be')
(gon, 'VERB', 'gon')
(na, 'PART', 'na')
(take, 'VERB', 'take')
(her, 'PRON', '-PRON-')
(anywaygotta, 'ADV', 'anywaygotta')
(keep, 'VERB', 'keep')
(searchin, 'NOUN', 'searchin')
(searchin, 'NOUN', 'searchin')
(find, 'VERB', 'find')
(a, 'DET', 'a')
(place, 'NOUN', 'place')
(to, 'PART', 'to')
(stay, 'VERB', 'stay')
(searchin, 'NOUN', 'searchin')
(searchin, 'NOUN', 'searchin')
(every, 'DET', 'every')
(night, 'NOUN', 'night')
(and, 'CCONJ', 'and')
(day, 'NOUN', 'day')
(if, 'ADP', 'if')
(we, 'PRON', '-PRON-')
(got, 'VERB', 'get')
(ta, 'PART', 'ta')
(keep, 'VERB', 'keep')
(on, 'ADP', 'on')
(the, 'DET', 'the')
(run, 'NOUN', 'run')
(well, 'ADV', 'well')
(follow, 'VERB', 'follow')
(the, 'DET', 'the')
(sunah, 'NOUN', 'sunah')
(weeooh, 'NOUN', 'weeooh')
(follow, 'VERB', 'follow')
(the, 'DET', 'the')
(sunah, 'NOUN', 'sunah')
(weeooh, 'NOUN', 'weeooh')

KNOWN ENTITIES

CARDINAL only one
TIME every night

### Tuning the Clean/Tokenization Process_____no_output_____On an initial sprint to vectorization, slang, phonetic representations, and the compound problem were potentially contributing to a messy bunch of vectors. I'll try to clean this up a bit more.

"Brown Eyed Girl" has a bunch of the -in' gerunds..._____no_output_____

browneyedgirl = lyrics_coll.find_one({"Song": "brown eyed girl"})
beg_lyrics = clean_it_up(browneyedgirl['Lyrics'])_____no_output_____print(beg_lyrics)hey where did we go days when the rains came down in the hollow playin a new gamelaughin and arunnin hey hey skippin and ajumpin in the misty mornin fog with our our hearts athumpinand you my brown eyed girl you my brown eyed girland whatever happened to tuesday and so slow going down the old man with a transistor radiostandin in the sunlight laughin hidin behind a rainbows wall slippin and aslidin all along the waterfallwith you my brown eyed girl you my brown eyed girldo you remember when we used to sing sha la la la la la la la la lla te da just like that sha la la la la la la la la lla te da la te daso hard to find my way now that im all on my own i saw you just the other day my how you have growncast my memory back there lord sometimes im overcome thinkin bout it makin love in the green grass behind the stadium with you my brown eyed girl you my brown eyed girldo you remember when we used to sing sha la la la la la la la la lla te da lyin in the green grass sha la la la la la la la la lla te da bit by bit by bit by bit by bit by bit sha la la la la la la la la lla te da sha la la la la la la la la la te da la te da la te da da da da sha la la la la la la la la lla te da

To take a closer look at how the various stemmer types perform: _____no_output_____

stemmers = [PorterStemmer(),
            LancasterStemmer(),
            SnowballStemmer('english')
           ]
stem_names = ['Porter', 'Lancaster', 'Snowball']

for name, stemmer in zip(stem_names, stemmers):
    lyric_list = []
    for word in beg_lyrics.split(' '):
        lyric_list.append(stemmer.stem(word))
    print(f"\n{name}:")
    print(lyric_list)
Porter:
['hey', 'where', 'did', 'we', 'go', 'day', 'when', 'the', 'rain', 'came', 'down', 'in', 'the', 'hollow', 'playin', 'a', 'new', 'gamelaughin', 'and', 'arunnin', 'hey', 'hey', 'skippin', 'and', 'ajumpin', 'in', 'the', 'misti', 'mornin', 'fog', 'with', 'our', 'our', 'heart', 'athumpinand', 'you', 'my', 'brown', 'eye', 'girl', 'you', 'my', 'brown', 'eye', 'girland', 'whatev', 'happen', 'to', 'tuesday', 'and', 'so', 'slow', 'go', 'down', 'the', 'old', 'man', 'with', 'a', 'transistor', 'radiostandin', 'in', 'the', 'sunlight', 'laughin', 'hidin', 'behind', 'a', 'rainbow', 'wall', 'slippin', 'and', 'aslidin', 'all', 'along', 'the', 'waterfallwith', 'you', 'my', 'brown', 'eye', 'girl', 'you', 'my', 'brown', 'eye', 'girldo', 'you', 'rememb', 'when', 'we', 'use', 'to', 'sing', 'sha', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'lla', 'te', 'da', 'just', 'like', 'that', 'sha', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'lla', 'te', 'da', 'la', 'te', 'daso', 'hard', 'to', 'find', 'my', 'way', 'now', 'that', 'im', 'all', 'on', 'my', 'own', 'i', 'saw', 'you', 'just', 'the', 'other', 'day', 'my', 'how', 'you', 'have', 'growncast', 'my', 'memori', 'back', 'there', 'lord', 'sometim', 'im', 'overcom', 'thinkin', 'bout', 'it', 'makin', 'love', 'in', 'the', 'green', 'grass', 'behind', 'the', 'stadium', 'with', 'you', 'my', 'brown', 'eye', 'girl', 'you', 'my', 'brown', 'eye', 'girldo', 'you', 'rememb', 'when', 'we', 'use', 'to', 'sing', 'sha', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'lla', 'te', 'da', 'lyin', 'in', 'the', 'green', 'grass', 'sha', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'lla', 'te', 'da', 'bit', 'by', 'bit', 'by', 'bit', 'by', 'bit', 'by', 'bit', 'by', 'bit', 'sha', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'lla', 'te', 'da', 'sha', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'te', 'da', 'la', 'te', 'da', 'la', 'te', 'da', 'da', 'da', 'da', 'sha', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'lla', 'te', 'da']

Lancaster:
['hey', 'wher', 'did', 'we', 'go', 'day', 'when', 'the', 'rain', 'cam', 'down', 'in', 'the', 'hollow', 'playin', 'a', 'new', 'gamelaughin', 'and', 'arunnin', 'hey', 'hey', 'skippin', 'and', 'ajumpin', 'in', 'the', 'misty', 'mornin', 'fog', 'with', 'our', 'our', 'heart', 'athumpinand', 'you', 'my', 'brown', 'ey', 'girl', 'you', 'my', 'brown', 'ey', 'girland', 'whatev', 'hap', 'to', 'tuesday', 'and', 'so', 'slow', 'going', 'down', 'the', 'old', 'man', 'with', 'a', 'transist', 'radiostandin', 'in', 'the', 'sunlight', 'laughin', 'hidin', 'behind', 'a', 'rainbow', 'wal', 'slippin', 'and', 'aslidin', 'al', 'along', 'the', 'waterfallwi', 'you', 'my', 'brown', 'ey', 'girl', 'you', 'my', 'brown', 'ey', 'girldo', 'you', 'rememb', 'when', 'we', 'us', 'to', 'sing', 'sha', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'lla', 'te', 'da', 'just', 'lik', 'that', 'sha', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'lla', 'te', 'da', 'la', 'te', 'daso', 'hard', 'to', 'find', 'my', 'way', 'now', 'that', 'im', 'al', 'on', 'my', 'own', 'i', 'saw', 'you', 'just', 'the', 'oth', 'day', 'my', 'how', 'you', 'hav', 'growncast', 'my', 'mem', 'back', 'ther', 'lord', 'sometim', 'im', 'overcom', 'thinkin', 'bout', 'it', 'makin', 'lov', 'in', 'the', 'green', 'grass', 'behind', 'the', 'stad', 'with', 'you', 'my', 'brown', 'ey', 'girl', 'you', 'my', 'brown', 'ey', 'girldo', 'you', 'rememb', 'when', 'we', 'us', 'to', 'sing', 'sha', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'lla', 'te', 'da', 'lyin', 'in', 'the', 'green', 'grass', 'sha', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'lla', 'te', 'da', 'bit', 'by', 'bit', 'by', 'bit', 'by', 'bit', 'by', 'bit', 'by', 'bit', 'sha', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'lla', 'te', 'da', 'sha', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'te', 'da', 'la', 'te', 'da', 'la', 'te', 'da', 'da', 'da', 'da', 'sha', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'lla', 'te', 'da']

Snowball:
['hey', 'where', 'did', 'we', 'go', 'day', 'when', 'the', 'rain', 'came', 'down', 'in', 'the', 'hollow', 'playin', 'a', 'new', 'gamelaughin', 'and', 'arunnin', 'hey', 'hey', 'skippin', 'and', 'ajumpin', 'in', 'the', 'misti', 'mornin', 'fog', 'with', 'our', 'our', 'heart', 'athumpinand', 'you', 'my', 'brown', 'eye', 'girl', 'you', 'my', 'brown', 'eye', 'girland', 'whatev', 'happen', 'to', 'tuesday', 'and', 'so', 'slow', 'go', 'down', 'the', 'old', 'man', 'with', 'a', 'transistor', 'radiostandin', 'in', 'the', 'sunlight', 'laughin', 'hidin', 'behind', 'a', 'rainbow', 'wall', 'slippin', 'and', 'aslidin', 'all', 'along', 'the', 'waterfallwith', 'you', 'my', 'brown', 'eye', 'girl', 'you', 'my', 'brown', 'eye', 'girldo', 'you', 'rememb', 'when', 'we', 'use', 'to', 'sing', 'sha', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'lla', 'te', 'da', 'just', 'like', 'that', 'sha', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'lla', 'te', 'da', 'la', 'te', 'daso', 'hard', 'to', 'find', 'my', 'way', 'now', 'that', 'im', 'all', 'on', 'my', 'own', 'i', 'saw', 'you', 'just', 'the', 'other', 'day', 'my', 'how', 'you', 'have', 'growncast', 'my', 'memori', 'back', 'there', 'lord', 'sometim', 'im', 'overcom', 'thinkin', 'bout', 'it', 'makin', 'love', 'in', 'the', 'green', 'grass', 'behind', 'the', 'stadium', 'with', 'you', 'my', 'brown', 'eye', 'girl', 'you', 'my', 'brown', 'eye', 'girldo', 'you', 'rememb', 'when', 'we', 'use', 'to', 'sing', 'sha', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'lla', 'te', 'da', 'lyin', 'in', 'the', 'green', 'grass', 'sha', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'lla', 'te', 'da', 'bit', 'by', 'bit', 'by', 'bit', 'by', 'bit', 'by', 'bit', 'by', 'bit', 'sha', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'lla', 'te', 'da', 'sha', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'te', 'da', 'la', 'te', 'da', 'la', 'te', 'da', 'da', 'da', 'da', 'sha', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'lla', 'te', 'da']

None of the stemmers catch the slang gerund form... one solution could be RE:_____no_output_____

st = REStemmer('in$', min=5)
st.stem('walkin')_____no_output_____

Inserting this into the spacy_tokenizer() function right before the main stemmer yields the following:_____no_output_____

beg_tokens = spacy_tokenizer(beg_lyrics)
print(beg_tokens)['hey', 'day', 'rain', 'came', 'hollow', 'play', 'new', 'gamelaugh', 'arunn', 'hey', 'hey', 'skipp', 'ajump', 'misti', 'morn', 'fog', 'heart', 'athumpinand', 'brown', 'eye', 'girl', 'brown', 'eye', 'girland', 'happen', 'tuesday', 'slow', 'go', 'old', 'man', 'transistor', 'radiostand', 'sunlight', 'laugh', 'hid', 'rainbow', 'wall', 'slipp', 'aslid', 'waterfallwith', 'brown', 'eye', 'girl', 'brown', 'eye', 'girldo', 'rememb', 'sing', 'sha', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'lla', 'te', 'da', 'like', 'sha', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'lla', 'te', 'da', 'la', 'te', 'daso', 'hard', 'find', 'way', 'm', 'saw', 'day', 'growncast', 'memori', 'lord', 'm', 'overcom', 'think', 'bout', 'mak', 'love', 'green', 'grass', 'stadium', 'brown', 'eye', 'girl', 'brown', 'eye', 'girldo', 'rememb', 'sing', 'sha', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'lla', 'te', 'da', 'lyin', 'green', 'grass', 'sha', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'lla', 'te', 'da', 'bit', 'bit', 'bit', 'bit', 'bit', 'bit', 'sha', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'lla', 'te', 'da', 'sha', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'te', 'da', 'la', 'te', 'da', 'la', 'te', 'da', 'da', 'da', 'da', 'sha', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'lla', 'te', 'da']
beg_tokens2 = spacy_tokenizer(beg_lyrics, stemmer=LancasterStemmer())
print(beg_tokens2)['hey', 'day', 'rain', 'cam', 'hollow', 'play', 'new', 'gamelaugh', 'arun', 'hey', 'hey', 'skip', 'ajump', 'misty', 'morn', 'fog', 'heart', 'athumpinand', 'brown', 'ey', 'girl', 'brown', 'ey', 'girland', 'hap', 'tuesday', 'slow', 'going', 'old', 'man', 'transist', 'radiostand', 'sunlight', 'laugh', 'hid', 'rainbow', 'wal', 'slip', 'aslid', 'waterfallwi', 'brown', 'ey', 'girl', 'brown', 'ey', 'girldo', 'rememb', 'sing', 'sha', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'lla', 'te', 'da', 'lik', 'sha', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'lla', 'te', 'da', 'la', 'te', 'daso', 'hard', 'find', 'way', 'm', 'saw', 'day', 'growncast', 'mem', 'lord', 'm', 'overcom', 'think', 'bout', 'mak', 'lov', 'green', 'grass', 'stad', 'brown', 'ey', 'girl', 'brown', 'ey', 'girldo', 'rememb', 'sing', 'sha', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'lla', 'te', 'da', 'lyin', 'green', 'grass', 'sha', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'lla', 'te', 'da', 'bit', 'bit', 'bit', 'bit', 'bit', 'bit', 'sha', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'lla', 'te', 'da', 'sha', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'te', 'da', 'la', 'te', 'da', 'la', 'te', 'da', 'da', 'da', 'da', 'sha', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'lla', 'te', 'da']

Lancaster seems to be a little more aggressive in this case, turning 'slipping' into 'slip' instead of 'slipp', as with Porter._____no_output_____#### A spell-checker?_____no_output_____Another solution could be a spell-checker/correcter which catches the -in' gerund cases and compound words, and attempts to correct them._____no_output_____

from nltk.metrics.distance import edit_distance
import enchant
eng_dict = enchant.Dict("en_US")_____no_output_____def show_ed(word):
    alts = eng_dict.suggest(word)
    ed = edit_distance(word, alts[0])
    print(f"{word} --> {alts[0]}: {str(ed)}")_____no_output_____eng_dict.check('running')_____no_output_____eng_dict.check('runnin')_____no_output_____show_ed('runnin')runnin --> running: 1
edit_distance('aslidin', 'sliding')_____no_output_____show_ed('aslidin')aslidin --> abiding: 3
eng_dict.suggest('aslidin')_____no_output_____show_ed('slidin')slidin --> sliding: 1
eng_dict.suggest('slidin')_____no_output_____# eng_dict = enchant.Dict("en_US")
# from nltk.metrics.distance import edit_distance

def spell_checker(word, min_ed=2, keep_fails=False):
    """
    Runs several spell-checking operations to clean up issues in the text.

    *** Must define eng_dict and edit_distance in the global space! ***
    ---
    IN
    word: word to be spell-checked (str)
    min_ed: minimum edit distance to replace word (int)
    keep_fails: if True, keep words even if none of the replacement methods
        have worked; if False, delete them (bool)
    OUT
    word_checked: list of checked words (list of strs)
    """

    # no further processes need to run if it's a recognizable word
    if eng_dict.check(word.lower()):
        return word

    # otherwise...
    try:
        alts = eng_dict.suggest(word.lower())
        ed = edit_distance(word, alts[0])

        # can word be edited in min edit distance?
        if ed <= min_ed:
            return alts[0].lower()
        # try with proper gerund form (e.g. 'runnin' > 'running')
        elif word[-1] == 'n':
            return spell_checker(word + 'g')
        # try with leading character removed
        elif ed == min_ed + 1 and len(word) > 1:
            return spell_checker(word[1:])
        elif keep_fails:
            return word
        else:
            return None

    except IndexError:
        print(f"IndexError while trying to check '{word}'")
        return None

def check_word_list(words, keep_fails=False):
    """
    Feeds word list to spell checker, reassembles, returns.

    *** Must import chain from collections! ***
    ---
    IN
    words: words (strs) to be spell-checked (list)
    keep_fails: if True, keep words even if none of the replacement methods
        have worked; if False, delete them (bool)
    OUT
    checked: list of checked/corrected words (list)
    """

    checked = []

    for word in words:
        new = spell_checker(word, keep_fails=keep_fails)
        if new:
            checked = list(chain(checked, new.split(' ')))

    return checked

def check_word_string(doc, keep_fails=False):
    """
    Wrapper for check_word_list(), takes and returns a string of
    component words.

    *** Must import chain from collections! ***
    ---
    IN
    doc: words to be spell-checked (str)
    keep_fails: if True, keep words even if none of the replacement methods
        have worked; if False, delete them (bool)
    OUT
    string of checked/corrected words (str)
    """

    checked = check_word_list(doc.split(' '), keep_fails=keep_fails)

    return ' '.join(checked)

def split_word(word, min_len=3):
    """
    Attempts to find a meaningful split of a falsely compounded word.
    ---
    IN
    word: word to split (str)
    min_len: minimum length of first word (int)
    OUT
    word: new string, either original word or successfully-split word (str)
    """
    pass_____no_output_____beg_checked = check_word_list(beg_lyrics.split(' '))
print("Raw length:", len(beg_lyrics.split(' ')))
print("Checked length:", len(beg_checked))
print(beg_checked)Raw length: 271
Checked length: 276
['hey', 'where', 'did', 'we', 'go', 'days', 'when', 'the', 'rains', 'came', 'down', 'in', 'the', 'hollow', 'plain', 'a', 'new', 'game', 'laughing', 'and', 'running', 'hey', 'hey', 'skipping', 'and', 'jumping', 'in', 'the', 'misty', 'morning', 'fog', 'with', 'our', 'our', 'hearts', 'you', 'my', 'brown', 'eyed', 'girl', 'you', 'my', 'brown', 'eyed', 'garland', 'whatever', 'happened', 'to', 'tuesday', 'and', 'so', 'slow', 'going', 'down', 'the', 'old', 'man', 'with', 'a', 'transistor', 'radio', 'standing', 'in', 'the', 'sunlight', 'laughing', 'hiding', 'behind', 'a', 'rainbows', 'wall', 'slipping', 'and', 'sliding', 'all', 'along', 'the', 'waterfall', 'with', 'you', 'my', 'brown', 'eyed', 'girl', 'you', 'my', 'brown', 'eyed', 'girl', 'do', 'you', 'remember', 'when', 'we', 'used', 'to', 'sing', 'ha', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'all', 'ch', 'da', 'just', 'like', 'that', 'ha', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'all', 'ch', 'da', 'la', 'ch', 'so', 'hard', 'to', 'find', 'my', 'way', 'now', 'that', 'mi', 'all', 'on', 'my', 'own', 'i', 'saw', 'you', 'just', 'the', 'other', 'day', 'my', 'how', 'you', 'have', 'grown', 'cast', 'my', 'memory', 'back', 'there', 'lord', 'sometimes', 'mi', 'overcome', 'thinking', 'bout', 'it', 'akin', 'love', 'in', 'the', 'green', 'grass', 'behind', 'the', 'stadium', 'with', 'you', 'my', 'brown', 'eyed', 'girl', 'you', 'my', 'brown', 'eyed', 'girl', 'do', 'you', 'remember', 'when', 'we', 'used', 'to', 'sing', 'ha', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'all', 'ch', 'da', 'yin', 'in', 'the', 'green', 'grass', 'ha', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'all', 'ch', 'da', 'bit', 'by', 'bit', 'by', 'bit', 'by', 'bit', 'by', 'bit', 'by', 'bit', 'ha', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'all', 'ch', 'da', 'ha', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'ch', 'da', 'la', 'ch', 'da', 'la', 'ch', 'da', 'da', 'da', 'da', 'ha', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'all', 'ch', 'da']

Much cleaner after the spell check. Five words lost because the spell checker couldn't parse them. A couple obvious mistakes, such as 'garland' instead of 'girland', and 'mi' for 'im.' To the former, 'garland' was the first option in the suggestion list despite having the same edit distance as 'girl and,' a mistake bound to be made if edit distance is the only criteria for substitution. To the latter, I'll add it to stopwords or create an exceptions list in the chcker, since it'll be fairly common, as I'll do with anything that seems to throw off the vectors later on._____no_output_____

# nlp = spacy.load('en')

new_stopwords = ['im', 'mi']

for new in new_stopwords:
    STOP_WORDS.add(new)

def spacy_tokenizer(text,
                    spell_check=True,
                    stemmer=LancasterStemmer(),
                    stopwords=STOP_WORDS,
                    punctuations=''):
    """
    Basic tokenizer based on Spacy doc object.

    *** Must spawn nlp object from spacy in global space! ***
    ---
    IN
    text: string of text to tokenize (str)
    spell_check: perform spell check function if True (bool)
    stemmer: stemmer object of choice
    stopwords: stopwords to omit from final list of tokens (set, list)
    punctuations: punctuation to omit from final tokens (set, list)
    OUT
    tokens: final list of word tokens
    """

    tokens = nlp(text)

#     tokens = [tok.lemma_.lower().strip() if tok.lemma_ != "-PRON-"
#                                          else tok.lower_
#                                          for tok in tokens]
    # or simply make list of words
    tokens = [tok.lower_ for tok in tokens]
    # spell check to clean up gerund and compound issues
    if spell_check:
        tokens = check_word_list(tokens)
    # remove if a stopword or punctuation
    tokens = [tok for tok in tokens if (tok not in stopwords and tok not in punctuations)]
    # stem remaining words
    tokens = [stemmer.stem(tok) for tok in tokens]

    return tokens_____no_output_____beg_tokens = spacy_tokenizer(beg_lyrics)
print(beg_tokens)['hey', 'day', 'rain', 'cam', 'hollow', 'plain', 'new', 'gam', 'laugh', 'run', 'hey', 'hey', 'skip', 'jump', 'misty', 'morn', 'fog', 'heart', 'brown', 'ey', 'girl', 'brown', 'ey', 'garland', 'hap', 'tuesday', 'slow', 'going', 'old', 'man', 'transist', 'radio', 'stand', 'sunlight', 'laugh', 'hid', 'rainbow', 'wal', 'slip', 'slid', 'waterfal', 'brown', 'ey', 'girl', 'brown', 'ey', 'girl', 'rememb', 'sing', 'ha', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'ch', 'da', 'lik', 'ha', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'ch', 'da', 'la', 'ch', 'hard', 'find', 'way', 'm', 'saw', 'day', 'grown', 'cast', 'mem', 'lord', 'm', 'overcom', 'think', 'bout', 'akin', 'lov', 'green', 'grass', 'stad', 'brown', 'ey', 'girl', 'brown', 'ey', 'girl', 'rememb', 'sing', 'ha', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'ch', 'da', 'yin', 'green', 'grass', 'ha', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'ch', 'da', 'bit', 'bit', 'bit', 'bit', 'bit', 'bit', 'ha', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'ch', 'da', 'ha', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'ch', 'da', 'la', 'ch', 'da', 'la', 'ch', 'da', 'da', 'da', 'da', 'ha', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'la', 'ch', 'da']

**NOTE:** I will be moving spell check to the cleaning function for the next notebooks, given the time it takes to do so and the ability to save cleaned lyrics in each MongoDB document._____no_output_____## 5. Vectorization_____no_output_____### Sample Preparation_____no_output_____

sample_lyrics, sample_ids = pull_and_clean_lyrics(lyrics_coll, size='all')_____no_output_____for i in range(5):
    print("Song ID:", sample_ids[i])
    print(sample_lyrics[i][:200], '\n')Song ID: 59f903140f3e05a46c01d788
sugar pie honey bunch you know that i love you i cant help myself i love you and nobody elsein and out my life you come and you go leaving just your picture behind and i kissed it a thousand timeswhen

Song ID: 59f903140f3e05a46c01d789
you never close your eyes anymore when i kiss your lips and theres no tenderness like before in your fingertips youre trying hard not to show it but baby baby i know ityou lost that lovin feelin whoa

Song ID: 59f903140f3e05a46c01d78a
sam the sham miscellaneous wooly bully wooly bully sam the sham the pharaohs domingo samudio uno dos one two tres quatro matty told hatty about a thing she saw had two big horns and a wooly jaw wooly

Song ID: 59f903140f3e05a46c01d78b
help i need somebody help not just anybody help you know i need someone help when i was younger so much younger than today i never needed anybodys help in any way but now these days are gone im not so

Song ID: 59f903140f3e05a46c01d78c
carterlewis every time i see you lookin my way baby baby cant you hear my heartbeat in the car or walking down the highway baby baby cant you hear my heartbeat when you move up closer to me i get a fe

### Let's Vectorize_____no_output_____

def return_vec_data(text, vectorizer):
    """
    IF a Pandas dataframe is needed...
    Vectorizes a list of text strings with the vectorizer of choice,
    returns the sparse matrix created by the vectorizer, a pandas
    dataframe, and the fit vectorizer object.
    ---
    IN
    text: list of text strings (list)
    vectorizer: pre-spawned vectorizer object
    """

    vec_data = vectorizer.fit_transform(text)
    vec_df = (pd.DataFrame(vec_data.toarray(),
                       columns=vectorizer.get_feature_names()))

    return vec_data, vec_df, vectorizer_____no_output_____# if not set above
# nlp = spacy.load('en')_____no_output_____sample_cv = CountVectorizer(tokenizer=spacy_tokenizer,
                                   ngram_range=(1,2),
                                   # max_features=1000,
                                   # stop_words='english',
                                   # token_pattern="\\b[a-z][a-z]+\\b",
                                   # lowercase=True,
                                   binary=True,
                                   max_df=0.6
                                   #min_df=0.02
                           )

# sample_tf = TfidfVectorizer(tokenizer=spacy_tokenizer,
#                                    ngram_range=(1,2),
#                                    max_features=1000,
#                                    # stop_words='english',
#                                    # token_pattern="\\b[a-z][a-z]+\\b",
#                                    # lowercase=True,
#                                    max_df = 0.6)_____no_output_____sample_data_cv = sample_cv.fit_transform(sample_lyrics)
#sample_data_tf = sample_tf.fit_transform(sample_lyrics)IndexError while trying to check '̢shotgun'
IndexError while trying to check 'hey̢'
IndexError while trying to check '̢buy'
IndexError while trying to check 'now̢'
IndexError while trying to check '̢shotgun'
IndexError while trying to check 'hey̢'
IndexError while trying to check '̢its'
IndexError while trying to check 'time̢'
IndexError while trying to check '̢its'
IndexError while trying to check 'time̢'
IndexError while trying to check '̢its'
IndexError while trying to check 'time̢'
IndexError while trying to check 'slaycrewetook'
IndexError while trying to check 'eyesahahahahah'
IndexError while trying to check 'shadeahahahahah'
IndexError while trying to check 'ooooohbut'
IndexError while trying to check 'ooooohcause'
IndexError while trying to check 'ooooohhow'
IndexError while trying to check 'ahhhhhhhhhhhhhhhhhhhhhhhh'
IndexError while trying to check 'ahhhhhhhhhhhhhhhhhhhhhhhhh'
IndexError while trying to check 'tmazanec1junocom'
IndexError while trying to check 'ronhontzworldnetattnet'
IndexError while trying to check 'weeoohdoesnt'
IndexError while trying to check 'nightdoobeedoobeedoo'
IndexError while trying to check 'dahdahdahdahdah'
IndexError while trying to check 'me̢'
IndexError while trying to check 'avodeeohdoe'
IndexError while trying to check 'avodeeohdoe'
IndexError while trying to check 'avodeeohdoedeeohdoe'
IndexError while trying to check 'owoooooooo'
IndexError while trying to check 'owoooooooo'
IndexError while trying to check 'owoooooooo'
IndexError while trying to check 'owoooooooo'
IndexError while trying to check 'owoooooooo'
IndexError while trying to check 'baaaaaa'
IndexError while trying to check 'bopbopbopbop'
IndexError while trying to check 'bopbopbopbop'
IndexError while trying to check 'bopbopbopbop'
IndexError while trying to check 'mmmmmm'
IndexError while trying to check 'ahhhhhh'
IndexError while trying to check 'aheyheyhey'
IndexError while trying to check 'carm�_n'
IndexError while trying to check 'carm�_n'
IndexError while trying to check 'b5b5b6b6'
IndexError while trying to check 'uhhmm'
IndexError while trying to check 'uhhmm'
IndexError while trying to check 'itawwyoure'
IndexError while trying to check 'ummmmmthis'
IndexError while trying to check 'ditditditdo'
IndexError while trying to check 'ditditditdo'
IndexError while trying to check 'ditditditdo'
IndexError while trying to check 'ditditditditditditditdit'
IndexError while trying to check 'ditditditditditditditdit'
IndexError while trying to check 'ditditditditditditditdit'
IndexError while trying to check 'mindchorusyeah'
IndexError while trying to check 'mindchorusbaby'
IndexError while trying to check '̢'
IndexError while trying to check 'tommyjamesandtheshondells'
IndexError while trying to check 'ahhhhhhh'
IndexError while trying to check 'ahhhh'
IndexError while trying to check 'ohhhdo'
IndexError while trying to check 'ohhhhhhh'
IndexError while trying to check 'awayayayaystop'
IndexError while trying to check 'nooooooshort'
IndexError while trying to check 'x11oh'
IndexError while trying to check 'uhyaoooooo'
IndexError while trying to check 'boopboopbaboopboop'
IndexError while trying to check 'oooooooo'
IndexError while trying to check 'oooooooowhy'
IndexError while trying to check 'oooooooo'
IndexError while trying to check 'oooooooowhy'
IndexError while trying to check 'luvmycomcsaolcom'
IndexError while trying to check 'fermintvservidorunammx'
IndexError while trying to check '̢im'
IndexError while trying to check 'will̢'
IndexError while trying to check 'ooooooooh'
IndexError while trying to check 'ooooooooh'
IndexError while trying to check 'yeahyeahyeahlookahere'
IndexError while trying to check '̢cause'
IndexError while trying to check '̢cause'
IndexError while trying to check '̢cause'
IndexError while trying to check '̢cause'
IndexError while trying to check 'deeeeeepall'
IndexError while trying to check 'deeeeeepmy'
IndexError while trying to check 'locutusimsaedu'
IndexError while trying to check 'desireahahahahahahah'
IndexError while trying to check 'dahdodahdahdah'
IndexError while trying to check 'chhchhwhen'
IndexError while trying to check 'dahdodahdahdah'
IndexError while trying to check 'gbh333'
IndexError while trying to check 'stayayayarizona'
IndexError while trying to check 'bendyeahohcatch'
IndexError while trying to check 'that̢s'
IndexError while trying to check 'that̢s'
IndexError while trying to check 'i̢m'
IndexError while trying to check '̢cause'
IndexError while trying to check 'can̢t'
IndexError while trying to check 'i̢m'
IndexError while trying to check 'i̢m'
IndexError while trying to check 'can̢t'
IndexError while trying to check 'that̢s'
IndexError while trying to check 'ain̢t'
IndexError while trying to check 'doing̢'
IndexError while trying to check 'good̢s'
IndexError while trying to check 'good̢s'
IndexError while trying to check 'that̢s'
IndexError while trying to check 'mcnamarabarrycretecos'
IndexError while trying to check 'delfi4110aolcom'
IndexError while trying to check 'sonofagunjoy'
IndexError while trying to check 'lookingfordonnatime'
IndexError while trying to check 'lookingfordonnatime'
IndexError while trying to check 'lookingfordonnatime'
IndexError while trying to check 'lookingfordonnatime'
IndexError while trying to check 'jbaer10134aolcom'
IndexError while trying to check 'yoyowoah'
IndexError while trying to check 'yoyowoah'
IndexError while trying to check 'yoyowoah'
IndexError while trying to check 'yoyowoah'
IndexError while trying to check 'tmazanec1junocom'
IndexError while trying to check 'yeaaaaaaaaaaaaaaaaaaaaaaaaah'
IndexError while trying to check 'deweumumaway'
IndexError while trying to check 'weumumaway'
IndexError while trying to check 'deweumumaway'
IndexError while trying to check 'weumumaway'
IndexError while trying to check '̢i'
IndexError while trying to check 'boy̢'
IndexError while trying to check 'wwwgoldinccomedwise'
IndexError while trying to check 'ronhontzworldnetattnet'
IndexError while trying to check 'shambalachorushow'
IndexError while trying to check 'shambalachorushow'
IndexError while trying to check 'kabenn000aolcom'
IndexError while trying to check '̫i'
IndexError while trying to check 'or̫r̫i'
IndexError while trying to check 'r̫r̫r̫i'
IndexError while trying to check 'r̫r̫r̫i'
IndexError while trying to check 'r̫r̫r̫i'
IndexError while trying to check 'r̫r̫r̫i'
IndexError while trying to check 'ohror̫'
IndexError while trying to check 'pop̫'
IndexError while trying to check 'y̫i'
IndexError while trying to check 'or̫r̫i'
IndexError while trying to check 'r̫r̫r̫i'
IndexError while trying to check 'r̫r̫r̫i'
IndexError while trying to check 'r̫r̫r̫i'
IndexError while trying to check 'r̫r̫r̫i'
IndexError while trying to check 'ohror̫'
IndexError while trying to check 'pop̫'
IndexError while trying to check 'ooooooooh'
IndexError while trying to check '̫i'
IndexError while trying to check 'or̫r̫i'
IndexError while trying to check 'r̫r̫r̫i'
IndexError while trying to check 'r̫r̫r̫i'
IndexError while trying to check 'r̫r̫r̫i'
IndexError while trying to check 'r̫r̫r̫i'
IndexError while trying to check 'ohror̫'
IndexError while trying to check 'pop̫'
IndexError while trying to check 'y̫i'
IndexError while trying to check 'or̫r̫i'
IndexError while trying to check 'r̫r̫r̫i'
IndexError while trying to check 'r̫r̫r̫i'
IndexError while trying to check 'r̫r̫r̫i'
IndexError while trying to check 'r̫r̫r̫i'
IndexError while trying to check 'ohror̫'
IndexError while trying to check 'boump̫'
IndexError while trying to check 'ooooooooh'
IndexError while trying to check 'tatrrrep̫tetretrepiec̫̣̩̩̫ḥh̩nh̩n'
IndexError while trying to check '̫i'
IndexError while trying to check 'tr̩gueregu̩d̫i'
IndexError while trying to check 'det̻ro'
IndexError while trying to check 'degued̫'
IndexError while trying to check 'tegueregued̻ro'
IndexError while trying to check 'd̻dow'
IndexError while trying to check '̫̩hr̫hr̻hr̻hr��hr�'
IndexError while trying to check 'h̩'
IndexError while trying to check 'ḥ'
IndexError while trying to check '̣i'
IndexError while trying to check 'er̻ṛi'
IndexError while trying to check 'ṛṛr̫i'
IndexError while trying to check 'r̫r̫r̫i'
IndexError while trying to check 'r̫r̫r̫i'
IndexError while trying to check 'r̫r̫r̫i'
IndexError while trying to check 'ohror̫'
IndexError while trying to check 'pop̫'
IndexError while trying to check 'y̫i'
IndexError while trying to check 'or̫r̫i'
IndexError while trying to check 'r̫r̫r̫i'
IndexError while trying to check 'r̫r̫r̫i'
IndexError while trying to check 'r̫r̫r̫i'
IndexError while trying to check 'r̫r̫r̫i'
IndexError while trying to check 'ohror̫m'
IndexError while trying to check 'pomp̫'
sample_data_cv.shape_____no_output_____

## 6. Dimensionality Reduction_____no_output_____

def display_topics(topic_model, feature_names, no_top_words, topic_names=None):
    for ind, topic in enumerate(topic_model.components_):
        if not topic_names or not topic_names[ind]:
            print("\nTopic ", ind)
        else:
            print("\nTopic: '",topic_names[ind],"'")
        print(", ".join([feature_names[i]
                         for i in topic.argsort()[:-no_top_words - 1:-1]]))_____no_output_____

Eventually we'll try LSA/NMF, and maybe LDA, but now, we start with LSA._____no_output_____

test = TruncatedSVD(n_components=10, random_state=23)_____no_output_____n_comp = 50_____no_output_____lsa_cv = TruncatedSVD(n_components=n_comp, random_state=23)
# nmf_cv = NMF(n_components=n_comp, random_state=23)
# lsa_tf = TruncatedSVD(n_components=n_comp, random_state=23)
# nmf_tf = NMF(n_components=n_comp, random_state=23)_____no_output_____lsa_cv_data = lsa_cv.fit_transform(sample_data_cv)
# nmf_cv_data = nmf_cv.fit_transform(sample_data_cv)_____no_output_____display_topics(lsa_cv, sample_cv.get_feature_names(), 10)
Topic  0
lov, know, s, m, got, oh, lik, tim, e, com

Topic  1
away, man, cam, look, night, day, old, sing, hom, left

Topic  2
got, yeah, ai nt, ai, right, man, ta, m, got ta, said

Topic  3
oh, baby, yeah, good, sweet, said, girl, dant, tel, oh yeah

Topic  4
e, know, tel, got, away, caus, e got, m, d, heart

Topic  5
want, ey, lov, said, lik, look, leav, tel, d, right

Topic  6
let, com, e, lik, think, tel, quo, ll, quo nt, lov

Topic  7
com, tim, yeah, try, il, e, long, feel, hard, right

Topic  8
s, liv, ai nt, ai, world, lif, lov, tel, tim, think

Topic  9
e, got, e got, lov, ai, ai nt, lif, ey, hold, heart

Topic  10
quo, quo nt, oh, tim, long, cry, light, right, ey, ai nt

Topic  11
e, oh, lik, ey, good, e got, tim, feel, d, let

Topic  12
know, s, girl, littl, got, ta, com, got ta, day, baby

Topic  13
oh, find, man, quo nt, quo, liv, il, lov, girl, got

Topic  14
know, let, lik, long, away, girl, nt know, tim, d, quo nt
display_topics(nmf_cv, sample_cv.get_feature_names(), 10)
Topic  0
lov, know, tel, want, heart, m, il, nee, tim, s

Topic  1
away, day, way, hom, left, stay, cam, leav, night, run

Topic  2
got, e, e got, ta, got ta, m, know, ai, ai nt, let

Topic  3
play, oh, mus, word, night, good, dant, deep, went, cam

Topic  4
lik, look, said, m, s, man, know, got, littl, right

Topic  5
caus, knew, ey, leav, yeah, thought, said, run, try, hat

Topic  6
baby, oh, yeah, feel, m, oh yeah, hey, good, right, ooh

Topic  7
s, world, liv, find, day, lif, mind, plac, peopl, tim

Topic  8
com, tim, right, s, il, m, long, try, look, night

Topic  9
long, oh, car, caus, sur, il, going, mind, tim, high

Topic  10
lov, lik, ey, dream, night, hand, hold, clos, sweet, walk

Topic  11
ai nt, ai, s, let, night, light, nt know, know, stop, shin

Topic  12
quo, quo nt, oh, let, know, light, lov, ll, find, s

Topic  13
song, sing, man, know, old, rock, rol, ai nt, ai, nt know

Topic  14
caus, ai nt, ai, friend, night, man, hom, right, thing, way
# getting an error...
# display_topics(lsa_tf, sample_tf.get_feature_names(), 10)
Topic  0
x = np.arange(n_comp)
y_lsa = lsa_cv.explained_variance_ratio_
# y_nmf = nmf_cv.explained_variance_ratio_

# fig, ax = plt.subplots(1, 2, figsize=(16,9))

# fig.suptitle("Explained Variance per Topic", fontsize=24)
# fig.subtitle("Topics", fontsize=16)
# fig.ylabel("Proportion of Variance")

# ax[0].bar(x, y_lsa, color="cyan")
# ax[0].plot(np.cumsum(y_lsa),'b')
# ax[0].set_title("LSA")

# ax[1].bar(x, y_nmf, color="cyan")
# ax[1].plot(np.cumsum(y_nmf),'b')
# ax[1].set_title("NMF")

plt.figure(figsize=(16,9))
plt.bar(x, y, color="cyan")
plt.plot(np.cumsum(y),'r')
plt.xlabel('Topics', fontsize=16)
plt.ylabel('Variance Ratio', fontsize=16)
plt.title('Explained Variance (LSA)', fontsize=24);
#plt.grid()  fig1.text(0.80, .8, 'p = {:1f}'.format(pdr[1]), fontsize=12);_____no_output_____print(plt.style.available)['_classic_test', 'bmh', 'classic', 'dark_background', 'fivethirtyeight', 'ggplot', 'grayscale', 'seaborn-bright', 'seaborn-colorblind', 'seaborn-dark-palette', 'seaborn-dark', 'seaborn-darkgrid', 'seaborn-deep', 'seaborn-muted', 'seaborn-notebook', 'seaborn-paper', 'seaborn-pastel', 'seaborn-poster', 'seaborn-talk', 'seaborn-ticks', 'seaborn-white', 'seaborn-whitegrid', 'seaborn']

## 7. Reduce to 3D and Cluster for Viz_____no_output_____Becuase it does not make sense to cluster in 3D space to get an idea of what clustering might be like in higher dimensional space, none of this was used in the final presentation. However, it was a good exercise in 3D plotting from several angles..._____no_output_____

lsa_cv_data.shape_____no_output_____lsa_cv_3d = TruncatedSVD(n_components=3)
lsa_3d_data = lsa_cv_3d.fit_transform(lsa_cv_data)_____no_output_____for col in lsa_3d_data.T:
    print('\nMin:', col.min())
    print('Max:', col.max())
    print('Mean:', col.mean())
    print('SD:', col.std())
Min: -9.57189229323e-22
Max: 6.14588112516
Mean: 2.18550564046
SD: 0.907628136929

Min: -2.30980264504
Max: 8.60994284072
Mean: 0.0102385774699
SD: 0.817280670486

Min: -3.56521691527
Max: 3.76279197311
Mean: 0.0497642135535
SD: 0.758742414415
# ss_3d = StandardScaler()
# lsa_3d_scaled = ss_3d.fit_transform(lsa_3d_data)_____no_output_____def plot_clusters_3D(X, y, figx, figy, azimuth=130):

    fig = plt.figure(1, figsize=(figx, figy))
    plt.style.use('seaborn-whitegrid')
    # this is the 'right' way to do it, so they say
    # ax = fig.add_subplot(111, projection='3d')
    ax = Axes3D(fig, elev=-160, azim=azimuth)
    ax.scatter(X[:, 0],
               X[:, 1],
               X[:, 2],
               c=y,
               cmap=plt.cm.Set1
              )
    ax.set_title("Lyrics in 3D")
    ax.set_xlabel("X1")
    ax.w_xaxis.set_ticklabels([])
    ax.set_ylabel("X2")
    ax.w_yaxis.set_ticklabels([])
    ax.set_zlabel("X3")
    ax.w_zaxis.set_ticklabels([])

    plt.show()_____no_output_____# this does not work, sadly

# def plot_clusters_3Dx4(X, y, figx, figy, az_start=30):

#     fig, ax = plt.subplots(2, 2, figsize=(figx, figy))
#     plt.style.use('seaborn-whitegrid')
#     az = az_start
#     for i in range(2):
#         for j in range(2):
#             fig = plt.figure(1, figsize=(figx/2, figy/2))
#             ax[i][j] = Axes3D(fig, elev=-160, azim=az)
#             ax[i][j].scatter(X[:, 0],
#                              X[:, 1],
#                              X[:, 2],
#                              c=y,
#                              cmap=plt.cm.Set1
#                             )
#             ax[i][j].set_title("Lyrics in 3D")
#             ax[i][j].set_xlabel("X1")
#             ax[i][j].w_xaxis.set_ticklabels([])
#             ax[i][j].set_ylabel("X2")
#             ax[i][j].w_yaxis.set_ticklabels([])
#             ax[i][j].set_zlabel("X3")
#             ax[i][j].w_zaxis.set_ticklabels([])
#             az += 90_____no_output_____for n in range(1,2):
    display.clear_output()
    agg = Agg(n_clusters=n)
    clusters_3d = agg.fit_predict(lsa_3d_data)
    print(f"{n} Clusters:")
    print(np.bincount(clusters_3d))
#     plot_clusters_3Dx4(lsa_3d_data, clusters_3d, 16, 16, az_start=30)
    for az in [30, 120, 210, 300]:
        plot_clusters_3D(lsa_3d_data, clusters_3d, 8, 8, azimuth=az)
    input('Hit enter to continue')
1 Clusters:
[1026]

## 8. Let's Make Cluster_____no_output_____A first pass to see what clusters "look" like in topic space._____no_output_____

ss_lsa = StandardScaler()
ss_nmf = StandardScaler()_____no_output_____lsacv_scaled = ss_lsa.fit_transform(lsa_cv_data)
nmfcv_scaled = ss_nmf.fit_transform(nmf_cv_data)_____no_output_____agg_5_lsa = Agg(n_clusters=5)
sample_clusters_lsa = agg_5_lsa.fit_predict(lsacv_scaled)
np.bincount(sample_clusters_lsa)_____no_output_____agg_5_nmf = Agg(n_clusters=5)
sample_clusters_nmf = agg_5_nmf.fit_predict(nmfcv_scaled)
np.bincount(sample_clusters_nmf)_____no_output_____

### Dendrogram?_____no_output_____

from scipy.cluster.hierarchy import dendrogram, linkage_____no_output_____def draw_dendro(data, l_method='single', t_mode='mlab'):
    Z = linkage(data, method=l_method)
    plt.figure(figsize=(16,10), dpi=200)
    dendrogram(Z,truncate_mode=t_mode,);_____no_output_____draw_dendro(lsacv_scaled)_____no_output_____

A reminder of what a dendrogram looks like with well-defined clusters..._____no_output_____

from sklearn.datasets.samples_generator import make_blobs_____no_output_____centers = [[-3,3], [3,3], [3,-3], [-3,-3]]
Xdata, _ = make_blobs(n_samples=100, centers=centers, cluster_std=0.7)

draw_dendro(Xdata)_____no_output_____