# Notebook from jideedu/SkillVet
Path: CreateNewFullDataset-MapTraceabilityToNewSkills.ipynb

# '''
this script loads the *.json files for skills of each market and maps the excel with the traceablity to each of the set of skills
It also filters repetitions and skills without name if any
'''_____no_output_____

#find all file.json files

import os
def getFiles(dirName):
    listOfFile = os.listdir(dirName)
    completeFileList = list()
    for file in listOfFile:
        completePath = os.path.join(dirName, file)
        if os.path.isdir(completePath):
            completeFileList = completeFileList + getFiles(completePath)
        elif 'file.json' in completePath:
            completeFileList.append(completePath.replace('\\', '/'))

    return completeFileList

allf = getFiles( "skills/")
print(allf)['skills/US/file.json', 'skills/IN/file.json', 'skills/IT/file.json', 'skills/CA/file.json', 'skills/UK/file.json', 'skills/AU/file.json', 'skills/MX/file.json', 'skills/DE/file.json', 'skills/FR/file.json', 'skills/ES/file.json', 'skills/JP/file.json']
#load all objects in the json files in a dictoinary with key = filepath
import json
def loadObjects(jsonPath):
    lines = open(jsonPath, 'r').readlines()
    allobj = []
    for line in lines:
        if(len(line.strip()) > 0):
            try:
                allobj.append(json.loads(line))
            except:
                pass
    return allobj

#NOTICE allo might contain repetitoins! (we filter them later)
allo = {}
for f in allf:
    allo[f] = loadObjects(f)
#list all markets
for key,value in allo.items():
    print(key)
    print(len(value))skills/US/file.json
90281
skills/IN/file.json
35467
skills/IT/file.json
5271
skills/CA/file.json
42429
skills/UK/file.json
38707
skills/AU/file.json
37071
skills/MX/file.json
2308
skills/DE/file.json
31880
skills/FR/file.json
2575
skills/ES/file.json
17623
skills/JP/file.json
16054

### Load and clean the csv containing the traceability analysis_____no_output_____

# This combine manual traceability file from NDSS with SKillvet manual Traceability
# import pandas as pd
# #Read csv in pandas
# dfold = pd.read_csv("Traceability.csv", header=0, encoding='utf-8')

# header_lists = ['Skills', 'Developer', 'Permission', 'Traceability', 'Category',
#        'sub-category', 'Account Linking', 'customer reviews', 'Rating',
#        'Cost',"d","y"]
# dfnew = pd.read_csv('ManualTraceability/data/Manual_Traceability3_15.10_21.46.csv', header=1,encoding='utf-8', names= header_lists)

# dfnew = dfnew.drop(['d','y'], axis=1)
# dfnew.head(2)
# dfold.head(3)

# for index, row in dfnew.iterrows():
#     for index2, row2 in dfold.iterrows():
#         if row['Skills']==row2['Skills']:
#             dfnew.iloc[index, 4] =  dfold.iloc[index2, 4]
#             dfnew.iloc[index, 5] =  dfold.iloc[index2, 5]
#             dfnew.iloc[index, 6] =  dfold.iloc[index2, 6]
#             dfnew.iloc[index, 7] =  dfold.iloc[index2, 7]
#             dfnew.iloc[index, 8] =  dfold.iloc[index2, 8]
#             dfnew.iloc[index, 9] =  dfold.iloc[index2, 9]
#             break
# dfnew.head(10)

# dfnew.to_csv('Manual_Traceability.csv')_____no_output_____import pandas as pd

#Read Manual Traceability csv in pandas
df = pd.read_csv('Traceability/data/Manual_Traceability.csv',header=0, encoding='utf-8')
df.head(n=10)
# df.columns_____no_output_____

### Map the traceability to each skill_____no_output_____

#filter repetitoins and skills without name or developer
import json, re
def _createSkillDict(oldd):
    '''
    example json obj
    {'Main Category': 'Business & Finance', 'SubCategory': 'Business & Finance', 'Sample_Invocation_Utterances': ["“'Alexa, Open Current Bitcoin'”", "“'Alexa, Ask Current Bitcoin for Prices'”", "“'Alexa, Ask Current Bitcoin what the price is'”"], 'Name': 'CurrentBitcoin', 'Developer': 'by Sarim Studios', 'Skill_permission': [], 'Account_linking': '', 'Review_Count': None, 'Rating': '', 'Total_customer_that_rate_the_skill': '0', 'Total_Customers_Reviews': 'No customer reviews', 'Cost': 'Free to Enable', 'In_skill_purchase': None, 'Skill_description': "Description\nAn 'easy to use' Alexa skill that will provide you with the latest price for a bitcoin. This skill uses BitStamp's latest prices, which are always up-to-date. All you have to do to begin using the skill is ask Alexa, Ask Current Bitcoin for Prices. Price is given in USD.", 'Skill_link': 'https://www.amazon.ca/Sarim-Studios-CurrentBitcoin/dp/B01N9SS2LI/ref=sr_1_635?dchild=1&fst=as%3Aoff&qid=1595709096&rnid=16286270011&s=alexa-skills&sr=1-635', 'privacy_policy and Terms_of_use specified': 'No privacy_policy and Terms_of_use specified'}
    '''
    newd = {}
    for obj in oldd:
        if len(obj) == 0:
            continue

        name = ''
        if 'Name' in obj:
            name = obj['Name']
        else:
            continue
            #print('[!] No name found', obj)

        skill_id = ''
        if 'Skill_link' in obj:
            try:
                skill_url = obj['Skill_link']
                skill_id = re.search(r"\b[B]+[0-9]+(?:[A-Z0-9]+/)+", skill_url).group().replace('/','')
            except:
                pass
        else:
            continue
            #print('[!] No name found', obj)

        dev = ''
        if 'Developer' in obj:
            dev = obj['Developer']
        else:
            continue
            #print('[!] No dev found', obj)

        maincat = ''
        if 'Main Category' in obj:
            maincat = obj['Main Category']

        subcat = ''
        if 'SubCategory' in obj:
            subcat = obj['SubCategory']

        perm = ''
        if 'Skill_permission' in obj:
            perm = obj['Skill_permission']

        linking = ''
        if 'Account_linking' in obj:
            linking = obj['Account_linking']

        inskill = ''
        if 'In_skill_purchase' in obj:
            inskill = obj['In_skill_purchase']

        policy = ''
        if 'privacy_policy' in obj:
            policy = obj['privacy_policy']

        newd[skill_id] = {'skill_id':skill_id, 'name': name, 'dev':dev, 'perm':perm, 'acc_linking':linking, 'in_skill_purchase':inskill,'policy':policy ,'cat':maincat, 'subcat':subcat}

    return newd

#Create a dictionary of unqiue skills per market
for f in allf:
    print('********************')
    market_name = f[7:9]
    print(market_name)
    skillsdict = _createSkillDict(allo[f])
    print('#skills', len(skillsdict))

    skillsfound = []
    count_skills = 0
    for sk in skillsdict:
        skill  = skillsdict[sk]
        skname = skill['name']
        skdev = skill['dev']
        skill_id = skill['skill_id']

        cols = df.loc[ (df['Skills'] == skname) &  (df['Developer'] == skdev),['Permission', 'Traceability',
       'Account Linking', 'customer reviews', 'Rating','Cost']]
        if(len(cols) > 0):
            res = cols.values.tolist()[0]
            skill['permission'] = res[0]
            skill['traceability'] = res[1].strip()
            skill['acc_linking'] = res[2]
            skill['reviews'] = res[3]
            skill['stars'] = res[4]
            skill['free'] = res[5]
            count_skills+=1
            skillsfound.append( skname )
        else:
            skill['traceability'] = 'n'
#             skill['reviews'] = None
#             skill['stars'] = None
#             skill['free'] = None

    print('[*]', count_skills, 'were identified and updated their traceablity status from the excel file')
    #print('skills found traceablity', skillsfound)

    #save analysis in NewFullDataset json
    path = 'NewFullDataset/'+market_name+'.json'
    with open(path, 'w') as json_file:
        print('Saving json in ',path)
        json.dump(skillsdict, json_file)
            ********************
US
#skills 61362
[*] 1172 were identified and updated their traceablity status from the excel file
Saving json in  NewFullDataset/US.json
********************
IN
#skills 29344
[*] 217 were identified and updated their traceablity status from the excel file
Saving json in  NewFullDataset/IN.json
********************
IT
#skills 4203
[*] 0 were identified and updated their traceablity status from the excel file
Saving json in  NewFullDataset/IT.json
********************
CA
#skills 26027
[*] 230 were identified and updated their traceablity status from the excel file
Saving json in  NewFullDataset/CA.json
********************
UK
#skills 32822
[*] 333 were identified and updated their traceablity status from the excel file
Saving json in  NewFullDataset/UK.json
********************
AU
#skills 23909
[*] 181 were identified and updated their traceablity status from the excel file
Saving json in  NewFullDataset/AU.json
********************
MX
#skills 1972
[*] 0 were identified and updated their traceablity status from the excel file
Saving json in  NewFullDataset/MX.json
********************
DE
#skills 9096
[*] 0 were identified and updated their traceablity status from the excel file
Saving json in  NewFullDataset/DE.json
********************
FR
#skills 2288
[*] 0 were identified and updated their traceablity status from the excel file
Saving json in  NewFullDataset/FR.json
********************
ES
#skills 4759
[*] 0 were identified and updated their traceablity status from the excel file
Saving json in  NewFullDataset/ES.json
********************
JP
#skills 3513
[*] 0 were identified and updated their traceablity status from the excel file
Saving json in  NewFullDataset/JP.json

# 1 .Load New Full dataset and perform traceablity analysis_____no_output_____

#number of developer involved in
#broken b and unqiue developers with broken policies,
#complete c -- developer, partial p -- developer, total skills -- total developer

#broken, complete, and partial, and total per main categories
#broken, complete, and partial, and total per main subcategories_____no_output_____import os
import json

def getFiles(dirName):
    listOfFile = os.listdir(dirName)
    completeFileList = list()
    for file in listOfFile:
        completePath = os.path.join(dirName, file)
        if os.path.isdir(completePath):
            completeFileList = completeFileList + getFiles(completePath)
        elif '.json' in completePath:
            completeFileList.append(completePath.replace('\\', '/'))

    return completeFileList

allf = getFiles( "NewFullDataset/")
print(allf)

#create dictionary of markets and skills
skillsmarketd = {}
for f in allf:
    marketname = f.split('/')[1].split('.json')[0]
    with open(f, 'r') as json_file:
        skillsmarketd[marketname] = json.load(json_file)

print()
print('Total unique skills per market')
for key,value in skillsmarketd.items():
    print(key, ": ", len(value))['NewFullDataset/DE.json', 'NewFullDataset/UK.json', 'NewFullDataset/MX.json', 'NewFullDataset/US.json', 'NewFullDataset/IT.json', 'NewFullDataset/AU.json', 'NewFullDataset/IN.json', 'NewFullDataset/FR.json', 'NewFullDataset/JP.json', 'NewFullDataset/CA.json', 'NewFullDataset/ES.json']

Total unique skills per market
DE :  9096
UK :  32822
MX :  1972
US :  61362
IT :  4203
AU :  23909
IN :  29344
FR :  2288
JP :  3513
CA :  26027
ES :  4759

### Get how many skills have broken, partial and complete policies (over the total) per skill category and per market_____no_output_____

from collections import Counter
print("### 1.2 Get how many skills have broken, partial and complete policies (over the total) per skill category and per market. skills with one or more permissions")

#first join all dictionaries to create a total english market market
#z = {**x, **y}
englishmarkets = ['UK', 'US', 'CA', 'IN', 'AU']
allm = {}
for market in englishmarkets:
    value = skillsmarketd[market]
    allm = {**allm, **value}
    #print('adding market', market, len(allm))
skillsmarketd['ALLENGLISH']  = allm

for key in englishmarkets+['ALLENGLISH']:
    print()
    print('************')
    print('Market', key)
    print('************')
    value = skillsmarketd[key]

    t = {'b':[], 'p':[], 'c':[], 'd':[], 'n':[], 't':[]}  #b, p, c,d,n total policies  -- traceablity policies category
    st = {'b':[], 'p':[], 'c':[], 'd':[], 'n':[], 't':[]}  #b, p, c,d,n total policies  -- traceablity policies subcategory

    for skill, skillobj in value.items():
        '''
        skillobj example:
        {'name': 'Wake on Lan (WoL)', 'dev': 'by Oscar Penelo', 'perm': [], 'acc_linking': '', 'policy': 'https://www.amazon.com.au/gp/help/customer/display.html?nodeId=201602230', 'cat': 'Smart Home', 'subcat': 'Smart Home', 'traceability': 'd', 'reviews': None, 'stars': None, 'free': None}
        '''
        trac = skillobj['traceability'].lower().strip()
        cat = skillobj['cat']
        scat = skillobj['subcat']
        n_perm = len(skillobj['perm'])
        #only consdier these skills with 1 or more permissions
        if(n_perm < 1):
            continue
        t[trac].append(cat)
        st[trac].append(scat)
        #add for the total
        t['t'].append(cat) #set of developers
        st['t'].append(scat)

    #print the data for CATEGORIES
    print('**CATEGORIES \t   #Skills')
    print('**Broken     \n   {} '.format(Counter(t['b'])) )
    print('**Partial    \n   {} '.format(Counter(t['p'])) )
    print('**Complete   \n   {} '.format(Counter(t['c'])) )
    print('**Dead Link  \n   {} '.format(Counter(t['d'])) )
    print('**No Policy  \n   {} '.format(Counter(t['n'])) )
    print('**Total      \n   {} '.format(Counter(t['t'])) )

    print()
    print()
    #print the data for SUBCATEGORIES
    print('**SUBCATEGORIES \t   #Skills')
    print('**Broken     \n   {} '.format(Counter(t['b'])) )
    print('**Partial    \n   {} '.format(Counter(t['p'])) )
    print('**Complete   \n   {} '.format(Counter(t['c'])) )
    print('**Dead Link  \n   {} '.format(Counter(t['d'])) )
    print('**No Policy  \n   {} '.format(Counter(t['n'])) )
    print('**Total      \n   {} '.format(Counter(t['t'])) )
    ### 1.2 Get how many skills have broken, partial and complete policies (over the total) per skill category and per market. skills with one or more permissions

************
Market UK
************
**CATEGORIES 	   #Skills
**Broken
   Counter({'': 19, 'Games': 6, 'Organizers & Assistants': 6, 'Navigation & Trip Planners': 6, 'Religion & Spirituality': 5, 'Home Services': 4, 'Cooking & Recipes': 3, 'Schools': 3, 'Knowledge & Trivia': 2, 'To-Do Lists & Notes': 2, 'Event Finders': 2, 'Calendars & Reminders': 2, 'Wine & Beverages': 1, 'Restaurant Booking, Info & Reviews': 1, 'Game Info & Accessories': 1, 'Self Improvement': 1, 'Movie Info & Reviews': 1, 'Streaming Services': 1, 'Podcasts': 1, 'Social Networking': 1})
**Partial
   Counter({'': 15, 'Games': 9, 'Home Services': 7, 'Public Transportation': 5, 'Streaming Services': 3, 'Organizers & Assistants': 3, 'Knowledge & Trivia': 2, 'Fashion & Style': 2, 'Music Info, Reviews & Recognition Services': 2, 'Navigation & Trip Planners': 2, 'Delivery & Takeout': 1, 'Cooking & Recipes': 1, 'Wine & Beverages': 1, 'Religion & Spirituality': 1, 'TV Guides': 1, 'Calendars & Reminders': 1, 'Self Improvement': 1, 'Social Networking': 1, 'Exercise & Workout': 1, 'Device Tracking': 1})
**Complete
   Counter({'': 61, 'Home Services': 18, 'Knowledge & Trivia': 17, 'Games': 10, 'To-Do Lists & Notes': 8, 'Organizers & Assistants': 8, 'Accessories': 7, 'Streaming Services': 7, 'Navigation & Trip Planners': 7, 'Religion & Spirituality': 6, 'Event Finders': 6, 'Public Transportation': 6, 'Calendars & Reminders': 6, 'Communication': 5, 'Cooking & Recipes': 5, 'Pets & Animals': 4, 'Podcasts': 4, 'Wine & Beverages': 3, 'Fashion & Style': 3, 'Self Improvement': 3, 'Astrology': 2, 'Taxi & Ridesharing': 2, 'Hotel Finders': 2, 'Friends & Family': 1, 'Movie Info & Reviews': 1, 'Device Tracking': 1})
**Dead Link
   Counter()
**No Policy
   Counter({'': 93, 'Communication': 74, 'Games': 19, 'Knowledge & Trivia': 14, 'Organizers & Assistants': 11, 'Streaming Services': 5, 'Self Improvement': 4, 'Cooking & Recipes': 3, 'To-Do Lists & Notes': 3, 'Game Info & Accessories': 2, 'Pets & Animals': 2, 'Public Transportation': 2, 'Podcasts': 2, 'News': 2, 'Calendars & Reminders': 2, 'Delivery & Takeout': 1, 'Restaurant Booking, Info & Reviews': 1, 'Home Services': 1, 'Music & Audio': 1, 'Education & Reference': 1, 'Religion & Spirituality': 1, 'Fashion & Style': 1, 'Astrology': 1, 'Friends & Family': 1, 'TV Guides': 1, 'Exercise & Workout': 1, 'Navigation & Trip Planners': 1})
**Total
   Counter({'': 188, 'Communication': 79, 'Games': 44, 'Knowledge & Trivia': 35, 'Home Services': 30, 'Organizers & Assistants': 28, 'Streaming Services': 16, 'Navigation & Trip Planners': 16, 'Religion & Spirituality': 13, 'To-Do Lists & Notes': 13, 'Public Transportation': 13, 'Cooking & Recipes': 12, 'Calendars & Reminders': 11, 'Self Improvement': 9, 'Event Finders': 8, 'Podcasts': 7, 'Accessories': 7, 'Pets & Animals': 6, 'Fashion & Style': 6, 'Wine & Beverages': 5, 'Game Info & Accessories': 3, 'Astrology': 3, 'Schools': 3, 'Delivery & Takeout': 2, 'Restaurant Booking, Info & Reviews': 2, 'Friends & Family': 2, 'Taxi & Ridesharing': 2, 'TV Guides': 2, 'Movie Info & Reviews': 2, 'Music Info, Reviews & Recognition Services': 2, 'News': 2, 'Social Networking': 2, 'Exercise & Workout': 2, 'Hotel Finders': 2, 'Device Tracking': 2, 'Music & Audio': 1, 'Education & Reference': 1})

**SUBCATEGORIES 	   #Skills
**Broken
   Counter({'': 19, 'Games': 6, 'Organizers & Assistants': 6, 'Navigation & Trip Planners': 6, 'Religion & Spirituality': 5, 'Home Services': 4, 'Cooking & Recipes': 3, 'Schools': 3, 'Knowledge & Trivia': 2, 'To-Do Lists & Notes': 2, 'Event Finders': 2, 'Calendars & Reminders': 2, 'Wine & Beverages': 1, 'Restaurant Booking, Info & Reviews': 1, 'Game Info & Accessories': 1, 'Self Improvement': 1, 'Movie Info & Reviews': 1, 'Streaming Services': 1, 'Podcasts': 1, 'Social Networking': 1})
**Partial
   Counter({'': 15, 'Games': 9, 'Home Services': 7, 'Public Transportation': 5, 'Streaming Services': 3, 'Organizers & Assistants': 3, 'Knowledge & Trivia': 2, 'Fashion & Style': 2, 'Music Info, Reviews & Recognition Services': 2, 'Navigation & Trip Planners': 2, 'Delivery & Takeout': 1, 'Cooking & Recipes': 1, 'Wine & Beverages': 1, 'Religion & Spirituality': 1, 'TV Guides': 1, 'Calendars & Reminders': 1, 'Self Improvement': 1, 'Social Networking': 1, 'Exercise & Workout': 1, 'Device Tracking': 1})
**Complete
   Counter({'': 61, 'Home Services': 18, 'Knowledge & Trivia': 17, 'Games': 10, 'To-Do Lists & Notes': 8, 'Organizers & Assistants': 8, 'Accessories': 7, 'Streaming Services': 7, 'Navigation & Trip Planners': 7, 'Religion & Spirituality': 6, 'Event Finders': 6, 'Public Transportation': 6, 'Calendars & Reminders': 6, 'Communication': 5, 'Cooking & Recipes': 5, 'Pets & Animals': 4, 'Podcasts': 4, 'Wine & Beverages': 3, 'Fashion & Style': 3, 'Self Improvement': 3, 'Astrology': 2, 'Taxi & Ridesharing': 2, 'Hotel Finders': 2, 'Friends & Family': 1, 'Movie Info & Reviews': 1, 'Device Tracking': 1})
**Dead Link
   Counter()
**No Policy
   Counter({'': 93, 'Communication': 74, 'Games': 19, 'Knowledge & Trivia': 14, 'Organizers & Assistants': 11, 'Streaming Services': 5, 'Self Improvement': 4, 'Cooking & Recipes': 3, 'To-Do Lists & Notes': 3, 'Game Info & Accessories': 2, 'Pets & Animals': 2, 'Public Transportation': 2, 'Podcasts': 2, 'News': 2, 'Calendars & Reminders': 2, 'Delivery & Takeout': 1, 'Restaurant Booking, Info & Reviews': 1, 'Home Services': 1, 'Music & Audio': 1, 'Education & Reference': 1, 'Religion & Spirituality': 1, 'Fashion & Style': 1, 'Astrology': 1, 'Friends & Family': 1, 'TV Guides': 1, 'Exercise & Workout': 1, 'Navigation & Trip Planners': 1})
**Total
   Counter({'': 188, 'Communication': 79, 'Games': 44, 'Knowledge & Trivia': 35, 'Home Services': 30, 'Organizers & Assistants': 28, 'Streaming Services': 16, 'Navigation & Trip Planners': 16, 'Religion & Spirituality': 13, 'To-Do Lists & Notes': 13, 'Public Transportation': 13, 'Cooking & Recipes': 12, 'Calendars & Reminders': 11, 'Self Improvement': 9, 'Event Finders': 8, 'Podcasts': 7, 'Accessories': 7, 'Pets & Animals': 6, 'Fashion & Style': 6, 'Wine & Beverages': 5, 'Game Info & Accessories': 3, 'Astrology': 3, 'Schools': 3, 'Delivery & Takeout': 2, 'Restaurant Booking, Info & Reviews': 2, 'Friends & Family': 2, 'Taxi & Ridesharing': 2, 'TV Guides': 2, 'Movie Info & Reviews': 2, 'Music Info, Reviews & Recognition Services': 2, 'News': 2, 'Social Networking': 2, 'Exercise & Workout': 2, 'Hotel Finders': 2, 'Device Tracking': 2, 'Music & Audio': 1, 'Education & Reference': 1})

************
Market US
************
**CATEGORIES 	   #Skills
**Broken
   Counter({'Business & Finance': 26, 'Lifestyle': 26, 'News': 24, 'Food & Drink': 23, 'Health & Fitness': 22, 'Games & Trivia': 21, 'Music & Audio': 17, 'Weather': 17, 'Travel & Transportation': 13, 'Education & Reference': 11, 'Productivity': 11, 'Smart Home': 11, 'Shopping': 9, 'Social': 6, 'Utilities': 5, 'Connected Car': 4, 'Novelty & Humor': 3, 'Sports': 3, 'Kids': 1, 'Movies & TV': 1})
**Partial
   Counter({'Games & Trivia': 34, 'Lifestyle': 32, 'Health & Fitness': 21, 'Food & Drink': 19, 'Travel & Transportation': 17, 'Business & Finance': 13, 'Weather': 13, 'Education & Reference': 11, 'Shopping': 9, 'Productivity': 8, 'Social': 8, 'Utilities': 5, 'Music & Audio': 4, 'News': 4, 'Novelty & Humor': 4, 'Smart Home': 4, 'Connected Car': 3, 'Sports': 2, 'Kids': 1, 'Local': 1, 'Movies & TV': 1})
**Complete
   Counter({'Music & Audio': 219, 'Shopping': 87, 'Business & Finance': 50, 'Lifestyle': 50, 'Education & Reference': 45, 'Health & Fitness': 37, 'Food & Drink': 30, 'Productivity': 30, 'Games & Trivia': 28, 'Smart Home': 26, 'Travel & Transportation': 21, 'Weather': 18, 'Connected Car': 11, 'News': 10, 'Social': 9, 'Utilities': 8, 'Novelty & Humor': 7, 'Local': 3, 'Movies & TV': 3, 'Sports': 3})
**Dead Link
   Counter()
**No Policy
   Counter({'Social': 90, 'Music & Audio': 76, 'Games & Trivia': 54, 'Education & Reference': 42, 'Sports': 41, 'Lifestyle': 35, 'Business & Finance': 34, 'Health & Fitness': 33, 'Productivity': 30, 'Smart Home': 28, 'Shopping': 15, 'Food & Drink': 12, 'Utilities': 11, 'Travel & Transportation': 9, 'Kids': 7, 'Connected Car': 5, 'News': 5, 'Novelty & Humor': 4, 'Movies & TV': 2, 'Weather': 2})
**Total
   Counter({'Music & Audio': 316, 'Lifestyle': 143, 'Games & Trivia': 137, 'Business & Finance': 123, 'Shopping': 120, 'Health & Fitness': 113, 'Social': 113, 'Education & Reference': 109, 'Food & Drink': 84, 'Productivity': 79, 'Smart Home': 69, 'Travel & Transportation': 60, 'Weather': 50, 'Sports': 49, 'News': 43, 'Utilities': 29, 'Connected Car': 23, 'Novelty & Humor': 18, 'Kids': 9, 'Movies & TV': 7, 'Local': 4})

**SUBCATEGORIES 	   #Skills
**Broken
   Counter({'Business & Finance': 26, 'Lifestyle': 26, 'News': 24, 'Food & Drink': 23, 'Health & Fitness': 22, 'Games & Trivia': 21, 'Music & Audio': 17, 'Weather': 17, 'Travel & Transportation': 13, 'Education & Reference': 11, 'Productivity': 11, 'Smart Home': 11, 'Shopping': 9, 'Social': 6, 'Utilities': 5, 'Connected Car': 4, 'Novelty & Humor': 3, 'Sports': 3, 'Kids': 1, 'Movies & TV': 1})
**Partial
   Counter({'Games & Trivia': 34, 'Lifestyle': 32, 'Health & Fitness': 21, 'Food & Drink': 19, 'Travel & Transportation': 17, 'Business & Finance': 13, 'Weather': 13, 'Education & Reference': 11, 'Shopping': 9, 'Productivity': 8, 'Social': 8, 'Utilities': 5, 'Music & Audio': 4, 'News': 4, 'Novelty & Humor': 4, 'Smart Home': 4, 'Connected Car': 3, 'Sports': 2, 'Kids': 1, 'Local': 1, 'Movies & TV': 1})
**Complete
   Counter({'Music & Audio': 219, 'Shopping': 87, 'Business & Finance': 50, 'Lifestyle': 50, 'Education & Reference': 45, 'Health & Fitness': 37, 'Food & Drink': 30, 'Productivity': 30, 'Games & Trivia': 28, 'Smart Home': 26, 'Travel & Transportation': 21, 'Weather': 18, 'Connected Car': 11, 'News': 10, 'Social': 9, 'Utilities': 8, 'Novelty & Humor': 7, 'Local': 3, 'Movies & TV': 3, 'Sports': 3})
**Dead Link
   Counter()
**No Policy
   Counter({'Social': 90, 'Music & Audio': 76, 'Games & Trivia': 54, 'Education & Reference': 42, 'Sports': 41, 'Lifestyle': 35, 'Business & Finance': 34, 'Health & Fitness': 33, 'Productivity': 30, 'Smart Home': 28, 'Shopping': 15, 'Food & Drink': 12, 'Utilities': 11, 'Travel & Transportation': 9, 'Kids': 7, 'Connected Car': 5, 'News': 5, 'Novelty & Humor': 4, 'Movies & TV': 2, 'Weather': 2})
**Total
   Counter({'Music & Audio': 316, 'Lifestyle': 143, 'Games & Trivia': 137, 'Business & Finance': 123, 'Shopping': 120, 'Health & Fitness': 113, 'Social': 113, 'Education & Reference': 109, 'Food & Drink': 84, 'Productivity': 79, 'Smart Home': 69, 'Travel & Transportation': 60, 'Weather': 50, 'Sports': 49, 'News': 43, 'Utilities': 29, 'Connected Car': 23, 'Novelty & Humor': 18, 'Kids': 9, 'Movies & TV': 7, 'Local': 4})

************
Market CA
************
**CATEGORIES 	   #Skills
**Broken
   Counter({'Lifestyle': 9, 'Education & Reference': 4, 'Games & Trivia': 4, 'Business & Finance': 3, 'Novelty & Humour': 3, 'Productivity': 3, 'Travel & Transportation': 3, 'Weather': 3, 'Food & Drink': 2, 'Health & Fitness': 2, 'Social': 2, 'Music & Audio': 1, 'Smart Home': 1})
**Partial
   Counter({'Games & Trivia': 9, 'Health & Fitness': 5, 'Education & Reference': 4, 'Lifestyle': 4, 'Business & Finance': 3, 'Music & Audio': 2, 'News': 2, 'Novelty & Humour': 2, 'Productivity': 2, 'Travel & Transportation': 2, 'Food & Drink': 1, 'Movies & TV': 1, 'Sports': 1, 'Weather': 1})
**Complete
   Counter({'Education & Reference': 27, 'Music & Audio': 20, 'Games & Trivia': 16, 'Lifestyle': 16, 'Productivity': 15, 'Health & Fitness': 9, 'Smart Home': 9, 'Business & Finance': 8, 'Travel & Transportation': 7, 'Food & Drink': 5, 'Shopping': 5, 'Social': 5, 'Weather': 3, 'News': 2, 'Utilities': 2})
**Dead Link
   Counter()
**No Policy
   Counter({'Social': 74, 'Games & Trivia': 19, 'Education & Reference': 8, 'Productivity': 7, 'Smart Home': 6, 'Lifestyle': 5, 'Health & Fitness': 4, 'Food & Drink': 3, 'Utilities': 3, 'Kids': 2, 'Music & Audio': 2, 'Business & Finance': 1, 'Novelty & Humour': 1, 'Sports': 1, 'Travel & Transportation': 1, 'Weather': 1})
**Total
   Counter({'Social': 81, 'Games & Trivia': 48, 'Education & Reference': 43, 'Lifestyle': 34, 'Productivity': 27, 'Music & Audio': 25, 'Health & Fitness': 20, 'Smart Home': 16, 'Business & Finance': 15, 'Travel & Transportation': 13, 'Food & Drink': 11, 'Weather': 8, 'Novelty & Humour': 6, 'Shopping': 5, 'Utilities': 5, 'News': 4, 'Kids': 2, 'Sports': 2, 'Movies & TV': 1})

**SUBCATEGORIES 	   #Skills
**Broken
   Counter({'Lifestyle': 9, 'Education & Reference': 4, 'Games & Trivia': 4, 'Business & Finance': 3, 'Novelty & Humour': 3, 'Productivity': 3, 'Travel & Transportation': 3, 'Weather': 3, 'Food & Drink': 2, 'Health & Fitness': 2, 'Social': 2, 'Music & Audio': 1, 'Smart Home': 1})
**Partial
   Counter({'Games & Trivia': 9, 'Health & Fitness': 5, 'Education & Reference': 4, 'Lifestyle': 4, 'Business & Finance': 3, 'Music & Audio': 2, 'News': 2, 'Novelty & Humour': 2, 'Productivity': 2, 'Travel & Transportation': 2, 'Food & Drink': 1, 'Movies & TV': 1, 'Sports': 1, 'Weather': 1})
**Complete
   Counter({'Education & Reference': 27, 'Music & Audio': 20, 'Games & Trivia': 16, 'Lifestyle': 16, 'Productivity': 15, 'Health & Fitness': 9, 'Smart Home': 9, 'Business & Finance': 8, 'Travel & Transportation': 7, 'Food & Drink': 5, 'Shopping': 5, 'Social': 5, 'Weather': 3, 'News': 2, 'Utilities': 2})
**Dead Link
   Counter()
**No Policy
   Counter({'Social': 74, 'Games & Trivia': 19, 'Education & Reference': 8, 'Productivity': 7, 'Smart Home': 6, 'Lifestyle': 5, 'Health & Fitness': 4, 'Food & Drink': 3, 'Utilities': 3, 'Kids': 2, 'Music & Audio': 2, 'Business & Finance': 1, 'Novelty & Humour': 1, 'Sports': 1, 'Travel & Transportation': 1, 'Weather': 1})
**Total
   Counter({'Social': 81, 'Games & Trivia': 48, 'Education & Reference': 43, 'Lifestyle': 34, 'Productivity': 27, 'Music & Audio': 25, 'Health & Fitness': 20, 'Smart Home': 16, 'Business & Finance': 15, 'Travel & Transportation': 13, 'Food & Drink': 11, 'Weather': 8, 'Novelty & Humour': 6, 'Shopping': 5, 'Utilities': 5, 'News': 4, 'Kids': 2, 'Sports': 2, 'Movies & TV': 1})

************
Market IN
************

# Extra. Perform analysis of how many skills talk about certain websites_____no_output_____

interesting_links = {'voiceskillsinc.com': set([]), 'topvoiceapps.com':set([]),'voxogenic.com':set([]) }
for key,value in allo.items():
    print('*market ', key)
    count = 0
    for skill in value:
        #print(skill)
        if('Skill_description' in skill):
            count+=1
            desc = skill['Skill_description'].lower()
            for iw in interesting_links.keys():
                if iw in desc:
                    interesting_links[iw].add(( skill['Name'], skill['Developer'], skill['Skill_description'] ) )
    print('processed ', count, 'skills with description')

print()
for k,v in interesting_links.items():
    print(k, ' ', len(v))

print()
print('****SKILLS with this word in description')
for k,v in interesting_links.items():
    print()
    print('*', k,'\n')
    for i in range(8):
        print('\t', list(v)[i])
*market  skills/US/file.json
processed  88419 skills with description
*market  skills/IN/file.json
processed  35005 skills with description
*market  skills/IT/file.json
processed  5269 skills with description
*market  skills/CA/file.json
processed  42429 skills with description
*market  skills/UK/file.json
processed  38705 skills with description
*market  skills/AU/file.json
processed  36794 skills with description
*market  skills/MX/file.json
processed  2304 skills with description
*market  skills/DE/file.json
processed  31879 skills with description
*market  skills/FR/file.json
processed  2562 skills with description
*market  skills/ES/file.json
processed  17588 skills with description
*market  skills/JP/file.json
processed  16054 skills with description

voiceskillsinc.com   9
topvoiceapps.com   15
voxogenic.com   42

****SKILLS with this word in description

* voiceskillsinc.com

	 ('VSI Today in Innovation', 'by VoiceSkillsInc', 'Description\nToday in Innovation reports innovative news stories from various fields of technology.  These fields include voice, artificial intelligence, cloud computing, emerging technologies and more.  This flash briefing was created by VoiceSkillsInc.com')
	 ('Space Dust', 'by VoiceSkillsInc', 'Description\nThe team at VoiceSkillsInc is proud to bring you Space Dust.\xa0\xa0A skill that is fully optimized for the Show and Spot devices using the new Alexa Presentation Language .\xa0\xa0You’ll be presented with a full screen image of the NASA image of the day with a description.\xa0\xa0For users without a display, the image description is read and the image is sent to the Alexa companion app.\xa0\xa0Happy viewing and please leave us a review. \nIf you would like us to build you a skill just click the contact us button on our website www.voiceskillsinc.com\nContact Info:\nE-mail: nick@voiceskillsinc.com\nWeb: https://voiceskillsinc.com\nWeb: https://gcnes.podbean.com')
	 ('VSI Weird News Daily', 'by VoiceSkillsInc', 'Description\nVoiceSkillsInc is a voice skills development company based in the USA.  We delight in bringing you skills you will use and enjoy.  Our newest skill is a flash briefing skill named VSI Weird News Daily.  Listen and laugh at the latest weirdness from around the globe.  Find us on the web at www.voiceskillsinc.com')
	 ('A Few Minutes In Voice', 'by VoiceSkillsInc', 'Description\nThis Podcast Flash Briefing provides up to date information pertaining to everything Alexa. Also included reviews of Skills, devices, and developer tips.\nYou can call or text us a message at 360-830-6662 for questions, comments, skill requests, and concerns about the show. Also, you can still contact us via e-mail at nick@voiceskillsinc.com.\nSkills and Contact Info:\nPhone: 360-830-6662\nE-mail: nick@voiceskillsinc.com\nWeb: https://voiceskillsinc.com\nWeb: https://gcnes.podbean.com')
	 ('VSI Gadget News', 'by VoiceSkillsInc', 'Description\nVoice Skills Inc or VSI brings you another flash briefing for your morning news feed.  VSI Gadget news features the latest buzz from the tech industry.  You can find VSI on the web at www.voiceskillsinc.com.')
	 ('Space Dust', 'by Voice Spark', 'Description\nThe team at VoiceSkillsInc is proud to bring you the space picture of the day.\xa0\xa0A skill that is fully optimized for the Show and Spot devices.\xa0\xa0You’ll be presented with a full screen image of today’s astronomy picture with a complete description.\xa0\xa0For users without a display, the image description is read and the image is sent to the Alexa companion app.\xa0\xa0Happy viewing and please leave us a review. \nIf you would like us to build you a skill just click the contact us button on our website www.voiceskillsinc.com\nContact Info:\nE-mail: nick@voiceskillsinc.com\nWeb: https://voiceskillsinc.com\nWeb: https://gcnes.podbean.com')
	 ('Camera Clicks', 'by VoiceSkillsInc', 'Description\nThe team at VoiceSkillsInc is proud to bring you Camera Clicks.\xa0\xa0A skill that is fully optimized for the Show and Spot devices.\xa0\xa0You’ll be presented with a full screen image of the picture of the day.\xa0\xa0For users without a display, the image description will be read and the image is sent to the Alexa companion app.\xa0\xa0Happy viewing and please leave us a review. \nIf you would like us to build you a skill just click the contact us button on our website www.voiceskillsinc.com\nContact Info:\nE-mail: nick@voiceskillsinc.com\nWeb: https://voiceskillsinc.com\nWeb: https://gcnes.podbean.com')
	 ('Fast Facts Hiking', 'by VoiceSkillsInc', 'Description\nLearn good hiking techniques on your daily flash briefing.  Content provided by Instagram Influencer, Patch of Wild.  The skill was created by VoiceSkillsInc.com')

* topvoiceapps.com

	 ('Elevator Pitch', 'by TopVoiceApps.com', "Description\nREHEARSE YOUR ELEVATOR PITCH! \n30 seconds of realistic elevator music for you to practice your pitch.\nLike this skill? Please leave a review. Don't like this skill? Tell us why at hello@topvoiceapps.com")
	 ('Alarm Sounds', 'by TopVoiceApps.com', 'Description\nDisclaimer: this skill is a prank skill for entertainment purposes only.\n*** AWESOME 4 MINUTE LONG ALARM SOUNDS !!! ***\nScare off intruders!\nWake up lazy teenagers!\nPrank your friends!\nPut the house on Red Alert!\nSay "Alexa, start Alarm Sounds" to hear a random alarm sound.\nYou can also say "Alexa, ask Alarm Sounds to play [alarm name]"\nCurrently available alarms:\n- Fire/Smoke Alarm\n- Car Alarm\n- Burglar Alarm\n- Air Raid Siren\n- Emergency Alarm\n- Alarm Clock\nIf you\'d like to see more alarms please share your suggestions with us by emailing hello@topvoiceapps.com\nIf you like this skill, please leave a 5* rating and review to help us make more amazing Alexa Skills.')
	 ('Musical Games', 'by TopVoiceApps.com', "Description\nMusical Chairs, Freeze Dancers, Musical Bumps and More!\nBored? This compendium of active and super fun family games will keep everyone entertained for hours. Dance and sing to catchy tunes and throw your best shapes. Go on, go crazy! \nMusical games currently available include:\nMusical Chairs (Trip to Jerusalem)\nFreeze Dancers (Musical Statues)\nMusical Bumps (We All Fall Down)\nGames coming soon include:\nAnimal Dancers\nIf You're Happy and You Know It\nCopy Clap\nName the Instrument\nSpin The Microphone\nDizzy Dancers\nMusical Clothes\nHot Potato\nPass The Parcel\nRing Around The Rosie\nDance, hop, clap, walk, crawl, jump, sing and shout to lots of different music tracks that everyone will love dancing to.\nIf you love Musical Games, please let us know!\nFeedback is always welcome at hello@topvoiceapps.com")
	 ('Musical Games', 'by TopVoiceApps.com', "Description\nMusical Chairs, Musical Statues, Musical Bumps and More!\nBored? This compendium of active and super fun family games will keep everyone entertained for hours. Dance and sing to catchy tunes and throw your best shapes. Go on, go crazy! \nMusical games currently available include:\nMusical Chairs (Trip to Jerusalem)\nMusical Statues (Freeze Dancers)\nMusical Bumps (We All Fall Down)\nGames coming soon include:\nAnimal Dancers\nIf You're Happy and You Know It\nCopy Clap\nName the Instrument\nSpin The Microphone\nDizzy Dancers\nMusical Clothes\nHot Potato\nPass The Parcel\nRing Around The Rosie\nDance, hop, clap, walk, crawl, jump, sing and shout to lots of different music tracks that everyone will love dancing to.\nIf you love Musical Games, please let us know!\nFeedback is always welcome at hello@topvoiceapps.com")
	 ('The Picture Quiz', 'by TopVoiceApps.com', 'Description\n*** Play using the Alexa App or a screen enabled Alexa device like the Echo Show or Echo Spot ***\nCAN YOU GUESS THE PICTURE?\nThe less guesses you have - the more points you get! Sounds easy right? Play this addictive game and become The Picture Quiz Champion of the month!\nGetting started is easy...\nJust say: "Alexa, Play The Picture Quiz"\nThe game is best played on a screen enabled Alexa Device such as the Echo Show, Echo Spot, Facebook Portal or Fire Tablet. \nDon\'t have one of these devices? No problem! You can also play using the Alexa App...look out for the cards on your home screen.\nIf you have any questions, comments, feedback or requests then please email hello@topvoiceapps.com\nIf you LOVE this game, it will mean the world to us if you could leave a 5* review using the Alexa App or on the Amazon site. THANK YOU!')
	 ('Musical Bumps', 'by TopVoiceApps.com', "Description\nRoll Up! Roll Up! It's time to play Musical Bumps with Alexa.\nWhen the music starts, it's time to go crazy and throw some shapes! Dance like you've never danced before.\nWhen the music stops....BUMP! Everybody to the ground! The last person to sit down is out.\nKeep dancing and bumping until you're left with a winner!\nKids and grown-ups love to play this game. Similar to musical chairs and musical statues it's perfect for parties or playing amongst friends on a rainy indoor day.\nIf you love Musical Bumps, please take the time to leave a 5 star review so we can add more features!\nFeedback is always welcome at hello@topvoiceapps.com")
	 ('My Prank', 'by TopVoiceApps.com', 'Description\nDisclaimer: this skill is a prank skill for entertainment purposes only\n�� My Prank is the easiest way to prank your friends and family using Alexa! ��\nSet a delay (up to 6 mins). Hide. Watch as your unsuspecting victim gets a surprise from Alexa.\nReady to design your prank? Visit the following website to continue:\n�� topvoiceapps.com/prank\nIf you ❤ My Prank and want us to add new pranks and features please leave us a 5* rating and review!\nThank you! ��')
	 ('ASMR Sleep Sounds', 'by TopVoiceApps.com', 'Description\nListen to soothing ASMR sounds and whispers to help you relax, meditate and sleep.\nLOOPING\nThese ASMR sounds will automatically loop by default.\nABOUT THIS SKILL\nThe ASMR Sleep Sounds Alexa Skill provides comforting whispers to make you feel relaxed, stress free and help you fall asleep.\nLet Alexa be your friend and trigger your ASMR tingles or simply find beauty and peace and let this Skill be your secret island of relaxation and meditation.\nWHAT IS ASMR?\nASMR, stands for Autonomous Sensory Meridian Response and is the technical term for what some people call "head orgasms" or "brain tingles". It describes a feeling of euphoric tingling and relaxation that can come over someone when he or she hears certain sounds or triggers. \nWhat are these sounds? It might surprise you, but the sounds are of people doing very simple, quiet, calming tasks, such as folding towels, brushing their hair, or flipping magazine pages.  Audio clips often consist of voices whispering nice things like "You are appreciated", or contain the sound of tapping, scratching, or rain.\nMany people describe the feeling as “tingles” that run through the back of someone’s head and spine. Others say the feeling is deeply relaxing, and can cause them to fall asleep.\nThe feeling isn’t normally sexual. Although some people are triggered by sounds that appear sexual, other people say that the tingles and feelings of relaxation have nothing to do with sex. \nSUPPORT\nFor questions or feedback please email hello@topvoiceapps.com ..... if it takes a little while for us to get back to you, it means we\'ve fallen asleep to our ASMR sounds :)')

* voxogenic.com

	 ('Moon Encyclopedia', 'by Voxogenic', "Description\nMoon Encyclopedia by Voxogenic Technologies is the one stop solution for all your queries related to the moon. It contains a large amount of information and can answer almost all of your queries.\nIf you enjoy the skill, please do leave us a 5 star review as it helps other people find it and enjoy it too.\nIf you'd like to give us feedback or have ideas for new features, please email us at contact@voxogenic.com.")
	 ('Bulls & Cows', 'by Voxogenic', 'Description\n*Available across all Alexa devices. [ Echo Dot / Echo Spot / Echo Show / FireTV ]\nBulls and Cows is a code breaking mind game. The objective of the game to help you sharpen your mind and memory.\nThe game will help you build your logical reasoning skills which will be very helpful in your future.\nHow to Play:\n============================================================================================\n*Before starting the skill keep an notebook and an pen in front of you. When playing the game, keep on writing your responses and my analysis. This will help you in the game.\n-> Say Alexa, open Bulls and Cows.\nThis is will open the skill and then it will ask you whether you want to Play the game or understand the rules.\nTo play the game say, Start Game.\nTo Understand the rules say, Rules or Help.\n-> When the game starts, I will think of a secret 4 Digit number and will ask you to guess my number.\n-> Now you have to tell me your guess. \n-> After listening to your guess, I will analyse it and tell you the number of bulls and cows in your answer.\nBull : A Bull means that there is a digit in your guess which matches a digit in my number and is at the same position also.\nCow: A Cow means that there is a digit in your guess which matches a digit in my number, but is not at the correct position.\n-> Now you will get 10 Seconds to analyse my response and come up with another number.\nThe same process will continue until you are able to guess my number.\n============================================================================================\nGame Example:\n============================================================================================\nLets assume that the my secret 4 digit number is 1734. Below are the example user guesses and my analysis with respect to them.\nYour Guess -------------------------------------- My Analysis\n6457 --------------------------------------------- 2 Cows, 0 Bulls [ 4 and 7 are Cows ]\n1289 --------------------------------------------- 0 Cows, 1 Bulls [ 1 is Bulls ]\n3471 --------------------------------------------- 4 Cows, 0 Bulls [ All the digits are Cows ]\n1234 --------------------------------------------- 0 Cows, 3 Bulls [ 1,3 and 4 are Bulls ]\n1734 --------------------------------------------- 0 Cows, 4 Bulls [ Number Matched , You Won]\n* You will only be told the number of bulls and cows in your guess. Which number is a bull and which is cow will not be told. You have to analyse and find out.\n============================================================================================\nWhile you are in the skill and are not playing the game you can ask Alexa to tell you the leader board and listen to the stats of the player leading the race to become the "Bell Cow".\nThe race to become the "Bell Cow" has already started. Link your Facebook account with the skill to enter the Race. Whenever you create a personal best, you will be prompted to Link your account.\nYou can also link you account by visiting the game\'s settings in the Alexa app.\nTo see the leaderboard visit: https://voxogenic.com/bullsandcows\nWhile not playing the game you can ask Alexa your rank.\nLiked the game?\nIf you like the game, please leave us a review. It would motivate us further to bring about more exiciting features and content in the future.\nHave a feedback?\nWe would love to hear from you about your experience and thoughts on how we can improve it further. If you have any feedback or suggestions mail us at contact@voxogenic.com\nFor more information about what we do visit: www.voxogenic.com')
	 ('Game of Thrones Countdown | India', 'by VOXOGENIC TECHNOLOGIES PVT. LTD.', "Description\nThis is the Game Of Thrones Countdown Flash briefing skill which reminds you of the commencement of the upcoming final season of Game of Thrones. For those of you who don't know, GOT is an American fantasy drama series created by David Benioff and D. B. Weiss. You can get to know more amazing facts about the show in this skill.\nTo start the skill please say:  Alexa, open Game of Thrones Countdown\nIf you enjoy the skill, please do leave us a 5-star review as it helps other people find it and enjoy it too.\nIf you'd like to give us feedback or have ideas for new features, please email us at contact@voxogenic.com.")
	 ('Bitcoin Pizza', 'by VOXOGENIC TECHNOLOGIES PVT. LTD.', 'Description\nOn 22nd May 2010, Laszlo Hanyecz bought a pizza for 10,000 bitcoins. This skill tells you the current value of that pizza in U.S. dollars.\nThings you can try:\nHow much is the Bitcoin pizza worth today?\nAsk Bitcoin Pizza, how much is it worth today?\nLiked the game?\nIf you like the game, please leave us a five-star review. It would motivate us further to bring about more exciting features and content in the future.\nHave feedback?\nWe would love to hear from you about your experience and thoughts on how we can improve it further. If you have any feedback or suggestions mail us at contact@voxogenic.com')
	 ('आज का सुविचार', 'by Voxogenic', 'Description\nआज का सुविचार जो हर रोज़ आपको एक नया सुविचार सुनाएगा. जिसको सुनके आप बहुत अच्छा महसूस करेंगे.\nसुविचार सुनाने के बाद ये आपसे पूछेगा के आप क्या करना चाहते है उस सुविचार को रिपीट करना चाहते है या अगला सुविचार सुनना चाहते है. अगर आप रिपीट करो बोलोगे तो ये आपको सुनाया हुआ सुविचार फिर से सुनाएगा अगर आपने अगला सुविचार सुनाओ बोला तो फिर आपको अगला सुविचार सनाया जायेगा.\nइस स्किल को शुरू करने के लिए कहे : अलेक्सा, आज का सुविचार खोलो\nअगर आपको ये स्किल का इस्तेमाल करके अच्छा लगा  तो कृपया करके रिव्यु में हमें पांच सितारे दे. और आप हमें कोई राय देना चाहते हो या कोई सुझाव देना चाहते हो तो हमे मेल करे: contact@voxogenic.com')
	 ('Next Holiday | Australia', 'by VOXOGENIC TECHNOLOGIES PVT. LTD.', 'Description\nAre you one of those people who is always scanning the calendar for the next public holiday? \nIf yes, you have come to the right place. This simple flash briefing skill will give you the information about the next coming holiday. In case the next event is less than two days away I will provide a detailed description of the holiday too. \nPlease leave a 5-star review if you love this skill, or send us an email with your suggestions at contact@voxogenic.com\nYour 5-star reviews encourage us to keep making more great skills, thanks!')
	 ('Indian National Pledge', 'by VOXOGENIC TECHNOLOGIES PVT. LTD.', "Description\nIndian pledge is an oath that we take while we live in this country. It reminds us of our responsibilities towards our country and motherland.  In my childhood, we recited it daily during the school assembly but sadly this practice is no longer in trend anymore.  In view of the aforesaid, the oath under discussion assumes a great significance and ought to be given due importance. \nIndian national pledge is a symbol of fraternity, unity in India and resembles the moral values of our eternal culture. Let's roll back to times when we used to recite this and reminisce.\nTo start the skill please say: Alexa, open Indian national pledge\nIf you enjoy the skill, please do leave us a 5-star review as it helps other people find it and enjoy it too.\nIf you'd like to give us feedback or have ideas for new features, please email us at contact@voxogenic.com.")
	 ('Next Holiday | United States', 'by VOXOGENIC TECHNOLOGIES PVT. LTD.', 'Description\nAre you one of those people who is always scanning the calendar for the next public holiday? \nIf yes, you have come to the right place. This simple flash briefing skill will give you the information about the next coming holiday. In case the next event is less than two days away I will provide a detailed description of the holiday too. \nPlease leave a 5-star review if you love this skill, or send us an email with your suggestions at contact@voxogenic.com\nYour 5-star reviews encourage us to keep making more great skills, thanks!')


# Notebook from katewomack/NuPyCEE
Path: DOC/Teaching/.ipynb_checkpoints/Section2.1-checkpoint.ipynb

# Section 2.1: Tracing the origin of C_____no_output_____### Result: Identification of which star is responsible for the origin of C_____no_output_____

import matplotlib.pyplot as plt
import sygma
import omega
import stellab_____no_output_____#loading the observational data module STELLAB
stellab = stellab.stellab()_____no_output_____

## Simulation of the Milky Way_____no_output_____

# OMEGA parameters for MW
mass_loading = 1     # How much mass is ejected from the galaxy per stellar mass formed
nb_1a_per_m = 3.0e-3  # Number of SNe Ia per stellar mass formed
sfe = 0.005           # Star formation efficiency, which sets the mass of gas
table = 'yield_tables/isotope_yield_table_MESA_only_ye.txt' # Yields for AGB and massive stars
#milky_way

o_mw = omega.omega(galaxy='milky_way',Z_trans=-1, table=table,sfe=sfe, DM_evolution=True,\
                  mass_loading=mass_loading, nb_1a_per_m=nb_1a_per_m, special_timesteps=60)OMEGA run in progress..
..Time refinement..
   OMEGA run completed - Run time: 6.29s

# Comparison of chemical evolution prediction with observation_____no_output_____

# Choose abundance ratios
%matplotlib nbagg
xaxis = '[Fe/H]'
yaxis = '[C/Fe]'

# Plot observational data points (Stellab)
stellab.plot_spectro(xaxis=xaxis, yaxis=yaxis,norm='Grevesse_Noels_1993',galaxy='milky_way',show_err=False)

# Extract the numerical predictions (OMEGA)
xy_f = o_mw.plot_spectro(fig=3,xaxis=xaxis,yaxis=yaxis,return_x_y=True)

# Overplot the numerical predictions (they are normalized according to Grevesse & Noels 1993)
plt.plot(xy_f[0],xy_f[1],linewidth=4,color='w')
plt.plot(xy_f[0],xy_f[1],linewidth=2,color='k',label='OMEGA')

# Update the existing legend
plt.legend(loc='center left', bbox_to_anchor=(1.01, 0.5), markerscale=0.8, fontsize=13)

# Choose X and Y limits
plt.xlim(-4.5,0.5)
plt.ylim(-1.4,1.6)_____no_output_____

# Tracing back to simple stellar populations._____no_output_____

s0p0001=sygma.sygma(iniZ=0.0001)
s0p006=sygma.sygma(iniZ=0.006)SYGMA run in progress..
   SYGMA run completed - Run time: 0.58s
SYGMA run in progress..
   SYGMA run completed - Run time: 0.54s

## What is [C/Fe] for two SSPs at Z=0.006 and Z=0.0001?_____no_output_____

elem='[C/Fe]'
s0p0001.plot_spectro(fig=3,yaxis=elem,marker='D',color='b',label='Z=0.0001')
s0p006.plot_spectro(fig=3,yaxis=elem,label='Z=0.006')_____no_output_____

## Now lets focus on C. What is the evolution of the total mass of C?_____no_output_____

# Plot the ejected mass of a certain element
elem='C'
s0p0001.plot_mass(fig=4,specie=elem,marker='D',color='b',label='Z=0.0001')
s0p006.plot_mass(fig=4,specie=elem,label='Z=0.006')_____no_output_____

## Which stars contribute the most to C?_____no_output_____

elem='C'
s0p0001.plot_mass_range_contributions(specie=elem,marker='D',color='b',label='Z=0.0001')
s0p006.plot_mass_range_contributions(specie=elem,label='Z=0.006')_____no_output_____

## Which stellar yields are the most?_____no_output_____

s0p0001.plot_table_yield(fig=6,iniZ=0.0001,table='yield_tables/isotope_yield_table.txt',yaxis='C-12',
                         masses=[1.0, 1.65, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0],marker='D',color='b',)
s0p006.plot_table_yield(fig=6,iniZ=0.006,table='yield_tables/isotope_yield_table.txt',yaxis='C-12',
                         masses=[1.0, 1.65, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0])_____no_output_____


# Notebook from jrlivesey/PHYS-434
Path: HW_2.ipynb

# Homework 2

**Joseph Livesey**

**October 14, 2021**_____no_output_____

import numpy as np
import matplotlib.pyplot as plt
from scipy import stats, signal_____no_output_____

## Problem 1_____no_output_____There is only one way to roll snake eyes: rolling two 1s. There are six ways to roll sevens: 1 + 6, 2 + 5, 3 + 4, 4 + 3, 5 + 2, and 1 + 6. Thus, the probability of rolling sevens is six times larger.

$$ \frac{P(\text{sevens})}{P(\text{snake eyes})} = 6 $$_____no_output_____## Problem 2_____no_output_____The probability mass function for one roll of a die is

$$ f_1(k) = \begin{cases} \frac{1}{6}, & k \in \{ 1, 2, 3, 4, 5, 6 \}, \\ 0, & \text{otherwise}. \end{cases} $$

So, to find the probability distribution for two dice, we convolve $f_1(k)$ with itself as follows.

$$ f_2(k) = f_1(k) * f_1(k) = \sum_t f_1(t) f_1(k - t) $$

First, we do this analytically.

$$
f_2(1) = \sum_t f_1(t) f_1(1 - t) = 0 \\
f_2(2) = \sum_t f_1(t) f_1(2 - t) = f_1(1) f_1(1) = \frac{1}{36} \\
f_2(3) = \sum_t f_1(t) f_1(3 - t) = f_1(1) f_1(2) + f_1(2) f_1(1) = \frac{1}{18} \\
f_2(4) = \sum_t f_1(t) f_1(4 - t) = f_1(1) f_1(3) + f_1(2) f_1(2) + f_1(3) f_1(1) = \frac{1}{12} \\
f_2(5) = \sum_t f_1(t) f_1(5 - t) = f_1(1) f_1(4) + f_1(2) f_1(3) + f_1(3) f_1(2) + f_1(4) f_1(1) = \frac{1}{9} \\
f_2(6) = \sum_t f_1(t) f_1(6 - t) = f_1(1) f_1(5) + f_1(2) f_1(4) + f_1(3) f_1(3) + f_1(4) f_1(2) + f_1(5) f_1(1) = \frac{5}{36} \\
f_2(7) = \sum_t f_1(t) f_1(7 - t) = f_1(1) f_1(6) + f_1(2) f_1(5) + f_1(3) f_1(4) + f_1(4) f_1(3) + f_1(5) f_1(2) + f_1(6) f_1(1) = \frac{1}{6} \\
f_2(8) = \sum_t f_1(t) f_1(8 - t) = f_1(2) f_1(6) + f_1(3) f_1(5) + f_1(4) f_1(4) + f_1(5) f_1(3) + f_1(6) f_1(2) = \frac{5}{36} \\
f_2(9) = \sum_t f_1(t) f_1(9 - t) = f_1(3) f_1(6) + f_1(4) f_1(5) + f_1(5) f_1(4) + f_1(6) f_1(3) = \frac{1}{9} \\
f_2(10) = \sum_t f_1(t) f_1(10 - t) = f_1(4) f_1(6) + f_1(5) f_1(5) + f_1(6) f_1(4) = \frac{1}{12} \\
f_2(11) = \sum_t f_1(t) f_1(11 - t) = f_1(5) f_1(6) + f_1(6) f_1(5) = \frac{1}{18} \\
f_2(12) = \sum_t f_1(t) f_1(12 - t) = f_1(6) f_1(6) = \frac{1}{36}
$$

We must now verify that these probabilities sum to unity.

$$ \sum_k f_2(k) = \frac{1}{36} + \frac{1}{18} + \frac{1}{12} + \frac{1}{9} + \frac{5}{36} + \frac{1}{6} + \frac{5}{36} + \frac{1}{9} + \frac{1}{12} + \frac{1}{18} + \frac{1}{36} = 1 $$

Now, we do it numerically._____no_output_____

def f1(k):
    if k >= 1 and k <= 6:
        prob = 1.0/6
    else:
        prob = 0
    return prob

def f2(k):
    prob = 0
    for t in range(k):
        prob += f1(t) * f1(k - t)
    return prob

for k in np.arange(1,13):
    print('f_2(' + str(k) + ')=' + str(f2(k)))f_2(1)=0.0
f_2(2)=0.027777777777777776
f_2(3)=0.05555555555555555
f_2(4)=0.08333333333333333
f_2(5)=0.1111111111111111
f_2(6)=0.1388888888888889
f_2(7)=0.16666666666666669
f_2(8)=0.1388888888888889
f_2(9)=0.1111111111111111
f_2(10)=0.08333333333333333
f_2(11)=0.05555555555555555
f_2(12)=0.027777777777777776

These values match the ones we derived analytically._____no_output_____## Problem 3_____no_output_____The mean of the discrete probability distribution $f_2(k)$ is

$$ \mu = E(k) = \sum_k k f_2(k). $$

The variance is

$$ \sigma^2 = \text{Var}(k) = E(k^2) - E(k)^2 = \sum_k k^2 f_2(k) - \left ( \sum_k k f_2(k) \right )^2. $$_____no_output_____

mean = sum([k * f2(k) for k in np.arange(1,13)])
var  = sum([k**2 * f2(k) for k in np.arange(1,13)]) - mean**2

print(mean, var)7.0 5.833333333333336

The mean of the distribution is $\mu=7.0$, while the variance is $\sigma^2 \approx 5.83$._____no_output_____## Problem 4_____no_output_____Let $f_{10}(k)$ be the probability density function for the sum of 10 rolls of dice. We compute this function by convolving $f_1(k)$ with itself 10 times._____no_output_____

# Array representing the pmf of 1 roll
f_1 = [f1(k) for k in np.arange(1, 7)]

# Convolution to find the pmf of 10 rolls
def f10():
    f = [f_1[i] for i in range(len(f_1))]
    for _ in range(9):
        f = np.convolve(f, f_1)
    return f_____no_output_____f10()_____no_output_____

Here is a plot of the average, which we find by taking the values of $k$ and dividing by the number of trials, 10._____no_output_____

k = np.arange(10, 61)

fig, ax = plt.subplots(1, 2, figsize=(10,6))
ax[0].bar(k/10, f10())
ax[1].scatter(k/10, np.log(f10()))
ax[1].plot(k/10, -(1.4*(k/10 - 3.5))**2 - 2.7, c='orange', lw=3);_____no_output_____

This appears to be a Gaussian distribution, because the semilog plot of the data is well-fit by a parabola._____no_output_____## Problem 5_____no_output_____We can make two different Gaussian distributions in `scipy` and combine them using a convolution._____no_output_____

xx = np.linspace(0, 10, 1000)

f = [stats.norm.pdf(x, loc=5.0, scale=1.0) for x in xx]

ff = signal.convolve(f, f)_____no_output_____fig, ax = plt.subplots(1, 2, figsize=(12, 8))
ax[0].bar(xx, f/sum(f), label=r'$f$')
ax[0].bar(np.linspace(0, 20, len(ff))/2, ff/sum(ff/2), label=r'$f \star f$')
ax[0].legend(loc=0)
ax[1].scatter(np.linspace(0, 20, len(ff))/2, np.log(ff/sum(ff)))
ax[1].plot(xx, -(xx - 5.0)**2 - 6.0, c='y', lw=3)
ax[1].set_xlim(0, 10);_____no_output_____

We have verified that the convolution (sum of the Gaussian with itself, and therefore also the average) is indeed a Gaussian distribution, since its logarithm is well-fit by a parabola. We note that the mean remains the same as that of the original distribution, with the standard deviation having decreased. This is important because it means we can diminish uncertainty in our measurements by averaging the background distribution over time._____no_output_____