# Notebook from dongxulee/lifeCycle
Path: 20201016/.ipynb_checkpoints/simulationOwning-checkpoint.ipynb

<code>
%pylab inline
from scipy.interpolate import interpn
from multiprocessing import Pool
from constant import *
import warnings
import math
warnings.filterwarnings("ignore")
np.printoptions(precision=2)Populating the interactive namespace from numpy and matplotlib
# wealth discretization
ws = np.array([10,25,50,75,100,125,150,175,200,250,500,750,1000,1500,3000])
w_grid_size = len(ws)
# 401k amount discretization
ns = np.array([1, 5, 10, 15, 25, 50, 100, 150, 400, 1000])
n_grid_size = len(ns)
# Mortgage amount
Ms = np.array([0.01*H,0.05*H,0.1*H,0.2*H,0.3*H,0.4*H,0.5*H,0.8*H]) * pt
M_grid_size = len(Ms)
points = (ws,ns,Ms)_____no_output_____import quantecon as qe
import timeit
mc = qe.MarkovChain(Ps)

#Vgrid = np.load("Vgrid_i.npy")
cgrid = np.load("cgrid" + str(H)+ ".npy")
bgrid = np.load("bgrid" + str(H)+ ".npy")
kgrid = np.load("kgrid" + str(H)+ ".npy")
qgrid = np.load("qgrid" + str(H)+ ".npy")

def action(t, x):
    w, n, M, e, s, z = x
    c = interpn(points, cgrid[:,:,:,e,s,z,t], x[:3], method = "nearest", bounds_error = False, fill_value = None)[0]
    b = interpn(points, bgrid[:,:,:,e,s,z,t], x[:3], method = "nearest", bounds_error = False, fill_value = None)[0]
    k = interpn(points, kgrid[:,:,:,e,s,z,t], x[:3], method = "nearest", bounds_error = False, fill_value = None)[0]
    q = interpn(points, qgrid[:,:,:,e,s,z,t], x[:3], method = "nearest", bounds_error = False, fill_value = None)[0]
    if q <= 0.75:
        q = 0.5
    else:
        q = 1
    return (c,b,k,q)

#Define the earning function, which applies for both employment and unemployment, 27 state
def y(t, x):
    w, n, M, e, s, z = x
    if t <= T_R:
        return detEarning[t] * (1+gGDP[int(s)]) * e + (1-e) * welfare
    else:
        return detEarning[t]

#Define the evolution of the amount in 401k account
def gn(t, n, x, r_k):
    w, n, M, e, s, z = x
    if t <= T_R and e == 1:
        # if the person is employed, then yi portion of his income goes into 401k
        n_cur = n + y(t, x) * yi
    elif t <= T_R and e == 0:
        # if the perons is unemployed, then n does not change
        n_cur = n
    else:
        # t > T_R, n/discounting amount will be withdraw from the 401k
        n_cur = n - n/Dt[t]
        # the 401 grow as the same rate as the stock
    return (1+r_k)*n_cur

def transition(x, a, t, s_next):
    '''
    Input: state and action and time
    Output: possible future states and corresponding probability
    '''
    w, n, M, e, s, z = x
    c,b,k,q = a
    # variables used to collect possible states and probabilities
    x_next = []
    m = M/D[T_max-t]
    M_next = M*(1+rh) - m
    # transition of z
    if z == 1:
        z_next = 1
    else:
        if k == 0:
            z_next = 0
        else:
            z_next = 1

    n_next = gn(t, n, x, r_k[s_next])
    w_next =  b*(1+r_b[int(s)]) + k*(1+r_k[s_next])
    if t >= T_R:
        return [w_next, n_next, M_next, 0, s_next, z_next]
    else:
        for e_next in [0,1]:
            x_next.append([w_next, n_next, M_next, e_next, s_next, z_next])
        prob_next = [Pe[s], 1 - Pe[s]]
        return x_next[np.random.choice(len(prob_next), 1, p = prob_next)[0]]_____no_output_____'''
    Start with:
    Ms = H * 0.8 * 0.25
    w = 20
    n  = 0
    e = 1
    s = 1
    z = 0
    1000 agents for 1 economy, 100 economies.

    use numpy array to contain the variable change:

    wealth, rFund, Mortgage, employment, sState, participation
    salary, consumption, hConsumption, bond, stock, hPercentage,
    life. Shape: (T_max-T_min, numAgents*numEcons)
'''
x0 = [20, 0, H * 0.8 * pt, 1, 1, 0]

numAgents = 1000
numEcons = 500_____no_output_____import random as rd
EconStates = [mc.simulate(ts_length=T_max - T_min, init=0) for _ in range(numEcons)]
def simulation(i):
    track = np.zeros((T_max - T_min,13))
    econState = EconStates[i//numAgents]
    alive = True
    x = x0
    for t in range(1,len(econState)-1):
        if rd.random() > Pa[t]:
            alive = False
        if alive:
            track[t, 0] = x[0]
            track[t, 1] = x[1]
            track[t, 2] = x[2]
            track[t, 3] = x[3]
            track[t, 4] = x[4]
            track[t, 5] = x[5]
            track[t, 6] = y(t,x)
            a = action(t, x)
            track[t, 7] = a[0]
            track[t, 9] = a[1]
            track[t, 10] = a[2]
            track[t, 11] = a[3]
            track[t, 12] = 1
            # calculate housing consumption
            if a[3] == 1:
                Vh = (1+kappa)*H
            else:
                Vh = (1-kappa)*(H/2)
            track[t, 8] = Vh
            s_next = econState[t+1]
            x = transition(x, a, t, s_next)
    return track_____no_output_____%%time
pool = Pool()
agentsHistory = pool.map(simulation, list(range(numAgents*numEcons)))
pool.close()_____no_output_____len(agentsHistory)_____no_output_____np.save("agents", np.array(agentsHistory))
agents = np.load("agents.npy")_____no_output_____#numpy array used to store
attributesNames = ['wealth', 'rFund', 'Mortgage','employment','sState', 'participation',
                   'salary', 'consumption', 'hConsumption', 'bond', 'stock','hPer', 'life']
attributes = [np.zeros((T_max-T_min, numAgents*numEcons)) for i in range(len(attributesNames))]
#shape the collected stats into the right format
def separateAttributes(agents):
    for i in range(numAgents*numEcons):
        for j in range(len(attributesNames)):
            attributes[j][:,i] = agents[i][:,j]
separateAttributes(agents)
#save the data
for i in range(len(attributes)):
    np.save(attributesNames[i], attributes[i])_____no_output_____wealth = np.load("wealth.npy")
rFund = np.load("rFund.npy")
Mortgage = np.load("Mortgage.npy")
employment = np.load("employment.npy")
sState = np.load("sState.npy")
salary = np.load("salary.npy")
consumption = np.load("consumption.npy")
hConsumption = np.load("hConsumption.npy")
bond = np.load("bond.npy")
stock = np.load("stock.npy")
hPer = np.load("hPer.npy")
life = np.load("life.npy")
participation = np.load("participation.npy")_____no_output_____# Population during the entire simulation period
plt.plot(np.mean(life, axis = 1))_____no_output_____def quantileForPeopleWholive(attribute, quantiles = [0.25, 0.5, 0.75]):
    qList = []
    for i in range(69):
        if len(np.where(life[i,:] == 1)[0]) == 0:
            qList.append(np.array([0] * len(quantiles)))
        else:
            qList.append(np.quantile(attribute[i, np.where(life[i,:] == 1)], q = quantiles))
    return np.array(qList)

def meanForPeopleWholive(attribute):
    means = []
    for i in range(69):
        if len(np.where(life[i,:] == 1)[0]) == 0:
            means.append(np.array([0]))
        else:
            means.append(np.mean(attribute[i, np.where(life[i,:] == 1)]))
    return np.array(means)_____no_output_____# plot participation ratio
plt.plot(meanForPeopleWholive(participation))_____no_output_____# plot the 0.25, 0.5, 0.75 quantiles of hConsumption
plt.plot(quantileForPeopleWholive(hConsumption))_____no_output_____# plot the 0.25, 0.5, 0.75 quantiles of wealth
plt.plot(quantileForPeopleWholive(wealth))_____no_output_____# plot the 0.25, 0.5, 0.75 quantiles of wealth
plt.plot(quantileForPeopleWholive(rFund))_____no_output_____# plot the 0.25, 0.5, 0.75 quantiles of wealth
plt.plot(quantileForPeopleWholive(Mortgage))_____no_output_____# plot the 0.25, 0.5, 0.75 quantiles of wealth
plt.plot(quantileForPeopleWholive(consumption))_____no_output_____# plot the 0.25, 0.5, 0.75 quantiles of wealth
plt.plot(quantileForPeopleWholive(bond))_____no_output_____# plot the 0.25, 0.5, 0.75 quantiles of wealth
plt.plot(quantileForPeopleWholive(stock))_____no_output_____# plot the 0.25, 0.5, 0.75 quantiles of wealth
plt.plot(quantileForPeopleWholive(hPer))_____no_output_____# plot the 0.25, 0.5, 0.75 quantiles of wealth
plt.figure(figsize = [14,8])
plt.plot(meanForPeopleWholive(wealth), label = "wealth")
plt.plot(meanForPeopleWholive(consumption), label = "Consumption")
plt.plot(meanForPeopleWholive(bond), label = "Bond")
plt.plot(meanForPeopleWholive(stock), label = "Stock")
plt.plot(meanForPeopleWholive(rFund), label = "401k")
plt.legend()_____no_output_____
</code>


# Notebook from jyw7664/M5_Idol_lyrics
Path: Crawling/url_crawling/.ipynb_checkpoints/melon_idol_lyrics_crawling-checkpoint.ipynb

!pip install html5lib beautifulsoup4
!pip install selenium

from selenium import webdriver
from bs4 import BeautifulSoup
from time import sleep
import requests
import json
import re
import pandas as pdRequirement already satisfied: html5lib in c:\users\pje17\anaconda3\lib\site-packages (1.0.1)
Requirement already satisfied: beautifulsoup4 in c:\users\pje17\anaconda3\lib\site-packages (4.6.0)
Requirement already satisfied: six>=1.9 in c:\users\pje17\anaconda3\lib\site-packages (from html5lib) (1.11.0)
Requirement already satisfied: webencodings in c:\users\pje17\anaconda3\lib\site-packages (from html5lib) (0.5.1)
Requirement already satisfied: selenium in c:\users\pje17\anaconda3\lib\site-packages (3.14.0)
Requirement already satisfied: urllib3 in c:\users\pje17\anaconda3\lib\site-packages (from selenium) (1.22)
from urllib import request
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.common.exceptions import NoSuchElementException_____no_output_____from selenium.webdriver.common.keys import Keys_____no_output_____import numpy as np_____no_output_____# 인코딩이 euc-kr입니다. 저걸 지정하지 않으면 csv가 안 열려요!
# idol_list_withid를 불러오셔야 합니다. 멜론 url에 가수를 찾기 위한 id입니다.

idol_list = pd.read_csv('idol_list_withid.csv', encoding='euc-kr')
idol_list.head()_____no_output_____idol_list.shape_____no_output_____# 멜론에 정보가 없어서 아예 페이지가 없었던 가수들이 몇몇 있습니다.
# 이런 경우 id가 공란이기에 수집이 불가합니다. 모두 드랍하겠습니다.

idol_list = idol_list[np.isfinite(idol_list['idol_id'])]_____no_output_____idol_list.shape_____no_output_____# id가 flaot으로 저장되어 있습니다. int로 바꿔주고 저장하겠습니다.
idol_list['idol_id'] = idol_list['idol_id'].astype(int)
idol_list.info()<class 'pandas.core.frame.DataFrame'>
Int64Index: 333 entries, 0 to 338
Data columns (total 7 columns):
artist         333 non-null object
artist_eng     332 non-null object
member_num     333 non-null object
leader_name    276 non-null object
gender         332 non-null object
agency         314 non-null object
idol_id        333 non-null int32
dtypes: int32(1), object(6)
memory usage: 19.5+ KB
# 수집을 위한 빈 리스트 생성

title = []
album = []
release_date = []
song_genre = []
like = []
lyrics = []
creator = []
artist = []
artist_m = []
debut_y = []
debut_t = []
agency = []
is_title = []_____no_output_____# 곡마다 가수 이름을 붙여주어야 하기에 가수 이름 리스트로 받아오기
names = idol_list.artist
names[0]_____no_output_____# 크롤러가 링크를 찾을 때 매번 바꿔주는 값. for문에 사용됩니다.
ids = idol_list.idol_id
ids[0]_____no_output_____# tests = ['861430', '789331']_____no_output_____

## 크롤러_____no_output_____

driver = webdriver.Chrome('C:/Users/pje17/Desktop/Lyrics/Crawlling/chromedriver')
driver.get("https://www.melon.com")_____no_output_____# 곡 정보를 찾아들어가기 위한 url을 분해해서 변수 지정해놓은 모습입니다.
# l3 뒤에 붙는 숫자가 멜론 서버에 저장된 해당 가수의 해당 곡 번호입니다.
# 레드벨벳의 32번째 곡은 뒤에 숫자가 32인 경우 곡 리스트에 첫번째로 보여집니다.
# 즉 url을 바꿔가되 항상 리스트의 첫곡의 정보를 가져오면 모든 곡을 돌 수 있습니다.

l1 = 'https://www.melon.com/artist/song.htm?artistId='
# idol_id
l2 = '#params%5BlistType%5D=A&params%5BorderBy%5D=ISSUE_DATE&params%5BartistId%5D='
# idol_id
l3 = '&po=pageObj&startIndex='
# song number
# l1 + 'idol_id' + l2 + 'idol_id' + l3 + number

# 가수 이름을 넣기 위한 변수
i = 0

for id in ids:

    n = 1
    while True:

        #sleep(3)
        # 해당 가수의 곡 리스트 n번째로 가기

        driver.get(l1+str(id)+l2+str(id)+l3+str(n))
        sleep(2)

        # 타이틀 여부 체크. 타이틀이 아니라면 NaN
        # 곡 세부 정보를 보기 전에 타이틀 태그를 확인해야함
        # 타이틀곡은 '타이틀 곡'이라고 수집이 됩니다.
        try:
            is_title.append(driver.find_element_by_xpath('//*[@id="frm"]/div/table/tbody/tr[1]/td[3]/div/div/span').text)
        except NoSuchElementException as e:
            is_title.append('')

        # 곡 정보로 들어가기
        try:
            driver.find_element_by_xpath('//*[@id="frm"]/div/table/tbody/tr[1]/td[3]/div/div/a[1]/span').click()

            # 제목 크롤링(제목은 없지 않기 때문에 예외 없음)
            title.append(driver.find_element_by_xpath('//*[@id="downloadfrm"]/div/div/div[2]/div[1]/div[1]').text)

            # 앨범 크롤링
            try:
                album.append(driver.find_element_by_xpath('//*[@id="downloadfrm"]/div/div/div[2]/div[2]/dl/dd[1]/a').text)
            except NoSuchElementException as e:
                album.append('')

            # 발매날짜 크롤링
            try:
                release_date.append(driver.find_element_by_xpath('//*[@id="downloadfrm"]/div/div/div[2]/div[2]/dl/dd[2]').text)
            except NoSuchElementException as e:
                release_date.append('')

            try:
                song_genre.append(driver.find_element_by_xpath('//*[@id="downloadfrm"]/div/div/div[2]/div[2]/dl/dd[3]').text)
            except NoSuchElementException as e:
                song_genre.append('')

            # 좋아요 수 크롤링(좋아요도 0인 경우만 있고 없지는 않기에 예외 없음)
            like.append(driver.find_element_by_xpath('//*[@id="d_like_count"]').text)

            # 가사 크롤링
            try:
                lyrics.append(driver.find_element_by_xpath('//*[@id="d_video_summary"]').text)
            except NoSuchElementException as e:
                lyrics.append('')

            # 작사 작곡 내역이 없는 경우
            try:
                creator.append(driver.find_element_by_xpath('//*[@id="conts"]/div[3]/ul').text)
            except NoSuchElementException as e:
                creator.append('')

            n += 1

            # 가수 이름 붙이기
            artist.append(names[i])

            # 디버깅
            print(len(title), title[len(title)-1])

        except NoSuchElementException as e:
            i += 1
            break

        4643 기분 좋은 날
4644 제발
4645 제발
4646 제발(Inst.)
4647 제발
4648 제발 (Inst.)
4649 오늘 뭐 먹지
4650 오늘 뭐 먹지 (Inst.)
4651 바람이 분다
4652 바람이 분다 (Inst.)
4653 떠나지마
4654 11시 11분
4655 11시 11분 (Inst.)
4656 기분좋은날
4657 사랑은 없다
4658 행복해라
4659 피도 눈물도 없이
4660 So Long
4661 어차피
4662 해&해
4663 Forget About Love
4664 사랑은 유치하게
4665 방랑자
4666 Who-Who
4667 유일무이
4668 미안해
4669 행복해라 (Remix)
4670 No Time To Lose
4671 You
4672 막상막하
4673 시간은 가도...
4674 미안해
4675 Why
4676 슬픈 약속
4677 인연
4678 Sexy Lady
4679 Deep Down
4680 Window Pain
4681 Rose
4682 난 단지 그대가
4683 초록비
4684 Tulip
4685 사랑해
4686 너의 집 앞에서
4687 Conflict In Me
4688 My Girl
4689 지킬수 없는 약속
4690 너의 향기를 느끼며
4691 오! 사랑
4692 초록비 (Inst.)
4693 다시
4694 Boy Club
4695 The Guy
4696 Always
4697 Mam
4698 Night
4699 Beautiful Day
4700 추억의 끝에 서서
4701 첫눈에 반한 사랑
4702 이젠
4703 Love Takes Time
4704 Love Takes Time (Inst.)
4705 돌아와줘요
4706 돌아와줘요 (Inst.)
4707 My Valentine
4708 My Valentine (Inst.)
4709 Lie
4710 Lie (Inst.)
4711 Because Of You
4712 Because Of You (Inst.)
4713 심장에 쥐가나서
4714 Still Beautiful
4715 Still Beautiful (Inst.)
4716 More Than Words (2014 Ver.)
4717 Love Love Love
4718 Love Love Love (Acoustic Ver.)
4719 Love Love Love (Inst.)
4720 메시지가 도착했습니다
4721 메시지가 도착했습니다 (Inst.)
4722 Only 4 U
4723 Only 4 U (Inst.)
4724 사계절 정원 (Narr. 카인)
4725 우리 결혼까지 하자
4726 Papillon
4727 Deep Slow
4728 우리 결혼까지 하자 (Narr. 현영)
4729 우리 결혼까지 하자 (Inst.)
4730 Papillon (Inst.)
4731 Deep Slow (Inst.)
4732 우리 결혼까지 하자
4733 우리 결혼까지 하자 (inst.)
4734 하늘이 준 사랑
4735 하늘이 준 사랑
4736 하늘이 준 사랑 (Inst.)
4737 내 사람인가 봐
4738 우리 다시 사랑하자 (Acoustic Ver.)
4739 내 사람인가 봐 (Inst.)
4740 우리 다시 사랑하자 (Acoustic Ver.)
4741 우리 다시 사랑하자
4742 우리 다시 사랑하자 (Inst.)
4743 겨울 선물
4744 Hollyday
4745 사랑이 다시 온다면 (우일 추천)
4746 More Than Words (오병진 추천)
4747 할 수 만 있다면 (유석호 추천)
4748 Happy (손일권 추천)
4749 Hollyday (김찬민 추천)
4750 24/7 (김민성 추천)
4751 The Fool
4752 사랑이 다시 온다면
4753 남자
4754 할수만 있다면
4755 가지 말라고
4756 Pain
4757 Hollyday
4758 바보
4759 나보다
4760 야후 (夜後)
4761 남자 (Inst.)
4762 겨울 선물
4763 Again
4764 24/7
4765 I Wish U
4766 Come 2 Me My Love
4767 I Miss U
4768 겨울 선물
4769 Eternity
4770 Everythin`
4771 감사
4772 Atta Boy!
4773 U & I Dream
4774 You End Me
4775 진달래
4776 Feel(必)
4777 More Than Words
4778 이제 우리
4779 Always
4780 Bye Bye
4781 She`s Gone
4782 My Girl
4783 With You
4784 말해
4785 Happy
4786 미로
4787 Good Night
4788 벽
4789 Without You
4790 24/7
4791 Intro
4792 돌아와줘
4793 Love Virus
4794 Party
4795 날개가 되어줘
4796 Paradise
4797 Tell Me Why
4798 선택
4799 Born Twice
4800 Sacrifice
4801 돌아와줘 (MR)
4802 Bonus Data
4803 Intro
4804 돌아와줘
4805 Love Virus
4806 Party
4807 날개가 되어줘
4808 Paradise
4809 Tell Me Why
4810 선택
4811 Born Twice
4812 Sacrifice
4813 돌아와줘 (MR)
4814 Flying
4815 영화처럼
4816 추억의 향기
4817 Lover Boys
4818 So Long....
4819 단 한번만
4820 그날
4821 July
4822 Do U Know
4823 수요일에는 빨간 장미를...
4824 Misty...
4825 집착
4826 Life
4827 Dear My Love
4828 Epilogue (outro)
4829 Y
4830 젊음
4831 신기루
4832 Joy
4833 매력
4834 The Color Of Love
4835 정열
4836 로망스
4837 4U
4838 낙원
4839 뛰어
4840 추상
4841 정열
4842 그림자
4843 Game
4844 0.5
4845 The Pain
4846 시나브로
4847 Comedy
4848 Smile Again
4849 연인
4850 Only You
4851 도플갱어
4852 소름
4853 케세라세라
4854 Love Hacker
4855 Yo DJ (Feat. 이하늘 Of DJ DOC)
4856 Yo DJ (Inst.)
4857 Everything
4858 Actually (Feat. 이지혜)
4859 Everything (Inst.)
4860 Actually (Inst.)
4861 Everything (English Ver.)
4862 Somebody (Feat. 싸이)
4863 Only You (Feat. 시기)
4864 모든 것을 너에게 (Remix) (Feat. Hoody)
4865 Somebody (Inst.)
4866 Intro
4867 Go
4868 연인
4869 What
4870 Real Love
4871 희망
4872 친구일 테니까
4873 Yeh!
4874 Outsider
4875 Rush
4876 처음
4877 Crazy
4878 헤아릴 수 없을 만큼
4879 왜
4880 모든것을 너에게
4881 원
4882 사랑한다는 것
4883 Good Life
4884 후회
4885 I`ll Tell U
4886 상처
4887 그대로
4888 샤인
4889 Cowboy
4890 모든것을 너에게
4891 원
4892 핫 앤 콜드 (Hot & Cold)
4893 Call My Name
4894 핫 앤 콜드 (Hot & Cold) (Inst.)
4895 Rhythm HA!!!
4896 룩앳미 (Look At Me)
4897 Party Rebel
4898 Me Too
4899 Single Single
4900 룩앳미 (Look At Me) (Inst.)
4901 운명같은 사랑
4902 운명같은 사랑 (Inst.)
4903 Step
4904 Step (Inst.)
4905 Shooting Star
4906 Shooting Star (Inst.)
4907 Pass
4908 Pass (Inst.)
4909 Back It Up
4910 Back It Up (Inst.)
4911 Bounce
4912 Love Story
4913 Vari2ty
4914 Butterfly
4915 Strong Girl
4916 Oops
4917 Fantastic4
4918 Sweet Boy
4919 Be Rocker
4920 Rally
4921 Vari2ty (Inst.)
4922 Rally (Inst.)
4923 Strong Girl
4924 Bounce
4925 Butterfly
4926 Vari2ty
4927 Oops
4928 Fantastic4
4929 Sweet Boy
4930 Be Rocker
4931 Rally
4932 Vari2ty (Inst.)
4933 Rally (Inst.)
4934 Rally (Feat. 낯선)
4935 Rally (Inst.)
4936 병원에 가다 (사랑..) (Song By 은정)
4937 모두 다 쉿!
4938 One More Time (Remix)
4939 손을 잡아요
4940 나는 나 너는 너
4941 ..Holic
4942 One More Time
4943 모를까봐서
4944 The Cradle Song
4945 Black Diamond
4946 모두 다 쉿!
4947 Stay With Me
4948 Don`t Know Why
4949 모를까봐서 (Inst.)
4950 One More Time (Inst.)
4951 Twenty
4952 Super Star
4953 진실게임
4954 Passion
4955 단장
4956 너를 보내며
4957 KIN
4958 Liar
4959 시간에게 (I`ll Be There)
4960 Sunny Day
4961 Twenty (Inst.)
4962 Superstar (Inst.)
4963 아름다운 날들
4964 Delight Sweet Life (Singing Individual Part Ver.)
4965 Delight Sweet Life (Inst.)
4966 Delight Sweet Life
4967 Delight Sweet Life (M-Oz Club Mix Edit)
4968 I Believe
4969 君と約束した優しいあの場所まで / Kimi To Yakusoku Shita Yasashii Ano Basyou Made (너와 약속했던 정다운 그 곳까지)
4970 Delight Sweet Life (Enjoy Club Remix)
4971 胸いっぱいのこの愛を 誰より君に / Mune Itpaino Kono Aiwo Dareyori Kimini (가슴 가득 이 사랑을 누구보다 너에게) (Inst.)
4972 Passionate Wave
4973 戶惑う戀ゴコロ / Tomadou Koigokoro (흔들리는 사랑의 마음)
4974 胸いっぱいのこの愛を 誰より君に / Mune Itpaino Kono Aiwo Dareyori Kimini (가슴 가득 이 사랑을 누구보다 너에게) (Nishi Azabu YELLOW `Korean Night` Support Remix)
4975 胸いっぱいのこの愛を 誰より君に / Mune Itpaino Kono Aiwo Dareyori Kimini (가슴 가득 이 사랑을 누구보다 너에게)
4976 胸いっぱいのこの愛を 誰より君に / Mune Itpaino Kono Aiwo Dareyori Kimini (가슴 가득 이 사랑을 누구보다 너에게) (M-Oz Club Mix Edit)
4977 胸いっぱいのこの愛を 誰より君に / Mune Itpaino Kono Aiwo Dareyori Kimini (가슴 가득 이 사랑을 누구보다 너에게) (Singing Individual Part Ver.)
4978 ココロが止まらない / Kokoroga Tomaranai (마음이 멈추지 않아)
4979 Stop The Music, Give Me A Chance
4980 ひらり夢一夜 / Hirari Yume Ichiya (흔들리는 하룻밤 꿈)
4981 ココロが止まらない / Kokoroga Tomaranai (마음이 멈추지 않아) (M-Oz Club Radio Mix)
4982 ココロが止まらない / Kokoroga Tomaranai (마음이 멈추지 않아) (Inst.)
4983 Be My Love
4984 My Life
4985 Amore (부제 : 아름다운 날들)
4986 Dear My Heart
4987 니가 참 좋아
4988 Fantasy
4989 끝
4990 고백 (Duet 박정아, 최현준)
4991 바보야
4992 사랑이란 이유로
4993 So Long
4994 유혹
4995 Cool Girl
4996 Again
4997 One Summer Night
4998 To You
4999 Tonight
5000 선배
5001 How Are You?
5002 말하지 못한 절반의 사랑
5003 Sad
5004 1492...(ㅜ.ㅜ;)
5005 Maybe
5006 이별은 모르게...
5007 눈물 (Piano Ver.)
5008 하늘같은 사랑
5009 사랑해
5010 이젠
5011 달콤한 상상
5012 작은 바램
5013 Say
5014 Heart
5015 Say Yes
5016 Forever
5017 소나기
5018 날 믿어요
5019 Happy End
5020 몽
5021 슈퍼스타
5022 슈퍼스타
5023 Passion (Summer Ver.)
5024 슈퍼스타
5025 Passion
5026 달콤한 상상
5027 하늘같은 사랑
5028 달콤한 상상
5029 하늘같은 사랑
5030 달콤한 상상
5031 하늘같은 사랑
5032 `게임 쉬리` OST (`이젠` Metal Ver.) 와 동영상 수록
5033 가
5034 봐
5035 Run Away
5036 Touch
5037 그렇게 넌
5038 Crazy
5039 Summer Love
5040 천사의 눈물
5041 Myself
5042 한번만
5043 I`ll Be There
5044 Singing My Life Away
5045 Stellate
5046 Stellate (Fredi Vega Remix)
5047 Stellate (Fredi Vega Remix)
5048 Stellate
5049 Stellate (Fredi Vega Remix)
5050 i Feel The Same (Original Mix)
5051 i Feel The Same (Original Mix)
5052 Feeling
5053 Miracle
5054 경고
5055 Be Free
5056 오늘은.. (To Day..)
5057 Secret Of Love
5058 Come To Me
5059 Brand New Day
5060 Give And Take
5061 돌아와
5062 You & Me
5063 질투후애
5064 Goodbye
5065 Intro
5066 하루만
5067 깻잎소녀
5068 Show Me Love
5069 Lucky
5070 산
5071 Bye Bye
5072 두근 두근
5073 어떻게
5074 Voixco Dance
5075 하루만 (Remix)
5076 Show Me Love
5077 산 (Inst.)
5078 Intro
5079 Ice Baby
5080 어떻게
5081 하루만
5082 Bye Bye
5083 두근두근
5084 왔어
5085 Breaking My Heart
5086 산
5087 깻잎소녀
5088 Voixco Dance
5089 Show Me Love
5090 Rock And Roll All Nite
5091 Rock And Roll All Nite
5092 Calling Dr. Love
5093 Rock And Roll All Nite
5094 Deuce
5095 Strutter
5096 Cold Gin
5097 She
5098 Firehouse
5099 Nothin` to Lose
5100 100,000 Years
5101 Black Diamond
5102 Detroit Rock City
5103 Cold Gin
5104 Creatures of the Night
5105 Fits Like a Glove
5106 Heaven`s on Fire
5107 Thrills in the Night
5108 Paul Stanley Guitar Solo
5109 Under the Gun
5110 War Machine
5111 Eric Carr Drum Solo
5112 Young and Wasted
5113 Gene Simmons Bass Solo
5114 I Love It Loud
5115 I Still Love You
5116 Love Gun
5117 Black Diamond
5118 Oh Susanna

## Dataframe에 크롤링된 데이터 저장하기_____no_output_____

test_list = [title, album, release_date, song_genre, is_title, like, creator, lyrics]
column_names = ['title', 'artist', 'album', 'release_date', 'song_genre', 'is_title', 'like', 'creator', 'lyrics']
column_v = [title, artist, album, release_date, song_genre, is_title, like, creator, lyrics]

test_df = pd.DataFrame(column_v)

# 이렇게 집어넣었더니 행이 9개인 데이터프레임이 나와버립니다. 행렬을 바꾸어줍니다. _____no_output_____test_p = test_df.head(9).T
test_p.head()_____no_output_____test_p.columns = column_names
test_p.tail(350)_____no_output_____test_p.shape_____no_output_____# 간혹 이상하게 수집된 곡들이 많습니다. 이런 곡들은 전처리에서 해결해줍시다.
test_p['title'].isnull().sum()_____no_output_____test_p.to_csv('song_data.csv')_____no_output_____

### JTL 이후로 알 수 없는 오류가 발생했기 때문에 앞의 50명을 날리고 재시작하기 위한 전처리_____no_output_____

# idol_list = idol_list.drop(idol_list.index[0:51]).reset_index(drop=True)
# idol_list.head()_____no_output_____


# Notebook from kimdon-ethz/Quantitative-Big-Imaging-2022
Path: Exercises/03-ImageEnhancement/ImageEnhancementPlayground.ipynb

<code>
from warnings import warn
from skimage.morphology import disk, octagon as oct_func, star, square
from skimage.restoration import denoise_nl_means, denoise_wavelet
from scipy import ndimage as ndi
from skimage.filters import gaussian, median
import ipywidgets as ipw
from IPython.display import display, update_display, clear_output
from skimage.color import rgb2gray
import skimage.data
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline
# To prevent automatic figure display when execution of the cell ends
%config InlineBackend.close_figures = False

def horizontal_line(n):
    return np.pad(np.ones((1, 2*n+1)), [[n, n], [0, 0]], mode='constant', constant_values=0).astype(int)

def vertical_line(n):
    return horizontal_line(n).T

def cross(n):
    return ((horizontal_line(n)+vertical_line(n)) > 0).astype(int)

def octagon(n):
    return oct_func(n, n)_____no_output_____def clean_img(x):
    if isinstance(x, tuple):
        x = x[0]
    y = rgb2gray(x)
    if y.dtype == np.uint8:
        y = y/255.0
    if y.dtype == bool:
        y = y/1.0
    return y

sample_images = {c_id: clean_img(getattr(skimage.data, c_id)()) for c_id in ['astronaut',
                                                                             'camera',
                                                                             'checkerboard',
                                                                             'chelsea',
                                                                             'clock',
                                                                             'coffee',
                                                                             'coins',
                                                                             'horse',
                                                                             'hubble_deep_field',
                                                                             'immunohistochemistry',
                                                                             'moon',
                                                                             'page',
                                                                             'text',
                                                                             'rocket',
                                                                             'stereo_motorcycle']}

def salt_and_pepper(x, strength):
    y = x.copy()
    y[np.random.uniform(0, 1, size=x.shape) < strength/2.0] = x.min()
    y[np.random.uniform(0, 1, size=x.shape) < strength/2.0] = x.max()
    return y

def additive_gaussian(x, strength):
    return x+np.random.normal(0, strength, size=x.shape)

def multiplicative_gaussian(x, strength):
    return x*np.random.normal(1, strength, size=x.shape)

def horizontally_streaky_salt(x, strength):
    y = x.copy()
    y[np.random.uniform(0, 1, size=x.shape[0]) < strength/2.0, :] = x.min()
    y[np.random.uniform(0, 1, size=x.shape[0]) < strength/2.0, :] = x.max()
    return y

def vertically_streaky_salt(x, strength):
    y = x.copy()
    y[:, np.random.uniform(0, 1, size=x.shape[1]) < strength/2.0] = x.min()
    y[:, np.random.uniform(0, 1, size=x.shape[1]) < strength/2.0] = x.max()
    return y

def no_filter(x, **kwargs):
    return x

def gaussian_filter(x, size, neighborhood):
    # gaussian from skimage doesn't implement custom neighborhood
    base_kernel = neighborhood(size)
    impulse_img = np.zeros_like(base_kernel, dtype=np.float32)
    impulse_img[impulse_img.shape[0]//2, impulse_img.shape[1]//2] = 1.0
    g_kernel = gaussian(impulse_img, sigma=size)
    g_kernel = g_kernel*base_kernel
    g_kernel /= g_kernel.sum()
    return ndi.convolve(x, g_kernel, mode='constant', cval=0.0)

def median_filter(x, size, neighborhood):
    return median((x*255).clip(0, 255).astype(np.uint8), neighborhood(size))/255.0

def nonlocal_means(x, size, neighborhood):
    if neighborhood.__name__ != 'square':
        warn('Nonlocal means only uses square filters', RuntimeWarning)
    return denoise_nl_means(x, patch_size=size)

def wavelet_filter(x, size, neighborhood):
    if neighborhood.__name__ != 'square':
        warn('Wsvelet means only uses square filters', RuntimeWarning)
    return denoise_wavelet(x, sigma=size)_____no_output_____plt.ioff()
fig, m_axs = plt.subplots(2, 2, figsize=(12, 9), dpi=120)
mpl_out = ipw.Output()

def show_results(m_axs, in_img, noise_func, noise_strength, filter_func, filter_args):
    ((ax_img, ax_noise), (ax_filt, ax_diff)) = m_axs
    plot_args = dict(cmap='bone', vmin=0, vmax=1)
    ax_img.imshow(in_img, **plot_args)
    ax_img.axis('off')
    ax_img.set_title('Original Image')

    noise_img = noise_func(in_img, noise_strength)

    ax_noise.imshow(noise_img, **plot_args)
    ax_noise.set_title('Noise Image\n%s (%2.1f%%)\nMSE: %2.2f%%' % (noise_func.__name__, 100*noise_strength,
                                                                    100*np.mean(np.square(noise_img-in_img))))
    ax_noise.axis('off')

    filt_img = filter_func(noise_img, **filter_args)

    ax_filt.imshow(filt_img, **plot_args)
    f_args_str = '\n'.join(['{} = {}'.format(k, getattr(v, '__name__', v))
                            for k, v in filter_args.items()]) if len(filter_args) > 0 else ''
    ax_filt.set_title('{}\n{}'.format(filter_func.__name__, f_args_str))
    ax_filt.axis('off')

    diff_img = filt_img-in_img
    v_max = np.abs(diff_img).max()
    ax_diff.imshow(diff_img, cmap='RdBu', vmin=-v_max, vmax=v_max)
    ax_diff.set_title('Difference Image\nMSE: %2.2f%%' %
                      (100*np.mean(np.square(diff_img))))
    ax_diff.axis('off')_____no_output_____def fn_dict(x): return [(getattr(f, '__name__', f), f) for f in x]

image_name = ipw.Dropdown(options=sample_images.keys())
noise_func = ipw.Dropdown(options=fn_dict([salt_and_pepper, additive_gaussian,
                                           multiplicative_gaussian,
                                           vertically_streaky_salt,
                                           horizontally_streaky_salt]))
noise_level = ipw.FloatSlider(min=0, max=1, value=0.1, step=0.025)
filter_func = ipw.Dropdown(options=fn_dict(
    [no_filter, gaussian_filter, median_filter, nonlocal_means, wavelet_filter]))

neighbor_functions = [square, disk, cross,
                      horizontal_line, vertical_line, star, octagon]
filter_shape = ipw.Dropdown(options=fn_dict(neighbor_functions))
filter_size = ipw.IntSlider(min=1, max=50, value=3)

def update_image(*args):
    show_results(m_axs,
                 sample_images[image_name.value],
                 noise_func.value,
                 noise_level.value,
                 filter_func.value,
                 {'size': filter_size.value,
                  'neighborhood': filter_shape.value
                  })
    with mpl_out:
        clear_output(wait=True)
        display(fig)

update_image()_____no_output_____display(ipw.VBox([ipw.VBox([ipw.HBox([ipw.Label(value='Image Name:'), image_name]),
                            ipw.HBox(
                                [ipw.Label(value='Noise Type:'), noise_func]),
                            ipw.HBox(
                                [ipw.Label(value='Noise Level:'), noise_level]),
                            ipw.HBox(
                                [ipw.Label(value='Filter Name:'), filter_func]),
                            ipw.HBox(
                                [ipw.Label(value='Filter Shape:'), filter_shape]),
                            ipw.HBox(
                                [ipw.Label(value='Filter Size:'), filter_size]),
                            mpl_out
                            ])]))_____no_output_____# add callbacks
image_name.observe(update_image, names='value')
noise_func.observe(update_image, names='value')
noise_level.observe(update_image, names='value')
filter_func.observe(update_image, names='value')
filter_shape.observe(update_image, names='value')
filter_size.observe(update_image, names='value')_____no_output_____
</code>


# Notebook from tudev/Workshops-2020-2021
Path: Python YouTube Tutorials/Python_Strings!.ipynb

import numpy as np_____no_output_____

# **Strings**
**Strings are immutable sequences of Unicode code points. Text within Python can be represented by the string type (str).**

**Because strings are a sequence type, they support indexing and slicing.**

**Each character represents an element in the
sequence.**

Source: [Documentation for Python - Built-in Types](https://docs.python.org/3/library/stdtypes.html#textseq)

Source: [Python Data Structures and Algorithms](https://www.amazon.com/Python-Data-Structures-Algorithms-application-ebook/dp/B01IF7NLM8)_____no_output_____

#since it's immutable, we can't change the string object

our_planet = 'Earth'
our_planet[0] = 'e'_____no_output_____

Source: [Mutability and Immutability in Python by Wendy Leung](https://medium.com/datadriveninvestor/mutable-and-immutable-python-2093deeac8d9)_____no_output_____

list_ex = [0,1,1,1]
list_ex[0]
list_ex[0] = 1
list_ex_____no_output_____

## **Printing Text**_____no_output_____

#can print with single quotes or double quotes

print('Earth, Jupiter, Neptune, Mars, Saturn, Mercury, Venus, Uranus')
print("Earth, Jupiter, Neptune, Mars, Saturn, Mercury, Venus, Uranus")Earth, Jupiter, Neptune, Mars, Saturn, Mercury, Venus, Uranus
Earth, Jupiter, Neptune, Mars, Saturn, Mercury, Venus, Uranus
#multiple lines use triple quotation marks
print('''Earth,
Jupiter,
Neptune,
Mars,
Saturn,
Mercury,
Venus,
Uranus''')Earth,
Jupiter,
Neptune,
Mars,
Saturn,
Mercury,
Venus,
Uranus
#we can all use line breaks with single quotes
print("Earth,\nJupiter")Earth,
Jupiter

## **Print Formatting**_____no_output_____

planet_moon_nums = {'Earth':1, 'Jupiter':79, 'Neptune':14, 'Mars':2, 'Saturn':62, 'Mercury':0, 'Venus':0, 'Uranus':27}_____no_output_____

Source: [NASA -  How Many Moons Does Each Planet Have?](https://solarsystem.nasa.gov/moons/in-depth/)_____no_output_____

for key, value in planet_moon_nums.items():
  print(f'{key} has {value} moon(s)')Earth has 1 moon(s)
Jupiter has 79 moon(s)
Neptune has 14 moon(s)
Mars has 2 moon(s)
Saturn has 62 moon(s)
Mercury has 0 moon(s)
Venus has 0 moon(s)
Uranus has 27 moon(s)

# **Concatenating Strings**

**String concatenation is when strings are combined together**_____no_output_____

uranus_moons = ['Ariel','Umbriel','Titania','Oberon','Miranda','Cordelia','Ophelia','Bianca','Cressida','Desdemona','Juliet','Portia','Rosalind','Belinda','Puck',
'Caliban','Sycorax','Prospero','Setebos','Stephano','Trinculo','Francisco','Margaret','Ferdinand','Perdita','Mab','Cupid']_____no_output_____'Uranus - ' + 'Ariel'_____no_output_____['Uranus - ' + i for i in uranus_moons]_____no_output_____

# **Convert String to Integer or Float**_____no_output_____

avg_temp_planet_f = {'Mercury':'800', 'Venus':'880', 'Earth':'61', 'Mars':'-20', 'Jupiter':'-162', 'Saturn':'-218', 'Uranus': '-320', 'Neptune':'-331'}_____no_output_____

Source: [NASA -   Solar System Temperatures](https://solarsystem.nasa.gov/resources/681/solar-system-temperatures/)_____no_output_____

a = {key : int(value) for i in avg_temp_planet_f}_____no_output_____avg_temp_planet_f = {key:int(value) for (key,value) in avg_temp_planet_f.items()}_____no_output_____#alt 2-4-8
print(f'The median temperature within our solar system is: {np.median(list(avg_temp_planet_f.values()))} F°') The median temperature within our solar system is: -91.0 F°

# **String Functions**_____no_output_____

planets = ['earth', 'jupiter', 'neptune', 'mars', 'saturn', 'mercury', 'venus', 'uranus']
planets_____no_output_____

## **Capitalize**

**Return a copy of the string with its first character capitalized and the rest lowercased.**_____no_output_____

planets = [i.capitalize() for i in planets]
planets_____no_output_____

## **Lower**

**Return a copy of the string with all the cased characters converted to lowercase.**_____no_output_____

planets = [i.lower() for i in planets]
planets_____no_output_____

## **Upper**

**Return a copy of the string with all the cased characters converted to uppercase.**_____no_output_____

planets = [i.upper() for i in planets]
planets_____no_output_____

## **Split**

**Returns a list that seperates the string based on a passed in delimiter (sep).**_____no_output_____

planets_str = 'Earth,Jupiter,Neptune,Mars,Saturn,Mercury,Venus,Uranus'
planets_str_____no_output_____planets_str = planets_str.split(sep = ",")
planets_str_____no_output_____

## **Find**_____no_output_____

astronomy_article = '''The elusive axion particle is many times lighter than an electron, with properties that barely make an impression on ordinary matter. As such, the ghost-like particle is a leading contender as a component of dark matter — a hypothetical, invisible type of matter that is thought to make up 85 percent of the mass in the universe. Axions have so far evaded detection. Physicists predict that if they do exist, they must be produced within extreme environments, such as the cores of stars at the precipice of a supernova. When these stars spew axions out into the universe, the particles, on encountering any surrounding magnetic fields, should briefly morph into photons and potentially reveal themselves.
Now, MIT physicists have searched for axions in Betelgeuse, a nearby star that is expected to burn out as a supernova soon, at least on astrophysical timescales. Given its imminent demise, Betelgeuse should be a natural factory of axions, constantly churning out the particles as the star burns away. However, when the team looked for expected signatures of axions, in the form of photons in the X-ray band, their search came up empty. Their results rule out the existence of ultralight axions that can interact with photons over a wide range of energies. The findings set new constraints on the particle’s properties that are three times stronger than any previous laboratory-based axion-detecting experiments. “What our results say is, if you want to look for these really light particles, which we looked for, they’re not going to talk very much to photons,” says Kerstin Perez, assistant professor of physics at MIT. “We’re basically making everyone’s lives harder because we’re saying, ‘you’re going to have to think of something else that would give you an axion signal.’” Perez and her colleagues have published their results today in Physical Review Letters. Her MIT co-authors include lead author Mengjiao Xiao, Brandon Roach, and Melania Nynka, along with Maurizio Giannotti of Barry University, Oscar Straniero of the Abruzzo Astronomical Observatory, Alessandro Mirizzi of the National Institute for Nuclear Physics in Italy, and Brian Grefenstette of Caltech. A hunt for coupling
Many of the current experiments that search for axions are designed to look for them as a product of the Primakoff effect, a process that describes a theoretical “coupling” between axions and photons. Axions are not normally thought to interact with photons — hence their likelihood of being dark matter. However, the Primakoff effect predicts that, when photons are subjected to intense magnetic fields, such as in stellar cores, they could morph into axions. The center of many stars should therefore be natural axion factories. When a star explodes in a supernova, it should churn the axions out into the universe. If the invisible particles run into a magnetic field, for instance between the star and Earth, they should turn back into photons, presumably with some detectable energy. Scientists are hunting for axions through this process, for instance from our own sun. “But the sun also has flares and gives off X-rays all the time, and it’s hard to understand,” says Perez.
She and her colleagues instead looked for axions from Betelgeuse, a star that normally does not emit X-rays. The star is among those nearest to Earth that are expected to explode soon. “Betelgeuse is at a temperature and lifestage where you don’t expect to see X-rays coming out of it, through standard stellar astrophysics,” Perez explains. “But if axions do exist, and are coming out, we might see an X-ray signature. So that’s why this star is a nice object: If you see X-rays, it’s a smoking gun signal that it’s got to be axions.” “Data are data” The researchers looked for X-ray signatures of axions from Betelgeuse, using data taken by NuSTAR, NASA’s space-based telescope that focuses high-energy X-rays from astrophysical sources. The team obtained 50 kiloseconds of data from NuSTAR during the time the telescope was trained on Betelgeuse. The researchers then modeled a range of X-ray emissions that they might see from Betelgeuse if the star was spewing out axions. They considered a range of masses that an axion might be, as well as a range of likelihoods that the axions would “couple” to and reconvert into a photon, depending on the magnetic field strength between the star and Earth.
“Out of all that modeling, you get a range of what your X-ray signal of axions could possibly look like,” Perez says. When they searched for these signals in NuSTAR’s data, however, they found nothing above their expected background or outside of any ordinary astrophysical sources of X-rays. “Betelgeuse is probably in the late stages of evolution and in that case should have a big probability of converting into axions,” Xiao says. “But data are data.” Given the range of conditions they considered, the team’s null result rules out a large space of possibilities and sets an upper limit that is three times stronger than previous limits, from laboratory-based searches, for what an axion must be. In essence, this means that if axions are ultralight in mass, the team’s results show that the particles must be at least three times less likely to couple to photons and emit any detectable X-rays. “If axions have ultralight masses, we can definitely tell you their coupling has to be very small, otherwise we would have seen it,” Perez says. Ultimately, this means that scientists may have to look to other, less detectable energy bands for axion signals. However, Perez says the search for axions from Betelgeuse is not over. “What would be exciting would be if we see a supernova, which would ignite a huge amount of axions that wouldn’t be in X-rays, but in gamma rays,” Perez says. “If a star explodes and we don’t see axions, then we’ll get really stringent constraints on an axion’s coupling to photons. So everyone’s crossing their fingers for Betelgeuse to go off.” This research was supported, in part, by NASA.
''' _____no_output_____astronomy_article[0:3]_____no_output_____#returns index position of the first instance of "supernova"
astronomy_article.find('supernova')_____no_output_____astronomy_article[509:(509 + len('supernova'))]_____no_output_____

# **References and Additional Learning**_____no_output_____## **Websites**

- **[Python Documentation - Text Sequence Type — str](https://docs.python.org/3/library/stdtypes.html#str)**

- **[Mutability and Immutability in Python — Let’s Break It Down by Wendy Leung on Medium.com](https://medium.com/datadriveninvestor/mutable-and-immutable-python-2093deeac8d9)**_____no_output_____## **Textbook**

- **[Python Data Structures and Algorithms](https://www.amazon.com/Python-Data-Structures-Algorithms-application-ebook/dp/B01IF7NLM8) by Benjamin Baka**_____no_output_____# **Connect**
- **Feel free to connect with Adrian on [YouTube](https://www.youtube.com/channel/UCPuDxI3xb_ryUUMfkm0jsRA), [LinkedIn](https://www.linkedin.com/in/adrian-dolinay-frm-96a289106/), [Twitter](https://twitter.com/DolinayG) and [GitHub](https://github.com/ad17171717). Happy coding!**_____no_output_____


# Notebook from TBaltensperger/cvxportfolio
Path: examples/DataEstimatesRiskModel.ipynb

from __future__ import print_function
import pandas as pd
import numpy as np
import datetime as dt
import collections
import quandl
# quandl.ApiConfig.api_key = 'KN5JYVY7yw-4RcRdzN1v'_____no_output_____

# Data _____no_output_____

datadir='../data/'
assets=pd.read_csv(datadir+'SP500.txt', comment='#').set_index('Symbol')

QUANDL={
    ## Get a key (free) from quandl.com and copy it here
    'authtoken':"KN5JYVY7yw-4RcRdzN1v",
    'start_date':dt.date(2007, 1, 1),
    'end_date':dt.date(2016, 12, 31)
}
RISK_FREE_SYMBOL = "USDOLLAR"
data={}_____no_output_____

#### Download loop_____no_output_____If it stops because of Quandl error codes 503 or 504, try re-running it (it won't download data already downloaded). If Quandl complains about the speed of requests, try adding sleep time._____no_output_____

# download assets' data
from time import sleep

for ticker in assets.index:
    if ticker in data:
        continue
    print('downloading %s from %s to %s' %(ticker, QUANDL['start_date'], QUANDL['end_date']))
    try:
        data[ticker] = quandl.get(assets.Quandlcode[ticker], **QUANDL)
        sleep(0.1)
    except quandl.NotFoundError:
        print('\tInvalid asset code')07-01-01 to 2016-12-31
downloading BWA from 2007-01-01 to 2016-12-31
downloading BXP from 2007-01-01 to 2016-12-31
downloading BSX from 2007-01-01 to 2016-12-31
downloading BMY from 2007-01-01 to 2016-12-31
downloading BRCM from 2007-01-01 to 2016-12-31
downloading BF-B from 2007-01-01 to 2016-12-31
	Invalid asset code
downloading CHRW from 2007-01-01 to 2016-12-31
downloading CA from 2007-01-01 to 2016-12-31
downloading CVC from 2007-01-01 to 2016-12-31
downloading COG from 2007-01-01 to 2016-12-31
downloading CAM from 2007-01-01 to 2016-12-31
downloading CPB from 2007-01-01 to 2016-12-31
downloading COF from 2007-01-01 to 2016-12-31
downloading CAH from 2007-01-01 to 2016-12-31
downloading HSIC from 2007-01-01 to 2016-12-31
downloading KMX from 2007-01-01 to 2016-12-31
downloading CCL from 2007-01-01 to 2016-12-31
downloading CAT from 2007-01-01 to 2016-12-31
downloading CBG from 2007-01-01 to 2016-12-31
downloading CBS from 2007-01-01 to 2016-12-31
downloading CELG from 2007-01-01 to 2016-12-31
downloading CNP from 2007-01-01 to 2016-12-31
downloading CTL from 2007-01-01 to 2016-12-31
downloading CERN from 2007-01-01 to 2016-12-31
downloading CF from 2007-01-01 to 2016-12-31
downloading SCHW from 2007-01-01 to 2016-12-31
downloading CHK from 2007-01-01 to 2016-12-31
downloading CVX from 2007-01-01 to 2016-12-31
downloading CMG from 2007-01-01 to 2016-12-31
downloading CB from 2007-01-01 to 2016-12-31
downloading CI from 2007-01-01 to 2016-12-31
downloading XEC from 2007-01-01 to 2016-12-31
downloading CINF from 2007-01-01 to 2016-12-31
downloading CTAS from 2007-01-01 to 2016-12-31
downloading CSCO from 2007-01-01 to 2016-12-31
downloading C from 2007-01-01 to 2016-12-31
downloading CTXS from 2007-01-01 to 2016-12-31
downloading CLX from 2007-01-01 to 2016-12-31
downloading CME from 2007-01-01 to 2016-12-31
downloading CMS from 2007-01-01 to 2016-12-31
downloading COH from 2007-01-01 to 2016-12-31
downloading KO from 2007-01-01 to 2016-12-31
downloading CCE from 2007-01-01 to 2016-12-31
downloading CTSH from 2007-01-01 to 2016-12-31
downloading CL from 2007-01-01 to 2016-12-31
downloading CPGX from 2007-01-01 to 2016-12-31
downloading CMCSA from 2007-01-01 to 2016-12-31
downloading CMCSK from 2007-01-01 to 2016-12-31
downloading CMA from 2007-01-01 to 2016-12-31
downloading CSC from 2007-01-01 to 2016-12-31
downloading CAG from 2007-01-01 to 2016-12-31
downloading COP from 2007-01-01 to 2016-12-31
downloading CNX from 2007-01-01 to 2016-12-31
downloading ED from 2007-01-01 to 2016-12-31
downloading STZ from 2007-01-01 to 2016-12-31
downloading GLW from 2007-01-01 to 2016-12-31
downloading COST from 2007-01-01 to 2016-12-31
downloading CCI from 2007-01-01 to 2016-12-31
downloading CSX from 2007-01-01 to 2016-12-31
downloading CMI from 2007-01-01 to 2016-12-31
downloading CVS from 2007-01-01 to 2016-12-31
downloading DHI from 2007-01-01 to 2016-12-31
downloading DHR from 2007-01-01 to 2016-12-31
downloading DRI from 2007-01-01 to 2016-12-31
downloading DVA from 2007-01-01 to 2016-12-31
downloading DE from 2007-01-01 to 2016-12-31
downloading DLPH from 2007-01-01 to 2016-12-31
downloading DAL from 2007-01-01 to 2016-12-31
downloading XRAY from 2007-01-01 to 2016-12-31
downloading DVN from 2007-01-01 to 2016-12-31
downloading DO from 2007-01-01 to 2016-12-31
downloading DFS from 2007-01-01 to 2016-12-31
downloading DISCA from 2007-01-01 to 2016-12-31
downloading DISCK from 2007-01-01 to 2016-12-31
downloading DG from 2007-01-01 to 2016-12-31
downloading DLTR from 2007-01-01 to 2016-12-31
downloading D from 2007-01-01 to 2016-12-31
downloading DOV from 2007-01-01 to 2016-12-31
downloading DOW from 2007-01-01 to 2016-12-31
downloading DPS from 2007-01-01 to 2016-12-31
downloading DTE from 2007-01-01 to 2016-12-31
downloading DD from 2007-01-01 to 2016-12-31
downloading DUK from 2007-01-01 to 2016-12-31
downloading DNB from 2007-01-01 to 2016-12-31
downloading ETFC from 2007-01-01 to 2016-12-31
downloading EMN from 2007-01-01 to 2016-12-31
downloading ETN from 2007-01-01 to 2016-12-31
downloading EBAY from 2007-01-01 to 2016-12-31
downloading ECL from 2007-01-01 to 2016-12-31
downloading EIX from 2007-01-01 to 2016-12-31
downloading EW from 2007-01-01 to 2016-12-31
downloading EA from 2007-01-01 to 2016-12-31
downloading EMC from 2007-01-01 to 2016-12-31
downloading EMR from 2007-01-01 to 2016-12-31
downloading ENDP from 2007-01-01 to 2016-12-31
downloading ESV from 2007-01-01 to 2016-12-31
downloading ETR from 2007-01-01 to 2016-12-31
downloading EOG from 2007-01-01 to 2016-12-31
downloading EQT from 2007-01-01 to 2016-12-31
downloading EFX from 2007-01-01 to 2016-12-31
downloading EQIX from 2007-01-01 to 2016-12-31
downloading EQR from 2007-01-01 to 2016-12-31
downloading ESS from 2007-01-01 to 2016-12-31
downloading EL from 2007-01-01 to 2016-12-31
downloading ES from 2007-01-01 to 2016-12-31
downloading EXC from 2007-01-01 to 2016-12-31
downloading EXPE from 2007-01-01 to 2016-12-31
downloading EXPD from 2007-01-01 to 2016-12-31
downloading ESRX from 2007-01-01 to 2016-12-31
downloading XOM from 2007-01-01 to 2016-12-31
downloading FFIV from 2007-01-01 to 2016-12-31
downloading FB from 2007-01-01 to 2016-12-31
downloading FAST from 2007-01-01 to 2016-12-31
downloading FDX from 2007-01-01 to 2016-12-31
downloading FIS from 2007-01-01 to 2016-12-31
downloading FITB from 2007-01-01 to 2016-12-31
downloading FSLR from 2007-01-01 to 2016-12-31
downloading FE from 2007-01-01 to 2016-12-31
downloading FISV from 2007-01-01 to 2016-12-31
downloading FLIR from 2007-01-01 to 2016-12-31
downloading FLS from 2007-01-01 to 2016-12-31
downloading FLR from 2007-01-01 to 2016-12-31
downloading FMC from 2007-01-01 to 2016-12-31
downloading FTI from 2007-01-01 to 2016-12-31
downloading F from 2007-01-01 to 2016-12-31
downloading FOSL from 2007-01-01 to 2016-12-31
downloading BEN from 2007-01-01 to 2016-12-31
downloading FCX from 2007-01-01 to 2016-12-31
downloading FTR from 2007-01-01 to 2016-12-31
downloading GME from 2007-01-01 to 2016-12-31
downloading GPS from 2007-01-01 to 2016-12-31
downloading GRMN from 2007-01-01 to 2016-12-31
downloading GD from 2007-01-01 to 2016-12-31
downloading GE from 2007-01-01 to 2016-12-31
downloading GGP from 2007-01-01 to 2016-12-31
downloading GIS from 2007-01-01 to 2016-12-31
downloading GM from 2007-01-01 to 2016-12-31
downloading GPC from 2007-01-01 to 2016-12-31
downloading GNW from 2007-01-01 to 2016-12-31
downloading GILD from 2007-01-01 to 2016-12-31
downloading GS from 2007-01-01 to 2016-12-31
downloading GT from 2007-01-01 to 2016-12-31
downloading GWW from 2007-01-01 to 2016-12-31
downloading HAL from 2007-01-01 to 2016-12-31
downloading HBI from 2007-01-01 to 2016-12-31
downloading HOG from 2007-01-01 to 2016-12-31
downloading HAR from 2007-01-01 to 2016-12-31
downloading HRS from 2007-01-01 to 2016-12-31
downloading HIG from 2007-01-01 to 2016-12-31
downloading HAS from 2007-01-01 to 2016-12-31
downloading HCA from 2007-01-01 to 2016-12-31
downloading HCP from 2007-01-01 to 2016-12-31
downloading HP from 2007-01-01 to 2016-12-31
downloading HES from 2007-01-01 to 2016-12-31
downloading HPQ from 2007-01-01 to 2016-12-31
downloading HD from 2007-01-01 to 2016-12-31
downloading HON from 2007-01-01 to 2016-12-31
downloading HRL from 2007-01-01 to 2016-12-31
downloading HST from 2007-01-01 to 2016-12-31
downloading HCBK from 2007-01-01 to 2016-12-31
downloading HUM from 2007-01-01 to 2016-12-31
downloading HBAN from 2007-01-01 to 2016-12-31
downloading ITW from 2007-01-01 to 2016-12-31
downloading IR from 2007-01-01 to 2016-12-31
downloading INTC from 2007-01-01 to 2016-12-31
downloading ICE from 2007-01-01 to 2016-12-31
downloading IBM from 2007-01-01 to 2016-12-31
downloading IP from 2007-01-01 to 2016-12-31
downloading IPG from 2007-01-01 to 2016-12-31
downloading IFF from 2007-01-01 to 2016-12-31
downloading INTU from 2007-01-01 to 2016-12-31
downloading ISRG from 2007-01-01 to 2016-12-31
downloading IVZ from 2007-01-01 to 2016-12-31
downloading IRM from 2007-01-01 to 2016-12-31
downloading JEC from 2007-01-01 to 2016-12-31
downloading JBHT from 2007-01-01 to 2016-12-31
downloading JNJ from 2007-01-01 to 2016-12-31
downloading JCI from 2007-01-01 to 2016-12-31
downloading JPM from 2007-01-01 to 2016-12-31
downloading JNPR from 2007-01-01 to 2016-12-31
downloading KSU from 2007-01-01 to 2016-12-31
downloading K from 2007-01-01 to 2016-12-31
downloading KEY from 2007-01-01 to 2016-12-31
downloading GMCR from 2007-01-01 to 2016-12-31
downloading KMB from 2007-01-01 to 2016-12-31
downloading KIM from 2007-01-01 to 2016-12-31
downloading KMI from 2007-01-01 to 2016-12-31
downloading KLAC from 2007-01-01 to 2016-12-31
downloading KSS from 2007-01-01 to 2016-12-31
downloading KHC from 2007-01-01 to 2016-12-31
downloading KR from 2007-01-01 to 2016-12-31
downloading LB from 2007-01-01 to 2016-12-31
downloading LLL from 2007-01-01 to 2016-12-31
downloading LH from 2007-01-01 to 2016-12-31
downloading LRCX from 2007-01-01 to 2016-12-31
downloading LM from 2007-01-01 to 2016-12-31
downloading LEG from 2007-01-01 to 2016-12-31
downloading LEN from 2007-01-01 to 2016-12-31
downloading LVLT from 2007-01-01 to 2016-12-31
downloading LUK from 2007-01-01 to 2016-12-31
downloading LLY from 2007-01-01 to 2016-12-31
downloading LNC from 2007-01-01 to 2016-12-31
downloading LLTC from 2007-01-01 to 2016-12-31
downloading LMT from 2007-01-01 to 2016-12-31
downloading L from 2007-01-01 to 2016-12-31
downloading LOW from 2007-01-01 to 2016-12-31
downloading LYB from 2007-01-01 to 2016-12-31
downloading MTB from 2007-01-01 to 2016-12-31
downloading MAC from 2007-01-01 to 2016-12-31
downloading M from 2007-01-01 to 2016-12-31
downloading MNK from 2007-01-01 to 2016-12-31
downloading MRO from 2007-01-01 to 2016-12-31
downloading MPC from 2007-01-01 to 2016-12-31
downloading MAR from 2007-01-01 to 2016-12-31
downloading MMC from 2007-01-01 to 2016-12-31
downloading MLM from 2007-01-01 to 2016-12-31
downloading MAS from 2007-01-01 to 2016-12-31
downloading MA from 2007-01-01 to 2016-12-31
downloading MAT from 2007-01-01 to 2016-12-31
downloading MKC from 2007-01-01 to 2016-12-31
downloading MCD from 2007-01-01 to 2016-12-31
downloading MHFI from 2007-01-01 to 2016-12-31
downloading MCK from 2007-01-01 to 2016-12-31
downloading MJN from 2007-01-01 to 2016-12-31
downloading WRK from 2007-01-01 to 2016-12-31
downloading MDT from 2007-01-01 to 2016-12-31
downloading MRK from 2007-01-01 to 2016-12-31
downloading MET from 2007-01-01 to 2016-12-31
downloading KORS from 2007-01-01 to 2016-12-31
downloading MCHP from 2007-01-01 to 2016-12-31
downloading MU from 2007-01-01 to 2016-12-31
downloading MSFT from 2007-01-01 to 2016-12-31
downloading MHK from 2007-01-01 to 2016-12-31
downloading TAP from 2007-01-01 to 2016-12-31
downloading MDLZ from 2007-01-01 to 2016-12-31
downloading MON from 2007-01-01 to 2016-12-31
downloading MNST from 2007-01-01 to 2016-12-31
downloading MCO from 2007-01-01 to 2016-12-31
downloading MS from 2007-01-01 to 2016-12-31
downloading MOS from 2007-01-01 to 2016-12-31
downloading MSI from 2007-01-01 to 2016-12-31
downloading MUR from 2007-01-01 to 2016-12-31
downloading MYL from 2007-01-01 to 2016-12-31
downloading NDAQ from 2007-01-01 to 2016-12-31
downloading NOV from 2007-01-01 to 2016-12-31
downloading NAVI from 2007-01-01 to 2016-12-31
downloading NTAP from 2007-01-01 to 2016-12-31
downloading NFLX from 2007-01-01 to 2016-12-31
downloading NWL from 2007-01-01 to 2016-12-31
downloading NFX from 2007-01-01 to 2016-12-31
downloading NEM from 2007-01-01 to 2016-12-31
downloading NWSA from 2007-01-01 to 2016-12-31
downloading NWS from 2007-01-01 to 2016-12-31
downloading NEE from 2007-01-01 to 2016-12-31
downloading NLSN from 2007-01-01 to 2016-12-31
downloading NKE from 2007-01-01 to 2016-12-31
downloading NI from 2007-01-01 to 2016-12-31
downloading NBL from 2007-01-01 to 2016-12-31
downloading JWN from 2007-01-01 to 2016-12-31
downloading NSC from 2007-01-01 to 2016-12-31
downloading NTRS from 2007-01-01 to 2016-12-31
downloading NOC from 2007-01-01 to 2016-12-31
downloading NRG from 2007-01-01 to 2016-12-31
downloading NUE from 2007-01-01 to 2016-12-31
downloading NVDA from 2007-01-01 to 2016-12-31
downloading ORLY from 2007-01-01 to 2016-12-31
downloading OXY from 2007-01-01 to 2016-12-31
downloading OMC from 2007-01-01 to 2016-12-31
downloading OKE from 2007-01-01 to 2016-12-31
downloading ORCL from 2007-01-01 to 2016-12-31
downloading OI from 2007-01-01 to 2016-12-31
downloading PCAR from 2007-01-01 to 2016-12-31
downloading PH from 2007-01-01 to 2016-12-31
downloading PDCO from 2007-01-01 to 2016-12-31
downloading PAYX from 2007-01-01 to 2016-12-31
downloading PYPL from 2007-01-01 to 2016-12-31
downloading PNR from 2007-01-01 to 2016-12-31
downloading PBCT from 2007-01-01 to 2016-12-31
downloading POM from 2007-01-01 to 2016-12-31
downloading PEP from 2007-01-01 to 2016-12-31
downloading PKI from 2007-01-01 to 2016-12-31
downloading PRGO from 2007-01-01 to 2016-12-31
downloading PFE from 2007-01-01 to 2016-12-31
downloading PCG from 2007-01-01 to 2016-12-31
downloading PM from 2007-01-01 to 2016-12-31
downloading PSX from 2007-01-01 to 2016-12-31
downloading PNW from 2007-01-01 to 2016-12-31
downloading PXD from 2007-01-01 to 2016-12-31
downloading PBI from 2007-01-01 to 2016-12-31
downloading PCL from 2007-01-01 to 2016-12-31
downloading PNC from 2007-01-01 to 2016-12-31
downloading RL from 2007-01-01 to 2016-12-31
downloading PPG from 2007-01-01 to 2016-12-31
downloading PPL from 2007-01-01 to 2016-12-31
downloading PX from 2007-01-01 to 2016-12-31
downloading PCP from 2007-01-01 to 2016-12-31
downloading PCLN from 2007-01-01 to 2016-12-31
downloading PFG from 2007-01-01 to 2016-12-31
downloading PG from 2007-01-01 to 2016-12-31
downloading PGR from 2007-01-01 to 2016-12-31
downloading PLD from 2007-01-01 to 2016-12-31
downloading PRU from 2007-01-01 to 2016-12-31
downloading PEG from 2007-01-01 to 2016-12-31
downloading PSA from 2007-01-01 to 2016-12-31
downloading PHM from 2007-01-01 to 2016-12-31
downloading PVH from 2007-01-01 to 2016-12-31
downloading QRVO from 2007-01-01 to 2016-12-31
downloading PWR from 2007-01-01 to 2016-12-31
downloading QCOM from 2007-01-01 to 2016-12-31
downloading DGX from 2007-01-01 to 2016-12-31
downloading RRC from 2007-01-01 to 2016-12-31
downloading RTN from 2007-01-01 to 2016-12-31
downloading O from 2007-01-01 to 2016-12-31
downloading RHT from 2007-01-01 to 2016-12-31
downloading REGN from 2007-01-01 to 2016-12-31
downloading RF from 2007-01-01 to 2016-12-31
downloading RSG from 2007-01-01 to 2016-12-31
downloading RAI from 2007-01-01 to 2016-12-31
downloading RHI from 2007-01-01 to 2016-12-31
downloading ROK from 2007-01-01 to 2016-12-31
downloading COL from 2007-01-01 to 2016-12-31
downloading ROP from 2007-01-01 to 2016-12-31
downloading ROST from 2007-01-01 to 2016-12-31
downloading RCL from 2007-01-01 to 2016-12-31
downloading R from 2007-01-01 to 2016-12-31
downloading CRM from 2007-01-01 to 2016-12-31
downloading SNDK from 2007-01-01 to 2016-12-31
downloading SCG from 2007-01-01 to 2016-12-31
downloading SLB from 2007-01-01 to 2016-12-31
downloading SNI from 2007-01-01 to 2016-12-31
downloading STX from 2007-01-01 to 2016-12-31
downloading SEE from 2007-01-01 to 2016-12-31
downloading SRE from 2007-01-01 to 2016-12-31
downloading SHW from 2007-01-01 to 2016-12-31
downloading SIAL from 2007-01-01 to 2016-12-31
downloading SIG from 2007-01-01 to 2016-12-31
downloading SPG from 2007-01-01 to 2016-12-31
downloading SWKS from 2007-01-01 to 2016-12-31
downloading SLG from 2007-01-01 to 2016-12-31
downloading SJM from 2007-01-01 to 2016-12-31
downloading SNA from 2007-01-01 to 2016-12-31
downloading SO from 2007-01-01 to 2016-12-31
downloading LUV from 2007-01-01 to 2016-12-31
downloading SWN from 2007-01-01 to 2016-12-31
downloading SE from 2007-01-01 to 2016-12-31
downloading STJ from 2007-01-01 to 2016-12-31
downloading SWK from 2007-01-01 to 2016-12-31
downloading SPLS from 2007-01-01 to 2016-12-31
downloading SBUX from 2007-01-01 to 2016-12-31
downloading HOT from 2007-01-01 to 2016-12-31
downloading STT from 2007-01-01 to 2016-12-31
downloading SRCL from 2007-01-01 to 2016-12-31
downloading SYK from 2007-01-01 to 2016-12-31
downloading STI from 2007-01-01 to 2016-12-31
downloading SYMC from 2007-01-01 to 2016-12-31
downloading SYY from 2007-01-01 to 2016-12-31
downloading TROW from 2007-01-01 to 2016-12-31
downloading TGT from 2007-01-01 to 2016-12-31
downloading TEL from 2007-01-01 to 2016-12-31
downloading TE from 2007-01-01 to 2016-12-31
downloading TGNA from 2007-01-01 to 2016-12-31
downloading THC from 2007-01-01 to 2016-12-31
downloading TDC from 2007-01-01 to 2016-12-31
downloading TSO from 2007-01-01 to 2016-12-31
downloading TXN from 2007-01-01 to 2016-12-31
downloading TXT from 2007-01-01 to 2016-12-31
downloading HSY from 2007-01-01 to 2016-12-31
downloading TRV from 2007-01-01 to 2016-12-31
downloading TMO from 2007-01-01 to 2016-12-31
downloading TIF from 2007-01-01 to 2016-12-31
downloading TWX from 2007-01-01 to 2016-12-31
downloading TWC from 2007-01-01 to 2016-12-31
downloading TJX from 2007-01-01 to 2016-12-31
downloading TMK from 2007-01-01 to 2016-12-31
downloading TSS from 2007-01-01 to 2016-12-31
downloading TSCO from 2007-01-01 to 2016-12-31
downloading RIG from 2007-01-01 to 2016-12-31
downloading TRIP from 2007-01-01 to 2016-12-31
downloading FOXA from 2007-01-01 to 2016-12-31
downloading FOX from 2007-01-01 to 2016-12-31
downloading TSN from 2007-01-01 to 2016-12-31
downloading TYC from 2007-01-01 to 2016-12-31
downloading USB from 2007-01-01 to 2016-12-31
downloading UA from 2007-01-01 to 2016-12-31
downloading UNP from 2007-01-01 to 2016-12-31
downloading UAL from 2007-01-01 to 2016-12-31
downloading UNH from 2007-01-01 to 2016-12-31
downloading UPS from 2007-01-01 to 2016-12-31
downloading URI from 2007-01-01 to 2016-12-31
downloading UTX from 2007-01-01 to 2016-12-31
downloading UHS from 2007-01-01 to 2016-12-31
downloading UNM from 2007-01-01 to 2016-12-31
downloading URBN from 2007-01-01 to 2016-12-31
downloading VFC from 2007-01-01 to 2016-12-31
downloading VLO from 2007-01-01 to 2016-12-31
downloading VAR from 2007-01-01 to 2016-12-31
downloading VTR from 2007-01-01 to 2016-12-31
downloading VRSN from 2007-01-01 to 2016-12-31
downloading VRSK from 2007-01-01 to 2016-12-31
downloading VZ from 2007-01-01 to 2016-12-31
downloading VRTX from 2007-01-01 to 2016-12-31
downloading VIAB from 2007-01-01 to 2016-12-31
downloading V from 2007-01-01 to 2016-12-31
downloading VNO from 2007-01-01 to 2016-12-31
downloading VMC from 2007-01-01 to 2016-12-31
downloading WMT from 2007-01-01 to 2016-12-31
downloading WBA from 2007-01-01 to 2016-12-31
downloading DIS from 2007-01-01 to 2016-12-31
downloading WM from 2007-01-01 to 2016-12-31
downloading WAT from 2007-01-01 to 2016-12-31
downloading ANTM from 2007-01-01 to 2016-12-31
downloading WFC from 2007-01-01 to 2016-12-31
downloading HCN from 2007-01-01 to 2016-12-31
downloading WDC from 2007-01-01 to 2016-12-31
downloading WU from 2007-01-01 to 2016-12-31
downloading WY from 2007-01-01 to 2016-12-31
downloading WHR from 2007-01-01 to 2016-12-31
downloading WFM from 2007-01-01 to 2016-12-31
downloading WMB from 2007-01-01 to 2016-12-31
downloading WEC from 2007-01-01 to 2016-12-31
downloading WYN from 2007-01-01 to 2016-12-31
downloading WYNN from 2007-01-01 to 2016-12-31
downloading XEL from 2007-01-01 to 2016-12-31
downloading XRX from 2007-01-01 to 2016-12-31
downloading XLNX from 2007-01-01 to 2016-12-31
downloading XL from 2007-01-01 to 2016-12-31
downloading XYL from 2007-01-01 to 2016-12-31
downloading YHOO from 2007-01-01 to 2016-12-31
downloading YUM from 2007-01-01 to 2016-12-31
downloading ZBH from 2007-01-01 to 2016-12-31
downloading ZION from 2007-01-01 to 2016-12-31
downloading ZTS from 2007-01-01 to 2016-12-31
downloading USDOLLAR from 2007-01-01 to 2016-12-31

#### Computation _____no_output_____

keys=[el for el in assets.index if not el in (set(assets.index)-set(data.keys()))]

def select_first_valid_column(df, columns):
    for column in columns:
        if column in df.columns:
            return df[column]

# extract prices
prices=pd.DataFrame.from_dict(collections.OrderedDict([(k,select_first_valid_column(data[k], ["Adj. Close", "Close", "Value"]))
                                for k in keys]))

#compute sigmas
open_price=pd.DataFrame.from_dict(collections.OrderedDict([(k,select_first_valid_column(data[k], ["Open"])) for k in keys]))
close_price=pd.DataFrame.from_dict(collections.OrderedDict([(k,select_first_valid_column(data[k], ["Close"])) for k in keys]))
sigmas = np.abs(np.log(open_price.astype(float))-np.log(close_price.astype(float)))

# extract volumes
volumes=pd.DataFrame.from_dict(collections.OrderedDict([(k,select_first_valid_column(data[k], ["Adj. Volume", "Volume"])) for k in keys]))

# fix risk free
prices[RISK_FREE_SYMBOL]=10000*(1 + prices[RISK_FREE_SYMBOL]/(100*250)).cumprod()_____no_output_____

#### Filtering _____no_output_____

# filter NaNs - threshold at 2% missing values
bad_assets = prices.columns[prices.isnull().sum()>len(prices)*0.02]
if len(bad_assets):
    print('Assets %s have too many NaNs, removing them' % bad_assets)

prices = prices.loc[:,~prices.columns.isin(bad_assets)]
sigmas = sigmas.loc[:,~sigmas.columns.isin(bad_assets)]
volumes = volumes.loc[:,~volumes.columns.isin(bad_assets)]

nassets=prices.shape[1]

# days on which many assets have missing values
bad_days1=sigmas.index[sigmas.isnull().sum(1) > nassets*.9]
bad_days2=prices.index[prices.isnull().sum(1) > nassets*.9]
bad_days3=volumes.index[volumes.isnull().sum(1) > nassets*.9]
bad_days=pd.Index(set(bad_days1).union(set(bad_days2)).union(set(bad_days3))).sort_values()
print ("Removing these days from dataset:")
print(pd.DataFrame({'nan price':prices.isnull().sum(1)[bad_days],
                    'nan volumes':volumes.isnull().sum(1)[bad_days],
                    'nan sigmas':sigmas.isnull().sum(1)[bad_days]}))

prices=prices.loc[~prices.index.isin(bad_days)]
sigmas=sigmas.loc[~sigmas.index.isin(bad_days)]
volumes=volumes.loc[~volumes.index.isin(bad_days)]

# extra filtering
print(pd.DataFrame({'remaining nan price':prices.isnull().sum(),
                    'remaining nan volumes':volumes.isnull().sum(),
                    'remaining nan sigmas':sigmas.isnull().sum()}))
prices=prices.fillna(method='ffill')
sigmas=sigmas.fillna(method='ffill')
volumes=volumes.fillna(method='ffill')
print(pd.DataFrame({'remaining nan price':prices.isnull().sum(),
                    'remaining nan volumes':volumes.isnull().sum(),
                    'remaining nan sigmas':sigmas.isnull().sum()}))Assets Index(['ABBV', 'ACE', 'ADT', 'GAS', 'ARG', 'AA', 'ALLE', 'GOOG', 'ALTR',
       'AVGO', 'BXLT', 'BRCM', 'CVC', 'CAM', 'CPGX', 'CMCSK', 'DLPH', 'DAL',
       'DFS', 'DISCK', 'DG', 'DPS', 'EMC', 'FB', 'GM', 'HCA', 'HCBK', 'GMCR',
       'KMI', 'KHC', 'LYB', 'MNK', 'MPC', 'MHFI', 'MJN', 'WRK', 'KORS', 'NAVI',
       'NWSA', 'NWS', 'NLSN', 'PYPL', 'POM', 'PM', 'PSX', 'PCL', 'PPG', 'PCP',
       'QRVO', 'SNDK', 'SNI', 'SIAL', 'HOT', 'TEL', 'TE', 'TDC', 'TRV', 'TWC',
       'TRIP', 'TYC', 'UA', 'VRSK', 'V', 'XYL', 'ZTS'],
      dtype='object') have too many NaNs, removing them
Removing these days from dataset:

#### Save _____no_output_____

# make volumes in dollars
volumes = volumes*prices

# compute returns
returns = (prices.diff()/prices.shift(1)).fillna(method='ffill').ix[1:]

bad_assets = returns.columns[((-.5>returns).sum()>0)|((returns > 2.).sum()>0)]
if len(bad_assets):
    print('Assets %s have dubious returns, removed' % bad_assets)

prices = prices.loc[:,~prices.columns.isin(bad_assets)]
sigmas = sigmas.loc[:,~sigmas.columns.isin(bad_assets)]
volumes = volumes.loc[:,~volumes.columns.isin(bad_assets)]
returns = returns.loc[:,~returns.columns.isin(bad_assets)]

# remove USDOLLAR
prices = prices.iloc[:,:-1]
sigmas = sigmas.iloc[:,:-1]
volumes = volumes.iloc[:,:-1]

# save data
prices.to_csv(datadir+'prices.csv.gz', compression='gzip', float_format='%.3f')
volumes.to_csv(datadir+'volumes.csv.gz', compression='gzip', float_format='%d')
returns.to_csv(datadir+'returns.csv.gz', compression='gzip', float_format='%.3e')
sigmas.to_csv(datadir+'sigmas.csv.gz', compression='gzip', float_format='%.3e')_____no_output_____

# Estimates _____no_output_____

print("typical variance of returns: %g"%returns.var().mean())_____no_output_____sigma2_n=0.02
sigma2_r=0.0005

np.random.seed(0)
noise=pd.DataFrame(index=returns.index, columns=returns.columns,
                   data=np.sqrt(sigma2_n)*np.random.randn(*returns.values.shape))
return_estimate= (returns + noise)*sigma2_r/(sigma2_r+sigma2_n)
return_estimate.USDOLLAR = returns.USDOLLAR

return_estimate.to_csv(datadir+'return_estimate.csv.gz', compression='gzip', float_format='%.3e')_____no_output_____agree_on_sign=np.sign(returns.ix[:,:-1]) == np.sign(return_estimate.ix[:,:-1])
print("Return predictions have the right sign %.1f%% of the times"%
      (100*agree_on_sign.sum().sum()/(agree_on_sign.shape[0]*(agree_on_sign.shape[1]-1))))_____no_output_____volume_estimate=volumes.rolling(window=10, center=False).mean().dropna()
volume_estimate.to_csv(datadir+'volume_estimate.csv.gz', compression='gzip', float_format='%d')
sigmas_estimate = sigmas.rolling(window=10, center=False).mean().dropna()
sigmas_estimate.to_csv(datadir+'sigma_estimate.csv.gz', compression='gzip', float_format='%.3e')_____no_output_____

# Risk model _____no_output_____

start_t="2012-01-01"
end_t="2016-12-31"

first_days_month=\
    pd.date_range(start=returns.index[next(i for (i,el) in enumerate(returns.index >= start_t) if el)-1],
                                 end=returns.index[-1], freq='MS')

k=15
exposures, factor_sigma, idyos = {}, {},{}
for day in first_days_month:
    used_returns = returns.loc[(returns.index < day)&
                        (returns.index >= day-pd.Timedelta("730 days"))]
    second_moment=used_returns.values.T@used_returns.values/used_returns.values.shape[0]
    eival,eivec=np.linalg.eigh(second_moment)
    factor_sigma[day]=np.diag(eival[-k:])
    exposures[day]=pd.DataFrame(data=eivec[:,-k:],index=returns.columns)
    idyos[day]=pd.Series(data=np.diag(eivec[:,:-k]@np.diag(eival[:-k])@eivec[:,:-k].T),index=returns.columns)

pd.Panel(exposures).swapaxes(1,2).to_hdf(datadir+'risk_model.h5', 'exposures')
pd.DataFrame(idyos).T.to_hdf(datadir+'risk_model.h5', 'idyos')
pd.Panel(factor_sigma).to_hdf(datadir+'risk_model.h5', 'factor_sigma')

# load with
# store = pd.HDFStore('risk_model.h5')
# store.exposures, etc._____no_output_____


# Notebook from czbiohub/scRFE
Path: scripts/practiceScripts/scRFEsourceV2.ipynb

# scRFE source V2 -- kidney split by age_____no_output_____

<code>
# cleaning up code after meeting with Sevahn_____no_output_____# all code broken up into separate arguments for scRFE_____no_output_____# Imports
import numpy as np
import pandas as pd
import scanpy as sc
from anndata import read_h5ad
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
from sklearn.feature_selection import SelectFromModel
from sklearn.metrics import accuracy_score
from sklearn.feature_selection import RFE
from sklearn.feature_selection import RFECV_____no_output_____# read in data
adata = read_h5ad('/Users/madelinepark/Downloads/Kidney_facs.h5ad')
tiss = adata_____no_output_____# split data for training
def loc_split_function(tiss, feature='age'):
    tiss.obs['feature_type_of_interest'] = 'rest'
    for c in list(set(tiss.obs[feature])):
        feature_of_interest = c
        tiss.obs.loc[tiss.obs[tiss.obs[feature] == feature_of_interest].index,'feature_type_of_interest'] = feature_of_interest
        feat_labels = tiss.var_names
        X = tiss.X
        y = tiss.obs['feature_type_of_interest']
    return X, y, feature, feat_labels #this is returning only the last thing in the loop _____no_output_____# create random forest and selector, then train
def train_function(X, y, test_size, random_state):
    print('training...')
    loc_split = loc_split_function(tiss=tiss, feature='age')
    X = loc_split[0]
    y = loc_split[1]
    feat_labels = loc_split[3] #this should not be hardcoded #genes be argument
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.05, random_state=0)
    clf = RandomForestClassifier(n_estimators=1000, random_state=0, n_jobs=-1, oob_score=True)
    selector = RFECV(clf, step=0.2, cv=3, n_jobs=4)
    clf.fit(X_train, y_train)
    selector.fit(X_train, y_train)
    feature_selected = feat_labels[selector.support_]
    return selector, clf, feat_labels, feature_selected, selector.estimator_.feature_importances_, X_train, X_test, y_train, y_test_____no_output_____cd /Users/madelinepark/src3/scRFE/scRFE-results/Users/madelinepark/src3/scRFE/scRFE-results
# result write
def result_write (c, feature_selected, tiss, feature='age',test_size=0.05, random_state=0):
    results_df = pd.DataFrame()
    print('result writing...')
    loc_split = loc_split_function(tiss=tiss, feature= feature)
    X = loc_split[0]
    y = loc_split[1]
    train = train_function(X, y, test_size, random_state)
    feat_labels = train[0]
    feature_selected = train[3]
    gini_scores = train[4]
    column_headings = []
    column_headings.append(c)
    column_headings.append(c + '_gini')
    resaux = pd.DataFrame(columns=column_headings)
    resaux[c] = feature_selected
    resaux[c + '_gini'] = (gini_scores)
    results_df = pd.concat([results_df,resaux],axis=1)
    tiss.obs['feature_type_of_interest'] = 'rest'
    file_name = feature + c + ".csv"
    print(results_df)
    results_df.to_csv(file_name) #trying to save results as a csv
    return results_df_____no_output_____# gini_3_m = result_write (c='3m', feature_selected, tiss, feature='age',test_size=0.05, random_state=0)
# gini_24_m = result_write (c='24m', feature_selected, tiss, feature='age',test_size=0.05, random_state=0)_____no_output_____# combined functions
def scRFE (tiss=tiss, X=tiss.X, feature='age', n_estimators=1000, random_state=0, n_jobs=-1, oob_score=True, test_size = 0.05, step=0.2, cv=5) :
    all_trees = []
    for c in list(set(tiss.obs[feature])):
        print(c)
        loc_split = loc_split_function(tiss=tiss, feature= feature)
        X = loc_split[0] #change age to feature
        y = loc_split[1]
        feature = loc_split[2]
        feat_labels = loc_split[3]
        train = train_function(X, y, test_size, random_state)
        feature_selected = train[1]
        X_train = train[2]
        X_test = train[3]
        y_train = train[4]
        y_test = train[5]
        all_trees += [result_write(c, feature_selected, tiss, feature=feature,test_size=0.05, random_state=0)]
        #returns data frames from SCRFE in order
        tiss.obs['age_type_of_interest'] = 'rest'
    return all_trees
#         results_df.to_csv('scRFE-results.csv') #trying to save results as a csv - this didnt work_____no_output_____scRFE(tiss=tiss, feature='age', n_estimators=10, random_state=0, n_jobs=-1, oob_score=True, test_size = 0.05, step=0.2, cv=5)_____no_output_____makeAllTrees = scRFE(tiss=tiss, feature='age', n_estimators=10, random_state=0, n_jobs=-1, oob_score=True, test_size = 0.05, step=0.2, cv=5)
# age3 = makeAllTrees[1]
# age24 = makeAllTrees[0]_____no_output_____# run function
scRFE(tiss=tiss, feature='age', n_estimators=1000, random_state=0, n_jobs=-1, oob_score=True, test_size = 0.05, step=0.2, cv=5)_____no_output_____# how do i get it to append after so the last dataframe doesnt completely overwrite the previous one_____no_output_____# # combined functions
# def scRFE_save (tiss=tiss, X=tiss.X, feature='age', n_estimators=1000, random_state=0, n_jobs=-1, oob_score=True, test_size = 0.05, step=0.2, cv=5) :
#     for c in list(set(tiss.obs[feature])):
#         print(c)
#         X = loc_split(tiss=tiss, feature= feature)[0] #change age to feature
#         y = loc_split(tiss=tiss, feature= feature)[1]
#         feature = loc_split(tiss=tiss, feature= feature)[2]
#         feat_labels = loc_split(tiss=tiss, feature= feature)[3]
#         feature_selected = train(X, y, test_size, random_state)[1]
#         X_train = train(X, y, test_size, random_state)[2]
#         X_test = train(X, y, test_size, random_state)[3]
#         y_train = train(X, y, test_size, random_state)[4]
#         y_test = train(X, y, test_size, random_state)[5]
#         result_write(c, feature_selected, tiss, feature=feature,test_size=0.05, random_state=0)
#         tiss.obs['age_type_of_interest'] = 'rest'
# #         not sorted_____no_output_____
</code>


# Notebook from nlihin/my-binder
Path: projects/projects_summer_2021/3_Linkedin.ipynb

# Final Project - Linkedin

## Introduction to Data Analytics 2021

### Presented by Racheli Hayat, Einav Diar, Adar Achwal and Sapir Hastar

### Group #3

#### Data source: https://www.kaggle.com/killbot/linkedin-profiles-and-jobs-data
#### Our GitHub: https://github.com/einavdiar/Linkedin_____no_output__________no_output_____## We've chosen to analyze information about Linkedin, an online social network designed to create professional and business connections between its users.
## During the project, we analyzed the network data in order to find the factors that help users reach efficient use of the network.
## As students and future engineers, over the next few years we will be looking for a job, therefore, it's important for us to learn about the best way to utalize this platform.

_____no_output__________no_output_____

import pandas as pd
import numpy as np
import datetime
import matplotlib.pyplot as plt
import matplotlib.colors as mcolors
import seaborn as sns_____no_output_____data = pd.read_csv('https://raw.githubusercontent.com/einavdiar/Linkedin/main/Linkedin.csv')_____no_output_____

# Wrangling data:
_____no_output_____## 1. Handling missing values:_____no_output_____

data=data.dropna(subset =['genderEstimate'])
data['hasPicture'].fillna('no picture', inplace = True)
data['companyHasLogo'].fillna('no logo', inplace = True)
data=data.dropna(subset = ['companyName'])_____no_output_____

#### 1) We removed missing data rows from "companyName" and "genderEstimate".
#### 2) From missing values under "hasPicture" and "companyHasLogo" we filled "no picture / logo"._____no_output_____

missing=data[['genderEstimate','hasPicture','companyHasLogo','companyName']].isnull().sum()
pd.DataFrame(missing)_____no_output_____

## 2. Fix columns:_____no_output_____

data[['genderEstimate','hasPicture','companyHasLogo','companyName','followersCount','ageEstimate']].info()<class 'pandas.core.frame.DataFrame'>
Int64Index: 35565 entries, 0 to 39536
Data columns (total 6 columns):
 #   Column          Non-Null Count  Dtype
---  ------          --------------  -----
 0   genderEstimate  35565 non-null  object
 1   hasPicture      35565 non-null  object
 2   companyHasLogo  35565 non-null  object
 3   companyName     35565 non-null  object
 4   followersCount  35565 non-null  int64
 5   ageEstimate     35565 non-null  int64
dtypes: int64(2), object(4)
memory usage: 1.9+ MB

#### All our Dtypes were correct.
## _____no_output_____# What do you think?
### Who is more likely to use linkedin?
### Male or Female?
_____no_output__________no_output_____

plt.figure(figsize=(15, 6))
plt.subplot(1,2,1)
plt.title("Male/Female Percentage",fontsize=25)
plotpie=data['genderEstimate'].value_counts().plot.pie(autopct='%1.1f%%',colors = ['mediumaquamarine', 'plum'],fontsize=15)
plt.legend(fontsize=15)
plt.subplot(1,2,2)
plt.title("Male/Female Count",fontsize=25)
plt.xlabel("Gender Estimate",fontsize=20)
plt.ylabel("Count",fontsize=20)
sns.countplot( x="genderEstimate",data=data , edgecolor = 'black', palette = 'PiYG_r', hue='genderEstimate')
plt.show()_____no_output_____

## On these charts we can see that there are more male users than females
## _____no_output_____## We would like to examine if there's a connection between Gender and Followers Count_____no_output_____

plt.figure(figsize=(20, 6))
type_df = data[["genderEstimate", "followersCount"]]
sns.catplot(data=type_df, kind="bar", x="genderEstimate", y = "followersCount",height=5,aspect=1,edgecolor = 'black',palette = 'PiYG_r')
plt.title("Male/Female Followers Count",fontsize=25)
plt.xlabel("Gender",fontsize=20)
plt.ylabel("Followers Count",fontsize=20)_____no_output_____

## Although the number of females is significantly lower than the number of males,
## Their average followers are close to the average followers of males.
## Despite their low number, we can  see that they have a high exposure and number of followers (almost the same as males).
## Therefore, it can be assumed that the LinkedIn network is recommended for females.

_____no_output_____# _____no_output_____## We would like to examine the connection between Logo, Picture and Followers Count_____no_output__________no_output_____

plt.figure(figsize=(10, 5))
data.loc[data['hasPicture'].str.contains('jpg'), 'hasPicture'] = 'has picture'
data.loc[data['hasPicture'].str.contains('A'), 'hasPicture'] = 'has picture'
data.loc[data['companyHasLogo'].str.contains('png'), 'companyHasLogo'] = 'has logo'
data.loc[data['companyHasLogo'].str.contains('jpg'), 'companyHasLogo'] = 'has logo'
data.loc[data['companyHasLogo'].str.contains('e'), 'companyHasLogo'] = 'has logo'
data.loc[data['companyHasLogo'].str.contains('A'), 'companyHasLogo'] = 'has logo'_____no_output_____

### To analyze the data, we replaced values under these columns _____no_output_____

plt.figure(figsize=(15, 2.4))
data.groupby(['companyHasLogo','hasPicture'])['followersCount'].count().plot.bar(edgecolor = 'black',color=['mediumvioletred','deeppink', 'hotpink', 'pink'])
plt.xticks(rotation=60,fontsize=12)
plt.title("Logo/Picture effect on Followers Count",fontsize=22)
plt.xlabel("Logo vs Picture",fontsize=15)
plt.ylabel("Followers Count",fontsize=15)_____no_output_____

## The graph clearly shows that users should upload both logo and image.
## A logo has a higher impact on the number of followers than an image, and it shows that an image without a logo is less effective.
## As you can see, users without logo and image have a significantly lower amount of followers._____no_output_____

plt.figure(figsize=(15, 5))
plt.subplot(1,2,1)
plt.title("Logo Percentage",fontsize=30)
plotpie=data['companyHasLogo'].value_counts().plot.pie(autopct='%1.2f%%',colors = ['orchid', 'lightpink'],fontsize=15)
plt.legend(fontsize=15)
plt.subplot(1,2,2)
plt.title("Pictures Percentage",fontsize=30)
plotpie=data['hasPicture'].value_counts().plot.pie(autopct='%1.2f%%',colors = ['orchid', 'lightpink'],fontsize=15)
plt.legend(fontsize=15)
plt.show()_____no_output_____

## As you can see, most of the LinkedIn users have logo and picture.
## _____no_output_____## We would like to examine the connection between Age and Followers Count_____no_output__________no_output_____

age = data['ageEstimate']
plt.figure(figsize=(10, 5))
plt.hist(age, bins = range(0, 100, 10) , edgecolor = 'black', color = "tab:purple")
plt.xticks(range(0, 100, 10))
plt.title("Followes Count by Age",fontsize=25)
plt.xlabel('Age Estimate',fontsize=20)
plt.ylabel('Followers Count',fontsize=20)_____no_output_____

## Base on the above graph, users between the ages of 30 to 50 have the highest number of followers.
## We can assume that the high number of followers for this age group is attributed to their  experience and seniority._____no_output_____

maxage=data.groupby('followersCount')[['ageEstimate']].max()
maxage.tail()_____no_output_____

## As you can see, the LinkedIn users with the highest number of followers are in the 30-50 age group._____no_output_____## _____no_output_____## In conclusion, in our research we wanted to examine the dependency between the following variables : gender, logo & image, age and followers count.

## Our dependent variable is the followers count, and our independent variables are gender, logo & image and age._____no_output__________no_output_____
## We found that the most useful ways to use LinkedIn is:
## 1. Although the number of females using the network is lower then the number of males, the  followers count percent was almost the same for both.
## 2. Using image is importand for user exposure. However, including a logo has a greater impact.
## 3. The users with the highest number of followers are in the age group of 30-50 ._____no_output_____

maxage=data.groupby('followersCount')[['genderEstimate','hasPicture','companyHasLogo','ageEstimate']].max()
maxage.tail()_____no_output_____

## As you can see, the findings from the table suitable to our expectations.
_____no_output__________no_output_____


# Notebook from StevenPZChan/ml_dl_coursera_Andrew_Ng
Path: machine-learning-python/machine-learning-ex8/ex8.ipynb

## Exercise 8 | Anomaly Detection and Collaborative Filtering
### ================== Part 1: Load Example Dataset  ===================_____no_output_____

from ex8 import *
%matplotlib inline
print('Visualizing example dataset for outlier detection.\n')

#  The following command loads the dataset. You should now have the
#  variables X, Xval, yval in your environment
from scipy import io as sio
data = sio.loadmat('ex8data1.mat')
X = data['X']
Xval = data['Xval']
yval = data['yval'].reshape(-1)

#  Visualize the example dataset
plt.plot(X[:, 0], X[:, 1], 'bx')
plt.axis([0, 30, 0, 30])
plt.xlabel('Latency (ms)')
plt.ylabel('Throughput (mb/s)')
plt.show()Visualizing example dataset for outlier detection.

### ================== Part 2: Estimate the dataset statistics ===================
$$\mu_j = \frac{1}{m}\sum_{i=1}^{m}{x_j^{(i)}}$$
$$\sigma_j^2 = \frac{1}{m}\sum_{i=1}^{m}{\left(x_j^{(i)} - \mu_j\right)^2}$$
$$p(x;\mu,\sigma^2) = \frac{1}{\sqrt{2\pi}\sigma}e^{-\frac{(x - \mu)^2}{2\sigma^2}}$$_____no_output_____

print('Visualizing Gaussian fit.\n')

#  Estimate my and sigma2
mu, sigma2 = estimateGaussian(X)

#  Returns the density of the multivariate normal at each data point (row)
#  of X
p = multivariateGaussian(X, mu, sigma2)

#  Visualize the fit
visualizeFit(X,  mu, sigma2)
plt.xlabel('Latency (ms)')
plt.ylabel('Throughput (mb/s)')
plt.show()Visualizing Gaussian fit.

### ================== Part 3: Find Outliers ===================_____no_output_____

pval = multivariateGaussian(Xval, mu, sigma2)

epsilon, F1 = selectThreshold(yval, pval)
print(f'Best epsilon found using cross-validation: {epsilon:e}')
print(f'Best F1 on Cross Validation Set:  {F1:f}')
print('   (you should see a value epsilon of about 8.99e-05)')
print('   (you should see a Best F1 value of  0.875000)\n')

#  Find the outliers in the training set and plot the
outliers = p < epsilon

#  Draw a red circle around those outliers
plt.figure()
visualizeFit(X,  mu, sigma2)
plt.xlabel('Latency (ms)')
plt.ylabel('Throughput (mb/s)')
plt.plot(X[outliers, 0], X[outliers, 1], 'ro', linewidth=2, markersize=10)
plt.show()Best epsilon found using cross-validation: 8.990853e-05
Best F1 on Cross Validation Set:  0.875000
   (you should see a value epsilon of about 8.99e-05)
   (you should see a Best F1 value of  0.875000)

### ================== Part 4: Multidimensional Outliers ===================_____no_output_____

#  Loads the second dataset. You should now have the
#  variables X, Xval, yval in your environment
data = sio.loadmat('ex8data2.mat')
X = data['X']
Xval = data['Xval']
yval = data['yval'].reshape(-1)

#  Apply the same steps to the larger dataset
mu, sigma2 = estimateGaussian(X)

#  Training set
p = multivariateGaussian(X, mu, sigma2)

#  Cross-validation set
pval = multivariateGaussian(Xval, mu, sigma2)

#  Find the best threshold
epsilon, F1 = selectThreshold(yval, pval)

print(f'Best epsilon found using cross-validation: {epsilon:e}')
print(f'Best F1 on Cross Validation Set:  {F1:f}')
print('   (you should see a value epsilon of about 1.38e-18)')
print('   (you should see a Best F1 value of 0.615385)')
print(f'# Outliers found: {(p < epsilon).sum()}')Best epsilon found using cross-validation: 1.377229e-18
Best F1 on Cross Validation Set:  0.615385
   (you should see a value epsilon of about 1.38e-18)
   (you should see a Best F1 value of 0.615385)
# Outliers found: 117

###### 以上部分代码在[ex8.py](https://github.com/StevenPZChan/ml_dl_coursera_Andrew_Ng/blob/master/machine-learning-python/machine-learning-ex8/ex8.py)中_____no_output_____### =============== Part 1: Loading movie ratings dataset ================_____no_output_____

from ex8_cofi import *
print('Loading movie ratings dataset.\n')

#  Load data
from scipy import io as sio
data = sio.loadmat('ex8_movies.mat')
Y = data['Y']
R = data['R']

#  Y is a 1682x943 matrix, containing ratings (1-5) of 1682 movies on
#  943 users
#
#  R is a 1682x943 matrix, where R(i,j) = 1 if and only if user j gave a
#  rating to movie i

#  From the matrix, we can compute statistics like average rating.
print(f'Average rating for movie 1 (Toy Story): {Y[0, R[0, :].astype(bool)].mean():f} / 5\n')

#  We can "visualize" the ratings matrix by plotting it with imagesc
plt.imshow(Y)
plt.ylabel('Movies')
plt.xlabel('Users')
plt.show()Loading movie ratings dataset.

Average rating for movie 1 (Toy Story): 3.878319 / 5

### ============ Part 2: Collaborative Filtering Cost Function ===========
$$J(x^{(1)},\ldots,x^{(n_m)},\theta^{(1)},\ldots,\theta^{(n_u)}) = \frac{1}{2}\sum_{(i,j):r(i,j)=1}{\left((\theta^{(j)})^Tx^{(i)} - y^{(i,j)}\right)^2 + \frac{\lambda}{2}\sum_{i=1}^{n_m}\sum_{k=1}^{n}{\left(x_k^{(i)}\right)^2} + \frac{\lambda}{2}\sum_{j=1}^{n_u}\sum_{k=1}^{n}{\left(\theta_k^{(i)}\right)^2}}$$_____no_output_____

#  Load pre-trained weights (X, Theta, num_users, num_movies, num_features)
data = sio.loadmat('ex8_movieParams.mat')
X = data['X']
Theta = data['Theta']
num_users = np.asscalar(data['num_users'])
num_movies = np.asscalar(data['num_movies'])
num_features = np.asscalar(data['num_features'])

#  Reduce the data set size so that this runs faster
num_users = 4
num_movies = 5
num_features = 3
X = X[:num_movies, :num_features]
Theta = Theta[:num_users, :num_features]
Y = Y[:num_movies, :num_users]
R = R[:num_movies, :num_users]

#  Evaluate cost function
J, _ = cofiCostFunc(np.concatenate([X.reshape(-1), Theta.reshape(-1)]), Y, R, num_users, num_movies, num_features, 0)

print(f'Cost at loaded parameters: {J:f} '
         '\n(this value should be about 22.22)')Cost at loaded parameters: 22.224604
(this value should be about 22.22)

### ============== Part 3: Collaborative Filtering Gradient ==============
$$\frac{\partial J}{\partial x_k^{(i)}} = \sum_{j:r(i,j)=1}{\left((\theta^{(j)})^Tx^{(i)} - y^{(i,j)}\right)\theta_k^{(j)} + \lambda x_k^{(i)}}$$
$$\frac{\partial J}{\partial\theta_k^{(j)}} = \sum_{i:r(i,j)=1}{\left((\theta^{(j)})^Tx^{(i)} - y^{(i,j)}\right)x_k^{(i)} + \lambda\theta_k^{(j)}}$$_____no_output_____

print('Checking Gradients (without regularization) ... ')

#  Check gradients by running checkNNGradients
checkCostFunction()Checking Gradients (without regularization) ...
[[ -4.007977  -4.007977]
 [  2.710351   2.710351]
 [  2.65428    2.65428 ]
 [ -0.049014  -0.049014]
 [  0.11209    0.11209 ]
 [  0.07776    0.07776 ]
 [  0.564534   0.564534]
 [-11.096384 -11.096384]
 [  4.779287   4.779287]
 [  0.992009   0.992009]
 [  0.092185   0.092185]
 [ -0.35919   -0.35919 ]
 [ -4.005086  -4.005086]
 [  6.281983   6.281983]
 [ -3.042467  -3.042467]
 [  1.147838   1.147838]
 [ -1.993641  -1.993641]
 [ -0.158091  -0.158091]
 [  2.454774   2.454774]
 [ -3.483138  -3.483138]
 [ -3.698394  -3.698394]
 [  3.871281   3.871281]
 [ -5.986774  -5.986774]
 [  5.496491   5.496491]
 [  0.033889   0.033889]
 [ -0.246756  -0.246756]
 [  0.509833   0.509833]]
['The above two columns you get should be very similar.\n(Left-Your Numerical Gradient, Right-Analytical Gradient)']
If your cost function implementation is correct, then
the relative difference will be small (less than 1e-9).

Relative Difference: 1.9003e-12

### ========= Part 4: Collaborative Filtering Cost Regularization ========_____no_output_____

#  Evaluate cost function
J, _ = cofiCostFunc(np.concatenate([X.reshape(-1), Theta.reshape(-1)]), Y, R, num_users, num_movies, num_features, 1.5)

print(f'Cost at loaded parameters (lambda = 1.5): {J:f} '
         '\n(this value should be about 31.34)')Cost at loaded parameters (lambda = 1.5): 31.344056
(this value should be about 31.34)

### ======= Part 5: Collaborative Filtering Gradient Regularization ======_____no_output_____

print('Checking Gradients (with regularization) ... ')

#  Check gradients by running checkNNGradients
checkCostFunction(1.5)Checking Gradients (with regularization) ...
[[-4.619530e-01 -4.619530e-01]
 [-1.050700e+00 -1.050700e+00]
 [ 1.612006e+01  1.612006e+01]
 [ 3.154325e-01  3.154325e-01]
 [ 2.677497e+00  2.677497e+00]
 [ 5.077390e+00  5.077390e+00]
 [-8.580218e-01 -8.580218e-01]
 [ 2.552349e+00  2.552349e+00]
 [ 3.537703e+00  3.537703e+00]
 [ 2.066214e+00  2.066214e+00]
 [-3.677363e+00 -3.677363e+00]
 [ 1.642278e+01  1.642278e+01]
 [-4.046519e-01 -4.046519e-01]
 [-3.083852e+00 -3.083852e+00]
 [-4.005439e+00 -4.005439e+00]
 [-3.797228e+00 -3.797228e+00]
 [ 2.835593e+00  2.835593e+00]
 [-1.318965e+01 -1.318965e+01]
 [-2.383072e+00 -2.383072e+00]
 [ 9.827656e-03  9.827656e-03]
 [-3.979154e+00 -3.979154e+00]
 [ 8.684664e-01  8.684664e-01]
 [ 2.169383e+00  2.169383e+00]
 [ 2.258721e+00  2.258721e+00]
 [-8.594728e-01 -8.594728e-01]
 [ 3.472921e+00  3.472921e+00]
 [ 1.809470e+00  1.809470e+00]]
['The above two columns you get should be very similar.\n(Left-Your Numerical Gradient, Right-Analytical Gradient)']
If your cost function implementation is correct, then
the relative difference will be small (less than 1e-9).

Relative Difference: 2.13093e-12

### ============== Part 6: Entering ratings for a new user ===============_____no_output_____

movieList = loadMovieList()

#  Initialize my ratings
my_ratings = np.zeros(1682)

# Check the file movie_idx.txt for id of each movie in our dataset
# For example, Toy Story (1995) has ID 1, so to rate it "4", you can set
my_ratings[0] = 4

# Or suppose did not enjoy Silence of the Lambs (1991), you can set
my_ratings[97] = 2

# We have selected a few movies we liked / did not like and the ratings we
# gave are as follows:
my_ratings[6] = 3
my_ratings[11] = 5
my_ratings[53] = 4
my_ratings[63] = 5
my_ratings[65] = 3
my_ratings[68] = 5
my_ratings[182] = 4
my_ratings[225] = 5
my_ratings[354] = 5

print('New user ratings:')
for i in range(len(my_ratings)):
    if my_ratings[i] > 0:
        print(f'Rated {my_ratings[i]:g} for {movieList[i]}')New user ratings:
Rated 4 for Toy Story (1995)
Rated 3 for Twelve Monkeys (1995)
Rated 5 for Usual Suspects, The (1995)
Rated 4 for Outbreak (1995)
Rated 5 for Shawshank Redemption, The (1994)
Rated 3 for While You Were Sleeping (1995)
Rated 5 for Forrest Gump (1994)
Rated 2 for Silence of the Lambs, The (1991)
Rated 4 for Alien (1979)
Rated 5 for Die Hard 2 (1990)
Rated 5 for Sphere (1998)

### ================== Part 7: Learning Movie Ratings ====================_____no_output_____

print('Training collaborative filtering...')

#  Load data
data = sio.loadmat('ex8_movies.mat')
Y = data['Y']
R = data['R']

#  Y is a 1682x943 matrix, containing ratings (1-5) of 1682 movies by
#  943 users
#
#  R is a 1682x943 matrix, where R(i,j) = 1 if and only if user j gave a
#  rating to movie i

#  Add our own ratings to the data matrix
Y = np.column_stack([my_ratings, Y])
R = np.column_stack([(my_ratings != 0).astype(int), R])

#  Normalize Ratings
Ynorm, Ymean = normalizeRatings(Y, R)

#  Useful Values
num_users = Y.shape[1]
num_movies = Y.shape[0]
num_features = 10

# Set Initial Parameters (Theta, X)
X = np.random.randn(num_movies, num_features)
Theta = np.random.randn(num_users, num_features)

initial_parameters = np.concatenate([X.reshape(-1), Theta.reshape(-1)])

# Set options for fmincg
options = {'disp': True, 'maxiter': None}

# Set Regularization
from scipy import optimize as opt
lambda_ = 10
res = opt.minimize(lambda t: cofiCostFunc(t, Ynorm, R, num_users, num_movies, num_features, lambda_)[0], initial_parameters,
                   method='CG', jac=lambda t: cofiCostFunc(t, Ynorm, R, num_users, num_movies, num_features, lambda_)[1], options=options)
theta = res.x

# Unfold the returned theta back into U and W
X = theta[:num_movies * num_features].reshape((num_movies, num_features))
Theta = theta[num_movies * num_features:].reshape((num_users, num_features))

print('Recommender system learning completed.')Training collaborative filtering...
Warning: Desired error not necessarily achieved due to precision loss.
         Current function value: 38951.847560
         Iterations: 371
         Function evaluations: 555
         Gradient evaluations: 554
Recommender system learning completed.

### ================== Part 8: Recommendation for you ====================_____no_output_____

p = np.matmul(X, Theta.transpose())
my_predictions = p[:, 0] + Ymean

movieList = loadMovieList()

ix = my_predictions.argsort()[::-1]
print('Top recommendations for you:')
for i in range(10):
    j = ix[i]
    print(f'Predicting rating {my_predictions[j]:.1f} for movie {movieList[j]}')

print('\n\nOriginal ratings provided:')
for i in range(len(my_ratings)):
    if my_ratings[i] > 0:
        print(f'Rated {my_ratings[i]:g} for {movieList[i]}')Top recommendations for you:
Predicting rating 5.0 for movie Prefontaine (1997)
Predicting rating 5.0 for movie Marlene Dietrich: Shadow and Light (1996)
Predicting rating 5.0 for movie Saint of Fort Washington, The (1993)
Predicting rating 5.0 for movie Entertaining Angels: The Dorothy Day Story (1996)
Predicting rating 5.0 for movie Star Kid (1997)
Predicting rating 5.0 for movie Santa with Muscles (1996)
Predicting rating 5.0 for movie They Made Me a Criminal (1939)
Predicting rating 5.0 for movie Great Day in Harlem, A (1994)
Predicting rating 5.0 for movie Someone Else's America (1995)
Predicting rating 5.0 for movie Aiqing wansui (1994)

Original ratings provided:
Rated 4 for Toy Story (1995)
Rated 3 for Twelve Monkeys (1995)
Rated 5 for Usual Suspects, The (1995)
Rated 4 for Outbreak (1995)
Rated 5 for Shawshank Redemption, The (1994)
Rated 3 for While You Were Sleeping (1995)
Rated 5 for Forrest Gump (1994)
Rated 2 for Silence of the Lambs, The (1991)
Rated 4 for Alien (1979)
Rated 5 for Die Hard 2 (1990)
Rated 5 for Sphere (1998)

跟说明文档里面的结果不太一样，估计是因为数据太少吧。
###### 以上部分代码在[ex8_cofi.py](https://github.com/StevenPZChan/ml_dl_coursera_Andrew_Ng/blob/master/machine-learning-python/machine-learning-ex8/ex8_cofi.py)中_____no_output_____### 总结：矩阵范数需要用到`numpy.linalg.norm`，求逆是`numpy.linalg.pinv`，电影列表的文本文档居然要用`ISO-8859-1`编码才能正确读取。
---
### 机器学习的8次作业完结撒花_____no_output_____