# Notebook from ctwardy/jacobs-vault
Path: nbs/05_HitTest.ipynb

# default_exp hit_test
%load_ext autoreload
%autoreload 2The autoreload extension is already loaded. To reload it, use:
  %reload_ext autoreload

# HitTest

> VAULT Proposal python module for addressing Technical Scenario objective 1_____no_output_____Objective 1:
> 1) Determine the “hits” where a satellite has geodetic overlap of any vessel(s) at any point(s) in time. For simplicity, it may be assumed that a satellite has full view of half the earth (regardless of satellite type or its elevation above the earth). However, additional accuracy models with rationale is allowed.

A satellite can see the ship only if the ship can see the satellite. So instead of the half-earth assumption, we define a "hit" as the satellite being above the horizon (defined as "alt" ≥0º in a ship-based alt-azimuth coordinate system).

According to [exactEarth](http://www.marinetraffic.org/exactearth/), a more realistic approximation is that AIS satellites at 650 km have a 5000 km diameter Field of View. That corresponds to a horizon of about 𝜃=14.6º, as shown here:

                             🛰
                             : `
                             :     `
                         650 :         `
                          km :             `
                             :                 `
                             :    2500km        𝜃  `
                             .........................`🛳_____no_output_____

import math
𝜃 = math.degrees(math.atan(650/2500))
print(f'{𝜃:.1f}')14.6
# export

# Requires modules in ../jacobs_vault be available. Either:
#    ln -s nbs/jacobs_vault -> jacobs_vault
# or:
#    in each notebook `import sys; sys.path.append('..')`
# or:
#    add .. to PYTHONPATH.

from jacobs_vault.starmap import starmap  # Plotting fn

from datetime import datetime
from dateutil import tz
from skyfield.api import EarthSatellite
from skyfield.api import Topos, load
import math
import pandas as pd
import numpy as np_____no_output_____# hide
#from nbdev.showdoc import *_____no_output_____

### Define Schema and Path

Save memory by using appropriate data types for the TLE elements. (Determined by peeking at the data file or reading the TLE format.) This saves quite a bit, and may save more as Pandas optimizes the `string` class vs. default `object`. Regardless, it makes columns single-typed._____no_output_____

#export
COLUMNS = ["satellite", "day_dt", "day", "tle_dt", "tle_ts", "line1", "line2"]
# DTYPES = [str, str, int, str, int, str, str]
DTYPES = {'satellite': 'uint16', # observed values are ints in 5..41678, so 0..65535 is good
          'day_dt': 'str',       # here a single date, but generally datetime: PARSE
          'day': 'uint16',       # here a single value 6026, too big for uint8, but 16 is good
          'tle_dt': 'str',       # again, PARSE AS DATETIME
          'tle_ts': 'uint32',    # large ints, but < 4294967295. We could compress more, but... meh
          'line1': 'string',     # 12K unique 80-char TLE strings. Category wd give tiny compression.
          'line2': 'string'}     # In theory "string" is better than "object". Not seeing it here.

DATE_COLS = ['day_dt', 'tle_dt']

# Where to look for the TLE dayfiles.
# Symlink ../data to the actual data.
DAY_FILE_PATH="../data/VAULT_Data/TLE_daily"
_____no_output_____

### Quality Values
_____no_output_____

# export

# Set horizon in degrees. Suggested: 0º or 14.6º.
HORIZON = 14.6

# Define cutoffs for TLE track quality, as TLE age in days
EXCELLENT, GOOD, POOR = 2, 14, 56

def get_qvals(𝚫t: int, alt: float, 𝜃:float=HORIZON):
    """Get quality vals for raw 𝚫t [days].

    Returns Series (alt, az, 𝚫t) in units (º,º, days)

    Params
    ------
        𝚫t - age of TLE in days. Int or flot.
        alt - altitude above horizon
        𝜃 - minimum alt in degrees to count as a hit (Default HORIZON)

    """

    if 𝚫t <= EXCELLENT:
        if alt.degrees > 𝜃:
            qvals = ["Excellent", math.nan]
        else:
            qvals = [math.nan, "Excellent"]
    elif 𝚫t <= GOOD:
        if alt.degrees > 0.0:
            qvals = ["Good", math.nan]
        else:
            qvals = [math.nan, "Good"]
    elif 𝚫t <= POOR:
        if alt.degrees > 0.0:
            qvals = ["Poor", math.nan]
        else:
            qvals = [math.nan, "Poor"]
    else:
        qvals = [math.nan, "Stale"]

    return qvals

#_____no_output_____

### HitTest Class

During data posturing we created single-day TLE files, so each day contains the most recent TLE for all 12K satellites.

The `HitTest` constructor takes a date, loads the TLE dayfile, and returns the corresponding DataFrame.

_TODO_: Most of these satellites are useless. Speed by ignoring._____no_output_____

#export

class HitTest:
    """ Counts the satellites that are visible at a given point on the globe at a
    given time, and returns counts classified by data quality and
    latitude, azimuth, hit_quality, radius for visible satellites
    """
    def __init__(self, dt:datetime,
                 day_file_base_path:str=DAY_FILE_PATH,
                 𝜃:float=HORIZON):
        """Look for and load TLE datafile for date {dt}."""

        df_path = "%s/%4d/%02d/%02d.tab.gz" % (day_file_base_path, dt.year, dt.month, dt.day)
        print(f"Trying to load {df_path}")
        df = pd.read_csv(df_path,
                         names=COLUMNS, sep='\t', compression='gzip',
                         dtype=DTYPES,
                         parse_dates=DATE_COLS,
                         infer_datetime_format=True)
        self.df_day_tle = df.drop_duplicates()
        self.𝜃 = 𝜃
        self.dt = dt
    #

    def satellite_alt_az_days(self, _t0: datetime, lat: float, lon: float):
        '''Load tracks for day {_t0} and return altº, azº, and 𝚫t [days]
        for each row.

        Usage eg: satellite_alt_az_days(datetime(2016, 6, 30), 45.0, -176.0)

        '''
        earth_position = Topos(lat, lon)

        ts = load.timescale()
        t = ts.utc(_t0.replace(tzinfo=tz.tzutc()))

        def eval_tle(row):
            '''Extract satellite info from line1/line2/tle_dt.

            Returns alt, az, and (days between dt and each row).
            Inherits {ts}, {t}, and {earth_position} values at function definition.

            TODO: Currently only works for `apply(raw=False)`.

            '''
            try:
                satellite = EarthSatellite(row['line1'], row['line2'], 'x', ts)
                𝚫t = abs(_t0 - row['tle_dt']).days
            except IndexError:
                # `apply(raw=True)` sends arrays instead of Series
                satellite = EarthSatellite(row[5], row[6], 'x', ts)
                𝚫t = abs(_t0 - row[3]).days
            topocentric = (satellite - earth_position).at(t)
            alt, az, distance = topocentric.altaz()
            qvals = get_qvals(𝚫t, alt)
            return pd.Series([alt.degrees, az.degrees, 𝚫t] + qvals)

        _ = self.df_day_tle.apply(eval_tle, axis=1, raw=False)
        df_alt_az_days = pd.DataFrame(_)
        df_alt_az_days.columns = ["altitude", "azimuth", "days", "hit", "miss"]
        #df_alt_az_days.reindex()
        return df_alt_az_days

    def invoke(self, dt: datetime, lat: float, lon: float):
        ''' Main logic for satellite hit-testing service

            Returns 2 DataFrames:
             - df_hit_miss_table :       The hit,miss stats table
             - df_alt_az_days_visible :  The information on the visible satellites for star-map plotting

        '''
        df_alt_az_days = self.satellite_alt_az_days(dt, lat, lon)

        # "invert" altitude for polar plotting.  Doing this thousands of times
        #  more than necessary (really just want R for the df_alt_az_days_visible slice)
        #  but pandas does not like apply on a slice.
        df_alt_az_days.loc["R"] = 90.0 - df_alt_az_days["altitude"]

        def apply_quality_str(row, col):
            q = ""
            if row[col] == Q_EXCELLENT:
                q = "Excellent"
            elif row[col] == Q_GOOD:
                q = "Good"
            elif row[col] == Q_POOR:
                q = "Poor"
            elif row[col] == Q_STALE:
                q = "Stale"
            # no-else ... leave the NaNs alone
            return q
        #

        df_hit_miss_table = pd.concat([
            df_alt_az_days["hit"].value_counts(sort=False),
            df_alt_az_days["miss"].value_counts()],
            axis=1, sort=False)

        df_alt_az_days_visible = df_alt_az_days[df_alt_az_days["altitude"] > self.𝜃]

        return df_hit_miss_table, df_alt_az_days_visible
    #

    def web_invoke(self, dt, lat, lon):
        ''' Main support function for satellite hit-testing service

            returns a json object having two objects:
            {
                "hitmiss": The hit,miss stats table
                "visible": The information on the visible satellites
            }
        '''
        df_hit_miss_table, df_alt_az_days_visible = self.invoke(dt, lat, lon)
        result = {
            "hitmiss": df_hit_miss_table.to_dict(),
            "visible": df_alt_az_days_visible.to_dict()
        }
        return json.dumps(result)
    #_____no_output_____

### Execute for a given day

2016-06-30 for starters_____no_output_____

dt = datetime(2016, 6, 30)
ht = HitTest(dt)
hitmiss, rows = ht.invoke(dt, 45.0, -176.0)Trying to load ../data/VAULT_Data/TLE_daily/2016/06/30.tab.gz
hitmiss_____no_output_____rows.sample(5)_____no_output_____

## Visualize the results

Generate a polar alt/az plot of the qualifying satellites
* Excellent = blue
* Good = red
* Else = yellow

**TODO:** Is "0" here 0 altitude? That would be on the horizon, which is counter-intuitive.

Note the band of satellites at southern bearings -- this ship was in the Northern hemisphere. _____no_output_____

fig = starmap(rows)_____no_output_____fig.write_image('images/starmap_new.pdf')_____no_output_____


# Notebook from younthu/photomanager
Path: notebooks/FileUtils.ipynb

<code>
import os
from pathlib import Path
import socket
import getpass
from collections import namedtuple
import time
import datetime

File = namedtuple('File', ['folder','name', 'type','created_at','modified_at','size','md5'])_____no_output_____print(os.environ['HOME'])
print(os.path.expandvars('$HOME'))
print(os.path.expanduser('~'))
print(os.path.expanduser('~/Documents'))/Users/i532140
/Users/i532140
/Users/i532140
/Users/i532140/Documents
user_name = getpass.getuser()  #用户名
hostname = socket.gethostname() #主机名

print(type(user_name))
print(hostname)
print(user_name)<class 'str'>
C02C22CTMD6R
i532140
#'''把时间戳转化为时间: 1479264792 to 2016-11-16 10:53:12'''
def TimeStampToTime(timestamp):
  timeStruct = time.localtime(timestamp)
  return time.strftime('%Y-%m-%d %H:%M:%S',timeStruct)

'''获取文件的大小,结果保留两位小数，单位为MB'''
def get_FileSize(filePath):
  fsize = os.path.getsize(filePath)
  fsize = fsize/float(1024*1024)
  return round(fsize,2)

'''获取文件的访问时间'''
def get_FileAccessTime(filePath):
  t = os.path.getatime(filePath)
  return TimeStampToTime(t)

'''获取文件的创建时间'''
def get_FileCreateTime(filePath):
  t = os.path.getctime(filePath)
  return TimeStampToTime(t)

'''获取文件的修改时间'''
def get_FileModifyTime(filePath):
  t = os.path.getmtime(filePath)
  return TimeStampToTime(t)

'''获取文件所在目录'''
def get_FileFolder(filePath):
    return os.path.dirname(filePath)

'''文件名'''
def get_FileName(filePath):
    return Path(filePath).name

'''MD5'''
def get_MD5(filePath):
    return _____no_output_____def get_file_attributes(path):
    file = File(folder=get_FileFolder(path), name=get_FileName(path), created_at=get_FileCreateTime(path),
               size=get_FileSize(path), type='', modified_at='', md5='')
    return file
def get_all_files_in_folder(folder):
    ret = []
    for current_folder, children_folders, files in os.walk(folder):
        for file in files:
            ret.append(get_file_attributes(current_folder+file))
    return c
def get_file_count_of_folder(folder):
    c = 0
    for current_folder, children_folders, files in os.walk(folder):
        print(current_folder, children_folders, files)
        c+=1
    return c_____no_output_____p = Path(".")
print([x for x in p.glob("**/*")])[PosixPath('FileUtils.ipynb'), PosixPath('.ipynb_checkpoints'), PosixPath('.ipynb_checkpoints/FileUtils-checkpoint.ipynb')]
print(get_file_count_of_folder(os.path.expanduser('~/Documents/[fuliba.net]U选1000')))/Users/i532140/Documents/[fuliba.net]U选1000 [] ['独角戏 - 许茹芸.mp3', '最近 - 李圣杰.mp3', '木乃伊 - 林俊杰.mp3', '私奔到月球 - 五月天&陈绮贞.mp3', '太委屈 - 陶晶莹.mp3', '执迷不悔 (粤语) - 王菲.mp3', '一人一半 - 伍家辉.mp3', '笑红尘 - 陈淑桦.mp3', '其实 - 薛之谦.mp3', '雪落下的声音 - 陆虎.mp3', '比我幸福 - 陈晓东.mp3', '天灰 - S.H.E.mp3', '你的眼睛背叛你的心 - 郑中基.mp3', '红尘有你 - 王杰.mp3', '往事随风 - 齐秦.mp3', "Men's Talk - 张清芳.mp3", '安静 - 周杰伦.mp3', '天高地厚 - 信乐团.mp3', '丝路 - 梁静茹.mp3', '做你的男人 - 张信哲.mp3', '恋上一个人 - 游鸿明.mp3', '醉赤壁 - 林俊杰.mp3', '我们 - 陈奕迅.mp3', '爱上一个不回家的人 - 林忆莲.mp3', '五月天 - S.H.E.mp3', '后来 - 刘若英.mp3', '暧昧 - 薛之谦.mp3', '匆匆那年 - 王菲.mp3', '无情的雨 无情的你 - 齐秦.mp3', '我很想爱他 - 林俊杰.mp3', '神啊 救救我 - 陈小春.mp3', '真心英雄 - 周华健&成龙&李宗盛.mp3', '说好不哭 - 周杰伦&五月天阿信.mp3', '握你的手 - 光良.mp3', '挥着翅膀的女孩 - 容祖儿.mp3', '举棋不定 - 李骥&张清芳.mp3', '烟火里的尘埃 - 华晨宇.mp3', '家家酒 - 家家.mp3', '哭砂 - 黄莺莺.mp3', '爱情傀儡 - 巫启贤.mp3', '十万毫升泪水 - 蔡健雅.mp3', '笑忘书 - 王菲.mp3', 'I BELIEVE - 范逸臣.mp3', '分享 - 伍思凯.mp3', '对你爱不完 - 郭富城.mp3', '步步 - 五月天.mp3', '我以为 - 品冠.mp3', '恋爱ing - 五月天.mp3', '唯一 - 王力宏.mp3', '用情 - 张信哲.mp3', '伴 - 黄小琥.mp3', '爱转角 - 罗志祥.mp3', '快乐崇拜 - 潘玮柏&张韶涵.mp3', '回忆的沙漏 (10周年版) - G.E.M. 邓紫棋.mp3', '大龄女子 - 彭佳慧.mp3', '气球 - 许哲珮.mp3', '好好 - 五月天.mp3', '爱的就是你 - 王力宏.mp3', '爱很简单 - 陶喆.mp3', '短发 - 梁咏琪.mp3', '城里的月光 - 许美静.mp3', '毕竟深爱过 - 六哲.mp3', '凡人歌 - 李宗盛.mp3', '潇洒走一回 - 叶蒨文.mp3', '不醉不会 - 田馥甄.mp3', '好想好想 - 古巨基.mp3', '哭过就好了 - 梁文音.mp3', '梦里花 - 张韶涵.mp3', '咖啡在等一个人 - 周慧敏.mp3', '关键词 - 林俊杰.mp3', '第一次 - 光良.mp3', '开不了口 - 周杰伦.mp3', '忘情森巴舞 - 草蜢.mp3', '这样爱你对不对 - 陈淑桦.mp3', '另一个天堂 - 王力宏&张靓颖.mp3', '爱就一个字 - 张信哲.mp3', '爱情字典 - 孙燕姿.mp3', '她说 - 林俊杰.mp3', '流言 - 周慧敏&林隆璇.mp3', '伤心的人更伤心 - 巫启贤.mp3', '我知道你很难过 - 蔡依林.mp3', '天天夜夜 - F.I.R..mp3', '对的人 - 戴爱玲.mp3', '想见你 - 无印良品.mp3', '少年 - 光良&曹格.mp3', '走在红毯那一天 - 彭佳慧.mp3', '超人 - 五月天.mp3', '转身之后 (国语) - 毕书尽.mp3', '爱得正好 - 苏永康&陈洁仪.mp3', '三生三世 - 张杰.mp3', '还是会 - 韦礼安.mp3', '青梅竹马 - 周治平.mp3', '看见什么吃什么 - 林宥嘉.mp3', '淋雨一直走 - 张韶涵.mp3', '戒情人 - 郑中基.mp3', '谢谢你的爱 - 刘德华.mp3', '我等到花儿也谢了 (国语) - 张学友.mp3', '别问我是谁 - 王馨平.mp3', '最浪漫的事 - 赵咏华.mp3', '剪爱 - 张惠妹.mp3', '问 - 陈淑桦.mp3', '姐妹 - 张惠妹.mp3', '怎么说我不爱你 - 萧敬腾.mp3', '不让我的眼泪陪我过夜 - 齐秦.mp3', '平凡的一天 - 毛不易.mp3', '少年 - 周华健.mp3', '夜太黑 - 林忆莲.mp3', '是你变了吗 - 无印良品.mp3', '我难过 - 5566.mp3', '隐形的翅膀 - 张韶涵.mp3', '如果你也听说 - 张惠妹.mp3', '时间煮雨 - 郁可唯.mp3', 'Love More - 毕书尽.mp3', '眼泪 - 范晓萱.mp3', '认识你真好 - 孙耀威.mp3', '黑暗骑士 - 林俊杰&五月天.mp3', '阴天 - 莫文蔚.mp3', '太想爱你 - 张信哲.mp3', '坏人 - 方泂镔.mp3', '我们都傻 - 杨丞琳.mp3', '可爱女人 - 周杰伦.mp3', '割爱 - 万芳.mp3', '心伤心痛不痛 - 陈晓东.mp3', '不想长大 - S.H.E.mp3', 'Tomorrow will be fine. - 苏打绿.mp3', '半句再见 - 孙燕姿.mp3', '肆无忌惮 - 薛之谦.mp3', '告白气球 - 周杰伦.mp3', '迷路在云端 - 萧煌奇&A-Lin.mp3', '崇拜 - 梁静茹.mp3', '有故事的人 - 周华健.mp3', 'Forever Young - 艾怡良.mp3', '亲亲 - 梁静茹.mp3', '因为爱情 - 陈奕迅&王菲.mp3', 'Super Woman - 曹格.mp3', '眼泪成诗 - 孙燕姿.mp3', '伤心太平洋 - 任贤齐.mp3', '年轮说 - 杨丞琳.mp3', '掌心 - 无印良品.mp3', '偶阵雨 - 梁静茹.mp3', '生命过客 - 童安格.mp3', '谢谢你爱我 - 文慧如.mp3', '像风一样 - 薛之谦.mp3', '很爱很爱你 - 刘若英.mp3', '我是真的 - 张信哲.mp3', '今生注定 - 高明骏&王馨平.mp3', '当冬夜渐暖 - 孙燕姿.mp3', '新不了情 - 万芳.mp3', '出卖 - 那英.mp3', '怎样 - 戴佩妮.mp3', '身后 - 张惠妹.mp3', '坚持到底 - 阿杜.mp3', '新的心跳 - G.E.M.邓紫棋.mp3', '天份 - 薛之谦.mp3', '爱久见人心 - 梁静茹.mp3', '不是每个恋曲都有美好回忆 - 林志颖.mp3', '稻香 - 周杰伦.mp3', '别在伤口洒盐 - 张惠妹.mp3', '想你的夜 - 关喆.mp3', '过客 - 周思涵.mp3', '我们很好 - 林俊杰.mp3', '跳舞的猫 - NEKO Highway（猫速公路）.mp3', '输了你赢了世界又如何 - 优客李林.mp3', '他一定很爱你 - 阿杜.mp3', '新不了情 - 萧敬腾.mp3', '给我你的爱 - Tank.mp3', '第三人称 - 蔡依林.mp3', '爱情教会我们的事 - 周兴哲.mp3', '不愿一个人 - 周华健.mp3', '冰雨 - 刘德华.mp3', '小夫妻 - 蔡淳佳&欧得洋.mp3', '当你孤单你会想起谁 - 张栋梁.mp3', '绿色 - 陈雪凝.mp3', '江南 - 林俊杰.mp3', '背对背拥抱 - 林俊杰.mp3', '存在 - 5566.mp3', '如果这都不算爱 - 张学友.mp3', '秋意浓 - 张学友.mp3', '听你听我 - 张惠妹.mp3', '月牙湾 - F.I.R..mp3', '一次就好 - 杨宗纬.mp3', '人质 - 张惠妹.mp3', '出界 - 郑秀文.mp3', '老鼠爱大米 - 郭美美.mp3', '依然是朋友 - 宇珩.mp3', '棋子 - 王菲.mp3', '那一场风花雪月的事 - 周治平.mp3', '如果我们不曾相遇 - 五月天.mp3', '空白格 - 蔡健雅.mp3', '起风了 - 吴青峰.mp3', '退后 - 周杰伦.mp3', '理想情人 - 杨丞琳.mp3', '鱼 - 陈绮贞.mp3', '英雄泪 - 王杰.mp3', '离人 - 林志炫.mp3', '人间 - 王菲.mp3', '掉了 - 张惠妹.mp3', '彩虹 - 周杰伦.mp3', '如果的事 - 张韶涵&范玮琪.mp3', '我可以 - 蔡旻佑.mp3', '一颗心交给谁 - 伍思凯.mp3', '男人KTV - 胡彦斌.mp3', '庆幸有你爱我 - 蔡淳佳.mp3', '关怀方式 - 蔡礼莲.mp3', '伟大的渺小 - 林俊杰.mp3', '大海 - 张雨生.mp3', '爱如潮水 - 张信哲.mp3', '如果没有你 - 莫文蔚.mp3', '爱那么重 - 巫启贤.mp3', '绿光 - 孙燕姿.mp3', '圣所 - 林俊杰.mp3', '和平分手 - 梁文音&萧煌奇.mp3', '童话 - 光良.mp3', '以前以后 - A-Lin.mp3', '你，好不好？ - 周兴哲.mp3', '平凡之路 - 朴树.mp3', '明明就 - 周杰伦.mp3', '老情歌 - 吕方.mp3', '不得不爱 - 潘玮柏&弦子.mp3', '花心 - 周华健.mp3', '差一点 - 阿杜.mp3', '来生缘 - 刘德华.mp3', '幸福太短 - A-Lin.mp3', '绅士 - 薛之谦.mp3', '好的事情 - 严爵.mp3', '忠孝东路走九遍 - 动力火车.mp3', '身不由己 - 丁当.mp3', '鸭子 - 苏慧伦.mp3', '直觉 - 张信哲.mp3', '中国话 - S.H.E.mp3', '成全 - 刘若英.mp3', 'Everybody Woohoo - 吴青峰&9m88.mp3', '一千个伤心的理由 - 张学友.mp3', '刚刚好 - 薛之谦.mp3', '除了爱你还能爱谁 - 动力火车.mp3', '每天爱你多一些 - 张学友.mp3', 'Play我呸 - 蔡依林.mp3', '手放开 - 李圣杰.mp3', '红色高跟鞋 - 蔡健雅.mp3', '麻雀衔竹枝 - 鼎艺团.mp3', '美丽新世界 - S.H.E.mp3', '听说爱情回来过 - 林忆莲.mp3', '勇气 - 梁静茹.mp3', '心动 - 林晓培.mp3', '靠岸 - 林宇中.mp3', '最爱的人不是你 - 郑中基.mp3', '连名带姓 - 张惠妹.mp3', '爱我别走 - 张震岳.mp3', 'Always On My Mind - S.H.E.mp3', '千纸鹤 - 邰正宵.mp3', '离歌 - 信乐团.mp3', '同类 - 孙燕姿.mp3', '下雨的夜晚 - 苏打绿.mp3', '一见钟情 - 蓝心湄.mp3', '家 - 陈洁仪.mp3', '天黑请闭眼 - 陈零九&邱锋泽.mp3', '寻人启事 - 徐佳莹.mp3', '一天一点爱恋 - 梁朝伟.mp3', '寂寞不痛 - A-Lin.mp3', 'Super Star - S.H.E.mp3', '独家记忆 - 陈小春.mp3', '耳朵 - 李荣浩.mp3', '拿走了什么 - A-Lin.mp3', '一人有一个梦想 - 黎瑞恩.mp3', '我吃得起苦 - 动力火车.mp3', '对的时间点 - 林俊杰.mp3', '一个像夏天一个像秋天 - 范玮琪.mp3', '一半 - 丁当.mp3', '一首简单的歌 - 王力宏.mp3', '爱一个人好难 - 苏永康.mp3', '爱 - 莫文蔚.mp3', '一起走到 - 孙燕姿.mp3', '你在烦恼什么 - 苏打绿.mp3', '寂寞边界 - 张栋梁.mp3', '我的歌声里 - 曲婉婷.mp3', '单身情歌 - 林志炫.mp3', '你们要快乐 - 李圣杰.mp3', '王妃 - 萧敬腾.mp3', '泪海 - 许茹芸.mp3', '忘情水 - 刘德华.mp3', '寂寞公路 - 伍思凯.mp3', '末班车 - 萧煌奇.mp3', '下雨天 - 南拳妈妈.mp3', '好难得 - 丁当.mp3', '浪花一朵朵 - 任贤齐&阿牛&光良.mp3', '你是我的眼 - 萧煌奇.mp3', '不要对他说 - 张信哲.mp3', '稳稳的幸福 - 陈奕迅.mp3', '伤心的话留到明天再说 - 郭富城.mp3', '我真的受伤了 - 张学友.mp3', '半点心 - 草蜢.mp3', '你的背包 - 陈奕迅.mp3', '消愁 - 毛不易.mp3', '爱情转移 - 陈奕迅.mp3', '今夜你会不会来 - 黎明.mp3', '飞云之下 - 韩红&林俊杰.mp3', '恋爱达人 - 罗志祥&徐熙娣.mp3', '海阔天空 - 信乐团.mp3', '青花瓷 - 周杰伦.mp3', '如果我变成回忆 - Tank.mp3', '可惜没如果 - 林俊杰.mp3', '一半人生 - 阿信.mp3', '容易受伤的女人 - 王菲.mp3', '一个人的我依然会微笑 - 林佳仪.mp3', '情书 - 张学友.mp3', '那些年 - 胡夏.mp3', '叫你一声MY LOVE - 小虎队.mp3', '寒鸦少年 - 华晨宇.mp3', '你的眼睛 - 熊天平&许茹芸.mp3', '夜夜夜夜 - 齐秦.mp3', '是非题 - 范玮琪.mp3', '离开我 - 陶晶莹.mp3', '第一天 - 孙燕姿.mp3', '我管你 - 华晨宇.mp3', '不值得 - 梦飞船.mp3', 'K歌之王 (粤语) - 陈奕迅.mp3', '笑场 - 薛之谦.mp3', '好的情人 - 严爵.mp3', '好想好好爱你 - 周蕙.mp3', '舞娘 - 蔡依林.mp3', '认错 - 优客李林.mp3', 'I Do - 莫文蔚.mp3', '黄昏 - 周传雄.mp3', '追梦人 - 凤飞飞.mp3', '观众 - 杨丞琳.mp3', '说谎 - 张信哲.mp3', '回家 - 王杰.mp3', '三天三夜 - 张惠妹.mp3', '梦一场 - 那英.mp3', '霍元甲 - 周杰伦.mp3', '傻子 - 林宥嘉.mp3', '爱情不能作比较 - 品冠.mp3', '情人 - Beyond.mp3', '她来听我的演唱会 - 张学友.mp3', '以父之名 - 周杰伦.mp3', '明天的明天的明天 - 动力火车.mp3', '一个人生活 - 林凡.mp3', '最重要的决定 - 范玮琪.mp3', '爱在身边 - 周兴哲.mp3', '亲爱的你怎么不在我身边 - 江美琪.mp3', '蝴蝶飞呀 - 小虎队.mp3', '心太软 - 任贤齐.mp3', '依然爱你 - 王力宏.mp3', '爱是你眼里的一首情歌 - 郭美美.mp3', '少女 - 林宥嘉.mp3', '特别的爱给特别的你 - 伍思凯.mp3', '寂寞无害 - 艾怡良.mp3', '抛物线 - 蔡健雅.mp3', '超快感 - 孙燕姿.mp3', '彩虹 - 动力火车.mp3', '真面目 - 苏慧伦.mp3', '等你下课 - 周杰伦&杨瑞代.mp3', '心有独钟 - 陈晓东.mp3', '天真 - 杜德伟.mp3', '恋爱频率 - 许慧欣&许志安.mp3', '心墙 - 郭静.mp3', '等一个人 - 林芯仪.mp3', '恋人未满 - S.H.E.mp3', '为你我受冷风吹 - 林忆莲.mp3', '九百九十九朵玫瑰 - 邰正宵.mp3', '命中注定 - 庾澄庆.mp3', '泡沫 - G.E.M. 邓紫棋.mp3', '干啦干啦 - 八三夭&五月天.mp3', '失恋阵线联盟 - 草蜢.mp3', '冲动 - 动力火车.mp3', 'May I Love You - 张智成.mp3', '北极雪 - 陈慧琳&冯德伦.mp3', '爱上你 - S.H.E.mp3', '遗憾 - 许美静.mp3', '多远都要在一起 - G.E.M. 邓紫棋.mp3', '爱的主打歌 - 萧亚轩.mp3', '爱着爱着就永远 - 田馥甄.mp3', '红日 (粤语) - 李克勤.mp3', '流星雨 - F4.mp3', '热带雨 - NEKO Highway（猫速公路）.mp3', '一眼万年 - S.H.E.mp3', '雨爱 - 杨丞琳.mp3', '如果这就是爱情 - 张靓颖.mp3', '柠檬草的味道 - 蔡依林.mp3', '慢慢 - 张学友.mp3', '你那么爱她 - 李圣杰&林隆璇.mp3', '无条件为你 - 梁静茹.mp3', '放心去飞 - 小虎队.mp3', '我们的爱 - F.I.R..mp3', 'Despacito 缓缓 - Luis Fonsi&林俊杰.mp3', '那个男人 - 杨宗纬.mp3', '句号 - G.E.M.邓紫棋.mp3', '你的酒馆对我打了烊 - 陈雪凝.mp3', '我要快乐 - 张惠妹.mp3', '珊瑚海 - 周杰伦&Lara梁心颐.mp3', '猜心 - 万芳.mp3', '天冷就回来 - 陈洁仪.mp3', '情网 - 张学友.mp3', '时间都去哪儿了 - 王铮亮.mp3', '喜欢你 - 陈洁仪.mp3', '别找我麻烦 - 蔡健雅.mp3', '好久不见 - 陈奕迅.mp3', '花香 - 许绍洋.mp3', '匿名的好友 - 杨丞琳.mp3', '只怕想家 - 黄美珍.mp3', '除此之外 - 范逸臣.mp3', '记事本 - 陈慧琳.mp3', '领悟 - 辛晓琪.mp3', '怎么能这样 - 曾沛慈.mp3', 'Lemon Tree - 苏慧伦.mp3', '你把我灌醉 - 黄大炜.mp3', '铁窗 - 许美静.mp3', '来互相伤害 - 八三夭&A-Lin.mp3', '第一天 - 五月天.mp3', '小时候的我们 - 周兴哲.mp3', 'Fly Away - F.I.R..mp3', '信仰 - 张信哲.mp3', '达尔文 - 蔡健雅.mp3', '差不多姑娘 - G.E.M.邓紫棋.mp3', '一天到晚游泳的鱼 - 张雨生.mp3', '让我留在你身边 - 陈奕迅.mp3', '光辉岁月 - Beyond.mp3', '只是太爱你 - 张敬轩.mp3', '我可以抱你吗 - 张惠妹.mp3', '爱你不是爱给别人看 - 叶良俊.m4a', '前面转个弯 - NEKO Highway.mp3', '雪人 - 范晓萱&王力宏.mp3', '什么歌 - 五月天.mp3', '带我走 - 杨丞琳.mp3', '有一点动心 - 张信哲&刘嘉玲.mp3', '谢谢你的爱1999 - 谢霆锋.mp3', '如果你是我的传说 - 刘德华.mp3', '最长的电影 - 周杰伦.mp3', '为了爱,梦一生 - 王杰.mp3', '心如刀割 - 张学友.mp3', '星光依旧灿烂 - 小虎队.mp3', '趁早 - 张惠妹.mp3', '最近还好吗 - S.H.E.mp3', '大城小爱 - 王力宏.mp3', '只对你说 (莎朗嘿唷) - 林俊杰.mp3', '忘记拥抱 - A-Lin.mp3', '我一个人住 - 苏慧伦.mp3', '为什么你背着我爱别人 - 许志安.mp3', '余波荡漾 - 田馥甄.mp3', '别让我走远 - 林宥嘉.mp3', '北风 - 张镐哲.mp3', '寓言 - 张韶涵.mp3', '纸飞机 - 林忆莲.mp3', '体面 - 于文文.mp3', '我愿意 - 王菲.mp3', '星晴 - 周杰伦.mp3', '红尘来去一场梦 - 巫启贤.mp3', '演员 - 薛之谦.mp3', '爱过你有多久就有多痛 - 萧秉治.mp3', '月亮惹的祸 - 张宇.mp3', '白色羽毛 - 芮恩.mp3', '戒烟 - 李荣浩.mp3', '雨一直下 - 张宇.mp3', '旧梦 - 彭佳慧.mp3', '到不了 - 范玮琪.mp3', '瘀青 - 张智成.mp3', '岩石里的花 - G.E.M. 邓紫棋.mp3', '候鸟 - S.H.E.mp3', '祝福 - 张学友.mp3', '相见恨晚 - 彭佳慧.mp3', '后会无期 - G.E.M. 邓紫棋.mp3', '最幸福的事 - 梁文音.mp3', '触电 - S.H.E.mp3', '最笨的人是我 - 方泂镔.mp3', '明天我要嫁给你 - 周华健.mp3', '海啸 - 庾澄庆.mp3', '是什么让我遇见这样的你 - 白安.mp3', '北京一夜 - 雪儿.mp3', '在,也不见 - 孙燕姿.mp3', '缺氧 - 杨丞琳.mp3', '滚滚红尘 - 陈淑桦.mp3', '你为什么说谎 - 丁当.mp3', '你被写在我的歌里 - 苏打绿&陈嘉桦.mp3', '爱你一万年(国) - 刘德华.mp3', '分手快乐 - 梁静茹.mp3', '风衣 - 孙燕姿.mp3', '我相信 - 杨培安.mp3', '小情歌 - 苏打绿.mp3', '制造浪漫 - 郑中基&陈慧琳.mp3', '无心伤害 - 杜德伟.mp3', '我是一只小小鸟 - 赵传.mp3', '月弯弯 - 陈洁仪.mp3', '等你等到我心痛 - 黎沸挥.mp3', '新加坡派2.0 (2015演唱会现场版) - 梁文福.mp3', '浪费 - 林宥嘉.mp3', '回家 - 顺子.mp3', '如果·爱 - 张学友.mp3', '跳舞的梵谷 - 孙燕姿.mp3', '伤心的人别听慢歌 (贯彻快乐) - 五月天.mp3', '说爱你 - 蔡依林.mp3', '如燕 - Olivia Ong.mp3', '爱情电影 - 许茹芸&熊天平.mp3', '马德里不思议 - 蔡依林.mp3', '叶子 - 阿桑.mp3', '慢慢喜欢你 - 莫文蔚.mp3', '回头太难 - 张学友.mp3', '无与伦比的美丽 - 苏打绿.mp3', '忘记你不如忘记自己 - 王杰.mp3', '春泥 - 庾澄庆.mp3', '深海 - 赵传.mp3', '大眠 - 王心凌.mp3', '还是要幸福 - 田馥甄.mp3', '世界唯一的你 - 曹格.mp3', '暖暖 - 梁静茹.mp3', '国王与乞丐 (Live) - 华晨宇.mp3', '忘记你我做不到 - 张学友.mp3', '原点 - 孙燕姿&蔡健雅.mp3', '当我知道你们相爱 - 郭富城.mp3', '来自天堂的魔鬼 - G.E.M.邓紫棋.mp3', '野子 - 苏运莹.mp3', '思念是一种病 - 张震岳&蔡健雅.mp3', '孤单北半球 - 欧得洋.mp3', '愚人码头 - 熊天平.mp3', '无底洞 - 蔡健雅.mp3', 'Kiss Goodbye - 王力宏.mp3', '欧若拉 - 张韶涵.mp3', '七里香 - 周杰伦.mp3', '再见 - G.E.M. 邓紫棋.mp3', '红蔷薇白玫瑰 - G.E.M. 邓紫棋.mp3', '你要的爱 - 戴佩妮.mp3', '因为爱所以爱 - 谢霆锋.mp3', '热带雨林 - S.H.E.mp3', '克卜勒 - 孙燕姿.mp3', '丑八怪 - 薛之谦.mp3', '不为谁而作的歌 - 林俊杰.mp3', '逃亡 - 孙燕姿.mp3', '心跳 - 王力宏.mp3', '你给我听好 - 陈奕迅.mp3', '天天看到你 - 阿杜.mp3', '鬼迷心窍 - 李宗盛.mp3', '给我一个理由忘记 - A-Lin.mp3', '我好吗 - 梁静茹.mp3', '你敢不敢 - 徐佳莹.mp3', '好朋友的祝福 - A-Lin.mp3', '可以了 - 陈奕迅.mp3', '我总是听你说 - 叶良俊.m4a', '哑巴 - 薛之谦.mp3', '挪威的森林 - 伍佰.mp3', '那些你很冒险的梦 - 林俊杰.mp3', '最近比较烦 - 周华健&品冠&李宗盛.mp3', 'Para Para Sakura - 郭富城.mp3', '你就不要想起我 - 田馥甄.mp3', '寂寞先生 - 曹格.mp3', '有点甜 - 汪苏泷&By2.mp3', '你是我的花朵 - 伍佰.mp3', '花又开好了 - S.H.E.mp3', '突然想爱你 - 许茹芸.mp3', '不是你的错 - 丁当.mp3', '我怀念的 - 孙燕姿.mp3', '不甘心不放手 - 动力火车.mp3', '原来 - 林俊杰.mp3', '蒙娜丽莎的眼泪 - 林志炫.mp3', '旅行的意义 - 陈绮贞.mp3', '你太猖狂 - 田馥甄.mp3', '爱与愁 - 伍思凯.mp3', '倒数 - G.E.M.邓紫棋.mp3', '不能说的秘密 - 周杰伦.mp3', '征服 - 那英.mp3', '爱了很久的朋友 - 田馥甄.mp3', '像我这样的人 - 毛不易.mp3', '想你想得好孤寂 - 邰正宵.mp3', '从开始到现在 - 张信哲.mp3', '为你的寂寞唱歌 - 家家.mp3', '一千零一夜 - 邰正宵.mp3', '被动 - 苏慧伦.mp3', '爱我还是他 - 陶喆.mp3', '风雨无阻 - 周华健.mp3', '刀剑如梦 - 周华健.mp3', 'P.S.我爱你 - A-Lin.mp3', '至少还有你 - 林忆莲.mp3', '男人女人 - 许茹芸&阿穆隆.mp3', '青春住了谁 - 杨丞琳.mp3', '山丘 - 李宗盛.mp3', '天后 - 陈势安.mp3', '听袁惟仁弹吉他 - S.H.E.mp3', '爱你不是两三天 - 梁静茹.mp3', '出嫁 - 张清芳&优客李林.mp3', '胆小鬼 - 梁咏琪.mp3', '情人 - 杜德伟.mp3', '说散就散- 袁娅维.mp3', '祈祷 - 王韵婵&王杰.mp3', '兜圈 - 林宥嘉.mp3', '如果有一天 - 梁静茹.mp3', '怎么办 - S.H.E.mp3', '男人不该让女人流泪 - 苏永康.mp3', '歌颂者 - 吴青峰.mp3', '凉凉 - 杨宗纬&张碧晨.mp3', '阳光总在风雨后 - 许美静.mp3', '眼底星空 - 李圣杰.mp3', '遗失的美好 - 张韶涵.mp3', '为情所困 - 梁朝伟.mp3', '终于了解自由 - 周兴哲.mp3', '记得 - 张惠妹.mp3', '伯乐 - 林宥嘉.mp3', '年少有为 - 李荣浩.mp3', '都是夜归人 - 许美静.mp3', '慢半拍 - 薛之谦.mp3', '小手拉大手 - 梁静茹.mp3', '心痛 - 陈洁仪.mp3', '喜剧之王 - 李荣浩.mp3', '洋葱 - 杨宗纬.mp3', '爱你 - 王心凌.mp3', '失落沙洲 - 徐佳莹.mp3', '存在 - G.E.M.邓紫棋.mp3', '选择 - 林子祥&叶蒨文.mp3', '宽容 - 张信哲.mp3', '豆浆油条 - 林俊杰.mp3', '末日之恋 - 张智成.mp3', '钟爱一生 - 杜德伟.mp3', '知己 - 周慧敏&邰正宵.mp3', '让我欢喜让我忧 - 周华健.mp3', '微光 - 华晨宇.mp3', '你不是真正的快乐 - 五月天.mp3', '远走高飞 - 李圣杰.mp3', '别爱我 - 郑中基.mp3', '爱和承诺 - 张学友&陈慧娴.mp3', '想太多 - 李玖哲.mp3', '想见你想见你想见你 - 八三夭乐团.mp3', '大地 (粤语) - BEYOND.mp3', '可惜不是你 - 梁静茹.mp3', '至少走得比你早 - 卢巧音.mp3', '舍得 - 郑秀文.mp3', 'Waiting For You - 胡彦斌.mp3', 'Piano - 范逸臣.mp3', '青春修炼手册 - TFBOYS.mp3', '天下有情人 - 周华健&齐豫.mp3', '明知道 - 许美静.mp3', '模特 - 李荣浩.mp3', '别说没爱过 - 韦礼安.mp3', '分手后不要做朋友 - 梁文音.mp3', '分手吧 - 张震岳.mp3', '命运 - 家家.mp3', '简简单单 - 林俊杰.mp3', '不只是朋友 - 黄小琥.mp3', '后来的我们 - 五月天.mp3', '十年 - 陈奕迅.mp3', '突然好想你 - 五月天.mp3', '听海 - 张惠妹.mp3', '天使 - 五月天.mp3', '男人哭吧不是罪 - 刘德华.mp3', '致青春 - 八三夭.mp3', '爱是永恒 (粤语) - 张学友.mp3', '最爱的人伤我最深 - 张雨生&张惠妹.mp3', '真的傻 - 徐佳莹.mp3', '失忆的金鱼 - 杨丞琳.mp3', '波斯猫 - S.H.E.mp3', '笨小孩 - 刘德华&柯受良&吴宗宪.mp3', '追光者 - 岑宁儿.mp3', '慢冷 - 梁静茹.mp3', '等一个晴天 - 蔡淳佳.mp3', '天空 - 蔡依林.mp3', '朋友 - 周华健.mp3', '不爱我就拉倒 - 周杰伦.mp3', '爱如空气 - 韦礼安.mp3', '红豆 - 王菲.mp3', '终于等到你 - 张靓颖.mp3', '小薇 - 黄品源.mp3', '麻雀 - 李荣浩.mp3', '没有如果 - 梁静茹.mp3', '痴心绝对 - 李圣杰.mp3', '真实 - 张惠妹.mp3', '约定 - 光良.mp3', '算什么男人 - 周杰伦.mp3', '背叛 - 曹格.mp3', '咖啡 - 张学友.mp3', '你还要我怎样 - 薛之谦.mp3', '他不懂 - 张杰.mp3', '天空 - 王菲.mp3', '大雨的夜里 - 张清芳.mp3', '李白 - 李荣浩.mp3', '爱相随 - 周华健.mp3', '我找你找了好久 - 无印良品.mp3', 'Better Man - 林忆莲.mp3', '旋木 - 王菲.mp3', '浪人情歌 - 伍佰.mp3', '黑夜问白天 - 林俊杰.mp3', '以后别做朋友 - 周兴哲.mp3', '曹操 - 林俊杰.mp3', '失恋无罪 - A Lin.mp3', '白月光 - 张信哲.mp3', '你不知道的事 - 王力宏.mp3', '我没那么爱你 - 丁当.mp3', '手掌心 - 丁当.mp3', '爱笑的眼睛 - 徐若瑄.mp3', '不痛 - 张韶涵.mp3', '味道 - 辛晓琪.mp3', '刀马旦 - 李玟&周杰伦.mp3', '左右为难 - 郑中基&张学友.mp3', '背叛情歌 - 动力火车.mp3', '一千年以后 - 林俊杰.mp3', '你是我心内的一首歌 - 王力宏&任家萱.mp3', '顽固 - 五月天.mp3', '也可以 - 阎奕格.mp3', '爱情证书 - 孙燕姿.mp3', '陪我看日出 - 蔡淳佳.mp3', '不过失去了一点点 - 曾沛慈.mp3', '好朋友只是朋友 - 郁可唯.mp3', '我们不一样 - 大壮.mp3', '猜不透 - 丁当.mp3', '夜空中最亮的星 - G.E.M.邓紫棋.mp3', 'Forever Love - 王力宏.mp3', '双栖动物 - 蔡健雅.mp3', '把你宠坏 - 杜德伟.mp3', '尽在不言中 - 周慧敏.mp3', '那女孩对我说 - 黄义达.mp3', '没那么简单 - 黄小琥.mp3', '阿五 - NEKO Highway（猫速公路）.mp3', '倒带 - 蔡依林.mp3', '寂寞寂寞就好 - 田馥甄.mp3', '守护永恒的爱 - 孙燕姿.mp3', '如果云知道 - 许茹芸.mp3', '无眠 - 苏打绿.mp3', '情歌 - 梁静茹.mp3', '爱情三十六计 - 蔡依林.mp3', '像天堂的悬崖 - 李佳薇.mp3', '我爱的人 - 陈小春.mp3', '再遇见 - 苏打绿.mp3', '我和我追逐的梦 - 刘德华.mp3', '断点 - 张敬轩.mp3', '情非得已 - 庾澄庆.mp3', '今天你要嫁给我 - 陶喆&蔡依林.mp3', '翅膀 - 林俊杰.mp3', '不仅仅是喜欢 - 萧全&孙语赛.mp3', '火柴天堂 - 熊天平.mp3', '偏爱 - 曾沛慈.mp3', '从你回眸那天开始 - 洪劭轩.mp3', '谁来剪月光 - 陈奕迅.mp3', '过火 - 张信哲.mp3', '凌晨三点钟 - 张智成.mp3', '温柔的慈悲 - 林良乐.mp3', '一个人想着一个人 - 曾沛慈.mp3', 'Wonderland - 林俊杰.mp3', '你知不知道 - 张学友.mp3', '我的爱 - 孙燕姿.mp3', '放不下的活着 - 插班生.mp3', '知足 - 五月天.mp3', '太空人 - 吴青峰.mp3', '心引力 - 王俊凯&蔡依林.mp3', '我是真的爱你 - 张信哲.mp3', '一百种孤独的理由 - 郭美美.mp3', '三天两夜 - 张学友.mp3', '想你就写信 - 浪花兄弟.mp3', '初恋 - 梁文音.mp3', '不该 (Live) - 周杰伦、aMEI.mp3', '第三者 - 梁静茹.mp3', '忘了时间的钟 - 古巨基.mp3', '矜持 - 王菲.mp3', '白天不懂夜的黑 - 那英.mp3', '发如雪 - 周杰伦.mp3', '镇守爱情 - 动力火车.mp3', '会呼吸的痛 - 梁静茹.mp3', '说好的幸福呢 - 周杰伦.mp3', '全世界失眠 - 陈奕迅.mp3', '约定 - 周蕙.mp3', '将故事写成我们 - 林俊杰.mp3', '伤痕 - 林忆莲.mp3', '我很忙 - A-Lin.mp3', '心碎了无痕 - 张学友.mp3', 'Lydia - F.I.R.飞儿乐团.mp3', '残缺的彩虹 - 陈绮贞.mp3', '风中的承诺 - 李翊君.mp3', '一言难尽 - 张宇.mp3', '下一个天亮 - 郭静.mp3', '别怕我伤心 - 张信哲.mp3', '爱情多恼河 - 熊天平.mp3', '燕尾蝶 - 梁静茹.mp3', '怎么了 - 周兴哲.mp3', '三寸天堂 - 严艺丹.mp3', '把你藏起来 - 杜德伟.mp3', '解脱 - 张惠妹.mp3', '永不失联的爱 - 周兴哲.mp3', '我要的幸福 - 孙燕姿.mp3', '祝你一路顺风 - 吴奇隆.mp3', '安静了 - S.H.E.mp3', '逆光 - 孙燕姿.mp3', '只能勇敢 - 萧煌奇.mp3', '亲爱的，那不是爱情 - 张韶涵.mp3', '漂洋过海来看你 - 娃娃.mp3', '给我一首歌的时间 - 周杰伦.mp3', '哭不出来 - 张惠妹.mp3', '雨天 - 孙燕姿.mp3', '听不到 - 梁静茹.mp3', '暧昧 - 杨丞琳.mp3', '我不知道 - 动力火车.mp3', '天使在唱歌 - S.H.E.mp3', '但愿人长久 - 王菲.mp3', '爱要怎么说出口 - 赵传.mp3', '温柔 - 五月天.mp3', '孤枕难眠 - 周华健.mp3', '我也很想他 - 孙燕姿.mp3', '甲乙丙丁 - 许志安&张学友&郑中基.mp3', '当你 - 王心凌.mp3', '谢谢你 - 李代沫.mp3', '我不愿 让你一个人 - 五月天.mp3', '温柔 #MaydayBlue20th - 五月天&孙燕姿.mp3', '木偶人 - 薛之谦.mp3', '花都开好了 - S.H.E.mp3', '普通朋友 - 陶喆.mp3', '广岛之恋 - 莫文蔚&张洪量.mp3', '飘向北方 - 黄明志&王力宏.mp3', '兄妹 - 陈奕迅.mp3', '晴天 - 周杰伦.mp3', '你怎么舍得我难过 - 黄品源.mp3', '对面的女孩看过来 - 阿牛.mp3', '亲密爱人 - 梅艳芳.mp3', '许愿 - 古巨基&梁咏琪.mp3', '难道你现在还不知道 - 张克帆.mp3', '我依然是你的情人 - 李玟.mp3', '当你老了 - 莫文蔚.mp3', '街角的祝福 - 戴佩妮.mp3', '爱情漫游 - 5566.mp3', '忽然之间 - 莫文蔚.mp3', '明明很爱你 - 品冠&梁静茹.mp3', '透明 - G.E.M.邓紫棋.mp3', '天使的指纹 - 孙燕姿.mp3', '小幸运 - 田馥甄.mp3', '不如这样 - 陈奕迅.mp3', '派对动物 - 五月天.mp3', '你知道我的迷惘 - Beyond.mp3', '爱的初体验 - 张震岳.mp3', '氧气 - 范晓萱.mp3', '每次都想呼喊你的名字 - 永邦.mp3', '天真有邪 - 林宥嘉.mp3', '魔鬼中的天使 - 田馥甄.mp3', '我不难过 - 孙燕姿.mp3', '其实不想走 - 周华健.mp3', '一笑而过 - 那英.mp3', '爱你的宿命 - 张信哲.mp3', '爱我的人和我爱的人 - 裘海正.mp3', '我好想你 - 苏打绿.mp3', '梁山伯与茱丽叶 - 卓文萱&曹格.mp3', '为你写诗 - 吴克群.mp3', '平凡天使 - G.E.M. 邓紫棋.mp3', '烂好人 - 阿杜.mp3', '沉默的羔羊 - 赵传.mp3', '一眼瞬间 - 萧敬腾&张惠妹.mp3', '幸福不难 - 龚芝怡.mp3', '我还是爱着你 - MP魔幻力量.mp3', 'Bad Boy - 张惠妹.mp3', '孤单的夜里我不孤单 - 陈晓东.mp3', '有一种悲伤 - A-Lin.mp3', '半途 - 蔡健雅.mp3', '冬季到台北来看雨 - 孟庭苇.mp3', '天天年年 - 孙燕姿.mp3', '亲爱的你 - 文慧如.mp3', '傻瓜 - 温岚.mp3', '当爱已成往事 - 李宗盛&林忆莲.mp3', '志明与春娇 - 五月天.mp3', '红蜻蜓 - 小虎队.mp3', 'Beautiful Love - 蔡健雅.mp3', '千里之外 - 周杰伦&费玉清.mp3', '听妈妈的话 - 周杰伦.mp3', '刚好遇见你 - 李玉刚.mp3', '光年之外 - G.E.M. 邓紫棋.mp3', '我相信 - 张靓颖.mp3', 'stay with you.mp3', '口是心非 - 张雨生.mp3', '太阳 (Live) - 邱振哲.mp3', '练习 - 刘德华.mp3', '姊姊妹妹站起来 - 陶晶莹.mp3', '路过人间 - 郁可唯.mp3', '你曾是少年 - S.H.E.mp3', '宁夏 - 梁静茹.mp3', '一路上有你 - 张学友.mp3', '想和你去吹吹风 - 张学友.mp3', '散场的拥抱 - 倪安东.mp3', '遇见 - 孙燕姿.mp3', '当爱已成往事 - 张国荣.mp3', '威尼斯的泪 - 永邦.mp3', '天黑 - 阿杜.mp3', '我知道 - BY2.mp3', '学猫叫 - 小潘潘（潘柚彤）&小峰峰（陈峰）.mp3', '我爱他 - 丁当.mp3', '烟火 - 吴奇隆.mp3', '难以抗拒你容颜 - 张信哲.mp3', '原来你什么都不要 - 张惠妹.mp3', '心痛 - 王杰.mp3', '原谅我 - 萧敬腾.mp3', '你是如此难以忘记 - 梁朝伟.mp3', '暗恋的代价 - 草蜢.mp3', '只对你有感觉 - 飞轮海&田馥甄.mp3', '听你说 - 郁可唯&林凡.mp3', '启程 - 范玮琪.mp3', '阿飞的小蝴蝶 - 萧敬腾.mp3', '我无所谓 - 张惠妹.mp3', '他还是不懂 - S.H.E.mp3', '我是不是该安静的走开 - 郭富城.mp3', '女人花 - 梅艳芳.mp3', '迷迭香 - 周杰伦.mp3', '每次醒来 - 刘德华.mp3', '亲亲我的宝贝 - 周华健.mp3', 'Fly Away - 梁静茹.mp3', '绝口不提!爱你 - 郑中基.mp3', '听见下雨的声音 - 魏如昀.mp3', '身骑白马 - 徐佳莹.mp3', '黑色幽默 - 周杰伦.mp3', '最熟悉的陌生人 - 萧亚轩.mp3', '无情的情书 - 动力火车.mp3', '爱你一万年 - 伍佰.mp3', '我的心太乱 - 周传雄.mp3', '喜欢寂寞 - 苏打绿.mp3', '怪美的 - 蔡依林.mp3', '我要给世界最悠长的湿吻 - 蔡健雅.mp3', '不必在乎我是谁 - 林忆莲.mp3', '你的微笑 - F.I.R..mp3', '旧爱还是最美 - 苏永康.mp3', '上弦月 - 许志安.mp3', '简单爱 - 周杰伦.mp3', '我最亲爱的 - 张惠妹.mp3', '有多少爱可以重来 - 迪克牛仔.mp3', '十七 - S.H.E.mp3', '黑白配 - 范玮琪.mp3', '脑公 - 蔡依林.mp3', '花言巧语 - 黄莺莺.mp3', '太傻 - 巫启贤.mp3', '年轮 - 张碧晨.mp3', '默 - 那英.mp3', '有一天我会 - 蔡淳佳.mp3', '擦肩而过 - 李圣杰.mp3', '囚鸟 - 彭羚.mp3', '非你莫属 - Tank.mp3', '爱你 - 陈芳语.mp3', '小酒窝 - 林俊杰&蔡卓妍.mp3', '不将就 - 李荣浩.mp3', '宠爱 - TFBOYS.mp3', '菊花台 - 周杰伦.mp3', '大雨 - 娃娃.mp3', '你是我最深爱的人 - 永邦.mp3', '在我生命中的每一天 - 成龙&苏慧伦.mp3', '学不会 - 林俊杰.mp3', '认真的雪 - 薛之谦.mp3', '落叶归根 - 王力宏.mp3', 'Shero - S.H.E.mp3', '想着你的感觉 - 巫启贤&邓淑娴&曾庆龄.mp3', '吻别 - 张学友.mp3', '把悲伤留给自己 - 赵传.mp3', '小时候 (Live) - 苏打绿.mp3', '胡思乱想 - 无印良品.mp3', '缺口 - 庾澄庆.mp3', '至少我还记得 - 周兴哲.mp3', '下雨了 - 薛之谦.mp3', '布拉格广场 - 蔡依林&周杰伦.mp3', '天意 - 刘德华.mp3', '妥协 - 蔡依林.mp3', '我为你伤心 - 苏永康.mp3', '当 - 动力火车.mp3', '担心 - 陈洁仪.mp3', '千年之恋 - F.I.R.飞儿乐团.mp3', '无名之辈 - 陈雪燃.mp3', '爱的代价 - 张艾嘉.mp3', '你要的全拿走 - 胡彦斌.mp3', '齐天 - 华晨宇.mp3', '女孩 - 韦礼安.mp3', '专属天使 - Tank.mp3', '羞答答的玫瑰静悄悄地开 - 胡夏.mp3', '宝贝对不起 - 草蜢.mp3', '修炼爱情 - 林俊杰.mp3', '你不在 - 王力宏.mp3', '用心良苦 - 张宇.mp3', '用尽一生的爱 - 张克帆.mp3', '子曰 - 卜学亮.mp3', '爱 - 小虎队.mp3', '明明白白我的心 - 成龙&陈淑桦.mp3', '进阶 - 林俊杰.mp3', '说谎 - 林宥嘉.mp3', '勇敢 - 张惠妹.mp3', '爱我的每个人 - 任家萱.mp3', '一个人记得 - 尤长靖.mp3', '背包 - 苏有朋.mp3', '错的人 - 萧亚轩.mp3', '话题 - 周蕙.mp3', '让我为你唱情歌 - 萧敬腾.mp3', '不搭 - 李荣浩.mp3', '大鱼 - 周深.mp3', '看我72变 - 蔡依林.mp3', '屋顶 - 周杰伦&温岚.mp3', '你最珍贵 - 张学友&高慧君.mp3', '死了都要爱 - 信乐团.mp3', '病态 - 薛之谦.mp3', '如果雨之后 - 周兴哲.mp3', '几分之几 (You Complete Me) - 卢广仲.mp3', '开始懂了 - 孙燕姿.mp3', '防…空洞 - 戴佩妮.mp3', '他不爱我 - 莫文蔚.mp3', '踮起脚尖爱 - 洪佩瑜.mp3', '祝我生日快乐 - 温岚.mp3', '摩天动物园 - G.E.M.邓紫棋.mp3', '离开地球表面 - 五月天.mp3', '倔强 - 五月天.mp3', '指望 - 郁可唯.mp3', '天黑黑 - 孙燕姿.mp3', '煎熬 - 李佳薇.mp3', '日不落 - 蔡依林.mp3', '人质 - 杨宗纬.mp3', '值得 - 郑秀文.mp3', '打扫 - 2个女生.mp3', '别让情两难 - 范文芳&张信哲.mp3', '淘汰 - 陈奕迅.mp3', '有没有一首歌会让你想起我 - 周华健.mp3', '那就这样吧 - 动力火车.mp3']
1
p = Path(os.path.expanduser('~/Documents/[fuliba.net]U选1000'))
print(p.glob('*'))<generator object Path.glob at 0x7fcfe81f27d0>

</code>


# Notebook from dongxulee/lifeCycle
Path: 20200922/simulationRent.ipynb

<code>
%pylab inline
from scipy.interpolate import interpn
from helpFunctions import surfacePlot
from constant import *
from multiprocessing import Pool
from functools import partial
import warnings
import math
warnings.filterwarnings("ignore")
np.printoptions(precision=2)Populating the interactive namespace from numpy and matplotlib
# load the policy funciton
Vgrid_renting = np.load("Vgrid_renting.npy")
cgrid_renting = np.load("cgrid_renting.npy")
bgrid_renting = np.load("bgrid_renting.npy")
kgrid_renting = np.load("kgrid_renting.npy")
hgrid_renting = np.load("hgrid_renting.npy")
Mgrid_renting = np.load("Mgrid_renting.npy")
Hgrid_renting = np.load("Hgrid_renting.npy")

V1000 = np.load("Vgrid1000.npy")
V1500 = np.load("Vgrid1500.npy")
V2000 = np.load("Vgrid2000.npy")
V750 = np.load("Vgrid750.npy")

cgrid1000 = np.load("cgrid1000.npy")
bgrid1000 = np.load("bgrid1000.npy")
kgrid1000 = np.load("kgrid1000.npy")
igrid1000 = np.load("igrid1000.npy")
qgrid1000 = np.load("qgrid1000.npy")

cgrid1500 = np.load("cgrid1500.npy")
bgrid1500 = np.load("bgrid1500.npy")
kgrid1500 = np.load("kgrid1500.npy")
igrid1500 = np.load("igrid1500.npy")
qgrid1500 = np.load("qgrid1500.npy")

cgrid2000 = np.load("cgrid2000.npy")
bgrid2000 = np.load("bgrid2000.npy")
kgrid2000 = np.load("kgrid2000.npy")
igrid2000 = np.load("igrid2000.npy")
qgrid2000 = np.load("qgrid2000.npy")

cgrid750 = np.load("cgrid750.npy")
bgrid750 = np.load("bgrid750.npy")
kgrid750 = np.load("kgrid750.npy")
igrid750 = np.load("igrid750.npy")
qgrid750 = np.load("qgrid750.npy")

indexH = [750, 1000, 1500, 2000]
cgrid = [cgrid750, cgrid1000, cgrid1500, cgrid2000]
bgrid = [bgrid750, bgrid1000, bgrid1500, bgrid2000]
kgrid = [kgrid750, kgrid1000, kgrid1500, kgrid2000]
igrid = [igrid750, igrid1000, igrid1500, igrid2000]
qgrid = [qgrid750, qgrid1000, qgrid1500, qgrid2000]_____no_output_____#Define the earning function, which applies for both employment and unemployment, good econ state and bad econ state
def y(t, x):
    if (len(x) == 8):
        w, n, M, g_lag, e, s, z, H = x
    else:
        w, n, e, s, z = x

    if t <= T_R:
        welfare = 5
        return detEarning[t] * earningShock[int(s)] * e + (1-e) * welfare
    else:
        return detEarning[t]

#Define the evolution of the amount in 401k account
def gn(t, n, x, s_next):
    if (len(x) == 8):
        w, n, M, g_lag, e, s, z, H = x
    else:
        w, n, e, s, z = x

    if t <= T_R and e == 1:
        # if the person is employed, then 5 percent of his income goes into 401k
        # i = 0.05
        n_cur = n + y(t, x) * yi
    elif t <= T_R and e == 0:
        # if the perons is unemployed, then n does not change
        n_cur = n
    else:
        # t > T_R, n/discounting amount will be withdraw from the 401k
        n_cur = n - n/Dt[t]
    return (1+r_k[int(s), s_next])*n_cur
_____no_output_____import quantecon as qe
import timeit
mc = qe.MarkovChain(Ps)

H_options = [0, 750, 1000, 1500, 2000]
M_options = [0.2, 0.5, 0.8]
# wealth discretization
ws = np.array([10,25,50,75,100,125,150,175,200,250,500,750,1000,1500,3000])
w_grid_size = len(ws)
# 401k amount discretization
ns = np.array([1, 5, 10, 15, 25, 40, 65, 100, 150, 300, 400,1000])
n_grid_size = len(ns)
# Improvement amount
gs = np.array([0,50,100,200,500,1500])
g_grid_size = len(gs)

def closest(lst, K):
    '''
        Find the closest value of K in a list lst
    '''
    lst = np.asarray(lst)
    idx = (np.abs(lst - K)).argmin()
    return lst[idx]

def action_rent(t, x):
    w, n, e, s, z = x
    points = (ws, ns)
    c = interpn(points, cgrid_renting[:,:,e,s,z,t], x[:2], method = "nearest", bounds_error = False, fill_value = None)[0]
    b = interpn(points, bgrid_renting[:,:,e,s,z,t], x[:2], method = "nearest", bounds_error = False, fill_value = None)[0]
    k = interpn(points, kgrid_renting[:,:,e,s,z,t], x[:2], method = "nearest", bounds_error = False, fill_value = None)[0]
    if e == 1:
        H = interpn(points, Hgrid_renting[:,:,e,s,z,t], x[:2], method = "nearest", bounds_error = False, fill_value = None)[0]
        # decide to buy or not to buy a house
        H = H_options[np.sum(H>=np.array(H_options))-1]
        if H == 0:
            h = interpn(points, hgrid_renting[:,:,e,s,z,t], x[:2], method = "nearest", bounds_error = False, fill_value = None)[0]
            return (c,b,k,h)
        else:
            M = interpn(points, Mgrid_renting[:,:,e,s,z,t], x[:2], method = "nearest", bounds_error = False, fill_value = None)[0]
            M = closest(H*M_options, M)
            return (c,b,k,M,H)
    else:
        h = interpn(points, hgrid_renting[:,:,e,s,z,t], x[:2], method = "nearest", bounds_error = False, fill_value = None)[0]
        return (c,b,k,h)

def action_own(t, x):
    w, n, M, g_lag, e, s, z, H = x
    # Mortgage amount, * 0.25 is the housing price per unit
    Ms = np.array([0.01*H,0.05*H,0.1*H,0.2*H,0.3*H,0.4*H,0.5*H,0.6*H,0.7*H,0.8*H]) * pt
    points = (ws,ns,Ms,gs)
    c = interpn(points, cgrid[indexH.index(H)][:,:,:,:,e,s,z,t], x[:4], method = "nearest", bounds_error = False, fill_value = None)[0]
    b = interpn(points, bgrid[indexH.index(H)][:,:,:,:,e,s,z,t], x[:4], method = "nearest", bounds_error = False, fill_value = None)[0]
    k = interpn(points, kgrid[indexH.index(H)][:,:,:,:,e,s,z,t], x[:4], method = "nearest", bounds_error = False, fill_value = None)[0]
    q = interpn(points, qgrid[indexH.index(H)][:,:,:,:,e,s,z,t], x[:4], method = "nearest", bounds_error = False, fill_value = None)[0]
    if (1-q) <= 0.01:
        q = 1
        i = 0
    else:
        i = interpn(points, igrid[indexH.index(H)][:,:,:,:,e,s,z,t], x[:4], method = "nearest", bounds_error = False, fill_value = None)[0]
    return (c,b,k,i,q)

def transition_to_rent(x,a,t,s_next):
    '''
        imput a is np array constains all possible actions
        from x = [w, n, e, s] to x = [w, n, e, s]
    '''
    w, n, e, s, z = x
    c, b, k, h = a
    # transition of z
    if z == 1:
        z_next = 1
    else:
        if k == 0:
            z_next = 0
        else:
            z_next = 1
    # variables used to collect possible states and probabilities
    x_next = []
    prob_next = []
    w_next =  b*(1+r_b[int(s)]) + k*(1+r_k[int(s), s_next])
    n_next = gn(t, n, x, s_next)
    if t >= T_R:
        e_next = 0
        return [w_next, n_next, e_next, s_next, z_next]
    else:
        for e_next in [0,1]:
            x_next.append([w_next, n_next, e_next, s_next, z_next])
            prob_next.append(Pe[int(s),s_next,int(e),e_next])
    return x_next[np.random.choice(len(prob_next), p = prob_next)]

def transition_to_own(x,a,t,s_next):
    '''
        imput a is np array constains all possible actions
        from x = [w, n, e, s] to x = [w, n, M, g=0, e, s, H]
    '''
    w, n, e, s, z = x
    # variables used to collect possible states and probabilities
    x_next = []
    prob_next = []
    c, b, k, M, H = a
    # transition of z
    if z == 1:
        z_next = 1
    else:
        if k == 0:
            z_next = 0
        else:
            z_next = 1
    M_next = M*(1+rh)
    w_next =  b*(1+r_b[int(s)]) + k*(1+r_k[int(s), s_next])
    n_next = gn(t, n, x, s_next)
    if t >= T_R:
        e_next = 0
        return [w_next, n_next, M_next, 0, e_next, s_next, z_next, H]
    else:
        for e_next in [0,1]:
            x_next.append([w_next, n_next, M_next, 0, e_next, s_next, z_next, H])
            prob_next.append(Pe[int(s),s_next,int(e),e_next])
    return x_next[np.random.choice(len(prob_next), p = prob_next)]

def transition_after_own(x, a, t, s_next):
    '''
    Input: state and action and time
    Output: possible future states and corresponding probability
    '''
    w, n, M, g_lag, e, s, z, H = x
    c,b,k,i,q = a
    # transition of z
    if z == 1:
        z_next = 1
    else:
        if k == 0:
            z_next = 0
        else:
            z_next = 1
    # variables used to collect possible states and probabilities
    x_next = []
    prob_next = []
    m = M/D[T_max-t]
    M_next = M*(1+rh) - m
    if q == 1:
        g = (1-delta)*g_lag + i
    else:
        g = (1-delta)*g_lag

    w_next =  b*(1+r_b[int(s)]) + k*(1+r_k[int(s), s_next])
    n_next = gn(t, n, x, s_next)
    if t >= T_R:
        e_next = 0
        return [w_next, n_next, M_next, g, s_next, e_next, z_next, H]
    else:
        for e_next in [0,1]:
            x_next.append([w_next, n_next, M_next, g, s_next, e_next, z_next, H])
            prob_next.append(Pe[int(s),s_next,int(e),e_next])
    return x_next[np.random.choice(len(prob_next), p = prob_next)]_____no_output_____'''
    Start with renting:
    w = 5
    n  = 0
    e = 1
    s = 1
    1000 agents for 1 economy, 500 economies.
    use numpy array to contain the variable change:
'''
names = ['w', 'n', 'M', 'gs', 'e',
         's', 'yt', 'c', 'b', 'k',
         'i', 'q', 'h','live', 'H', "Z"]
x0 = [5, 0, 1, 1, 0]
numAgents = 1000
numEcons = 500_____no_output_____import quantecon as qe
import random as rd
mc = qe.MarkovChain(Ps)
EconStates = [mc.simulate(ts_length=T_max - T_min, init=0) for _ in range(numEcons)]

def simulation(i):
    track = np.zeros((T_max - T_min,len(names)))
    econState = EconStates[i//numAgents]
    alive = True
    x = x0
    for t in range(1,len(econState)-1):
        if rd.random() > Pa[t]:
            alive = False
        if alive:
            track[t, 13] = 1
            s_next = econState[t+1]
            # if is still renting
            if (len(x) == 5):
                a = action_rent(t,x)
                # continue to rent
                if (len(a) == 4):
                    # x = [w,n,e,s,z]
                    # a = [c,b,k,h]
                    track[t, 0] = x[0]
                    track[t, 1] = x[1]
                    track[t, 2] = 0
                    track[t, 3] = 0
                    track[t, 4] = x[2]
                    track[t, 5] = x[3]
                    track[t, 6] = y(t,x)
                    track[t, 7] = a[0]
                    track[t, 8] = a[1]
                    track[t, 9] = a[2]
                    track[t, 10] = 0
                    track[t, 11] = 0
                    track[t, 12] = a[3]
                    track[t, 15] = x[4]
                    x = transition_to_rent(x,a,t,s_next)
                # switch to own
                else:
                    # a = [c,b,k,M,H]
                    track[t, 0] = x[0]
                    track[t, 1] = x[1]
                    track[t, 2] = a[3]
                    track[t, 3] = 0
                    track[t, 4] = x[2]
                    track[t, 5] = x[3]
                    track[t, 6] = y(t,x)
                    track[t, 7] = a[0]
                    track[t, 8] = a[1]
                    track[t, 9] = a[2]
                    track[t, 10] = 0
                    track[t, 11] = 1
                    track[t, 12] = a[4]*(1+kappa)
                    track[t, 14] = a[4]
                    track[t, 15] = x[4]
                    x = transition_to_own(x,a,t,s_next)

            # if owning a house already
            else:
                # x = [w,n,M,gs,e,s,z,H]
                # a = [c,b,k,i,q]
                a = action_own(t,x)

                track[t, 0] = x[0]
                track[t, 1] = x[1]
                track[t, 2] = x[2]
                track[t, 3] = x[3]
                track[t, 4] = x[4]
                track[t, 5] = x[5]
                track[t, 6] = y(t,x)
                track[t, 7] = a[0]
                track[t, 8] = a[1]
                track[t, 9] = a[2]
                track[t, 10] = a[3]
                track[t, 11] = a[4]
                track[t, 14] = x[7]
                track[t, 15] = x[6]
                # calculate housing consumption
                if a[4] == 1:
                    h = x[7] + (1-delta)*x[3] + a[3]
                    Vh = (1+kappa)*h
                else:
                    h = x[7] + (1-delta)*x[3]
                    Vh = (1-kappa)*(h-(1-a[4])*x[7])
                track[t, 12] = Vh
                x = transition_after_own(x, a, t, s_next)
    return track_____no_output_____%%time
pool = Pool()
agents = pool.map(simulation, list(range(numAgents*numEcons)))
pool.close()CPU times: user 33 s, sys: 34.2 s, total: 1min 7s
Wall time: 32min 54s
ww = np.zeros((T_max-T_min, numAgents*numEcons))
nn = np.zeros((T_max-T_min, numAgents*numEcons))
MM = np.zeros((T_max-T_min, numAgents*numEcons))
gg = np.zeros((T_max-T_min, numAgents*numEcons))
ee = np.zeros((T_max-T_min, numAgents*numEcons))
ss = np.zeros((T_max-T_min, numAgents*numEcons))
ytyt = np.zeros((T_max-T_min, numAgents*numEcons))
cc = np.zeros((T_max-T_min, numAgents*numEcons))
hh = np.zeros((T_max-T_min, numAgents*numEcons))
bb = np.zeros((T_max-T_min, numAgents*numEcons))
kk = np.zeros((T_max-T_min, numAgents*numEcons))
ii = np.zeros((T_max-T_min, numAgents*numEcons))
qq = np.zeros((T_max-T_min, numAgents*numEcons))
live = np.zeros((T_max-T_min, numAgents*numEcons))
HH = np.zeros((T_max-T_min, numAgents*numEcons))
ZZ = np.zeros((T_max-T_min, numAgents*numEcons))_____no_output_____def separateAttributes(agents):
    for i in range(numAgents*numEcons):
        ww[:,i] = agents[i][:,0]
        nn[:,i] = agents[i][:,1]
        MM[:,i] = agents[i][:,2]
        gg[:,i] = agents[i][:,3]
        ee[:,i] = agents[i][:,4]
        ss[:,i] = agents[i][:,5]
        ytyt[:,i] = agents[i][:,6]
        cc[:,i] = agents[i][:,7]
        bb[:,i] = agents[i][:,8]
        kk[:,i] = agents[i][:,9]
        ii[:,i] = agents[i][:,10]
        qq[:,i] = agents[i][:,11]
        hh[:,i] = agents[i][:,12]
        live[:,i] = agents[i][:,13]
        HH[:,i] = agents[i][:,14]
        ZZ[:,i] = agents[i][:,15]
separateAttributes(agents)_____no_output_____def quantileForPeopleWholive(attribute, quantiles = [0.25, 0.5, 0.75]):
    qList = []
    for i in range(69):
        if len(np.where(live[i,:] == 1)[0]) == 0:
            qList.append(np.array([0] * len(quantiles)))
        else:
            qList.append(np.quantile(attribute[i, np.where(live[i,:] == 1)], q = quantiles))
    return np.array(qList)

def meanForPeopleWholive(attribute):
    means = []
    for i in range(69):
        if len(np.where(live[i,:] == 1)[0]) == 0:
            means.append(np.array([0]))
        else:
            means.append(np.mean(attribute[i, np.where(live[i,:] == 1)]))
    return np.array(means)_____no_output_____# Population during the entire simulation period
plt.plot(np.mean(live,axis = 1))_____no_output_____plt.plot(quantileForPeopleWholive(ww))_____no_output_____plt.plot(quantileForPeopleWholive(nn))_____no_output_____plt.plot(quantileForPeopleWholive(MM))_____no_output_____plt.plot(quantileForPeopleWholive(gg, quantiles = [0.1,0.25, 0.5, 0.75,0.99]))_____no_output_____plt.plot(quantileForPeopleWholive(cc))_____no_output_____plt.plot(quantileForPeopleWholive(bb))_____no_output_____plt.plot(quantileForPeopleWholive(kk))_____no_output_____plt.plot(quantileForPeopleWholive(ii))_____no_output_____plt.plot(quantileForPeopleWholive(qq))_____no_output_____plt.plot(quantileForPeopleWholive(hh))_____no_output_____plt.plot(quantileForPeopleWholive(HH))_____no_output_____plt.plot(quantileForPeopleWholive(ZZ))_____no_output_____# mean value of the key variables
plt.figure(figsize = [14,8])
plt.plot(meanForPeopleWholive(ww), label = "wealth")
plt.plot(meanForPeopleWholive(cc), label = "Consumption")
plt.plot(meanForPeopleWholive(bb), label = "Bond")
plt.plot(meanForPeopleWholive(kk), label = "Stock")
plt.plot(meanForPeopleWholive(nn), label = "401k")
plt.legend()_____no_output_____
</code>


# Notebook from z-haiman/MASSpy
Path: docs/education/sb2/model_construction/sb2_pentose_phosphate_pathway.ipynb

# Pentose Phosphate Pathway Model Construction

Based on Chapter 11 of Systems Biology: Simulation of Dynamic Network States

To construct a model of the pentose phosphate pathway (PPP), first we import **MASSpy** and other essential packages. Constants used throughout the notebook are also defined._____no_output_____

from os import path

import matplotlib.pyplot as plt

from cobra import DictList

from mass import (
    MassConfiguration, MassMetabolite, MassModel,
    MassReaction, Simulation, UnitDefinition)
from mass.io import json, sbml
from mass.util import qcqa_model

mass_config = MassConfiguration()

mass_config.irreversible_Keq = float("inf")_____no_output_____

## Model Construction

The first step of creating a model of the PPP is to define the `MassModel`. _____no_output_____

ppp = MassModel("PentosePhosphatePathway")Set parameter Username

### Metabolites

The next step is to define all of the metabolites using the `MassMetabolite` object. Some considerations for this step include the following:

1. It is important to use a clear and consistent format for identifiers and names when defining the `MassMetabolite` objects for various reasons, some of which include improvements to model clarity and utility, assurance of unique identifiers (required to add metabolites to the model), and consistency when collaborating and communicating with others.

2. In order to ensure our model is physiologically accurate, it is important to provide the `formula` argument with a string representing the chemical formula for each metabolite, and the `charge` argument with an integer representing the metabolite's ionic charge (Note that neutrally charged metabolites are provided with 0). These attributes can always be set later if necessary using the `formula` and `charge` attribute set methods.

3. To indicate that the cytosol is the cellular compartment in which the reactions occur, the string "c" is provided to the `compartment` argument.

This model will be created using identifiers and names found in the [BiGG Database](http://bigg.ucsd.edu/).

In this model, there are 17 metabolites inside the cytosol compartment. _____no_output_____

g6p_c = MassMetabolite(
    "g6p_c",
    name="D-Glucose 6-phosphate",
    formula="C6H11O9P",
    charge=-2,
    compartment="c",
    fixed=False)

_6pgl_c = MassMetabolite(
    "_6pgl_c",
    name="6-Phospho-D-gluco-1,5-lactone",
    formula="C6H9O9P",
    charge=-2,
    compartment="c",
    fixed=False)

_6pgc_c = MassMetabolite(
    "_6pgc_c",
    name="6-Phospho-D-gluconate",
    formula="C6H10O10P",
    charge=-3,
    compartment="c",
    fixed=False)

ru5p__D_c = MassMetabolite(
    "ru5p__D_c",
    name="D-Ribulose 5-phosphate",
    formula="C5H9O8P",
    charge=-2,
    compartment="c",
    fixed=False)

r5p_c = MassMetabolite(
    "r5p_c",
    name="Alpha-D-Ribose 5-phosphate",
    formula="C5H9O8P",
    charge=-2,
    compartment="c",
    fixed=False)

xu5p__D_c = MassMetabolite(
    "xu5p__D_c",
    name="D-Xylulose 5-phosphate",
    formula="C5H9O8P",
    charge=-2,
    compartment="c",
    fixed=False)

g3p_c = MassMetabolite(
    "g3p_c",
    name="Glyceraldehyde 3-phosphate",
    formula="C3H5O6P",
    charge=-2,
    compartment="c",
    fixed=False)

s7p_c = MassMetabolite(
    "s7p_c",
    name="Sedoheptulose 7-phosphate",
    formula="C7H13O10P",
    charge=-2,
    compartment="c",
    fixed=False)

f6p_c = MassMetabolite(
    "f6p_c",
    name="D-Fructose 6-phosphate",
    formula="C6H11O9P",
    charge=-2,
    compartment="c",
    fixed=False)

e4p_c = MassMetabolite(
    "e4p_c",
    name="D-Erythrose 4-phosphate",
    formula="C4H7O7P",
    charge=-2,
    compartment="c",
    fixed=False)

h_c = MassMetabolite(
    "h_c",
    name="H+",
    formula="H",
    charge=1,
    compartment="c",
    fixed=False)

nadp_c = MassMetabolite(
    "nadp_c",
    name="Nicotinamide adenine dinucleotide phosphate",
    formula="[NADP]-C21H25N7O17P3",
    charge=-3,
    compartment="c",
    fixed=False)

nadph_c = MassMetabolite(
    "nadph_c",
    name="Nicotinamide adenine dinucleotide phosphate - reduced",
    formula="[NADP]-C21H26N7O17P3",
    charge=-4,
    compartment="c",
    fixed=False)

h2o_c = MassMetabolite(
    "h2o_c",
    name="H2O",
    formula="H2O",
    charge=0,
    compartment="c",
    fixed=False)

gthox_c = MassMetabolite(
    "gthox_c",
    name="Oxidized glutathione",
    formula="C20H30N6O12S2",
    charge=-2,
    compartment="c",
    fixed=False)

gthrd_c = MassMetabolite(
    "gthrd_c",
    name="Reduced glutathione",
    formula="C10H16N3O6S",
    charge=-1,
    compartment="c",
    fixed=False)

co2_c = MassMetabolite(
    "co2_c",
    name="CO2",
    formula="CO2",
    charge=0,
    compartment="c",
    fixed=False)_____no_output_____

### Reactions

Once all of the `MassMetabolite` objects for each metabolite, the next step is to define all of the reactions that occur and their stoichiometry.

1. As with the metabolites, it is also important to use a clear and consistent format for identifiers and names when defining when defining the `MassReaction` objects.

2. To make this model useful for integration with other models, it is important to provide a string to the `subsystem` argument. By providing the subsystem, the reactions can be easily obtained even when integrated with a significantly larger model through the `subsystem` attribute.

3. After the creation of each `MassReaction` object, the metabolites are added to the reaction using a dictionary where keys are the `MassMetabolite` objects and values are the stoichiometric coefficients (reactants have negative coefficients, products have positive ones).

This model will be created using identifiers and names found in the [BiGG Database](http://bigg.ucsd.edu/).

In this model, there are 10 reactions occuring inside the cytosol compartment._____no_output_____

G6PDH2r = MassReaction(
    "G6PDH2r",
    name="Glucose 6-phosphate dehydrogenase",
    subsystem=ppp.id,
    reversible=True)
G6PDH2r.add_metabolites({
    g6p_c: -1,
    nadp_c: -1,
    _6pgl_c: 1,
    nadph_c: 1,
    h_c: 1})

PGL = MassReaction(
    "PGL",
    name="6-phosphogluconolactonase",
    subsystem=ppp.id,
    reversible=True)
PGL.add_metabolites({
    _6pgl_c: -1,
    h2o_c: -1,
    _6pgc_c: 1,
    h_c: 1})

GND = MassReaction(
    "GND",
    name="Phosphogluconate dehydrogenase",
    subsystem=ppp.id,
    reversible=True)
GND.add_metabolites({
    _6pgc_c: -1,
    nadp_c: -1,
    nadph_c: 1,
    co2_c: 1,
    ru5p__D_c: 1})

RPI = MassReaction(
    "RPI",
    name="Ribulose 5-Phosphate Isomerase",
    subsystem=ppp.id,
    reversible=True)
RPI.add_metabolites({
    ru5p__D_c: -1,
    r5p_c: 1})

RPE = MassReaction(
    "RPE",
    name="Ribulose 5-phosphate 3-epimerase",
    subsystem=ppp.id,
    reversible=True)
RPE.add_metabolites({
    ru5p__D_c: -1,
    xu5p__D_c: 1})

TKT1 = MassReaction(
    "TKT1",
    name="Transketolase",
    subsystem=ppp.id,
    reversible=True)
TKT1.add_metabolites({
    r5p_c: -1,
    xu5p__D_c: -1,
    g3p_c: 1,
    s7p_c: 1})

TALA = MassReaction(
    "TALA",
    name="Transaldolase",
    subsystem=ppp.id,
    reversible=True)
TALA.add_metabolites({
    g3p_c: -1,
    s7p_c: -1,
    e4p_c: 1,
    f6p_c: 1})

TKT2 = MassReaction(
    "TKT2",
    name="Transketolase",
    subsystem=ppp.id,
    reversible=True)
TKT2.add_metabolites({
    e4p_c: -1,
    xu5p__D_c: -1,
    f6p_c: 1,
    g3p_c: 1})

GTHOr = MassReaction(
    "GTHOr",
    name="Glutathione oxidoreductase",
    subsystem="Misc.",
    reversible=True)
GTHOr.add_metabolites({
    gthox_c: -1,
    h_c: -1,
    nadph_c: -1,
    gthrd_c: 2,
    nadp_c: 1})

GSHR = MassReaction(
    "GSHR",
    name="Glutathione-disulfide reductase",
    subsystem="Misc.",
    reversible=True)
GSHR.add_metabolites({
    gthrd_c: -2,
    gthox_c: 1,
    h_c: 2})_____no_output_____

After generating the reactions, all reactions are added to the model through the `MassModel.add_reactions` class method. Adding the `MassReaction` objects will also add their associated `MassMetabolite` objects if they have not already been added to the model. _____no_output_____

ppp.add_reactions([
    G6PDH2r, PGL, GND, RPI, RPE, TKT1, TALA, TKT2, GTHOr, GSHR])

for reaction in ppp.reactions:
    print(reaction)G6PDH2r: g6p_c + nadp_c <=> _6pgl_c + h_c + nadph_c
PGL: _6pgl_c + h2o_c <=> _6pgc_c + h_c
GND: _6pgc_c + nadp_c <=> co2_c + nadph_c + ru5p__D_c
RPI: ru5p__D_c <=> r5p_c
RPE: ru5p__D_c <=> xu5p__D_c
TKT1: r5p_c + xu5p__D_c <=> g3p_c + s7p_c
TALA: g3p_c + s7p_c <=> e4p_c + f6p_c
TKT2: e4p_c + xu5p__D_c <=> f6p_c + g3p_c
GTHOr: gthox_c + h_c + nadph_c <=> 2 gthrd_c + nadp_c
GSHR: 2 gthrd_c <=> gthox_c + 2 h_c

### Boundary reactions

After generating the reactions, the next step is to add the boundary reactions and boundary conditions (the concentrations of the boundary 'metabolites' of the system). This can easily be done using the `MassModel.add_boundary` method. With the generation of the boundary reactions, the system becomes an open system, allowing for the flow of mass through the biochemical pathways of the model. Once added, the model will be able to return the boundary conditions as a dictionary through the `MassModel.boundary_conditions` attribute.

All boundary reactions are originally created with the metabolite as the reactant. However, there are times where it would be preferable to represent the metabolite as the product. For these situtations, the `MassReaction.reverse_stoichiometry` method can be used with its `inplace` argument to create a new `MassReaction` or simply reverse the stoichiometry for the current `MassReaction.`

In this model, there are 7 boundary reactions that must be defined._____no_output_____

DM_f6p_c = ppp.add_boundary(
    metabolite=f6p_c, boundary_type="demand", subsystem="Pseudoreaction",
    boundary_condition=1)

DM_r5p_c = ppp.add_boundary(
    metabolite=r5p_c, boundary_type="demand", subsystem="Pseudoreaction",
    boundary_condition=1)

DM_g3p_c = ppp.add_boundary(
    metabolite=g3p_c, boundary_type="demand", subsystem="Pseudoreaction",
    boundary_condition=1)
SK_g6p_c = ppp.add_boundary(
    metabolite=g6p_c, boundary_type="sink", subsystem="Pseudoreaction",
    boundary_condition=1)
SK_g6p_c.reverse_stoichiometry(inplace=True)

SK_h_c = ppp.add_boundary(
    metabolite=h_c, boundary_type="sink", subsystem="Pseudoreaction",
    boundary_condition=6.30957e-05)

SK_h2o_c = ppp.add_boundary(
    metabolite=h2o_c, boundary_type="sink", subsystem="Pseudoreaction",
    boundary_condition=1)

SK_co2_c = ppp.add_boundary(
    metabolite=co2_c, boundary_type="sink", subsystem="Pseudoreaction",
    boundary_condition=1)

print("Boundary Reactions and Values\n-----------------------------")
for reaction in ppp.boundary:
    boundary_met = reaction.boundary_metabolite
    bc_value = ppp.boundary_conditions.get(boundary_met)
    print("{0}\n{1}: {2}\n".format(
        reaction, boundary_met, bc_value))Boundary Reactions and Values
-----------------------------
DM_f6p_c: f6p_c -->
f6p_b: 1.0

DM_r5p_c: r5p_c -->
r5p_b: 1.0

DM_g3p_c: g3p_c -->
g3p_b: 1.0

SK_g6p_c:  <=> g6p_c
g6p_b: 1.0

SK_h_c: h_c <=>
h_b: 6.30957e-05

SK_h2o_c: h2o_c <=>
h2o_b: 1.0

SK_co2_c: co2_c <=>
co2_b: 1.0

### Ordering of internal species and reactions

Sometimes, it is also desirable to reorder the metabolite and reaction objects inside the model to follow the physiology. To reorder the internal objects, one can use `cobra.DictList` containers and the `DictList.get_by_any` method with the list of object identifiers in the desirable order. To ensure all objects are still present and not forgotten in the model, a small QA check is also performed. _____no_output_____

new_metabolite_order = [
    "f6p_c", "g6p_c", "g3p_c", "_6pgl_c", "_6pgc_c",
    "ru5p__D_c", "xu5p__D_c", "r5p_c", "s7p_c", "e4p_c",
    "nadp_c", "nadph_c", "gthrd_c", "gthox_c", "co2_c",
    "h_c", "h2o_c"]

if len(ppp.metabolites) == len(new_metabolite_order):
    ppp.metabolites = DictList(
        ppp.metabolites.get_by_any(new_metabolite_order))

new_reaction_order = [
    "G6PDH2r", "PGL", "GND", "RPE", "RPI",
    "TKT1", "TKT2", "TALA", "GTHOr", "GSHR",
    "SK_g6p_c", "DM_f6p_c", "DM_g3p_c","DM_r5p_c",
    "SK_co2_c", "SK_h_c", "SK_h2o_c"]

if len(ppp.reactions) == len(new_reaction_order):
    ppp.reactions = DictList(
        ppp.reactions.get_by_any(new_reaction_order))

ppp.update_S(array_type="DataFrame", dtype=int)_____no_output_____

## Model Parameterization
### Steady State fluxes
Steady state fluxes can be computed as a summation of the MinSpan pathway vectors. Pathways are obtained using [MinSpan](https://github.com/SBRG/minspan).

Using these pathways and literature sources, independent fluxes can be defined in order to calculate the steady state flux vector. For this model, the flux of glucose-6-phosphate uptake is fixed at 0.21 mM/hr, and the flux of Alpha-D-Ribose 5-phosphate uptake is fixed at 0.01 mM/hr. The former value represents a typical flux through the pentose pathway while the latter will couple at a low flux level to the AMP pathways

With these pathays and numerical values, the steady state flux vector can be computed as the weighted sum of the corresponding basis vectors. The steady state flux vector is computed as an inner product:_____no_output_____

minspan_paths = [
    [1, 1, 1, 2/3, 1/3, 1/3, 1/3, 1/3, 2, 2, 1, 2/3, 1/3, 0, 1, 4, -1],
    [1, 1, 1,  0,   1,   0,   0,   0,  2, 2, 1,  0,   0,  1, 1, 4, -1]]
ppp.compute_steady_state_fluxes(
    pathways=minspan_paths,
    independent_fluxes={
        SK_g6p_c: 0.21,
        DM_r5p_c: 0.01},
    update_reactions=True)

print("Steady State Fluxes\n-------------------")
for reaction, steady_state_flux in ppp.steady_state_fluxes.items():
    print("{0}: {1:.6f}".format(reaction.flux_symbol_str, steady_state_flux))Steady State Fluxes
-------------------
v_G6PDH2r: 0.210000
v_PGL: 0.210000
v_GND: 0.210000
v_RPE: 0.133333
v_RPI: 0.076667
v_TKT1: 0.066667
v_TKT2: 0.066667
v_TALA: 0.066667
v_GTHOr: 0.420000
v_GSHR: 0.420000
v_SK_g6p_c: 0.210000
v_DM_f6p_c: 0.133333
v_DM_g3p_c: 0.066667
v_DM_r5p_c: 0.010000
v_SK_co2_c: 0.210000
v_SK_h_c: 0.840000
v_SK_h2o_c: -0.210000

### Initial Conditions

Once the network has been built, the concentrations can be added to the metabolites. These concentrations are also treated as the initial conditions required to integrate and simulate the model's ordinary differential equations (ODEs). The metabolite concentrations are added to each individual metabolite using the `MassMetabolite.initial_condition` (alias: `MassMetabolite.ic`) attribute setter methods. Once added, the model will be able to return the initial conditions as a dictionary through the `MassModel.initial_conditions` attribute._____no_output_____

g6p_c.ic = 0.0486
f6p_c.ic = 0.0198
g3p_c.ic = 0.00728
_6pgl_c.ic = 0.00175424
_6pgc_c.ic = 0.0374753
ru5p__D_c.ic = 0.00493679
xu5p__D_c.ic = 0.0147842
r5p_c.ic = 0.0126689
s7p_c.ic = 0.023988
e4p_c.ic = 0.00507507
nadp_c.ic = 0.0002
nadph_c.ic = 0.0658
gthrd_c.ic = 3.2
gthox_c.ic = 0.12
co2_c.ic = 1
h_c.ic = 0.0000714957
h2o_c.ic = 1

print("Initial Conditions\n------------------")
for metabolite, ic_value in ppp.initial_conditions.items():
    print("{0}: {1}".format(metabolite, ic_value))Initial Conditions
------------------
f6p_c: 0.0198
g6p_c: 0.0486
g3p_c: 0.00728
_6pgl_c: 0.00175424
_6pgc_c: 0.0374753
ru5p__D_c: 0.00493679
xu5p__D_c: 0.0147842
r5p_c: 0.0126689
s7p_c: 0.023988
e4p_c: 0.00507507
nadp_c: 0.0002
nadph_c: 0.0658
gthrd_c: 3.2
gthox_c: 0.12
co2_c: 1
h_c: 7.14957e-05
h2o_c: 1

### Equilibirum Constants

After adding initial conditions and steady state fluxes, the equilibrium constants are defined using the `MassReaction.equilibrium_constant` (alias: `MassReaction.Keq`) setter method. _____no_output_____

G6PDH2r.Keq = 1000
PGL.Keq = 1000
GND.Keq = 1000
RPE.Keq = 3
RPI.Keq = 2.57
TKT1.Keq = 1.2
TKT2.Keq = 10.3
TALA.Keq = 1.05
GTHOr.Keq = 100
GSHR.Keq = 2

SK_g6p_c.Keq = 1
SK_h_c.Keq = 1
SK_h2o_c.Keq = 1
SK_co2_c.Keq = 1

print("Equilibrium Constants\n---------------------")
for reaction in ppp.reactions:
    print("{0}: {1}".format(reaction.Keq_str, reaction.Keq))Equilibrium Constants
---------------------
Keq_G6PDH2r: 1000
Keq_PGL: 1000
Keq_GND: 1000
Keq_RPE: 3
Keq_RPI: 2.57
Keq_TKT1: 1.2
Keq_TKT2: 10.3
Keq_TALA: 1.05
Keq_GTHOr: 100
Keq_GSHR: 2
Keq_SK_g6p_c: 1
Keq_DM_f6p_c: inf
Keq_DM_g3p_c: inf
Keq_DM_r5p_c: inf
Keq_SK_co2_c: 1
Keq_SK_h_c: 1
Keq_SK_h2o_c: 1

### Calculation of PERCs
By defining the equilibrium constant and steady state parameters, the values of the pseudo rate constants (PERCs) can be calculated and added to the model using the `MassModel.calculate_PERCs` method._____no_output_____

ppp.calculate_PERCs(update_reactions=True)

print("Forward Rate Constants\n----------------------")
for reaction in ppp.reactions:
    print("{0}: {1:.6f}".format(reaction.kf_str, reaction.kf))Forward Rate Constants
----------------------
kf_G6PDH2r: 21864.589249
kf_PGL: 122.323112
kf_GND: 29287.807474
kf_RPE: 15284.677111
kf_RPI: 10564.620051
kf_TKT1: 1595.951975
kf_TKT2: 1092.246435
kf_TALA: 844.616138
kf_GTHOr: 53.329812
kf_GSHR: 0.041257
kf_SK_g6p_c: 0.220727
kf_DM_f6p_c: 6.734007
kf_DM_g3p_c: 9.157509
kf_DM_r5p_c: 0.789335
kf_SK_co2_c: 100000.000000
kf_SK_h_c: 100000.000000
kf_SK_h2o_c: 100000.000000

## QC/QA Model

Before simulating the model, it is important to ensure that the model is elementally balanced, and that the model can simulate. Therefore, the `qcqa_model` function from the `mass.util.qcqa` submodule is used to provide a report on the model quality and indicate whether simulation is possible and if not, what parameters and/or initial conditions are missing.

Generally, pseudoreactions (e.g. boundary exchanges, sinks, demands) are not elementally balanced. The `qcqa_model` function does not include elemental balancing of boundary reactions. However, some models contain pseudoreactions reprsenting a simplified mechanism, and show up in the returned report. The elemental imbalance of these pseudoreactions is therefore expected in certain reaction and should not be a cause for concern.

In this model of the PPP, the GSHR reaction is a simplified pseudoreaction that represents oxidative stress due to gluthathione and is not expected to be balanced._____no_output_____

qcqa_model(ppp, parameters=True, concentrations=True,
           fluxes=True, superfluous=True, elemental=True)╒══════════════════════════════════════════════╕
│ MODEL ID: PentosePhosphatePathway            │
│ SIMULATABLE: True                            │
│ PARAMETERS NUMERICALY CONSISTENT: True       │
╞══════════════════════════════════════════════╡
│ ============================================ │
│             CONSISTENCY CHECKS               │
│ ============================================ │
│ Elemental                                    │
│ -------------------                          │
│ GSHR: {charge: 2.0}                          │
│ ============================================ │
╘══════════════════════════════════════════════╛

From the results of the QC/QA test, it can be seen that the model can be simulated and is numerically consistent._____no_output_____## Steady State and Model Validation

To find the steady state of the model and perform simulations, the model must first be loaded into a `Simulation`. In order to load a model into a `Simulation`, the model must be simulatable, meaning there are no missing numerical values that would prevent the integration of the ODEs that comprise the model. The `verbose` argument can be used while loading a model to produce a message indicating the successful loading of a model, or why a model could not load.

Once loaded into a `Simulation`, the `find_steady_state` method can be used with the `update_values` argument in order to update the initial conditions and fluxes of the model to a steady state (if necessary). The model can be simulated using the `simulate` method by passing the model to simulate, and a tuple containing the start time and the end time. The number of time points can also be included, but is optional.

After a successful simulation, two `MassSolution` objects are returned. The first `MassSolution` contains the concentration results of the simulation, and the second contains the flux results of the simulation.

To visually validate the steady state of the model, concentration and flux solutions can be plotted using the `plot_time_profile` function from `mass.visualization`. Alternatively, the `MassSolution.view_time_profile` property can be used to quickly generate a time profile for the results._____no_output_____

# Setup simulation object
sim = Simulation(ppp, verbose=True)
# Simulate from 0 to 1000 with 10001 points in the output
conc_sol, flux_sol = sim.simulate(ppp, time=(0, 1e3))
# Quickly render and display time profiles
conc_sol.view_time_profile()Successfully loaded MassModel 'PentosePhosphatePathway' into RoadRunner.

### Storing information and references
#### Compartment
Because the character "c" represents the cytosol compartment, it is recommended to define and set the compartment in the `MassModel.compartments` attribute._____no_output_____

ppp.compartments = {"c": "Cytosol"}
print(ppp.compartments){'c': 'Cytosol'}

#### Units
All of the units for the numerical values used in this model are "Millimoles" for amount and "Liters" for volume (giving a concentration unit of 'Millimolar'), and "Hours" for time. In order to ensure that future users understand the numerical values for model, it is important to define the `MassModel.units` attribute.

The `MassModel.units` is a `cobra.DictList` that contains only `UnitDefinition` objects from the `mass.core.unit` submodule. Each `UnitDefinition` is created from `Unit` objects representing the base units that comprise the `UnitDefinition`. These `Units` are stored in the `list_of_units` attribute. Pre-built units can be viewed using the `print_defined_unit_values` function from the `mass.core.unit` submodule. Alternatively, custom units can also be created using the `UnitDefinition.create_unit` method. For more information about units, please see the module docstring for `mass.core.unit` submodule.

__Note:__ It is important to note that this attribute will NOT track units, but instead acts as a reference for the user and others so that they can perform necessary unit conversions._____no_output_____

# Using pre-build units to define UnitDefinitions
concentration = UnitDefinition("mM", name="Millimolar",
                               list_of_units=["millimole", "per_litre"])
time = UnitDefinition("hr", name="hour", list_of_units=["hour"])

# Add units to model
ppp.add_units([concentration, time])
print(ppp.units)[<UnitDefinition Millimolar "mM" at 0x7fd9f93fd2b0>, <UnitDefinition hour "hr" at 0x7fd9f93fd190>]

## Export

After validation, the model is ready to be saved. The model can either be exported as a ".json" file or as an ".sbml" (".xml") file using their repsective submodules in `mass.io`.

To export the model, only the path to the directory and the model object itself need to be specified._____no_output_____### Export using SBML_____no_output_____

sbml.write_sbml_model(mass_model=ppp, filename="SB2_" + ppp.id + ".xml")_____no_output_____

### Export using JSON_____no_output_____

json.save_json_model(mass_model=ppp, filename="SB2_" + ppp.id + ".json")_____no_output_____


# Notebook from lasofivec/dataflowr
Path: Notebooks/04_dogscast_fromcolab_emty.ipynb

# Adding a dense layer to VGG

The last layer of Vgg16 outputs a vector of 1000 categories, because that is the number of categories the competition asked for. Of these categories, some of them certainly correspond to cats and dogs, but at a much more granular level (specific breeds).

We will simply add a Dense layer on top of the imagenet layer, and train the model to map the imagenet classifications of input images of cats and dogs to cat and dog labels.

Note that this is not what we have been doing in the very first lecture!

Have a look at [CS231n: Linear Classification](http://cs231n.github.io/linear-classify/) for more precisions and especially to [CS231n: Softmax classifier](http://cs231n.github.io/linear-classify/#softmax) if you had trouble with logistic regression._____no_output_____## 1. Preparations_____no_output_____

from PIL import Image
import numpy as np
import matplotlib.pyplot as plt
import os
import torch
import torch.nn as nn
import torchvision
from torchvision import models,transforms,datasets
import bcolz
import time
%matplotlib inline_____no_output_____

We did precompute the outputs of Vgg16 model on our dataset (with Colab) and stored these values._____no_output_____

use_gpu = torch.cuda.is_available()
print('Using gpu: %s ' % use_gpu)

dtype = torch.FloatTensor
if use_gpu:
    dtype = torch.cuda.FloatTensor_____no_output_____def load_array(fname):
    return bcolz.open(fname)[:]_____no_output_____#where you stored your features
data_dir_colab = '/home/lelarge/courses/data/colab/'_____no_output_____feat_train = load_array(os.path.join(data_dir_colab,'vgg16','feat_train.bc'))
lbs_train = load_array(os.path.join(data_dir_colab,'vgg16','lbs_train.bc'))
feat_val = load_array(os.path.join(data_dir_colab,'vgg16','feat_val.bc'))
lbs_val = load_array(os.path.join(data_dir_colab,'vgg16','lbs_val.bc'))_____no_output_____

## 2. Linear model for VGG16 features_____no_output_____We are now ready to define our linear model.

For more details about the [cross entropy cost function](http://neuralnetworksanddeeplearning.com/chap3.html#the_cross-entropy_cost_function)_____no_output_____

lm = torch.nn.Sequential(
    torch.nn.Linear(1000, 2),
    torch.nn.LogSoftmax(dim = 1)
)
loss_fn = torch.nn.NLLLoss(size_average=False)
if use_gpu:
    lm = lm.cuda()_____no_output_____

Since our features are currently stacked in a _numpy ndarray_, we need to create a dataset of tensors and then a dataloader.

For the dataset, you can use _torch.from_numpy_, _torch.tensor_ and _zip_

For the dataloader, you should use _torch.utils.data.DataLoader______no_output_____

bs = 128

train_dataset = # your code
test_dataset = # your code
train_loader = torch.utils.data.DataLoader(#your code here)
test_loader = torch.utils.data.DataLoader(#your code here)_____no_output_____

### 2.1 Training_____no_output_____We define next a holistic training function (```train_model```) that will:
- run for a pre-defined number of epochs/iterations
- fetch training samples randomly during each epoch(all samples are used during an epoch)
- pass samples through network, compute error, gradients and updates network parameters
- keep and print training statistics: training loss, accuracy
_____no_output_____

def train_model(model,size,data_loader=None,epochs=1,optimizer=None):
    model.train()
    loss_t = np.zeros(epochs)
    acc_t = np.zeros(epochs)
    for epoch in range(epochs):

        running_loss = 0.0
        running_corrects = 0
        for inputs,classes in data_loader:

            #
            # your code
            #
            running_loss += # your code
            running_corrects += # your code

        epoch_loss = running_loss / size
        epoch_acc = running_corrects.data.item() / size

        loss_t[epoch] = epoch_loss
        acc_t[epoch] = epoch_acc
    print('Loss: {:.4f} Acc: {:.4f}'.format(epoch_loss, epoch_acc))
    return loss_t, acc_t_____no_output_____

We set our hyperparameters:
- learning rate
- optimizer to be used for gradient descent, here SGD (Stochastic Gradient Descent)_____no_output_____

learning_rate = 1e-4
optimizer_lm = torch.optim.SGD(lm.parameters(), lr=learning_rate)_____no_output_____dset_sizes = {'train': 23000, 'valid': 2000}_____no_output_____

We train our model for 100 epochs_____no_output_____

%%time
loss1, acc1 = train_model(model=lm,size=dset_sizes['train'],data_loader = train_loader ,epochs=100,optimizer=optimizer_lm)
#loss1, acc1 = (train_model(model=lm,size=dset_sizes['train'],feat=feat_train,labels=lbs_train, epochs=100,optimizer=optimizer_lm,batch_size = 64,shuffle=True))_____no_output_____

We plot the evolution of the training loss across epochs.

Ideally is should have a steep descent in the first epochs, then decrease smoothly._____no_output_____

plt.plot(loss1)_____no_output_____

We plot the evolution of the accuracy of our model on the training data. The behavior resembles globally to the one of the loss: big improvement at the beginning, then smaller improvements as training advances.
_____no_output_____

plt.plot(acc1)_____no_output_____

The __loss__ helps the network to learn and update the parameters according to the criterion that we give to the network.

The __accuracy__ on the other hand is a performance metric for the task for which we want to use the network for. In many cases the accuracy cannot be integrated as a loss/criterion function, so we need to identify or design loss functions that will guide the model towards the behavior we wish to have for our task._____no_output_____Next we let the model train for 100 additional epochs_____no_output_____

%%time
loss2, acc2 = train_model(model=lm,size=dset_sizes['train'],data_loader =train_loader ,epochs=100,optimizer=optimizer_lm)_____no_output_____

Again we plot the loss and accuracy for the current training interval: _epochs[100:200]_.
What changes do you notice? _____no_output_____

plt.plot(loss2)_____no_output_____plt.plot(acc2)_____no_output_____

We train the model train for 100 more epochs and plot the evolution of our training indicators.
How are they evolving comparing to the previous runs?
 _____no_output_____

%%time
loss3, acc3 = train_model(model=lm,size=dset_sizes['train'],data_loader =train_loader ,epochs=100,optimizer=optimizer_lm)_____no_output_____plt.plot(loss3)_____no_output_____plt.plot(acc3)_____no_output_____

### 2.2 Testing_____no_output_____We define next a holistic test function (```test_model```) that will:
- fetch test samples
- pass samples through network, compute error, accuracy and predictions
- keep and print test statistics: test loss, accuracy
_____no_output_____

def test_model(model,size,data_loader=None):
    model.eval()

    predictions = np.zeros(size)
    running_loss = 0.0
    running_corrects = 0
    count = 0
    for inputs,classes in data_loader:
        #
        # your code
        #
        count +=1

    print('Loss: {:.4f} Acc: {:.4f}'.format(running_loss / size, running_corrects.data.item() / size))
    return predictions, running_loss / size, running_corrects.data.item() / size_____no_output_____

We evaluate on the test data a snapshot of our model at _epoch #300______no_output_____

%%time
preds, loss_val, acc_val = test_model(model=lm,size=dset_sizes['valid'],data_loader=test_loader)_____no_output_____loss_val_____no_output_____

## 3. Quantitative analysis_____no_output_____We concatenate the training losses across the 300 training epochs and plot them along with the loss on the test data using a snapshot of our model at epoch #300.

What do you notice? _____no_output_____

plt.plot(np.concatenate((loss1, loss2, loss3)))
plt.plot([loss_val]*300)_____no_output_____

We illustrate a similar plot for the training loss values at _epochs[200:300]______no_output_____

plt.plot(loss3)
plt.plot([loss_val]*100)_____no_output_____

We now illustrate the aggregated training accuracies on epochs[0:300] along with the test accuracy for the model at epoch #300._____no_output_____

plt.plot(np.concatenate((acc1, acc2, acc3)))
plt.plot([acc_val]*300)_____no_output_____

We train our model for 1000 more epochs._____no_output_____

%%time
loss4, acc4 = train_model(model=lm,size=dset_sizes['train'],data_loader =train_loader ,epochs=1000,optimizer=optimizer_lm)_____no_output_____

We test the model snapshot at _epoch #1300_ and keep its statiscs and performance._____no_output_____

%%time
preds2, conf2, loss_val2, acc_val2 = (test_model(model=lm,size=dset_sizes['valid'],feat=feat_val,labels=lbs_val,batch_size=2000))_____no_output_____

We aggregate train loss values at _epochs[300:1300]_ and test loss at _epochs[300]_ and _epochs[1300]_.
Do you notice a trend?_____no_output_____

plt.plot(np.concatenate((loss3,loss4)))
plt.plot(np.concatenate(([loss_val]*100,[loss_val2]*1000)))_____no_output_____

A similar plot for the accuracy values_____no_output_____

plt.plot(np.concatenate((acc1, acc2, acc3, acc4)))
plt.plot(np.concatenate(([acc_val]*300,[acc_val2]*1000)))_____no_output_____acc_val_____no_output_____

## Exercise

What is happening?

Make better plots on which we see the evolution of the loss/accuracy on both the training and validation sets as a function of the number of epochs._____no_output_____## 4. Viewing model prediction (qualitative analysis)

The most important metrics for us to look at are for the validation set, since we want to check for over-fitting.

With our first model we should try to overfit before we start worrying about how to handle that - there's no point even thinking about regularization, data augmentation, etc if you're still under-fitting! (We'll be looking at these techniques after the 2 weeks break...)

As well as looking at the overall metrics, it's also a good idea to look at examples of each of:

   1. A few correct labels at random
   2. A few incorrect labels at random
   3. The most correct labels of each class (ie those with highest probability that are correct)
   4. The most incorrect labels of each class (ie those with highest probability that are incorrect)
   5. The most uncertain labels (ie those with probability closest to 0.5).

In general, these are particularly useful for debugging problems in the model. Since our model is very simple, there may not be too much to learn at this stage..._____no_output_____

# Number of images to view for each visualization task
n_view = 8_____no_output_____

Selecting correct predictions._____no_output_____

correct = np.where(preds==lbs_val)[0]_____no_output_____from numpy.random import random, permutation
idx = permutation(correct)[:n_view]_____no_output_____idx_____no_output_____def imshow(inp, title=None):
#   Imshow for Tensor.
    inp = inp.numpy().transpose((1, 2, 0))
    mean = np.array([0.485, 0.456, 0.406])
    std = np.array([0.229, 0.224, 0.225])
    inp = std * inp + mean
    plt.imshow(inp)
    if title is not None:
        plt.title(title)
    plt.pause(0.001)  # pause a bit so that plots are updated

data_dir = '/home/lelarge/courses/data/dogscats'
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])

prep1 = transforms.Compose([
                transforms.CenterCrop(224),
                transforms.ToTensor(),
                normalize,
            ])
dsets = {x: datasets.ImageFolder(os.path.join(data_dir, x), prep1)
         for x in ['train', 'valid']}

dataset_correct = torch.utils.data.DataLoader([dsets['valid'][x] for x in idx],batch_size = n_view,shuffle=True)_____no_output_____for data in dataset_correct:
    inputs_cor,labels_cor = data_____no_output_____# Make a grid from batch
out = torchvision.utils.make_grid(inputs_cor)

imshow(out, title=[x for x in labels_cor])_____no_output_____from IPython.display import Image, display
for x in idx:
    display(Image(filename=dsets['valid'].imgs[x][0], retina=True))_____no_output_____

Selecting incorrect predictions._____no_output_____

incorrect = np.where(preds!=lbs_val)[0]
for x in permutation(incorrect)[:n_view]:
    print(dsets['valid'].imgs[x][1])
    display(Image(filename=dsets['valid'].imgs[x][0], retina=True))_____no_output_____#3. The images we most confident were cats, and are actually cats
correct_cats = np.where((preds==0) & (preds==lbs_val))[0]
most_correct_cats = np.argsort(conf[correct_cats,1])[:n_view]_____no_output_____for x in most_correct_cats:
    display(Image(filename=dsets['valid'].imgs[correct_cats[x]][0], retina=True))_____no_output_____#3. The images we most confident were dogs, and are actually dogs
correct_dogs = np.where((preds==1) & (preds==lbs_val))[0]
most_correct_dogs = np.argsort(conf[correct_dogs,0])[:n_view]_____no_output_____for x in most_correct_dogs:
    display(Image(filename=dsets['valid'].imgs[correct_dogs[x]][0], retina=True))_____no_output_____

## Exercise

As seen in the first lecture, the last layer of Vgg16 is simply a dense layer that outputs 1000 elements. Therefore, it seems somewhat unreasonable to stack a dense layer meant to find cats and dogs on top of one that's meant to find imagenet categories, in that we're limiting the information available to us by first coercing the neural network to classify to imagenet before cats and dogs...

Instead, do finetuning, i.e remove that last layer and add on a new layer for cats and dogs.

Compare to what we did in the first lecture._____no_output_____