# Notebook from ababino/corona
Path: Binomial_L1_LR_AIC_states_all.ipynb

# LASSO Regresion on covid-19 cases in NYS

## Load modulues_____no_output_____

%matplotlib widget
import os
import requests
import urllib.parse
import json
import io
from zipfile import ZipFile
from datetime import datetime, timedelta

import pandas as pd
import numpy as np
import statsmodels.api as sm
import scipy

import seaborn as sns
import matplotlib.pyplot as plt
import matplotlib.dates as mdates
from statsmodels.sandbox.regression.predstd import wls_prediction_std
from sklearn import linear_model
from scipy import stats as sps
from sklearn.feature_selection import SelectFromModel
from IPython.display import display

locator = mdates.AutoDateLocator()
formatter = mdates.ConciseDateFormatter(locator)

sns.set_style("whitegrid")
GAMMA = 1/7.5

events = {}
df_dict = {}
dataset_info = {}
#may 15 last day_____no_output_____dsname = 'New York'
dfs = []
offset = 0
while offset >=0:
    url = "https://health.data.ny.gov/resource/xdss-u53e.csv/?$limit=5000&$offset={}".format(offset)
    df = pd.read_csv(url)
    dfs.append(df)
    if len(df)==5000:
        offset += 5000
    else:
        offset = -1
dfraw = pd.concat(dfs)
#'test_date=2020-03-15T00:00:00.000'

#dfraw = dfraw.rename(columns={'new_positives': 'Positives', 'total_number_of_tests': 'Tests', 'test_date': 'date'})
print(len(dfraw))
#dfraw['date'] = pd.to_datetime(dfraw['date'])

#counties = (df.groupby('date')['Tests']>0).count()
df = dfraw.groupby('test_date').sum()
dfraw.to_csv('NYS_all_data.csv')

8184

## Important dates_____no_output_____

# intervention dates
# 03/10/2020 school close in New Rochelle
# https://www.governor.ny.gov/news/during-novel-coronavirus-briefing-governor-cuomo-announces-new-mass-gatherings-regulations
# 03/12/2020 mass gathering reduced to 500 max. 50% occupancy

# 03/17/2020 close of gyms, restaurants and bars, movie theaters, mass gathering up to 50. https://www.governor.ny.gov/news/amid-lack-federal-direction-governor-cuomo-governor-murphy-and-governor-lamont-announce
bars = pd.to_datetime('03-16-2020 20:00', dayfirst=False)
# 03/18/2020 school clousure http://www.nysed.gov/news/2020/state-education-department-issues-updated-guidance-schools-regarding-novel-coronavirus
schools = pd.to_datetime('03-18-2020 20:00', dayfirst=False)

# https://www.governor.ny.gov/news/amid-ongoing-covid-19-pandemic-governor-cuomo-announces-deployment-1000-bed-hospital-ship-usns
# 03/20/2020 00:00 50% of the workforce
workforce_50 = pd.to_datetime('03-20-2020 20:00', dayfirst=False)
# 03/22/2020 20:00 ny_pause
ny_pause = pd.to_datetime('22-03-2020 00:00', dayfirst=True)
# CDC masks https://www.npr.org/sections/goatsandsoda/2020/04/10/829890635/why-there-so-many-different-guidelines-for-face-masks-for-the-public
masks_cdc = pd.to_datetime('03-04-2020 00:00', dayfirst=True)
mask_employers = pd.to_datetime('12-04-2020 00:00', dayfirst=True)
mask_public = pd.to_datetime('17-04-2020 00:00', dayfirst=True)

events_list = [bars, schools, workforce_50, ny_pause, masks_cdc, mask_employers, mask_public]
events['New York'] = events_list_____no_output_____

## Get data_____no_output_____

dsname = 'Alaska'
url = 'https://opendata.arcgis.com/datasets/72c6d13ea1e9420ea398724bdd10372f_0.geojson'
if os.path.isfile('data/alaska.csv'):
    r = requests.get(url)
    data = json.loads(r.content)
    df = pd.DataFrame([x['properties'] for x in data['features']])
    df['date'] = pd.to_datetime(df['Date_'].str[:-3], format='%Y/%m/%d 12:00:00')
    df = df.rename(columns={'daily_positive': 'Positives', 'daily_negative': 'Negatives', 'daily_tests': 'Tests'})
    df = df.set_index('date')
    #df.index = df.index.round('D')
    df['Date'] = df.index
    df.to_csv('data/alaska.csv')

else:
    df = pd.read_csv('data/alaska.csv', index_col='date', parse_dates=['date', 'Date'])
df.Positives = df.Positives.astype(float)
df.Tests = df.Tests.astype(float)
df.Negatives = df.Negatives.astype(float)
df = df[['Date', 'Positives', 'Negatives', 'Tests']]
df['Odds'] = df.Positives /df.Negatives
df = df[df['Date']>'2020-03-23']
df_dict[dsname] = df
dataset_info[dsname] = {'Link to source': url,
                        'Link to information': 'https://coronavirus-response-alaska-dhss.hub.arcgis.com/datasets/daily-test-positivity/data',
                        'Type': 'API'}_____no_output_____dsname = 'Colorado'
url = 'https://opendata.arcgis.com/datasets/566216cf203e400f8cbf2e6b4354bc57_0.geojson'
if not os.path.isfile('data/colorado.csv'):
    r = requests.get(url)
    data = json.loads(r.content)
    df = pd.DataFrame([x['properties'] for x in data['features']])
    #df = df.rename(columns={'Cases': 'Positives', 'Tested': 'Tests'})
    df['Date'] = pd.to_datetime(df['Date'])
    df['date'] = pd.to_datetime(df['Date'])
    df = df.set_index('date')
    df = df.sort_index()
    df.to_csv('data/colorado.csv')
else:
    df = pd.read_csv('data/colorado.csv', index_col='date', parse_dates=['date', 'Date'])
df['Tests'] = df['Tested'].diff()
df['Positives'] = df['Cases'].diff()
df['Negatives'] = df['Tests'] - df['Positives']
df = df[['Date', 'Positives', 'Negatives', 'Tests']]
df['Odds'] = df['Positives'] /df['Negatives']
df = df[df.Odds.notna()]
df_dict[dsname] = df
dataset_info[dsname] = {'Link to source': url, 'Link to information': 'https://data-cdphe.opendata.arcgis.com/datasets/cdphe-covid19-daily-state-statistics/data', 'Type': 'API'}_____no_output_____dsname = 'Connecticut'

# reopening phase 1 may 20
# https://portal.ct.gov/Coronavirus/Covid-19-Knowledge-Base/Reopen-plan

if not os.path.isfile('data/connecticut.csv'):
    dfs = []
    offset = 0
    while offset >=0:
        url = 'https://data.ct.gov/resource/qfkt-uahj.csv?$limit=5000&$offset={}'.format(offset)
        df = pd.read_csv(url)
        dfs.append(df)
        if len(df)==5000:
            offset += 5000
        else:
            offset = -1
    dfcounty = pd.concat(dfs)
    dfcounty = dfcounty.rename(columns={'number_of_positives': 'Positives', 'number_of_tests': 'Tests', 'number_of_negatives': 'Negatives'})
    dfcounty['Tests'] = dfcounty['Tests'] - dfcounty['number_of_indeterminates']
    dfcounty['date'] = pd.to_datetime(dfcounty['date'])
    df = dfcounty.groupby('date').sum()
    df['Odds'] = df.Positives / df.Negatives
    df['Date'] = pd.to_datetime(df.index)
    df.to_csv('data/connecticut.csv')
    #df = df[df['Date'] <= '2020-05-19']
    df = df[df['Date'] > '2020-03-19']
else:
    df = pd.read_csv('data/connecticut.csv', index_col='date', parse_dates=['date', 'Date'])
    #df = df[df['Date'] <= '2020-05-19']
    df = df[df['Date'] > '2020-03-19']
    #df = df.set_index('date')
df['Date'] = pd.to_datetime(df['Date'])
df.index = pd.to_datetime(df.index)

# last days are not accurate. See S.I.
df = df[df['Date']<datetime.today()-timedelta(7)]

# intervention dates
# https://portal.ct.gov/Coronavirus/Pages/Emergency-Orders-issued-by-the-Governor-and-State-Agencies
# 03/12/2020 no gatherings with more than 250
large_gatherings = pd.to_datetime('03-12-2020 00:00', dayfirst=False)
schools = pd.to_datetime('03-17-2020 00:00', dayfirst=False)
bars = pd.to_datetime('03-16-2020 20:00', dayfirst=False)

malls = pd.to_datetime('03-19-2020 20:00', dayfirst=False)
workforce100 = pd.to_datetime('03-23-2020 20:00', dayfirst=False)
# CDC masks https://www.npr.org/sections/goatsandsoda/2020/04/10/829890635/why-there-so-many-different-guidelines-for-face-masks-for-the-public
masks_cdc = pd.to_datetime('04-03-2020 00:00', dayfirst=False)
mask_public = pd.to_datetime('04-20-2020 00:00', dayfirst=False)

event_list = [large_gatherings, bars, schools, malls, workforce100, masks_cdc,  mask_public]
events[dsname] = event_list
df = df[['Date', 'Tests', 'Positives', 'Negatives', 'Odds']]
df_dict[dsname] = df
dataset_info[dsname] = {'Link to source': 'https://data.ct.gov/resource/qfkt-uahj.csv',
                        'Link to information': 'https://data.ct.gov/Health-and-Human-Services/COVID-19-PCR-Based-Test-Results-by-Date-of-Specime/qfkt-uahj',
                        'Type': 'API'}_____no_output_____dsname = 'Delaware'
url = 'https://myhealthycommunity.dhss.delaware.gov/locations/state/download_covid_19_data'
if not os.path.isfile('data/delaware.csv'):
    r = requests.get(url)
    df = pd.read_csv(io.BytesIO(r.content))
    df['date'] = pd.to_datetime(df[['Year', 'Month', 'Day']])
    df = df[df['Unit']=='people']
    #df = df[(df['Statistic']=='New Positive Cases') ]# | (df['Statistic'].str.startswith('Total Persons Tested'))
    df = df.pivot(index='date', columns='Statistic', values='Value')
    df['Date']  = df.index
    df.to_csv('data/delaware.csv')
else:
    df = pd.read_csv('data/delaware.csv', index_col='date', parse_dates=['date', 'Date'])
#print(df.columns)
#df = df[['New Positive Cases', 'Total Persons Tested']]
df['Tests'] = df['Total Persons Tested'].diff()
df['Positives']  = df['New Positive Cases']
#df['neg2'] = df['Tested Negative'].diff()
df = df[['Positives', 'Tests', 'Date']]
df['Negatives'] = df['Tests'] - df['Positives']
df['Odds'] = df.Positives / df.Negatives
df = df.dropna()
#df = df[df['Date']<datetime.today()-timedelta(7)]
df= df[df['Date']>'2020-03-22']
df_dict[dsname] = df
dataset_info[dsname] = {'Link to source': url,
                        'Link to information': 'https://myhealthycommunity.dhss.delaware.gov/locations/state',
                        'Type': 'File'}_____no_output_____dsname = 'Indiana'
url = 'https://hub.mph.in.gov/dataset/ab9d97ab-84e3-4c19-97f8-af045ee51882/resource/182b6742-edac-442d-8eeb-62f96b17773e/download/covid_report_date.xlsx'
if not os.path.isfile('data/indiana.csv'):
    r = requests.get(url)
    df =pd.read_excel(io.BytesIO(r.content))
    df['date'] = pd.to_datetime(df['DATE'])
    df = df.set_index('date')
    df = df.sort_index()
    df['Date'] = df.index
    df.to_csv('data/indiana.csv')
else:
    df = pd.read_csv('data/indiana.csv', index_col='date', parse_dates=['date', 'Date'])
df['Tests'] = df['COVID_TEST']
df['Positives'] = df['DAILY_BASE_CASES']
df = df[['Date', 'Tests', 'Positives']]
df['Negatives'] = df['Tests'] - df['Positives']
df['Odds'] = df.Positives / df.Negatives
df = df[df['Date'] > '2020-03-18']
df = df[df['Date']<datetime.today()-timedelta(7)]
df_dict[dsname] = df
dataset_info[dsname] = {'Link to source': url,
                        'Link to information': 'https://hub.mph.in.gov/dataset/covid-19-case-trend/resource/182b6742-edac-442d-8eeb-62f96b17773e',
                        'Type': 'API and file'}_____no_output_____dsname = 'Iowa'
cases = pd.read_csv('data/iowa_positive.csv')
tests = pd.read_csv('data/iowa_tested.csv')
df = tests.merge(cases, on='Date')
df['Tests'] = df['Total Individuals Tested']
df['Positives'] = df['Total Positive Cases']
df['date'] = pd.to_datetime(df['Date'])
df = df.set_index('date')
df['Date'] = df.index
df = df[['Date', 'Tests', 'Positives']]
df['Negatives'] = df['Tests'] - df['Positives']
df['Odds'] = df.Positives / df.Negatives
df = df[df['Date']>'2020-03-21']
df_dict[dsname] = df
dataset_info[dsname] = {'Link to source': '  Downloadable from dashboard',
                        'Link to information': 'https://coronavirus.iowa.gov/',
                        'Type': 'File'}_____no_output_____dsname = 'Massachusetts'
# massachusets reopening on May 18
# https://www.mass.gov/doc/reopening-massachusetts-may-18-2020/download
yesterday = datetime.today() - timedelta(1)
if not os.path.isfile('data/massachusetts.csv'):
    #yesterday = datetime.today() - timedelta(10)
    yesterday_str = datetime.strftime(yesterday, '%B-%d-%Y').lower()
    fn = 'data/{}.zip'.format(dsname)

    url2 = 'https://www.mass.gov/doc/covid-19-raw-data-june-10-2020/download'
    url = 'https://www.mass.gov/doc/covid-19-raw-data-{}/download'.format(yesterday_str)
    print(url)
    print(url2)
    myfile = requests.get(url, allow_redirects=True)
    #open(fn, 'wb').write(myfile.content)
    zf = ZipFile(io.BytesIO(myfile.content))
    csvf = zf.open('TestingByDate.csv')
    df = pd.read_csv(csvf)

    # https://www.mass.gov/doc/covid-19-raw-data-may-27-2020/download
    #df = pd.read_csv('data/massachusetts/COVID-19-Dashboard-Files-05-24-2020/TestingByDate.csv',
    #                 usecols=['Date', 'New', 'Positive'])

    df = df.rename(columns={'Molecular Positive New': 'Positives', 'Molecular New': 'Tests'})
    df['Negatives'] = df.Tests - df.Positives
    df = df.query('Tests>100')
    df['date'] = pd.to_datetime(df['Date'])
    df= df.set_index('date')
    df['Date'] = pd.to_datetime(df['Date'])
    df['Odds'] = df.Positives / df.Negatives
    #df = df[df['Date']<'2020-05-15']
    df.to_csv('data/massachusetts.csv')
    #df = df[df['Date']<'2020-05-18']
    df = df[df['Date']>'2020-03-15']
else:
    df = pd.read_csv('data/massachusetts.csv')
    df['Date'] = pd.to_datetime(df['Date'])
    df['date'] = pd.to_datetime(df['date'])
    df = df.set_index('date')
    #df = df[df['Date']<'2020-05-18']
    df = df[df['Date']>'2020-03-15']

df = df[df['Date']<datetime.today()-timedelta(7)]
# https://www.mass.gov/info-details/covid-19-state-of-emergency
schools = pd.to_datetime('03-22-2020 00:00', dayfirst=False)
saty_at_home = pd.to_datetime('03-24-2020 00:00', dayfirst=False)
masks_cdc = pd.to_datetime('04-03-2020 00:00', dayfirst=False)
masks_public = pd.to_datetime('05-06-2020 00:00', dayfirst=False)

event_list = [schools, saty_at_home, masks_cdc, masks_public]
events[dsname] = event_list
df = df[['Date', 'Tests', 'Positives', 'Negatives', 'Odds']]
df = df[df['Date']<df.Date.max()-timedelta(7)]
df_dict[dsname] = df
dataset_info[dsname] = {'Link to source': 'https://www.mass.gov/doc/covid-19-raw-data-june-20-2020/download',
                        'Link to information': 'https://www.mass.gov/info-details/covid-19-response-reporting',
                        'Type': 'API'}_____no_output_____#display(old_ma.tail(20))
#df.tail(20)_____no_output_____dsname = 'Michigan'
url = 'https://www.michigan.gov/documents/coronavirus/Diagnostic_Tests_by_Result_and_County_2020-06-16_693915_7.xlsx'
if os.path.isfile('data/michigan.csv'):
    df = pd.read_excel(url)
    df['date'] = pd.to_datetime(df['MessageDate'])
    df = df[df['COUNTY']!='Correctional']
    df = df.groupby('date').sum()
    df['Date'] = df.index
    df.to_csv('data/michigan.csv')
else:
    df = pd.read_csv('data/michigan.csv', index_col='date', parse_dates=['date', 'Date'])
df = df.rename(columns={'Negative': 'Negatives', 'Positive': 'Positives','Total': 'Tests'})
df['Odds'] = df.Positives / df.Negatives
df = df[df['Date']>'2020-03-18']
df = df[df['Date']<datetime.today()-timedelta(7)]
df_dict[dsname] = df
dataset_info[dsname] = {'Link to source': url,
                        'Link to information': 'https://www.michigan.gov/coronavirus/0,9753,7-406-98163_98173---,00.html',
                        'Type': 'File'}_____no_output_____dsname = 'Minnesota'
url = 'https://www.health.state.mn.us/diseases/coronavirus/situation.html'
if not os.path.isfile('data/minnesota.csv'):
    dfs = pd.read_html(url)
    tests = dfs[4]
    tests['Tests'] = tests['Completed tests reported from the MDH Public Health Lab (daily)'] + tests['Completed tests reported from external laboratories (daily)']
    tests['Date reported to MDH'] = tests['Date reported to MDH'] + '/2020'
    tests['date'] = pd.to_datetime(tests['Date reported to MDH'])
    tests = tests[['date', 'Tests']]
    cases = dfs[5]
    cases['Positives'] = cases['Positive cases']
    cases = cases.iloc[:len(cases)-2].copy()
    cases['Specimen collection date'] = cases['Specimen collection date'] + '/2020'
    cases['date'] = pd.to_datetime(cases['Specimen collection date'])
    cases = cases[['date', 'Positives']]
    #df_dict[dsname] = df
    df = tests.join(cases.set_index('date'), on='date', how='inner')
    df = df.set_index('date')
    df['Date'] = df.index
    df.to_csv('data/minnesota.csv')
else:
    df = pd.read_csv('data/minnesota.csv', index_col='date', parse_dates=['date', 'Date'])
df['Negatives'] = df.Tests - df.Positives
df['Odds'] = df.Positives / df.Negatives

df_dict[dsname] = df
dataset_info[dsname] = {'Link to source': 'Downloadable from dashboard',
                        'Link to information': url,
                        'Type': 'HTML table'}_____no_output_____dsname = 'New York'
url = "https://health.data.ny.gov/resource/xdss-u53e.csv"
if os.path.isfile('data/ny.csv'):
    dfs = []
    offset = 0
    while offset >=0:
        url = "https://health.data.ny.gov/resource/xdss-u53e.csv/?$limit=5000&$offset={}".format(offset)
        df = pd.read_csv(url, usecols=['test_date', 'total_number_of_tests', 'new_positives'])
        dfs.append(df)
        if len(df)==5000:
            offset += 5000
        else:
            offset = -1
    dfraw = pd.concat(dfs)
    #'test_date=2020-03-15T00:00:00.000'

    dfraw = dfraw.rename(columns={'new_positives': 'Positives', 'total_number_of_tests': 'Tests', 'test_date': 'date'})
    print(len(dfraw))
    dfraw['date'] = pd.to_datetime(dfraw['date'])
    #counties = (df.groupby('date')['Tests']>0).count()
    df = dfraw.groupby('date').sum()
    df['Odds'] = df.Positives / (df.Tests - df.Positives)
    df['Date'] = pd.to_datetime(df.index)
    df.to_csv('data/ny.csv')
    df = df[df['Date'] >= '2020-03-15']
    # last date of full NYS PAUSE
    #df = df[df['Date'] <= '2020-05-15']

else:
    df = pd.read_csv('data/ny.csv')
    df['date'] =pd.to_datetime(df['date'])
    df['Date'] = pd.to_datetime(df['Date'])
    df = df.set_index('date')
    df = df[df['Date'] >= '2020-03-15']
    # last date of full NYS PAUSE
    #df = df[df['Date'] <= '2020-05-15']
df['Negatives'] = df.Tests - df.Positives
df_dict[dsname] = df
dataset_info[dsname] = {'Link to source': url,
                        'Link to information': 'https://health.data.ny.gov/Health/New-York-State-Statewide-COVID-19-Testing/xdss-u53e',
                        'Type': 'API'}8184
dsname = 'Rhode Island'
#https://docs.google.com/spreadsheets/d/1n-zMS9Al94CPj_Tc3K7Adin-tN9x1RSjjx2UzJ4SV7Q/edit#gid=590763272
url = 'https://docs.google.com/spreadsheets/d/1n-zMS9Al94CPj_Tc3K7Adin-tN9x1RSjjx2UzJ4SV7Q/export?format=csv&id=1n-zMS9Al94CPj_Tc3K7Adin-tN9x1RSjjx2UzJ4SV7Q&gid=590763272'
df  = pd.read_csv(url,
                  usecols=['New positive labs', 'New negative labs', 'Date'])
#df  = pd.read_csv('data/COVID-19 Rhode Island Data - COVID Trends.csv',
#                  usecols=['New positive labs', 'New negative labs', 'Date'])

df = df.rename(columns={'New positive labs': 'Positives', 'New negative labs': 'Negatives'})
df['Tests'] = df.Positives + df.Negatives
df = df.query('Tests>100')
df['date'] = pd.to_datetime(df['Date'])
df= df.set_index('date')
df['Date'] = pd.to_datetime(df['Date'])
df['Odds'] = df.Positives / df.Negatives
df = df[df['Date']>='2020-04-01']
df_dict[dsname] = df
dataset_info[dsname] = {'Link to source': url,
                        'Link to information': 'https://docs.google.com/spreadsheets/d/1n-zMS9Al94CPj_Tc3K7Adin-tN9x1RSjjx2UzJ4SV7Q/edit#gid=590763272',
                        'Type': 'File'}_____no_output_____dsname = 'Tennessee'
url = 'https://www.tn.gov/content/dam/tn/health/documents/cedep/novel-coronavirus/datasets/Public-Dataset-Daily-Case-Info.XLSX'
if not os.path.isfile('data/tennessee.csv'):
    df = pd.read_excel(url,
                       usecols=['DATE', 'NEW_CASES', 'POS_TESTS', 'NEG_TESTS', 'TOTAL_TESTS', 'NEW_TESTS'])
    df['date'] = pd.to_datetime(df['DATE'])
    df = df.set_index('date')
    df['Date'] = df.index
    df.to_csv('data/tennessee.csv')
else:
    df = pd.read_csv('data/tennessee.csv', index_col='date', parse_dates=['date', 'Date'])
df['Positives'] = df['POS_TESTS'].diff()
df['Negatives'] = df['NEG_TESTS'].diff()
df['Tests'] = df.Positives + df.Negatives
df['Odds'] = df.Positives / df.Negatives
df = df[['Date', 'Positives', 'Negatives', 'Tests','Odds']]
df = df.dropna()

# data on 2020 06 12 unrelaible. see notes
df[df['Date']=='2020-06-12'] = np.nan
df_dict[dsname] = df
dataset_info[dsname] = {'Link to source': url,
                        'Link to information': 'https://www.tn.gov/health/cedep/ncov/data/downloadable-datasets.html',
                        'Type': 'File'}_____no_output_____dsname = 'Texas'
url = 'https://www.dshs.texas.gov/coronavirus/TexasCOVID19CaseCountData.xlsx'
if not os.path.isfile('data/texas.csv'):
    cases = pd.read_excel(url, sheet_name='Trends', skiprows=[0], skipfooter=2)
    tests = pd.read_excel(url, sheet_name='Tests by day', skiprows=[0], skipfooter=2)
    cases = cases[cases['Date']!='06/16/20-TDCJ*']
    cases = cases[cases['Date']!='06/16/2020-TDCJ*']
    cases['date'] = pd.to_datetime(cases['Date'])
    cases['Date'] = pd.to_datetime(cases['Date'])
    #tests['date'] = tests['Date']
    tests = tests.set_index('Date')
    tests = tests.loc[tests['Viral Tests']!='.']
    tests['Tests'] = tests['Viral Tests'].diff()
    cases['Positives'] = cases['Daily\nNew\nCases']
    df = cases.join(tests, on='date', how='inner')
    df = df.dropna()
    df = df.set_index('date')
    df.to_csv('data/texas.csv')
else:
    df = pd.read_csv('data/texas.csv', index_col='date', parse_dates=['date', 'Date'])
df['Negatives'] = df.Tests + df.Positives
df['Odds'] = df.Positives / df.Negatives
df = df[['Date', 'Positives', 'Negatives', 'Tests', 'Odds']]
df = df.dropna()
df_dict[dsname] = df
dataset_info[dsname] = {'Link to source': url,
                        'Link to information': 'https://www.dshs.texas.gov/coronavirus/',
                        'Type': 'File'}_____no_output_____dsname = 'Virginia'
if not os.path.isfile('data/virginia.csv'):
    #df  = pd.read_csv('https://www.vdh.virginia.gov/content/uploads/sites/182/2020/05/VDH-COVID-19-PublicUseDataset-Tests_by-LabReportDate.csv')
    df = pd.read_csv('data/VDH-COVID-19-PublicUseDataset-Tests_by-LabReportDate.csv')
    df = df.rename(columns={'Number of Positive PCR Tests': 'Positives', 'Number of PCR Testing Encounters': 'Tests'})
    df = df[df['Lab Report Date']!='Not Reported']
    df['date'] = pd.to_datetime(df['Lab Report Date'])
    df = df.groupby('date').sum()
    df['Date'] = df.index
    df.to_csv('data/virginia.csv')
else:
    df = pd.read_csv('data/virginia.csv', index_col='date', parse_dates=['date', 'Date'])
df = df[['Date', 'Tests', 'Positives']]
df['Negatives'] = df.Tests + df.Positives
df['Odds'] = df.Positives / df.Negatives
df = df[df['Date']>'2020-03-22']

# lab report day might be lagging
df = df[df['Date']<datetime.today()-timedelta(7)]
df_dict[dsname] = df
dataset_info[dsname] = {'Link to source': 'https://www.vdh.virginia.gov/content/uploads/sites/182/2020/05/VDH-COVID-19-PublicUseDataset-Tests_by-LabReportDate.csv',
                        'Link to information': 'https://www.vdh.virginia.gov/coronavirus/',
                        'Type': 'File'}_____no_output_____dsname = 'Wisconsin'
url = "https://opendata.arcgis.com/datasets/b913e9591eae4912b33dc5b4e88646c5_10.geojson?where=GEO%20%3D%20'State'"
if not os.path.isfile('data/wisconsin.csv'):
    r = requests.get(url)
    data = json.loads(r.content)
    df = pd.DataFrame([x['properties'] for x in data['features']])

    df['date'] = pd.to_datetime(df['DATE'].str[:-3], format='%Y/%m/%d 14:00:00')
    df = df.set_index('date')
    df['Date'] = df.index
    df.to_csv('data/wisconsin.csv')
else:
    df = pd.read_csv('data/wisconsin.csv', index_col='date', parse_dates=['date', 'Date'])
#print(df.head(20))
df = df.rename(columns={'POS_NEW': 'Positives', 'NEG_NEW': 'Negatives', 'TEST_NEW': 'Tests'})

df['Odds'] = df.Positives / df.Negatives
df = df[['Date', 'Positives', 'Negatives', 'Tests', 'Odds']]
df = df.dropna()
#df = df[df['Odds']>0]
# on the 30th they changed the definition of negative case
df = df[df['Date']>'2020-03-30']
df_dict[dsname] = df
dataset_info[dsname] = {'Link to source': url,
                        'Link to information': 'https://data.dhsgis.wi.gov/datasets/covid-19-historical-data-table/data?where=GEO%20%3D%20%27State%27',
                        'Type': 'API'}_____no_output_____dsname = 'Wyoming'
df = pd.read_csv('data/wyoming.csv', encoding='utf-16', sep='\t', thousands=',')
df = df.rename(columns={'All': 'Tests'})
df['date'] = pd.to_datetime(df['Spec Coll Dt'])
df = df.set_index('date')
pos = df.loc[df.Lab.isna(),'Positivity']
nwphl = df.loc[df.Lab=='Non-WPHL', 'Tests']
wphl = df.loc[df.Lab=='WPHL', 'Tests']
df['Tests'] = nwphl + wphl
df['Date'] = df.index

df['Positivity'] = df.Positivity.str[:-1].astype(float)/100
#df['Tests'] = df['Tests'].str.replace(',', '').astype(float)
df['Positives'] = (df['Tests'] * df['Positivity']).round(0)
df['Negatives'] = df.Tests - df.Positives
df['Odds'] = df.Positives / df.Negatives
df = df[['Date', 'Positives', 'Negatives', 'Tests', 'Odds']]
df = df.dropna()
df = df[df['Odds']>0]
df = df[df['Date']>'2020-03-16']
df_dict[dsname] = df
dataset_info[dsname] = {'Link to source': 'Downloadable from dashboard',
                        'Link to information': 'https://health.wyo.gov/publichealth/infectious-disease-epidemiology-unit/disease/novel-coronavirus/covid-19-testing-data/',
                        'Type': 'API'}_____no_output_____# df = pd.concat(df_dict).reset_index()

# df = df.rename(columns={'level_0': 'State'})
# df['Odds'] = df['Odds'].astype(float)
# df['Tests'] = df['Tests'].astype(float)
# df['Positives'] = df['Positives'].astype(float)
# df['Tests (right)'] = df['Tests']
_____no_output_____# dataset_info_df = pd.DataFrame(dataset_info, ).T.reset_index().rename(columns={'index': 'Dataset'})
# pd.options.display.max_colwidth = 150

# #df.loc[df['Link to source']=='Downloadable from dashboard', 'Link to source'] = df['Link to information']
# with open('dataset_table.txt', 'w') as fp:
#     dataset_info_df[['Dataset', 'Link to information', 'Type']].to_latex(fp, index=False, label='tab:datasets', caption='Information about the source of the datasets used in this work')_____no_output_____# pd.options.display.max_colwidth_____no_output_____# dsname = 'Argentina'
# df1 = pd.read_csv('data/argentina/argentina_tests.csv', parse_dates=[0],
#                   index_col=0)
# df2 = pd.read_csv('data/argentina/argentina_tests2.csv',
#                   parse_dates=[0],
#                   index_col=0)
# df1['Date'] = df1.index
# df2['Date'] = df2.index
# df2['Positives'] = df2.confirmed.diff()
# df = df1.merge(df2, on='Date', how='outer').fillna(0)
# df['Positives'] = df[['new_confirmed', 'Positives']].max(axis=1)
# df['Tests'] = df[['new_tests_x', 'new_tests_y']].max(axis=1)
# df = df[['Date', 'Positives', 'Tests']].set_index('Date')
# df['Date'] = df.index
# df = df.sort_index()

# df = df[df['Date'] > '2020-04-17']
# df['Odds'] = df.Positives / (df.Tests-df.Positives)

# ax = df.plot.scatter(x='Date', y='Odds')
# ax.set_yscale('log')
# df.tail()_____no_output_____

## Plots

I we plot the number of positive tests we can see that the data is noisy.
But, if we take into account the number of people tested each day, the data looks way more clean._____no_output_____

plt.close(1)

current_palette = sns.color_palette()

df = pd.concat(df_dict).reset_index()

df = df.rename(columns={'level_0': 'State'})
df['Odds'] = df['Odds'].astype(float)
df['Tests'] = df['Tests'].astype(float)
df['Positives'] = df['Positives'].astype(float)
df['Tests (right)'] = df['Tests']
#df = df[df['Odds']>0]

g = sns.FacetGrid(df, col='State', col_wrap=3, xlim=[df.Date.min(), df.Date.max()], sharey=False, aspect=1.3, )
legend_and_hand = []
def myplot(data, y=None, secondary_y=None, legend=None, color=None):
    ax = data.plot(x='Date', y=y[0], ax=plt.gca(), label=y[0])
    legend_and_hand.append(ax.get_legend_handles_labels())
    ax = data.plot(x='Date', y=y[1], secondary_y=secondary_y, ax=ax, label=y[1])
    legend_and_hand.append(ax.get_legend_handles_labels())
    return
g.map_dataframe(myplot, y=['Positives', 'Tests (right)'], secondary_y=['Tests (right)'])
g.axes[0].xaxis.set_major_locator(locator)
g.axes[0].xaxis.set_major_formatter(formatter)

g.add_legend({l[0]: h[0] for h,l in legend_and_hand[:2]}, loc='lower center', bbox_to_anchor=(0.7, 0), ncol=2)
g.set_titles('{col_name}')
plt.tight_layout()
plt.savefig('figs/all_state_testes.jpg', dpi=300)_____no_output_____

## Relationship between the total number of infected individuals and positive tests

As has been shown previously [[1]](https://journals.plos.org/plosone/article?id=10.1371/journal.pone.0002185), the number of new infected individuals in a given day $k_t$ is given by:
$$
k_t = k_{t-1} e^{(R_{t-1}-1)\gamma}
$$

where $R_t$ is the effective reproductuve number and $\gamma^{-1}$ is the infectious period estimated as 7 days accoring to [2].

The following derivation was suggested to my by Will Meierjurgen Farr on this GitHub [Issue](https://github.com/k-sys/covid-19/issues/45#issuecomment-623782130).
Since we do not have access to the total number of infected indiviudals, but only to the population being tested, we have to use some statisticals assumtions on this populations.
If we asume that the people being tested, in a given day, is a sample of the population with COVID-19-like sympoms we can state that:

$$
n_{t} = [P_t(CV|sympoms) P_t(sympoms) +P_t(not CV|sympoms)P_t(sympoms)]Nf_t
$$

where $n_{t}$ is the number of people tested, $P_t(CV|sympoms)$ is the probablity of a pacient being positive for coronavirus given that the she is sympomatic, $P_t(sympoms)$ is the probablity of having COVID-like sympoms, $P_t(not CV|sympoms)$ is the probability of a pacient being coronavirus negative given he has COVID-19-like sympoms, $N$ is the total population, and $f_t$ is the fraction of people with sympoms that is selected to be tested (this number can be different each day, for example if the number of tests availabes changes).
Also, note that the probability of a test being positive in a given day is $Positive_t=P_t(CV|sympoms) P_t(sympoms) N f_t$

Now, if we assume that $P_t(sympoms|CV)=cte$ we can use Bayes theorem to show that:

$$
P_t(CV|sympoms) P_t(sympoms) \propto P_t(CV) = \frac{k_t}{N}
$$

Then:
$$
P_t(CV|sympoms) P_t(sympoms) \propto k_t
$$

Finally, if we assume that $P_t(not CV|sympoms)P_t(sympoms)=cte$:
$$
Odds_t = \frac{P(CV|sympoms) P(sympoms)Nf_t}{P_t(not CV|sympoms)P_t(sympoms)Nf_t}
$$
$$
Odds_t = \frac{P(CV|sympoms) P(sympoms)}{P_t(not CV|sympoms)P_t(sympoms)}
$$
$$
Odds_t \propto k_t
$$

\begin{equation}
Odds_t = Odds_{t-1} e^{(R_{t-1}-1)\gamma} (1)
\end{equation}

We used three hypothesis. First, constant population $N$ (for $P_t \propto k_t$ and for the evolution of $k_t$). Second, that the tested population is a random sample from the population with COVID19-like sympoms ($n_t = [P_t(CV|sympoms) P_t(sympoms) +P_t(not CV|sympoms)P_t(sympoms)]Nf_t$) this is not the case when people is being tested based on contacts for example. And third, that $P_t(not CV|sympoms)P_t(sympoms)=cte$, this is equivalent to say that the number of people with covid-19-like sympoms but without the coronavirus (for example people with the flu) is constant, or at least it changes are negligible compared with the changes in the amount of sympomatic people with coronavirus.

## Linearization

Defining

$$
b_i =  e^{(R_{i-1}-1)\gamma} (2)
$$

We can write [1] as:

\begin{equation}
odd_i = b_{i-1} * odd_{i-1} (3)
\end{equation}

Now, instead of using $b_i$ as the parameters to estimate we decompose each $b$ as follows:

$$
b_i = \prod_{j=0}^{i} a_j (4)
$$

Now, the $a_j$ represent the rate of change of the variable $b_i$. Next, we replace the [4] in [3]
$$
odd_i = \prod_{j=0}^{i-1} a_j * odd_{i-1}
$$
$$
odd_i = \prod_{j=0}^{i-1} a_j * \prod_{j=0}^{i-2} a_j * odd_{i-2}
$$
$$
odd_i = \prod_{k=0}^{i-1}\prod_{j=0}^{k} a_j * odd_{0}
$$
$$
odd_i = \prod_{j=0}^{i-1} a_j^{i-j} * odd_{0}
$$

Now, we liniarize this result and we generalize it to the case where $i=0$ using the $max$ function:

$$
log(odd_i) = \sum_{j=0}^{max(i-1, 0)} (i-j)log(a_j)  +  log(odd_{0}) (5)
$$

We can write [5] as a linear problem with the following definitions:

$$
y = X \beta + \beta_0
$$

$$
y_i = log(odd_i)
$$

$$
X_{i,j} =  max(i-j, 0)
$$

$$
\beta_i =  log(a_i) (6)
$$

Now if we apply a LASSO regression we will find the solution that minimize the following cost function

$$
Err = \sum (y-\hat{y})^2 + \alpha |\beta|_1
$$

Hopefully, this solution will be sparse which means that most of the $\beta_i$ will be $0$, and hence $a_i=1$.
This is equivalent to say that the $b_i$ are almost constant except at the values whete $a_i \neq 1$.

[1] Bettencourt, L. M. A., & Ribeiro, R. M. (2008). Real time bayesian estimation of the epidemic potential of emerging infectious diseases. PLoS ONE, 3(5). https://doi.org/10.1371/journal.pone.0002185

[2] Sanche, S., Lin, Y. T., Xu, C., Romero-Severson, E., Hengartner, N., & Ke, R. (2020). High Contagiousness and Rapid Spread of Severe Acute Respiratory Syndrome Coronavirus 2. Emerging Infectious Diseases, 26(7). https://doi.org/10.3201/eid2607.200282

_____no_output_____## Classes

This cell contains the main class: LassoICSelector. Its main method is fit_best_alpha. It works as follows:
```
For each alpha value:
    1. Fits a lasso regression to the data
    2. Selectes the first non zero variable from each chunck
    3. Fits a linear regression with the selected variables
    4. Excludes all non sifgificative (p-value>0.05) variables and fits a linear model again

The linear model with less AIC from step 4 is selected.
```

_____no_output_____

class FirstInChunkSelector(object):
    '''Selects first element from each non zero chunk.'''

    def __init__(self, clf):
        self.clf = clf
        self.coef = None
        self.mask = None

    def select_coef(self):
        n_features = len(self.clf.coef_)
        no_zero = np.zeros(n_features+1)
        no_zero[1:] = self.clf.coef_ != 0
        #v = np.hstack([np.zeros(np.int(1/GAMMA-2)), np.ones(np.int(1/GAMMA-1))])
        #no_zero[1:] = np.convolve(self.clf.coef_ != 0, v, mode='same') > 0
        self.mask = np.diff(no_zero)>0
        self.mask[0] = True
        self.coef = self.clf.coef_[self.mask]
        return self.coef

    def transform(self, X):
        self.select_coef()
        return X[:, self.mask]

    def get_support(self):
        self.select_coef()
        return self.mask

    def get_number_of_features(self):
        self.select_coef()
        return sum(self.mask)

class LassoICSelector(object):
    """LASSO regression with FirstInChunk selector."""

    def __init__(self, X, y, criterion, alpha=0.05):
        self.lasso = linear_model.LassoLars(alpha=0, max_iter=100000)
        self.criterion = criterion
        self.selector = FirstInChunkSelector(self.lasso)
        self.OLS = sm.OLS
        #self.OLS = sm.RLM
        self.ols = self.OLS(y, X)

        self.ols_results = None
        self.X = X
        self.y = y
        self.final_ols = False
        self.alpha = alpha

    def transform_to_ols(self, X):
        '''Selects only the features of X are used by OLS.
        Also, adds a coloumn with ones for the intercept.
        '''

        X_new = self.selector.transform(X)
        if self.final_ols:
            X_new = X[:, self.support]
        X_new_with_cte = np.hstack([X_new, np.ones((X_new.shape[0], 1))])
        return X_new_with_cte

    def fit(self, X, y):
        '''Selects features and fits the OLS.'''

        # select features
        X_new = self.transform_to_ols(X)

        # fit ols
        self.ols = self.OLS(y, X_new)
        self.ols_results = self.ols.fit()

        # iteratively remove non signicative variables and fit again
        mask = self.ols_results.pvalues < self.alpha / len(self.ols_results.pvalues)
        mask[0] = True
        Xnew = self.transform_to_ols(X)
        Xnew = Xnew[:, mask]
        self.support = self.selector.get_support()
        self.ols = self.OLS(y, Xnew)
        self.ols_results = self.ols.fit()
        while any(self.ols_results.pvalues[1:] >= self.alpha / len(self.ols_results.pvalues)):
            mask.values[mask.values] = (self.ols_results.pvalues < self.alpha / len(self.ols_results.pvalues)).values
            mask[0] = True
            Xnew = self.transform_to_ols(X)
            Xnew = Xnew[:, mask]
            self.support = self.selector.get_support()
            self.ols = self.OLS(y, Xnew)
            self.ols_results = self.ols.fit()

        self.support[self.support] = mask[:-1]

    def fit_best_alpha(self, X, y):
        '''returns the model with the lowst cirterion.'''

        self.lasso.fit(X, y)
        alphas = self.lasso.alphas_
        self.criterions_ = np.zeros(len(alphas))
        self.log_liklehods = np.zeros(len(alphas))

        for i, alpha in enumerate(alphas):
            self.lasso.coef_ = self.lasso.coef_path_[:, i]
            self.fit(X, y)
            self.criterions_[i], self.log_liklehods[i] = self.get_criterion(self.ols.exog, y)

        # we use a list of tuples to find the minimum cirterion value.
        # If there are ties, we use the maximum alpha value.
        criterions_idx = list(zip(self.criterions_, alphas, range(len(alphas))))
        criterion, alpha, idx = min(criterions_idx, key=lambda x: (x[0], -x[1]))

        self.lasso.coef_ = self.lasso.coef_path_[:, idx]
        self.lasso.alpha = alpha
        self.fit(X, y)
        self.final_ols = True

    def predict(self, X):
        '''Predicts y useing the OLS fit.'''

        return self.ols.predict(self.ols_results.params, X)

    def log_liklihood(self, X, y):
        '''Computes the log liklihood assuming normally distributed errors.'''

        eps64 = np.finfo('float64').eps

        # residuals
        R = y - self.predict(X)
        sigma2 = np.var(R)

        loglike = -0.5 * len(R) * np.log(sigma2)
        loglike -= 0.5 * len(R) * np.log(2*np.pi) - 0.5*len(R) + 0.5
        return loglike

    def get_criterion(self, X, y):
        '''Computes AIC or BIC criterion.'''

        n_samples = X.shape[0]
        if self.criterion == 'aic':
            K = 2  # AIC
        elif self.criterion == 'bic':
            K = np.log(n_samples)
        else:
            raise ValueError('criterion should be either bic or aic')

        log_like = self.log_liklihood(X, y)
        df = X.shape[1]

        aic = K * df - 2*log_like
        self.criterion_ = aic

        return self.criterion_, log_like_____no_output_____

## Fit
Now, we create the linear system and fit the model_____no_output_____

plt.close('all')
lics_dict = {}
# fig, axes = plt.subplots(nrows=17, ncols=3, figsize=(12, 8))
gof_list = []
for i, state in enumerate(df.State.unique()):
    dfstate = df[df['State']==state]
    #dfstate = dfstate[(dfstate.Odds.notna()) & (dfstate.Odds!=0)]
    # create the independent and the dependent variables
    y = np.log(dfstate['Odds'])
    X = np.tri(len(y))
    X = np.cumsum(X, axis=0)[:, 1:]
    X = X[(dfstate.Odds.notna()) & (dfstate.Odds!=0), :]
    y = y[(dfstate.Odds.notna()) & (dfstate.Odds!=0)]

    # create lasso instance
    lics = LassoICSelector(X, y.values, 'bic', alpha=0.01)

    # fit
    lics.fit_best_alpha(X, y)
    lics_dict[state] = lics

    #     ax = sns.lineplot(lics.lasso.alphas_, lics.criterions_, ax=axes[i, 2])
    #     ax.vlines(lics.lasso.alpha, min(lics.criterions_), max(lics.criterions_))
    #     ax.set_ylabel('BIC')
    #     ax.set_xlabel('Alpha')
    #     ax.set_xscale('log')
    #     axes[i, 0].plot(lics.lasso.alphas_, lics.lasso.coef_path_.T)
    #     axes[i, 0].set_xscale('log')
    #     axes[i, 0].set_title(state)
    #     axes[i, 0].set_xlabel('Alpha')
    #     axes[i, 0].set_ylabel('Coefficient Value')
    #     axes[i, 0].vlines(lics.lasso.alpha, lics.lasso.coef_path_.min(), lics.lasso.coef_path_.max())
    #     axes[i, 1].plot(lics.lasso.coef_)
    #     axes[i, 1].scatter(np.arange(len(lics.lasso.coef_))[lics.support], lics.lasso.coef_[lics.support])
    #     axes[i, 1].set_xlabel('Coefficient #')
    #     axes[i, 1].set_ylabel('Coefficient Value')

    gof_list.append({"State":state, '$R^2$':lics.ols_results.rsquared, "N": int(lics.ols_results.nobs), "df_model": int(lics.ols_results.df_model), "fvalue":lics.ols_results.fvalue, "f_pvalue":lics.ols_results.f_pvalue})
    #print('{:<13} & {:.2f} & {:n} & {:n} & {:.2f} & {:.2e}'.format(state, lics.ols_results.rsquared, lics.ols_results.nobs, lics.ols_results.df_model, lics.ols_results.fvalue, lics.ols_results.f_pvalue))
    #print(state)
    #display(lics.ols_results.summary())
gof = pd.DataFrame(gof_list).sort_values('$R^2$', ascending=False).rename(columns={'df_model': 'D.F.', 'fvalue': 'F-statistic', 'f_pvalue': 'p-value'})
out = gof.to_latex(index=False,formatters={"$R^2$": "{:0.3f}".format, 'fvalue':"{:0.2f}".format, 'f_pvalue': "{:.2e}".format},
             bold_rows=True, escape=False, label='tab:gof', caption='Goodness of fit for each dataset.')
print(out)/home/andres/anaconda3/lib/python3.7/site-packages/pandas/core/series.py:679: RuntimeWarning: divide by zero encountered in log
  result = getattr(ufunc, method)(*inputs, **kwargs)
/home/andres/anaconda3/lib/python3.7/site-packages/pandas/core/series.py:679: RuntimeWarning: divide by zero encountered in log
  result = getattr(ufunc, method)(*inputs, **kwargs)
/home/andres/anaconda3/lib/python3.7/site-packages/pandas/core/series.py:679: RuntimeWarning: divide by zero encountered in log
  result = getattr(ufunc, method)(*inputs, **kwargs)

Lets copy the fitted values to a dataframe, and calculate the parameters and erros of the model._____no_output_____

data_list = []
for state in df.State.unique():
    print(state)
    lics = lics_dict[state]
    data = df[df['State']==state].copy()
    #data = data[(data.Odds.notna()) & (data.Odds!=0)]
    # yhat = lics.ols_results.fittedvalues
    y = np.log(data['Odds'])
    X = np.tri(len(y))
    X = np.cumsum(X, axis=0)[:, 1:]
    X = X[(data.Odds.notna()) & (data.Odds!=0), :]
    y = y[(data.Odds.notna()) & (data.Odds!=0)]
    data = data[(data.Odds.notna()) & (data.Odds!=0)]
    Xols = lics.transform_to_ols(X)
    yhat = lics.ols.predict(lics.ols_results.params, Xols)
    # from equation 5
    odds_hat = np.exp(yhat)

    # the error in yhat is
    # Xols = lics.transform_to_ols(X)
    (yhat_std, yhat_l, yhat_u) = wls_prediction_std(lics.ols_results, Xols)

    # propagation of errors
    #oddshat_std = np.array([exp_x_sigma(mu, s)[0] for mu, s in zip(yhat, yhat_std)])#odds_hat*yhat_std
    #oddshat_std = exp_x_sigma(yhat, yhat_std)
    oddshat_l = np.exp(yhat-2*yhat_std)
    oddshat_u = np.exp(yhat+2*yhat_std)
    data.loc[:, 'odds_hat'] = odds_hat
    #data.loc[:, 'oddshat_std'] = oddshat_std
    #data.loc[:, 'oddshat_l'] = odds_hat - 2*oddshat_std
    #data.loc[:, 'oddshat_u'] = odds_hat + 2*oddshat_std
    data.loc[:, 'oddshat_l'] = oddshat_l
    data.loc[:, 'oddshat_u'] = oddshat_u

    # use coefficients to calculate Rt
    coef = np.zeros(len(data))
    coef_std = np.zeros_like(coef) * np.nan
    ind = np.squeeze(np.argwhere(lics.support))

    # we do not use the last coefficient since it's the intercept (=log(odds_0))
    coef[ind] = lics.ols_results.params[:-1]

    # using equation 2, 4 and 6
    data.loc[:, 'R'] = np.cumsum(coef)/GAMMA+1

    # get covarinace matrix of coefficients
    cov = lics.ols_results.cov_params().values

    # since the values of Rts are a sum of variables, we use the formula
    # of the sum of gaussian variables with a known covariance matrix
    stds = [np.sqrt(cov[:n, :n].sum()) for n in range(1, cov.shape[0])]
    if len(stds)==1:
        stds = stds[0]
    coef_std[ind] = stds

    # error propagation formula
    data.loc[:, 'Rstd'] = coef_std / GAMMA

    data['Rstd'] = data['Rstd'].fillna(method='ffill')
    data['R_l'] = data['R'] - 2*data['Rstd']
    data['R_u'] = data['R'] + 2*data['Rstd']

    r_index = data.R.diff() != 0
    Rts = data.loc[r_index, ['Date', 'R', 'R_l', 'R_u']]
    Rts['delta'] = Rts['R_u'] - Rts['R_l']
    display(Rts)
    data_list.append(data)
data = pd.concat(data_list)Alaska
res = lics_dict['Delaware'].ols_results
dir(res)
#res.params
pd.Series(2*np.diag(res.cov_params())**0.5, index=res.cov_params().index)_____no_output_____plt.close(1)
stacked = data.set_index(['Date', 'State'])[['Odds', 'odds_hat', 'oddshat_u', 'oddshat_l']]
stacked = stacked.stack().reset_index().rename(columns={'level_2': 'variable', 0: 'val'})
stacked['Branch'] = 'Fit'
stacked.loc[stacked['variable'] == 'Odds', 'Branch'] = 'Data'
stacked = stacked.set_index(['Date', 'State', 'Branch', 'variable']).unstack().reset_index()
stacked.columns = ['Date', 'State', 'Branch', 'Odds', 'odds_hat', 'oddshat_u', 'oddshat_l']
stacked = stacked.sort_values(['State', 'Date'])
#stacked = stacked[stacked.State.isin(['Alaska', 'Colorado', 'Connecticut'])]

g1 = sns.FacetGrid(stacked, col='State', col_wrap=3, hue='Branch', aspect=1.5, dropna=False, hue_order=['Data', 'Fit'], sharey=False, col_order=gof.State)
legend_and_hand = []
def myplot(data, x=None, y=None, color=None, label=None):
    ax = plt.gca()
    ax = data.plot(x=x, y=y, ax=ax, label=label)
    legend_and_hand.append(ax.get_legend_handles_labels())
    return ax
#g1.map_dataframe(myplot, x='Date', y='odds_hat')
g1.map_dataframe(sns.lineplot, x='Date', y='odds_hat')
g1.map_dataframe(sns.scatterplot, x='Date', y='Odds')

g1.map(plt.fill_between, 'Date', 'oddshat_l', 'oddshat_u', alpha=0.1)
hl = g1.axes[0].get_legend_handles_labels()

handlers = [hl[0][1], hl[0][0]]
labels = [hl[1][1], hl[1][0]]

lh = {l:h for h,l in zip(handlers, labels)}
g1.add_legend(lh, loc='lower center', bbox_to_anchor=(0.7, 0), title='', ncol=3)#

# plt.yscale('log')
# plt.xlabel('')
g1.axes[0].xaxis.set_major_locator(locator)
g1.axes[0].xaxis.set_major_formatter(formatter)
g1.set(xlim=[data['Date'].min(), data.loc[data.odds_hat.notna(), 'Date'].max()], ylabel='Odds', yscale='log')
g1.set_titles("{col_name}")
plt.tight_layout()
plt.savefig('figs/all_states_OddsL1.jpg', dpi=300)
plt.show()_____no_output_____plt.close('all')

g1 = sns.FacetGrid(data.set_index('date'), col='State', col_wrap=3, aspect=1.5, dropna=False, ylim=[0, 2.5])
def myplot(data, y=None, color=None):
    data.plot(x='Date', y=y, ax=plt.gca())
g1.map_dataframe(sns.lineplot, x='Date', y='R')
#g1.map_dataframe(myplot, y='R')
g1.map(plt.fill_between, 'Date', 'R_u', 'R_l', alpha=0.2)
#ax = data.plot(x='Date', y='R', legend=False)
# lines = []
# for ax, dsname in zip(g1.axes[:, 0], data.State.unique()):
#     for line in events[dsname]:
#         if line < pd.to_datetime('03-04-2020 00:00', dayfirst=True):
#             color = current_palette[1]
#             label = 'Mobility restrictions'
#         elif line == pd.to_datetime('03-04-2020 00:00', dayfirst=True):
#             color = 'k'
#             label = 'Masks (CDC)'
#         else:
#             label = 'Masks (State)'
#             color = current_palette[2]
#         lines.append(ax.axvline(line, 0,1, linestyle='--', color=color, label=label))

g1.set(ylabel='$R_t$')
g1.set(yscale='linear')
g#1.axes[0, 0].legend([lines[0], lines[4], lines[5]], ['Mobility restrictions', 'Masks (CDC)', 'Masks (Local)'])
#print(g1.axes[0, 0].legend().get_legend_handler_map())
g1.axes[0].xaxis.set_major_locator(locator)
g1.axes[0].xaxis.set_major_formatter(formatter)
plt.xlabel('')
g1.set_titles('{col_name}')
plt.tight_layout()
plt.savefig('figs/all_states_Rt.jpg', dpi=300)_____no_output_____


# Notebook from saiddddd/statistical_analysis_python_tutorial
Path: 1. Introduction to Pandas.ipynb

# Introduction to Pandas

**pandas** is a Python package providing fast, flexible, and expressive data structures designed to work with *relational* or *labeled* data both. It is a fundamental high-level building block for doing practical, real world data analysis in Python.

pandas is well suited for:

- Tabular data with heterogeneously-typed columns, as in an SQL table or Excel spreadsheet
- Ordered and unordered (not necessarily fixed-frequency) time series data.
- Arbitrary matrix data (homogeneously typed or heterogeneous) with row and column labels
- Any other form of observational / statistical data sets. The data actually need not be labeled at all to be placed into a pandas data structure

Key features:

- Easy handling of **missing data**
- **Size mutability**: columns can be inserted and deleted from DataFrame and higher dimensional objects
- Automatic and explicit **data alignment**: objects can be explicitly aligned to a set of labels, or the data can be aligned automatically
- Powerful, flexible **group by functionality** to perform split-apply-combine operations on data sets
- Intelligent label-based **slicing, fancy indexing, and subsetting** of large data sets
- Intuitive **merging and joining** data sets
- Flexible **reshaping and pivoting** of data sets
- **Hierarchical labeling** of axes
- Robust **IO tools** for loading data from flat files, Excel files, databases, and HDF5
- **Time series functionality**: date range generation and frequency conversion, moving window statistics, moving window linear regressions, date shifting and lagging, etc._____no_output_____

from IPython.core.display import HTML
HTML("")_____no_output_____%matplotlib inline
import pandas as pd
import numpy as np

# Set some Pandas options
pd.set_option('html', False)
pd.set_option('max_columns', 30)
pd.set_option('max_rows', 20)_____no_output_____

## Pandas Data Structures_____no_output_____### Series

A **Series** is a single vector of data (like a NumPy array) with an *index* that labels each element in the vector._____no_output_____

counts = pd.Series([632, 1638, 569, 115])
counts_____no_output_____

If an index is not specified, a default sequence of integers is assigned as the index. A NumPy array comprises the values of the `Series`, while the index is a pandas `Index` object._____no_output_____

counts.values_____no_output_____counts.index_____no_output_____

We can assign meaningful labels to the index, if they are available:_____no_output_____

bacteria = pd.Series([632, 1638, 569, 115],
    index=['Firmicutes', 'Proteobacteria', 'Actinobacteria', 'Bacteroidetes'])

bacteria_____no_output_____

These labels can be used to refer to the values in the `Series`._____no_output_____

bacteria['Actinobacteria']_____no_output_____bacteria[[name.endswith('bacteria') for name in bacteria.index]]_____no_output_____[name.endswith('bacteria') for name in bacteria.index]_____no_output_____

Notice that the indexing operation preserved the association between the values and the corresponding indices.

We can still use positional indexing if we wish._____no_output_____

bacteria[0]_____no_output_____

We can give both the array of values and the index meaningful labels themselves:_____no_output_____

bacteria.name = 'counts'
bacteria.index.name = 'phylum'
bacteria_____no_output_____

NumPy's math functions and other operations can be applied to Series without losing the data structure._____no_output_____

np.log(bacteria)_____no_output_____

We can also filter according to the values in the `Series`:_____no_output_____

bacteria[bacteria>1000]_____no_output_____

A `Series` can be thought of as an ordered key-value store. In fact, we can create one from a `dict`:_____no_output_____

bacteria_dict = {'Firmicutes': 632, 'Proteobacteria': 1638, 'Actinobacteria': 569, 'Bacteroidetes': 115}
pd.Series(bacteria_dict)_____no_output_____

Notice that the `Series` is created in key-sorted order.

If we pass a custom index to `Series`, it will select the corresponding values from the dict, and treat indices without corrsponding values as missing. Pandas uses the `NaN` (not a number) type for missing values._____no_output_____

bacteria2 = pd.Series(bacteria_dict, index=['Cyanobacteria','Firmicutes','Proteobacteria','Actinobacteria'])
bacteria2_____no_output_____bacteria2.isnull()_____no_output_____

Critically, the labels are used to **align data** when used in operations with other Series objects:_____no_output_____

bacteria + bacteria2_____no_output_____

Contrast this with NumPy arrays, where arrays of the same length will combine values element-wise; adding Series combined values with the same label in the resulting series. Notice also that the missing values were propogated by addition._____no_output_____### DataFrame

Inevitably, we want to be able to store, view and manipulate data that is *multivariate*, where for every index there are multiple fields or columns of data, often of varying data type.

A `DataFrame` is a tabular data structure, encapsulating multiple series like columns in a spreadsheet. Data are stored internally as a 2-dimensional object, but the `DataFrame` allows us to represent and manipulate higher-dimensional data._____no_output_____

data = pd.DataFrame({'value':[632, 1638, 569, 115, 433, 1130, 754, 555],
                     'patient':[1, 1, 1, 1, 2, 2, 2, 2],
                     'phylum':['Firmicutes', 'Proteobacteria', 'Actinobacteria',
    'Bacteroidetes', 'Firmicutes', 'Proteobacteria', 'Actinobacteria', 'Bacteroidetes']})
data_____no_output_____

Notice the `DataFrame` is sorted by column name. We can change the order by indexing them in the order we desire:_____no_output_____

data[['phylum','value','patient']]_____no_output_____

A `DataFrame` has a second index, representing the columns:_____no_output_____

data.columns_____no_output_____

If we wish to access columns, we can do so either by dict-like indexing or by attribute:_____no_output_____

data['value']_____no_output_____data.value_____no_output_____type(data.value)_____no_output_____type(data[['value']])_____no_output_____

Notice this is different than with `Series`, where dict-like indexing retrieved a particular element (row). If we want access to a row in a `DataFrame`, we index its `ix` attribute.
_____no_output_____

data.ix[3]_____no_output_____

Alternatively, we can create a `DataFrame` with a dict of dicts:_____no_output_____

data = pd.DataFrame({0: {'patient': 1, 'phylum': 'Firmicutes', 'value': 632},
                    1: {'patient': 1, 'phylum': 'Proteobacteria', 'value': 1638},
                    2: {'patient': 1, 'phylum': 'Actinobacteria', 'value': 569},
                    3: {'patient': 1, 'phylum': 'Bacteroidetes', 'value': 115},
                    4: {'patient': 2, 'phylum': 'Firmicutes', 'value': 433},
                    5: {'patient': 2, 'phylum': 'Proteobacteria', 'value': 1130},
                    6: {'patient': 2, 'phylum': 'Actinobacteria', 'value': 754},
                    7: {'patient': 2, 'phylum': 'Bacteroidetes', 'value': 555}})_____no_output_____data_____no_output_____

We probably want this transposed:_____no_output_____

data = data.T
data_____no_output_____

Its important to note that the Series returned when a DataFrame is indexted is merely a **view** on the DataFrame, and not a copy of the data itself. So you must be cautious when manipulating this data:_____no_output_____

vals = data.value
vals_____no_output_____vals[5] = 0
vals_____no_output_____data_____no_output_____vals = data.value.copy()
vals[5] = 1000
data_____no_output_____

We can create or modify columns by assignment:_____no_output_____

data.value[3] = 14
data_____no_output_____data['year'] = 2013
data_____no_output_____

But note, we cannot use the attribute indexing method to add a new column:_____no_output_____

data.treatment = 1
data_____no_output_____data.treatment_____no_output_____

Specifying a `Series` as a new columns cause its values to be added according to the `DataFrame`'s index:_____no_output_____

treatment = pd.Series([0]*4 + [1]*2)
treatment_____no_output_____data['treatment'] = treatment
data_____no_output_____

Other Python data structures (ones without an index) need to be the same length as the `DataFrame`:_____no_output_____

month = ['Jan', 'Feb', 'Mar', 'Apr']
data['month'] = month_____no_output_____data['month'] = ['Jan']*len(data)
data_____no_output_____

We can use `del` to remove columns, in the same way `dict` entries can be removed:_____no_output_____

del data['month']
data_____no_output_____

We can extract the underlying data as a simple `ndarray` by accessing the `values` attribute:_____no_output_____

data.values_____no_output_____

Notice that because of the mix of string and integer (and `NaN`) values, the dtype of the array is `object`. The dtype will automatically be chosen to be as general as needed to accomodate all the columns._____no_output_____

df = pd.DataFrame({'foo': [1,2,3], 'bar':[0.4, -1.0, 4.5]})
df.values_____no_output_____

Pandas uses a custom data structure to represent the indices of Series and DataFrames._____no_output_____

data.index_____no_output_____

Index objects are immutable:_____no_output_____

data.index[0] = 15_____no_output_____

This is so that Index objects can be shared between data structures without fear that they will be changed._____no_output_____

bacteria2.index = bacteria.index_____no_output_____bacteria2_____no_output_____

## Importing data_____no_output_____A key, but often under-appreciated, step in data analysis is importing the data that we wish to analyze. Though it is easy to load basic data structures into Python using built-in tools or those provided by packages like NumPy, it is non-trivial to import structured data well, and to easily convert this input into a robust data structure:

    genes = np.loadtxt("genes.csv", delimiter=",", dtype=[('gene', '|S10'), ('value', '<f4')])

Pandas provides a convenient set of functions for importing tabular data in a number of formats directly into a `DataFrame` object. These functions include a slew of options to perform type inference, indexing, parsing, iterating and cleaning automatically as data are imported._____no_output_____Let's start with some more bacteria data, stored in csv format._____no_output_____

!cat data/microbiome.csvTaxon,Patient,Tissue,Stool
Firmicutes,1,632,305
Firmicutes,2,136,4182
Firmicutes,3,1174,703
Firmicutes,4,408,3946
Firmicutes,5,831,8605
Firmicutes,6,693,50
Firmicutes,7,718,717
Firmicutes,8,173,33
Firmicutes,9,228,80
Firmicutes,10,162,3196
Firmicutes,11,372,32
Firmicutes,12,4255,4361
Firmicutes,13,107,1667
Firmicutes,14,96,223
Firmicutes,15,281,2377
Proteobacteria,1,1638,3886
Proteobacteria,2,2469,1821
Proteobacteria,3,839,661
Proteobacteria,4,4414,18
Proteobacteria,5,12044,83
Proteobacteria,6,2310,12
Proteobacteria,7,3053,547
Proteobacteria,8,395,2174
Proteobacteria,9,2651,767
Proteobacteria,10,1195,76
Proteobacteria,11,6857,795
Proteobacteria,12,483,666
Proteobacteria,13,2950,3994
Proteobacteria,14,1541,816
Proteobacteria,15,1307,53
Actinobacteria,1,569,648
Actinobacteria,2,1590,4
Actinobacteria,3,25,2
Actinobacteria,4,259,300
Actinobacteria,5,568,7
Actinobacteria,6,1102,9
Actinobacteria,7,678,377
Actinobacteria,8,260,58
Actinobacteria,9,424,233
Actinobacteria,10,548,21
Actinobacteria,11,201,83
Actinobacteria,12,42,75
Actinobacteria,13,109,59
Actinobacteria,14,51,183
Actinobacteria,15,310,204
Bacteroidetes,1,115,380
Bacteroidetes,2,67,0
Bacteroidetes,3,0,0
Bacteroidetes,4,85,5
Bacteroidetes,5,143,7
Bacteroidetes,6,678,2
Bacteroidetes,7,4829,209
Bacteroidetes,8,74,651
Bacteroidetes,9,169,254
Bacteroidetes,10,106,10
Bacteroidetes,11,73,381
Bacteroidetes,12,30,359
Bacteroidetes,13,51,51
Bacteroidetes,14,2473,2314
Bacteroidetes,15,102,33
Other,1,114,277
Other,2,195,18
Other,3,42,2
Other,4,316,43
Other,5,202,40
Other,6,116,0
Other,7,527,12
Other,8,357,11
Other,9,106,11
Other,10,67,14
Other,11,203,6
Other,12,392,6
Other,13,28,25
Other,14,12,22
Other,15,305,32

This table can be read into a DataFrame using `read_csv`:_____no_output_____

mb = pd.read_csv("data/microbiome.csv")
mb_____no_output_____

Notice that `read_csv` automatically considered the first row in the file to be a header row.

We can override default behavior by customizing some the arguments, like `header`, `names` or `index_col`._____no_output_____

pd.read_csv("data/microbiome.csv", header=None).head()_____no_output_____

`read_csv` is just a convenience function for `read_table`, since csv is such a common format:_____no_output_____

mb = pd.read_table("data/microbiome.csv", sep=',')_____no_output_____

The `sep` argument can be customized as needed to accomodate arbitrary separators. For example, we can use a regular expression to define a variable amount of whitespace, which is unfortunately very common in some data formats:

    sep='\s+'_____no_output_____For a more useful index, we can specify the first two columns, which together provide a unique index to the data._____no_output_____

mb = pd.read_csv("data/microbiome.csv", index_col=['Taxon','Patient'])
mb.head()_____no_output_____

This is called a *hierarchical* index, which we will revisit later in the tutorial._____no_output_____If we have sections of data that we do not wish to import (for example, known bad data), we can populate the `skiprows` argument:_____no_output_____

pd.read_csv("data/microbiome.csv", skiprows=[3,4,6]).head()_____no_output_____

Conversely, if we only want to import a small number of rows from, say, a very large data file we can use `nrows`:_____no_output_____

pd.read_csv("data/microbiome.csv", nrows=4)_____no_output_____

Alternately, if we want to process our data in reasonable chunks, the `chunksize` argument will return an iterable object that can be employed in a data processing loop. For example, our microbiome data are organized by bacterial phylum, with 15 patients represented in each:_____no_output_____

data_chunks = pd.read_csv("data/microbiome.csv", chunksize=15)

mean_tissue = {chunk.Taxon[0]:chunk.Tissue.mean() for chunk in data_chunks}

mean_tissue_____no_output_____

Most real-world data is incomplete, with values missing due to incomplete observation, data entry or transcription error, or other reasons. Pandas will automatically recognize and parse common missing data indicators, including `NA` and `NULL`._____no_output_____

!cat data/microbiome_missing.csvTaxon,Patient,Tissue,Stool
Firmicutes,1,632,305
Firmicutes,2,136,4182
Firmicutes,3,,703
Firmicutes,4,408,3946
Firmicutes,5,831,8605
Firmicutes,6,693,50
Firmicutes,7,718,717
Firmicutes,8,173,33
Firmicutes,9,228,NA
Firmicutes,10,162,3196
Firmicutes,11,372,-99999
Firmicutes,12,4255,4361
Firmicutes,13,107,1667
Firmicutes,14,?,223
Firmicutes,15,281,2377
Proteobacteria,1,1638,3886
Proteobacteria,2,2469,1821
Proteobacteria,3,839,661
Proteobacteria,4,4414,18
Proteobacteria,5,12044,83
Proteobacteria,6,2310,12
Proteobacteria,7,3053,547
Proteobacteria,8,395,2174
Proteobacteria,9,2651,767
Proteobacteria,10,1195,76
Proteobacteria,11,6857,795
Proteobacteria,12,483,666
Proteobacteria,13,2950,3994
Proteobacteria,14,1541,816
Proteobacteria,15,1307,53
Actinobacteria,1,569,648
Actinobacteria,2,1590,4
Actinobacteria,3,25,2
Actinobacteria,4,259,300
Actinobacteria,5,568,7
Actinobacteria,6,1102,9
Actinobacteria,7,678,377
Actinobacteria,8,260,58
Actinobacteria,9,424,233
Actinobacteria,10,548,21
Actinobacteria,11,201,83
Actinobacteria,12,42,75
Actinobacteria,13,109,59
Actinobacteria,14,51,183
Actinobacteria,15,310,204
Bacteroidetes,1,115,380
Bacteroidetes,2,67,0
Bacteroidetes,3,0,0
Bacteroidetes,4,85,5
Bacteroidetes,5,143,7
Bacteroidetes,6,678,2
Bacteroidetes,7,4829,209
Bacteroidetes,8,74,651
Bacteroidetes,9,169,254
Bacteroidetes,10,106,10
Bacteroidetes,11,73,381
Bacteroidetes,12,30,359
Bacteroidetes,13,51,51
Bacteroidetes,14,2473,2314
Bacteroidetes,15,102,33
Other,1,114,277
Other,2,195,18
Other,3,42,2
Other,4,316,43
Other,5,202,40
Other,6,116,0
Other,7,527,12
Other,8,357,11
Other,9,106,11
Other,10,67,14
Other,11,203,6
Other,12,392,6
Other,13,28,25
Other,14,12,22
Other,15,305,32pd.read_csv("data/microbiome_missing.csv").head(20)_____no_output_____

Above, Pandas recognized `NA` and an empty field as missing data._____no_output_____

pd.isnull(pd.read_csv("data/microbiome_missing.csv")).head(20)_____no_output_____

Unfortunately, there will sometimes be inconsistency with the conventions for missing data. In this example, there is a question mark "?" and a large negative number where there should have been a positive integer. We can specify additional symbols with the `na_values` argument:
   _____no_output_____

pd.read_csv("data/microbiome_missing.csv", na_values=['?', -99999]).head(20)_____no_output_____

These can be specified on a column-wise basis using an appropriate dict as the argument for `na_values`._____no_output_____### Microsoft Excel

Since so much financial and scientific data ends up in Excel spreadsheets (regrettably), Pandas' ability to directly import Excel spreadsheets is valuable. This support is contingent on having one or two dependencies (depending on what version of Excel file is being imported) installed: `xlrd` and `openpyxl` (these may be installed with either `pip` or `easy_install`).

Importing Excel data to Pandas is a two-step process. First, we create an `ExcelFile` object using the path of the file:                                             _____no_output_____

mb_file = pd.ExcelFile('data/microbiome/MID1.xls')
mb_file_____no_output_____

Then, since modern spreadsheets consist of one or more "sheets", we parse the sheet with the data of interest:_____no_output_____

mb1 = mb_file.parse("Sheet 1", header=None)
mb1.columns = ["Taxon", "Count"]
mb1.head()_____no_output_____

There is now a `read_excel` conveneince function in Pandas that combines these steps into a single call:_____no_output_____

mb2 = pd.read_excel('data/microbiome/MID2.xls', sheetname='Sheet 1', header=None)
mb2.head()_____no_output_____

There are several other data formats that can be imported into Python and converted into DataFrames, with the help of buitl-in or third-party libraries. These include JSON, XML, HDF5, relational and non-relational databases, and various web APIs. These are beyond the scope of this tutorial, but are covered in [Python for Data Analysis](http://shop.oreilly.com/product/0636920023784.do)._____no_output_____## Pandas Fundamentals_____no_output_____This section introduces the new user to the key functionality of Pandas that is required to use the software effectively.

For some variety, we will leave our digestive tract bacteria behind and employ some baseball data._____no_output_____

baseball = pd.read_csv("data/baseball.csv", index_col='id')
baseball.head()_____no_output_____

Notice that we specified the `id` column as the index, since it appears to be a unique identifier. We could try to create a unique index ourselves by combining `player` and `year`:_____no_output_____

player_id = baseball.player + baseball.year.astype(str)
baseball_newind = baseball.copy()
baseball_newind.index = player_id
baseball_newind.head()_____no_output_____

This looks okay, but let's check:_____no_output_____

baseball_newind.index.is_unique_____no_output_____

So, indices need not be unique. Our choice is not unique because some players change teams within years._____no_output_____

pd.Series(baseball_newind.index).value_counts()_____no_output_____

The most important consequence of a non-unique index is that indexing by label will return multiple values for some labels:_____no_output_____

baseball_newind.ix['wickmbo012007']_____no_output_____

We will learn more about indexing below._____no_output_____We can create a truly unique index by combining `player`, `team` and `year`:_____no_output_____

player_unique = baseball.player + baseball.team + baseball.year.astype(str)
baseball_newind = baseball.copy()
baseball_newind.index = player_unique
baseball_newind.head()_____no_output_____baseball_newind.index.is_unique_____no_output_____

We can create meaningful indices more easily using a hierarchical index; for now, we will stick with the numeric `id` field as our index._____no_output_____### Manipulating indices

**Reindexing** allows users to manipulate the data labels in a DataFrame. It forces a DataFrame to conform to the new index, and optionally, fill in missing data if requested.

A simple use of `reindex` is to alter the order of the rows:_____no_output_____

baseball.reindex(baseball.index[::-1]).head()_____no_output_____

Notice that the `id` index is not sequential. Say we wanted to populate the table with every `id` value. We could specify and index that is a sequence from the first to the last `id` numbers in the database, and Pandas would fill in the missing data with `NaN` values:_____no_output_____

id_range = range(baseball.index.values.min(), baseball.index.values.max())
baseball.reindex(id_range).head()_____no_output_____

Missing values can be filled as desired, either with selected values, or by rule:_____no_output_____

baseball.reindex(id_range, method='ffill', columns=['player','year']).head()_____no_output_____baseball.reindex(id_range, fill_value='mr.nobody', columns=['player']).head()_____no_output_____

Keep in mind that `reindex` does not work if we pass a non-unique index series._____no_output_____We can remove rows or columns via the `drop` method:_____no_output_____

baseball.shape_____no_output_____baseball.drop([89525, 89526])_____no_output_____baseball.drop(['ibb','hbp'], axis=1)_____no_output_____

## Indexing and Selection

Indexing works analogously to indexing in NumPy arrays, except we can use the labels in the `Index` object to extract values in addition to arrays of integers._____no_output_____

# Sample Series object
hits = baseball_newind.h
hits_____no_output_____# Numpy-style indexing
hits[:3]_____no_output_____# Indexing by label
hits[['womacto01CHN2006','schilcu01BOS2006']]_____no_output_____

We can also slice with data labels, since they have an intrinsic order within the Index:_____no_output_____

hits['womacto01CHN2006':'gonzalu01ARI2006']_____no_output_____hits['womacto01CHN2006':'gonzalu01ARI2006'] = 5
hits_____no_output_____

In a `DataFrame` we can slice along either or both axes:_____no_output_____

baseball_newind[['h','ab']]_____no_output_____baseball_newind[baseball_newind.ab>500]_____no_output_____

The indexing field `ix` allows us to select subsets of rows and columns in an intuitive way:_____no_output_____

baseball_newind.ix['gonzalu01ARI2006', ['h','X2b', 'X3b', 'hr']]_____no_output_____baseball_newind.ix[['gonzalu01ARI2006','finlest01SFN2006'], 5:8]_____no_output_____baseball_newind.ix[:'myersmi01NYA2006', 'hr']_____no_output_____

Similarly, the cross-section method `xs` (not a field) extracts a single column or row *by label* and returns it as a `Series`:_____no_output_____

baseball_newind.xs('myersmi01NYA2006')_____no_output_____

## Operations

`DataFrame` and `Series` objects allow for several operations to take place either on a single object, or between two or more objects.

For example, we can perform arithmetic on the elements of two objects, such as combining baseball statistics across years:_____no_output_____

hr2006 = baseball[baseball.year==2006].xs('hr', axis=1)
hr2006.index = baseball.player[baseball.year==2006]

hr2007 = baseball[baseball.year==2007].xs('hr', axis=1)
hr2007.index = baseball.player[baseball.year==2007]_____no_output_____hr2006 = pd.Series(baseball.hr[baseball.year==2006].values, index=baseball.player[baseball.year==2006])
hr2007 = pd.Series(baseball.hr[baseball.year==2007].values, index=baseball.player[baseball.year==2007])_____no_output_____hr_total = hr2006 + hr2007
hr_total_____no_output_____

Pandas' data alignment places `NaN` values for labels that do not overlap in the two Series. In fact, there are only 6 players that occur in both years._____no_output_____

hr_total[hr_total.notnull()]_____no_output_____

While we do want the operation to honor the data labels in this way, we probably do not want the missing values to be filled with `NaN`. We can use the `add` method to calculate player home run totals by using the `fill_value` argument to insert a zero for home runs where labels do not overlap:_____no_output_____

hr2007.add(hr2006, fill_value=0)_____no_output_____

Operations can also be **broadcast** between rows or columns.

For example, if we subtract the maximum number of home runs hit from the `hr` column, we get how many fewer than the maximum were hit by each player:_____no_output_____

baseball.hr - baseball.hr.max()_____no_output_____

Or, looking at things row-wise, we can see how a particular player compares with the rest of the group with respect to important statistics_____no_output_____

baseball.ix[89521]["player"]_____no_output_____stats = baseball[['h','X2b', 'X3b', 'hr']]
diff = stats - stats.xs(89521)
diff[:10]_____no_output_____

We can also apply functions to each column or row of a `DataFrame`_____no_output_____

stats.apply(np.median)_____no_output_____stat_range = lambda x: x.max() - x.min()
stats.apply(stat_range)_____no_output_____

Lets use apply to calculate a meaningful baseball statistics, slugging percentage:

$$SLG = \frac{1B + (2 \times 2B) + (3 \times 3B) + (4 \times HR)}{AB}$$

And just for fun, we will format the resulting estimate._____no_output_____

slg = lambda x: (x['h']-x['X2b']-x['X3b']-x['hr'] + 2*x['X2b'] + 3*x['X3b'] + 4*x['hr'])/(x['ab']+1e-6)
baseball.apply(slg, axis=1).apply(lambda x: '%.3f' % x)_____no_output_____

## Sorting and Ranking

Pandas objects include methods for re-ordering data._____no_output_____

baseball_newind.sort_index().head()_____no_output_____baseball_newind.sort_index(ascending=False).head()_____no_output_____baseball_newind.sort_index(axis=1).head()_____no_output_____

We can also use `order` to sort a `Series` by value, rather than by label._____no_output_____

baseball.hr.order(ascending=False)_____no_output_____

For a `DataFrame`, we can sort according to the values of one or more columns using the `by` argument of `sort_index`:_____no_output_____

baseball[['player','sb','cs']].sort_index(ascending=[False,True], by=['sb', 'cs']).head(10)_____no_output_____

**Ranking** does not re-arrange data, but instead returns an index that ranks each value relative to others in the Series._____no_output_____

baseball.hr.rank()_____no_output_____

Ties are assigned the mean value of the tied ranks, which may result in decimal values._____no_output_____

pd.Series([100,100]).rank()_____no_output_____

Alternatively, you can break ties via one of several methods, such as by the order in which they occur in the dataset:_____no_output_____

baseball.hr.rank(method='first')_____no_output_____

Calling the `DataFrame`'s `rank` method results in the ranks of all columns:_____no_output_____

baseball.rank(ascending=False).head()_____no_output_____baseball[['r','h','hr']].rank(ascending=False).head()_____no_output_____

### Exercise

Calculate **on base percentage** for each player, and return the ordered series of estimates.

$$OBP = \frac{H + BB + HBP}{AB + BB + HBP + SF}$$_____no_output_____

# Write your answer here_____no_output_____

## Hierarchical indexing

In the baseball example, I was forced to combine 3 fields to obtain a unique index that was not simply an integer value. A more elegant way to have done this would be to create a hierarchical index from the three fields._____no_output_____

baseball_h = baseball.set_index(['year', 'team', 'player'])
baseball_h.head(10)_____no_output_____

This index is a `MultiIndex` object that consists of a sequence of tuples, the elements of which is some combination of the three columns used to create the index. Where there are multiple repeated values, Pandas does not print the repeats, making it easy to identify groups of values._____no_output_____

baseball_h.index[:10]_____no_output_____baseball_h.index.is_unique_____no_output_____baseball_h.ix[(2007, 'ATL', 'francju01')]_____no_output_____

Recall earlier we imported some microbiome data using two index columns. This created a 2-level hierarchical index:_____no_output_____

mb = pd.read_csv("data/microbiome.csv", index_col=['Taxon','Patient'])_____no_output_____mb.head(10)_____no_output_____mb.index_____no_output_____

With a hierachical index, we can select subsets of the data based on a *partial* index:_____no_output_____

mb.ix['Proteobacteria']_____no_output_____

Hierarchical indices can be created on either or both axes. Here is a trivial example:_____no_output_____

frame = pd.DataFrame(np.arange(12).reshape(( 4, 3)),
                  index =[['a', 'a', 'b', 'b'], [1, 2, 1, 2]],
                  columns =[['Ohio', 'Ohio', 'Colorado'], ['Green', 'Red', 'Green']])

frame_____no_output_____

If you want to get fancy, both the row and column indices themselves can be given names:_____no_output_____

frame.index.names = ['key1', 'key2']
frame.columns.names = ['state', 'color']
frame_____no_output_____

With this, we can do all sorts of custom indexing:_____no_output_____

frame.ix['a']['Ohio']_____no_output_____frame.ix['b', 2]['Colorado']_____no_output_____

Additionally, the order of the set of indices in a hierarchical `MultiIndex` can be changed by swapping them pairwise:_____no_output_____

mb.swaplevel('Patient', 'Taxon').head()_____no_output_____

Data can also be sorted by any index level, using `sortlevel`:_____no_output_____

mb.sortlevel('Patient', ascending=False).head()_____no_output_____

## Missing data

The occurence of missing data is so prevalent that it pays to use tools like Pandas, which seamlessly integrates missing data handling so that it can be dealt with easily, and in the manner required by the analysis at hand.

Missing data are represented in `Series` and `DataFrame` objects by the `NaN` floating point value. However, `None` is also treated as missing, since it is commonly used as such in other contexts (*e.g.* NumPy)._____no_output_____

foo = pd.Series([np.nan, -3, None, 'foobar'])
foo_____no_output_____foo.isnull()_____no_output_____

Missing values may be dropped or indexed out:_____no_output_____

bacteria2_____no_output_____bacteria2.dropna()_____no_output_____bacteria2[bacteria2.notnull()]_____no_output_____

By default, `dropna` drops entire rows in which one or more values are missing._____no_output_____

data_____no_output_____data.dropna()_____no_output_____

This can be overridden by passing the `how='all'` argument, which only drops a row when every field is a missing value._____no_output_____

data.dropna(how='all')_____no_output_____

This can be customized further by specifying how many values need to be present before a row is dropped via the `thresh` argument._____no_output_____

data.ix[7, 'year'] = np.nan
data_____no_output_____data.dropna(thresh=4)_____no_output_____

This is typically used in time series applications, where there are repeated measurements that are incomplete for some subjects._____no_output_____If we want to drop missing values column-wise instead of row-wise, we use `axis=1`._____no_output_____

data.dropna(axis=1)_____no_output_____

Rather than omitting missing data from an analysis, in some cases it may be suitable to fill the missing value in, either with a default value (such as zero) or a value that is either imputed or carried forward/backward from similar data points. We can do this programmatically in Pandas with the `fillna` argument._____no_output_____

bacteria2.fillna(0)_____no_output_____data.fillna({'year': 2013, 'treatment':2})_____no_output_____

Notice that `fillna` by default returns a new object with the desired filling behavior, rather than changing the `Series` or  `DataFrame` in place (**in general, we like to do this, by the way!**)._____no_output_____

data_____no_output_____

We can alter values in-place using `inplace=True`._____no_output_____

_ = data.year.fillna(2013, inplace=True)
data_____no_output_____

Missing values can also be interpolated, using any one of a variety of methods:_____no_output_____

bacteria2.fillna(method='bfill')_____no_output_____bacteria2.fillna(bacteria2.mean())_____no_output_____

## Data summarization

We often wish to summarize data in `Series` or `DataFrame` objects, so that they can more easily be understood or compared with similar data. The NumPy package contains several functions that are useful here, but several summarization or reduction methods are built into Pandas data structures._____no_output_____

baseball.sum()_____no_output_____

Clearly, `sum` is more meaningful for some columns than others. For methods like `mean` for which application to string variables is not just meaningless, but impossible, these columns are automatically exculded:_____no_output_____

baseball.mean()_____no_output_____

The important difference between NumPy's functions and Pandas' methods is that the latter have built-in support for handling missing data._____no_output_____

bacteria2_____no_output_____bacteria2.mean()_____no_output_____

Sometimes we may not want to ignore missing values, and allow the `nan` to propagate._____no_output_____

bacteria2.mean(skipna=False)_____no_output_____

Passing `axis=1` will summarize over rows instead of columns, which only makes sense in certain situations._____no_output_____

extra_bases = baseball[['X2b','X3b','hr']].sum(axis=1)
extra_bases.order(ascending=False)_____no_output_____

A useful summarization that gives a quick snapshot of multiple statistics for a `Series` or `DataFrame` is `describe`:_____no_output_____

baseball.describe()_____no_output_____

`describe` can detect non-numeric data and sometimes yield useful information about it._____no_output_____

baseball.player.describe()_____no_output_____

We can also calculate summary statistics *across* multiple columns, for example, correlation and covariance.

$$cov(x,y) = \sum_i (x_i - \bar{x})(y_i - \bar{y})$$_____no_output_____

baseball.hr.cov(baseball.X2b)_____no_output_____

$$corr(x,y) = \frac{cov(x,y)}{(n-1)s_x s_y} = \frac{\sum_i (x_i - \bar{x})(y_i - \bar{y})}{\sqrt{\sum_i (x_i - \bar{x})^2 \sum_i (y_i - \bar{y})^2}}$$_____no_output_____

baseball.hr.corr(baseball.X2b)_____no_output_____baseball.ab.corr(baseball.h)_____no_output_____baseball.corr()_____no_output_____

If we have a `DataFrame` with a hierarchical index (or indices), summary statistics can be applied with respect to any of the index levels:_____no_output_____

mb.head()_____no_output_____mb.sum(level='Taxon')_____no_output_____

## Writing Data to Files

As well as being able to read several data input formats, Pandas can also export data to a variety of storage formats. We will bring your attention to just a couple of these._____no_output_____

mb.to_csv("mb.csv")_____no_output_____

The `to_csv` method writes a `DataFrame` to a comma-separated values (csv) file. You can specify custom delimiters (via `sep` argument), how missing values are written (via `na_rep` argument), whether the index is writen (via `index` argument), whether the header is included (via `header` argument), among other options._____no_output_____An efficient way of storing data to disk is in binary format. Pandas supports this using Python’s built-in pickle serialization._____no_output_____

baseball.to_pickle("baseball_pickle")_____no_output_____

The complement to `to_pickle` is the `read_pickle` function, which restores the pickle to a `DataFrame` or `Series`:_____no_output_____

pd.read_pickle("baseball_pickle")_____no_output_____

As Wes warns in his book, it is recommended that binary storage of data via pickle only be used as a temporary storage format, in situations where speed is relevant. This is because there is no guarantee that the pickle format will not change with future versions of Python._____no_output_____


# Notebook from CLima86/Physics_5300_CDL
Path: 2020_week_3/.ipynb_checkpoints/bifurcation_diagram_sine_map-checkpoint.ipynb

# Bifurcation diagram of a chaotic dynamical system

Adapted from the Ipython Cookbook.

Here we consider a logistic map, which is an example of chaos arising from a simple nonlinear equation.
The logistic map models the evolution of a population including reproduction and mortality.

For more information, see Wikipedia articles on
chaos theory,
complex systems,
the logistic map,
iterated functions,
bifurcation diagrams._____no_output_____

%matplotlib inline

# standard numpy and matplotlib imports
import numpy as np
import matplotlib.pyplot as plt
_____no_output_____def logistic(r, x):
    """Function definition of logistic map.
           f_r(x) = r x(1-x)
    """
    return r*x*(1-x)

def sine_map(r, x):
    return r * np.sin(np.pi * x)

def sine_map_deriv(r, x):
    return r * np.pi * np.cos(np.pi * x)_____no_output_____x, step = np.linspace(0., 1., num=101, endpoint=True, retstep=True)
fig = plt.figure()
ax = fig.add_subplot(1,1,1)

r_param = 0.1
ax.plot(x, sine_map(r_param, x), color='black')
ax.set_xlabel('x')
ax.set_ylabel('f(x)')

fig.tight_layout()_____no_output_____def plot_system(r, x0, n, ax=None):
    """Plot the function and the y=x diagonal line."""
    t = np.linspace(0,1, num=101)
    ax.plot(t, sine_map(r,t), 'k', lw=2)  # black, linewidth 2
    ax.plot([0,1], [0,1], 'k', lw=2)  # x is an array of 0 and 1,
                                      # y is the same array, so plot
                                      # from (0,0) to (1,1)

    # Recursively apply y=f(x) and plot two lines:
    #   line from (x, x) to (x, y)
    #   line from (x, y) to (y, y)
    x = x0
    for i in range(n):
        y = sine_map(r, x)
        # Plot the two lines
        ax.plot([x,x], [x,y], 'k', lw=1)
        ax.plot([x,y], [y,y], 'k', lw=1)
        # Plot the positions with increasing opacity
        ax.plot([x], [y], 'ok', ms=10, alpha=(i+1)/n)
        x=y  # recursive: reset x to y
    ax.set_xlim(0,1)
    ax.set_ylim(0,1)
    ax.set_title(f"$r={r:.1f}, \, x_0={x0:.1f}$")

fig = plt.figure(figsize=(12,12))
ax1 = fig.add_subplot(2,2,1)
plot_system(.3, 0.1, 10, ax=ax1)  # start at 0.1, r parameter is 2.5, take n steps
# or fig, [[ax1, ax2], [ax3, ax4]] = plt.subplots(nrows=2, ncols=2, figsize=(12,12), sharey=True)
ax2 = fig.add_subplot(2,2,2)
plot_system(.5, 0.1, 10, ax=ax2)
ax3 = fig.add_subplot(2,2,3)
plot_system(.7, 0.1, 10, ax=ax3)
ax4 = fig.add_subplot(2,2,4)
plot_system(.9, 0.1, 20, ax=ax4)_____no_output_____n = 10000
r = np.linspace(0.6, 1.0, n)  # n values of r linearly spaced

iterations = 1000   # iterations of the map; keep last iterations
last = 100

x = 0.1 * np.ones(n)  # x_0 initial condition

lyapunov = np.zeros(n) # initialize vector for lyapunov exponents_____no_output_____# Display the bifurcation diagram with one pixel per point x_n^(r) for last iterations
fig = plt.figure(figsize=(12,12))
ax1 = fig.add_subplot(2,1,1)
ax2 = fig.add_subplot(2,1,2)

for i in range(iterations):
    x = sine_map(r,x)
    # Compute the partial sum of the Lyapunov exponent.
    lyapunov += np.log(abs(sine_map_deriv(r,x)))
    # Display the bifurcation diagram.
    if i >= (iterations-last):
        ax1.plot(r, x, ',k', alpha=0.25)
ax1.set_xlim(0.6, 1.)
ax1.set_title("Bifurcation")

# Display the Lyapunov exponent
# Add a zero line (lightened with alpha=0.5)
ax2.axhline(0, color='k', lw=0.5, alpha=0.5)
# Negative Lyapunov exponent
ax2.plot(r[lyapunov < 0],
         lyapunov[lyapunov < 0] / iterations,
         '.k', alpha=0.5, ms=0.5)
# Positive Lyapunov exponent
ax2.plot(r[lyapunov >= 0],
         lyapunov[lyapunov >= 0] / iterations,
         '.r', alpha=0.5, ms=0.5)

ax2.set_xlim(0.6, 1.)
#ax2.set_ylim(-2, 1)
ax2.set_title("Lyapunov exponent")

plt.tight_layout()_____no_output_____

We see there is a fixed point for $r < 3$, then two and four equilibria and a chaotic behavior when $r$ belongs to certain areas of the parameter space.

The Lyapunov exponent is positive when the system is chaotic (in red)._____no_output_____