# Notebook from dongxulee/lifeCycle
Path: 20200805/discreteOptimization.ipynb

### Agent state and action definition
* State Variable: x = [w, n, M, g_lag, e, s, (H, r, m), O_lag], actions variable a = [c, b, k, i, q] both of them are numpy array. _____no_output_____

%pylab inline
from scipy.interpolate import interpn
from sklearn.neighbors import KNeighborsRegressor as KN
from helpFunctions import surfacePlot
import numpy as np
from scipy import optimize
from multiprocessing import Pool
from functools import partial
import warnings
import math
from numba import jit, float64, int32, guvectorize
warnings.filterwarnings("ignore")
np.printoptions(precision=2)

# time line
T_min = 0
T_max = 70
T_R = 45
# discounting factor
beta = 1/(1+0.02)
# utility function parameter
gamma = 2
# relative importance of housing consumption and non durable consumption
alpha = 0.8
# parameter used to calculate the housing consumption
kappa = 0.3
# depreciation parameter
delta = 0.05
# housing parameter
chi = 0.3
# uB associated parameter
B = 2
# # minimum consumption
# c_bar = 3
# constant cost
c_h = 2
# All the money amount are denoted in thousand dollars
earningShock = [0.8,1.2]
# Define transition matrix of economical states
# GOOD -> GOOD 0.8, BAD -> BAD 0.6
Ps = np.array([[0.6, 0.4],[0.2, 0.8]])
# current risk free interest rate
r_b = np.array([0.01 ,0.03])
# stock return depends on current and future econ states
# r_k = np.array([[-0.2, 0.15],[-0.15, 0.2]])
r_k = np.array([[-0.15, 0.20],[-0.15, 0.20]])
# expected return on stock market
# r_bar = 0.0667
r_bar = 0.02
# probability of survival
Pa = np.load("prob.npy")
# deterministic income
detEarning = np.load("detEarning.npy")
# probability of employment transition Pe[s, s_next, e, e_next]
Pe = np.array([[[[0.3, 0.7], [0.1, 0.9]], [[0.25, 0.75], [0.05, 0.95]]],
               [[[0.25, 0.75], [0.05, 0.95]], [[0.2, 0.8], [0.01, 0.99]]]])
# tax rate before and after retirement
tau_L = 0.2
tau_R = 0.1

# constant state variables: Purchase value 250k, down payment 50k, mortgage 200k, interest rate 3.6%,
# 55 payment period, 8.4k per period. One housing unit is roughly 1 square feet. Housing price 0.25k/sf

# some variables associate with 401k amount
Nt = [np.sum(Pa[t:]) for t in range(T_max-T_min)]
Dt = [np.ceil(((1+r_bar)**N - 1)/(r_bar*(1+r_bar)**N)) for N in Nt]
# owning a house
O_lag = 1
# housing unit
H = 1000
# mortgate rate
rh = 0.036
# mortgate payment
m = 8.4
# housing price constant
pt = 250/1000
# 30k rent 1000 sf
pr = 30/1000Populating the interactive namespace from numpy and matplotlib
#Define the utility function
def u(c):
    # shift utility function to the left, so it only takes positive value
    return (np.float_power(c, 1-gamma) - 1)/(1 - gamma)

#Define the bequeath function, which is a function of wealth
def uB(tb):
    return B*u(tb)

#Calcualte HE
def calHE(x):
    w, n, M, g_lag, e, s = x
    HE = (H+(1-chi)*(1-delta)*g_lag)*pt - M
    return HE

#Calculate TB
def calTB(x):
    w, n, M, g_lag, e, s = x
    TB = w + n + calHE(x)
    return TB

def R(x, a):
    '''
    Input:
        state x: w, n, M, g_lag, e, s
        action a: c, b, k, i, q
    Output:
        reward value
    '''
    w, n, M, g_lag, e, s = x
    c, b, k, i, q = a
    if q == 1:
        h = H + (1-delta)*g_lag + i
        Vh = (1+kappa)*h
    else:
        h = H + (1-delta)*g_lag
        Vh = (1-kappa)*(h-(1-q)*H)
    C = np.float_power(c, alpha) * np.float_power(Vh, 1-alpha)
    return u(C)

#Define the earning function, which applies for both employment and unemployment, good econ state and bad econ state
def y(t, x):
    w, n, M, g_lag, e, s = x
    if t <= T_R:
        welfare = 5
        return detEarning[t] * earningShock[int(s)] * e + (1-e) * welfare
    else:
        return detEarning[t]

#Earning after tax and fixed by transaction in and out from 401k account
def yAT(t,x):
    yt = y(t, x)
    w, n, M, g_lag, e, s = x
    if t <= T_R and e == 1:
        # 5% of the income will be put into the 401k
        i = 0.05
        return (1-tau_L)*(yt * (1-i))
    if t <= T_R and e == 0:
        return yt
    else:
        # t > T_R, n/discounting amount will be withdraw from the 401k
        return (1-tau_R)*yt + n/Dt[t]

#Define the evolution of the amount in 401k account
def gn(t, n, x, s_next):
    w, n, M, g_lag, e, s = x
    if t <= T_R and e == 1:
        # if the person is employed, then 5 percent of his income goes into 401k
        i = 0.05
        n_cur = n + y(t, x) * i
    elif t <= T_R and e == 0:
        # if the perons is unemployed, then n does not change
        n_cur = n
    else:
        # t > T_R, n/discounting amount will be withdraw from the 401k
        n_cur = n - n/Dt[t]
    return (1+r_k[int(s), s_next])*n_cur

def transition(x, a, t):
    '''
    Input: state and action and time
    Output: possible future states and corresponding probability
    '''
    w, n, M, g_lag, e, s = x
    c, b, k, i, q = a
    # variables used to collect possible states and probabilities
    x_next = []
    prob_next = []
    M_next = M*(1+rh) - m
    if q == 1:
        g = (1-delta)*g_lag + i
    else:
        g = (1-delta)*g_lag
    for s_next in [0,1]:
        w_next =  b*(1+r_b[int(s)]) + k*(1+r_k[int(s), s_next])
        n_next = gn(t, n, x, s_next)
        if t >= T_R:
            e_next = 0
            x_next.append([w_next, n_next, M_next, g, s_next, e_next])
            prob_next.append(Ps[int(s),s_next])
        else:
            for e_next in [0,1]:
                x_next.append([w_next, n_next, M_next, g, s_next, e_next])
                prob_next.append(Ps[int(s),s_next] * Pe[int(s),s_next,int(e),e_next])
    return np.array(x_next), np.array(prob_next)_____no_output_____# Value function is a function of state and time t < T
def V(x, t, NN):
    w, n, M, g_lag, e, s = x
    yat = yAT(t,x)
    if t == T_max-1:
        # The objective functions of terminal state
        def obj1(cbki):
            # Not renting out case
            q = 1
            c,b,k,i = cbki
            a = [c, b, k, i, q]
            x_next, p_next  = transition(x, a, t)
            TB_next = list(map(calTB, x_next)) # conditional on being dead in the future
            uBTB = np.vectorize(uB)(TB_next)
            return R(x, a) + beta * np.dot(uBTB, p_next)

        def obj2(cbkq):
            # Renting out case
            i = 0
            c,b,k,q = cbkq
            a = [c, b, k, i, q]
            x_next, p_next  = transition(x, a, t)
            TB_next = list(map(calTB, x_next)) # conditional on being dead in the future
            uBTB = np.vectorize(uB)(TB_next)
            return R(x, a) + beta * np.dot(uBTB, p_next)

    else:
        # The objective functions are functions of actions
        def obj1(cbki):
            # Not renting out case
            q = 1
            c,b,k,i = cbki
            a = [c, b, k, i, q]
            x_next, p_next  = transition(x, a, t)
            V_tilda = NN.predict(x_next) # V_{t+1} conditional on being alive, approximation here
            TB_next = list(map(calTB, x_next)) # conditional on being dead in the future
            uBTB = np.vectorize(uB)(TB_next)
            return R(x, a) + beta * (Pa[t] * np.dot(V_tilda, p_next)) + (1 - Pa[t]) * np.dot(uBTB, p_next)

        def obj2(cbkq):
            # Renting out case
            i = 0
            c,b,k,q = cbkq
            a = [c, b, k, i, q]
            x_next, p_next  = transition(x, a, t)
            V_tilda = NN.predict(x_next) # V_{t+1} conditional on being alive, approximation here
            TB_next = list(map(calTB, x_next)) # conditional on being dead in the future
            uBTB = np.vectorize(uB)(TB_next)
            return R(x, a) + beta * (Pa[t] * np.dot(V_tilda, p_next)) + (1 - Pa[t]) * np.dot(uBTB, p_next)

    def obj1_solver(obj1):
    # Constrain: yat + w - m = c + b + k + (1+chi)*i*pt + I{i>0}*c_h
    # i_portion takes [0:0.05:0.95]
    # c_portion takes remaining [0:0.05:0.95]
    # b_portion takes reamining [0:0.05:0.95]
    # k is the remainder
        mfun = -np.inf
        ma = [0,0,0,0]
        for ip in np.linspace(0,0.9,10):
            budget1 = yat + w - m
            if ip*budget1 > c_h:
                i = (budget1*ip - c_h)/((1+chi)*pt)
                budget2 = budget1 * (1-ip)
            else:
                i = 0
                budget2 = budget1
            for cp in np.linspace(0,1,11):
                c = budget2*cp
                budget3 = budget2 * (1-cp)
                for bp in np.linspace(0,1,11):
                    b = budget3* bp
                    k = budget3 * (1-bp)
                    a = [c,b,k,i]
                    fun = obj1(a)
                    if fun > mfun:
                        mfun = fun
                        ma = a
        return fun, ma

    def obj2_solver(obj2):
    # Constrain: yat + w - m + (1-q)*H*pr = c + b + k
    # q takes value [0:0.05:0.95]
    # c_portion takes remaining [0:0.05:0.95]
    # b_portion takes reamining [0:0.05:0.95]
    # k is the remainder
        mfun = -np.inf
        ma = [0,0,0,1]
        for q in np.linspace(0,0.9,10):
            budget1 = yat + w - m + (1-q)*H*pr
            for cp in np.linspace(0,1,11):
                c = budget1*cp
                budget2 = budget1 * (1-cp)
                for bp in np.linspace(0,1,11):
                    b = budget2* bp
                    k = budget2 * (1-bp)
                    a = [c,b,k,q]
                    fun = obj2(a)
                    if fun > mfun:
                        mfun = fun
                        ma = a
        return fun, ma

    fun1, action1 = obj1_solver(obj1)
    fun2, action2 = obj2_solver(obj2)

    if fun1 > fun2:
        c, b, k, i = action1
        q = 1
        return np.array([fun1, [c,b,k,i,q]])
    else:
        c, b, k, q = action2
        i = 0
        return np.array([fun2, [c,b,k,i,q]])_____no_output_____# wealth discretization
# w_grid_size = 15
# w_lower = 10
# w_upper = 10000
# 401k amount discretization
# n_grid_size = 5
# n_lower = 10
# n_upper = 6000
# power = 2

# wealth discretization
ws = np.array([10,25,50,75,100,125,150,175,200,250,500,750,1000,1500,3000])
w_grid_size = len(ws)
# 401k amount discretization
ns = np.array([1,3,7,14,24,39,63,100,165,385,13255])
n_grid_size = len(ns)
# Mortgage amount
Ms = np.array([200, 400, 600, 800, 1000])
M_grid_size = len(Ms)
# Improvement amount
gs = np.array([0, 50, 100])
g_grid_size = len(gs)
# def powspace(start, stop, power, num):
#     start = np.power(start, 1/float(power))
#     stop = np.power(stop, 1/float(power))
#     return np.power(np.linspace(start, stop, num=num), power)
# ws = powspace(w_lower, w_upper, power, w_grid_size)
# ns = powspace(n_lower, n_upper, power, n_grid_size)
xgrid = np.array([[w, n, M, g_lag, e, s]
                            for w in ws
                            for n in ns
                            for M in Ms
                            for g_lag in gs
                            for e in [0,1]
                            for s in [0,1]
                            ]).reshape((w_grid_size, n_grid_size,M_grid_size,g_grid_size,2,2,6))

Vgrid = np.zeros((w_grid_size, n_grid_size,M_grid_size,g_grid_size,2,2, T_max))
cgrid = np.zeros((w_grid_size, n_grid_size,M_grid_size,g_grid_size,2,2, T_max))
bgrid = np.zeros((w_grid_size, n_grid_size,M_grid_size,g_grid_size,2,2, T_max))
kgrid = np.zeros((w_grid_size, n_grid_size,M_grid_size,g_grid_size,2,2, T_max))
igrid = np.zeros((w_grid_size, n_grid_size,M_grid_size,g_grid_size,2,2, T_max))
qgrid = np.zeros((w_grid_size, n_grid_size,M_grid_size,g_grid_size,2,2, T_max))_____no_output_____

### SLSQP with KNN approximation and multidimensional interpolation_____no_output_____

class iApproxy(object):
    def __init__(self, points, Vgrid):
        self.V = Vgrid
        self.p = points
    def predict(self, x_next):
        pvalues = []
        for xx in x_next:
            pvalues.append(interpn(self.p, self.V[:,:,:,:,int(xx[4]),int(xx[5])], xx[:4], bounds_error = False, fill_value = None)[0])
        return pvalues_____no_output_____

### Value iteration with interpolation approximation_____no_output_____

%%time
# value iteration part
xs = xgrid.reshape((w_grid_size*n_grid_size*M_grid_size*g_grid_size*2*2,6))
pool = Pool()
points = (ws,ns,Ms,gs)
for t in range(T_max-1, T_min-1, -1):
    print(t)
    if t == T_max - 1:
        f = partial(V, t = t, NN = None)
        results = np.array(pool.map(f, xs))
    else:
        approx = iApproxy(points,Vgrid[:,:,:,:,:,:,t+1])
        f = partial(V, t = t, NN = approx)
        results = np.array(pool.map(f, xs))
    Vgrid[:,:,:,:,:,:,t] = results[:,0].reshape((w_grid_size,n_grid_size,M_grid_size,g_grid_size,2,2))
    cgrid[:,:,:,:,:,:,t] = np.array([r[0] for r in results[:,1]]).reshape((w_grid_size,n_grid_size,M_grid_size,g_grid_size,2,2))
    bgrid[:,:,:,:,:,:,t] = np.array([r[1] for r in results[:,1]]).reshape((w_grid_size,n_grid_size,M_grid_size,g_grid_size,2,2))
    kgrid[:,:,:,:,:,:,t] = np.array([r[2] for r in results[:,1]]).reshape((w_grid_size,n_grid_size,M_grid_size,g_grid_size,2,2))
    igrid[:,:,:,:,:,:,t] = np.array([r[3] for r in results[:,1]]).reshape((w_grid_size,n_grid_size,M_grid_size,g_grid_size,2,2))
    qgrid[:,:,:,:,:,:,t] = np.array([r[4] for r in results[:,1]]).reshape((w_grid_size,n_grid_size,M_grid_size,g_grid_size,2,2))
pool.close()

np.save("Vgrid_i", Vgrid)
np.save("cgrid_i", cgrid)
np.save("bgrid_i", bgrid)
np.save("kgrid_i", kgrid)
np.save("igrid_i", igrid)
np.save("qgrid_i", qgrid)69
68
67
66
65
64
63
62
61
60
59
58
57
56
55
54
53
52
51
50
49
48
47
46
45
44
43
42
41
40
39
38
37
36
35
34
33
32
31
30
29
28
27
26
25
24
# Mortgage amount
Ms = np.array([50, 100, 150, 200])
# Improvement amount
gs = np.array([10, 20, 100, 200])
for i in range(len(Ms)):
    for j in range(len(gs)):
        print(Ms[i])
        print(Ms[j])
        plt.figure()
        surfacePlot(xgrid[:,:,i,j,1,1,0], xgrid[:,:,i,j,1,1,1], Vgrid[:,:,i,j,1,1,t])
        plt.show()_____no_output_____plt.plot(cgrid[4,3,2,2,1,1,:69])_____no_output_____plt.plot(bgrid[4,3,2,2,1,1,:69])_____no_output_____plt.plot(kgrid[4,3,2,2,1,1,:69])_____no_output_____plt.plot(igrid[4,3,2,2,1,1,:69])_____no_output_____plt.plot(qgrid[4,3,2,2,1,1,:69])_____no_output_____


# Notebook from dmayo0317/big-data-challenge
Path: Level-1/load_big_data_two.ipynb

import os
# Find the latest version of spark 3.0  from http://www.apache.org/dist/spark/ and enter as the spark version
# For example:
# spark_version = 'spark-3.0.3'
spark_version = 'spark-3.2.0'
os.environ['SPARK_VERSION']=spark_version

# Install Spark and Java
!apt-get update
!apt-get install openjdk-8-jdk-headless -qq > /dev/null
!wget -q http://www.apache.org/dist/spark/$SPARK_VERSION/$SPARK_VERSION-bin-hadoop2.7.tgz
!tar xf $SPARK_VERSION-bin-hadoop2.7.tgz
!pip install -q findspark

# Set Environment Variables
os.environ["JAVA_HOME"] = "/usr/lib/jvm/java-8-openjdk-amd64"
os.environ["SPARK_HOME"] = f"/content/{spark_version}-bin-hadoop2.7"

# Start a SparkSession
import findspark
findspark.init()
0% [Working]

Ign:1 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  InRelease

0% [Connecting to archive.ubuntu.com] [Connecting to security.ubuntu.com (91.18

Ign:2 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64  InRelease

0% [Connecting to archive.ubuntu.com] [Connecting to security.ubuntu.com (91.18

Hit:3 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  Release
Hit:4 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64  Release
Hit:5 http://security.ubuntu.com/ubuntu bionic-security InRelease
Hit:7 http://ppa.launchpad.net/c2d4u.team/c2d4u4.0+/ubuntu bionic InRelease
Hit:8 https://cloud.r-project.org/bin/linux/ubuntu bionic-cran40/ InRelease
Hit:9 http://archive.ubuntu.com/ubuntu bionic InRelease
Hit:11 http://ppa.launchpad.net/cran/libgit2/ubuntu bionic InRelease
Hit:12 http://archive.ubuntu.com/ubuntu bionic-updates InRelease
Hit:13 http://archive.ubuntu.com/ubuntu bionic-backports InRelease
Hit:14 http://ppa.launchpad.net/deadsnakes/ppa/ubuntu bionic InRelease
Hit:15 http://ppa.launchpad.net/graphics-drivers/ppa/ubuntu bionic InRelease
Reading package lists... Done
!wget https://jdbc.postgresql.org/download/postgresql-42.2.9.jar--2021-12-04 05:23:48--  https://jdbc.postgresql.org/download/postgresql-42.2.9.jar
Resolving jdbc.postgresql.org (jdbc.postgresql.org)... 72.32.157.228, 2001:4800:3e1:1::228
Connecting to jdbc.postgresql.org (jdbc.postgresql.org)|72.32.157.228|:443... connected.
HTTP request sent, awaiting response... 200 OK
Length: 914037 (893K) [application/java-archive]
Saving to: ‘postgresql-42.2.9.jar’

postgresql-42.2.9.j 100%[===================>] 892.61K  5.61MB/s    in 0.2s

2021-12-04 05:23:49 (5.61 MB/s) - ‘postgresql-42.2.9.jar’ saved [914037/914037]

from pyspark.sql import SparkSession
spark = SparkSession.builder.appName("big_data_challenge").config("spark.driver.extraClassPath","/content/postgresql-42.2.9.jar").getOrCreate()_____no_output_____# Read in data from S3 Buckets
from pyspark import SparkFiles
url="https://dbm-bucket-2021-utsa.s3.us-east-2.amazonaws.com/books.tsv.gz"
spark.sparkContext.addFile(url)
books_df = spark.read.csv(SparkFiles.get("books.tsv.gz"), sep="\t", header=True, inferSchema=True)

# Show DataFrame
books_df.show()+-----------+-----------+--------------+----------+--------------+--------------------+----------------+-----------+-------------+-----------+----+-----------------+--------------------+--------------------+-----------+
|marketplace|customer_id|     review_id|product_id|product_parent|       product_title|product_category|star_rating|helpful_votes|total_votes|vine|verified_purchase|     review_headline|         review_body|review_date|
+-----------+-----------+--------------+----------+--------------+--------------------+----------------+-----------+-------------+-----------+----+-----------------+--------------------+--------------------+-----------+
|         US|   12076615| RQ58W7SMO911M|0385730586|     122662979|Sisterhood of the...|           Books|          4|            2|          3|   N|                N|this book was a g...|this boook was a ...| 2005-10-14|
|         US|   12703090|  RF6IUKMGL8SF|0811828964|      56191234|The Bad Girl's Gu...|           Books|          3|            5|          5|   N|                N|           Fun Fluff|If you are lookin...| 2005-10-14|
|         US|   12257412|R1DOSHH6AI622S|1844161560|     253182049|Eisenhorn (A Warh...|           Books|          4|            1|         22|   N|                N| this isn't a review|never read it-a y...| 2005-10-14|
|         US|   50732546| RATOTLA3OF70O|0373836635|     348672532|Colby Conspiracy ...|           Books|          5|            2|          2|   N|                N|fine author on he...|Though she is hon...| 2005-10-14|
|         US|   51964897|R1TNWRKIVHVYOV|0262181533|     598678717|The Psychology of...|           Books|          4|            0|          2|   N|                N|Execellent cursor...|Review based on a...| 2005-10-14|
|         US|   31048862|R2F53LI9KK9MOY|0316769487|     862964341|The Catcher in th...|           Books|          4|            2|          2|   N|                N|   Interesting, fun.|My only complaint...| 2005-10-14|
|         US|   53000124|R1KJ6MB7MRSQFF|0805076069|     145341889|Bait and Switch: ...|           Books|          4|            9|         11|   N|                N|I viewed this the...|This book is chil...| 2005-10-14|
|         US|   29732693|R2XIM9LT335WHE|1581603681|     640542054|Opening Combinati...|           Books|          4|            3|         10|   N|                N|No Frills - Just ...|When looking for ...| 2005-10-14|
|         US|   48541186|R1VE0FQQ0QTQJN|0300108834|     915891133|A Little History ...|           Books|          5|           16|         20|   N|                Y|Simple, entertain...|Never been much f...| 2005-10-14|
|         US|   15280864|R1VKEE2NWSWDRU|0446531081|     880645124|Hour Game (King &...|           Books|          4|            0|          0|   N|                N|        A good read!|If you enjoy a we...| 2005-10-14|
|         US|   24209213|R2UP6XSVYJBJ2H|0976822105|     731931430|     Faith is a Verb|           Books|          5|            0|          0|   N|                N|Made me Question ...|I thoroughly enjo...| 2005-10-14|
|         US|   52253037|R21SYDQ70ILUC0|1580085695|     586052746|Furry Logic: A Gu...|           Books|          5|            1|          1|   N|                Y|    The Perfect Gift|If you are stumpe...| 2005-10-14|
|         US|   27925116| R7M06Z88PD7SX|0029148510|      72387289|Acts of War: Beha...|           Books|          4|           14|         14|   N|                N|Solid Book About ...|Whether intention...| 2005-10-14|
|         US|   15005044| RRS38KZ4WB5O2|1592285570|     702385650|Temple to the Win...|           Books|          5|            1|          1|   N|                N|A riveting accoun...|Temple to the win...| 2005-10-14|
|         US|   52534781|R2YDYRSLGNHPHR|0811848833|     102333549|The Star Wars Pos...|           Books|          4|           16|         19|   N|                Y|Comprehensive vis...|At last... a comp...| 2005-10-14|
|         US|   26217071|R1BMPM18O6VZOR|0782144276|     971452783|Photoshop for Nat...|           Books|          4|           13|         14|   N|                N|          Great Book|I find \\"Photosh...| 2005-10-14|
|         US|   37684582|R1AABFZH0J0C0C|0375757996|     760991156|The Basic Works o...|           Books|          3|           17|         35|   N|                N|Problems with thi...|I love Aristotle ...| 2005-10-14|
|         US|   45232107|R3SJYLG07EHNE9|013146308X|     332774292|The Pterosaurs: F...|           Books|          2|           16|         34|   N|                N|Warmed over lefto...|To his discredit,...| 2005-10-14|
|         US|   15437121|R2TIO43ND87XVO|0375701907|     635714194|Straight Man: A N...|           Books|          5|            1|          1|   N|                N|       Funniest Book|Hands down, the f...| 2005-10-14|
|         US|   52157117|R18TY4WD19OUAO|141378240X|     656010659|   Lipstick's Legacy|           Books|          5|            1|          1|   N|                N|          Great Work|I read this book ...| 2005-10-14|
+-----------+-----------+--------------+----------+--------------+--------------------+----------------+-----------+-------------+-----------+----+-----------------+--------------------+--------------------+-----------+
only showing top 20 rows

# Drop null values
dropna_df = books_df.dropna()
dropna_df.show(10)+-----------+-----------+--------------+----------+--------------+--------------------+----------------+-----------+-------------+-----------+----+-----------------+--------------------+--------------------+-----------+
|marketplace|customer_id|     review_id|product_id|product_parent|       product_title|product_category|star_rating|helpful_votes|total_votes|vine|verified_purchase|     review_headline|         review_body|review_date|
+-----------+-----------+--------------+----------+--------------+--------------------+----------------+-----------+-------------+-----------+----+-----------------+--------------------+--------------------+-----------+
|         US|   12076615| RQ58W7SMO911M|0385730586|     122662979|Sisterhood of the...|           Books|          4|            2|          3|   N|                N|this book was a g...|this boook was a ...| 2005-10-14|
|         US|   12703090|  RF6IUKMGL8SF|0811828964|      56191234|The Bad Girl's Gu...|           Books|          3|            5|          5|   N|                N|           Fun Fluff|If you are lookin...| 2005-10-14|
|         US|   12257412|R1DOSHH6AI622S|1844161560|     253182049|Eisenhorn (A Warh...|           Books|          4|            1|         22|   N|                N| this isn't a review|never read it-a y...| 2005-10-14|
|         US|   50732546| RATOTLA3OF70O|0373836635|     348672532|Colby Conspiracy ...|           Books|          5|            2|          2|   N|                N|fine author on he...|Though she is hon...| 2005-10-14|
|         US|   51964897|R1TNWRKIVHVYOV|0262181533|     598678717|The Psychology of...|           Books|          4|            0|          2|   N|                N|Execellent cursor...|Review based on a...| 2005-10-14|
|         US|   31048862|R2F53LI9KK9MOY|0316769487|     862964341|The Catcher in th...|           Books|          4|            2|          2|   N|                N|   Interesting, fun.|My only complaint...| 2005-10-14|
|         US|   53000124|R1KJ6MB7MRSQFF|0805076069|     145341889|Bait and Switch: ...|           Books|          4|            9|         11|   N|                N|I viewed this the...|This book is chil...| 2005-10-14|
|         US|   29732693|R2XIM9LT335WHE|1581603681|     640542054|Opening Combinati...|           Books|          4|            3|         10|   N|                N|No Frills - Just ...|When looking for ...| 2005-10-14|
|         US|   48541186|R1VE0FQQ0QTQJN|0300108834|     915891133|A Little History ...|           Books|          5|           16|         20|   N|                Y|Simple, entertain...|Never been much f...| 2005-10-14|
|         US|   15280864|R1VKEE2NWSWDRU|0446531081|     880645124|Hour Game (King &...|           Books|          4|            0|          0|   N|                N|        A good read!|If you enjoy a we...| 2005-10-14|
+-----------+-----------+--------------+----------+--------------+--------------------+----------------+-----------+-------------+-----------+----+-----------------+--------------------+--------------------+-----------+
only showing top 10 rows

# Load in a sql function to use columns
from pyspark.sql.functions import col

# Filter for only columns with active users
cleaned_df = dropna_df.filter(col("active_user")  == True)
cleaned_df.show()+-------------+---+----------+-----------+-----------+--------------------+--------------------+----------+--------------------+
|     username| id|first_name|  last_name|active_user|      street_address|               state|billing_id|        cc_encrypted|
+-------------+---+----------+-----------+-----------+--------------------+--------------------+----------+--------------------+
|   fstappard5|  6|    Fraser|    Korneev|       true|  76084 Novick Court|           Minnesota|         6|a799fcafe47d7fb19...|
|   lhambling6|  7|    Demott|     Rapson|       true|    86320 Dahle Park|District of Columbia|         7|a799fcafe47d7fb19...|
|    wheinerte| 15|   Sadella|      Jaram|       true|7528 Waxwing Terrace|         Connecticut|        15|a799fcafe47d7fb19...|
| droughsedgeg| 17|    Hewitt|    Trammel|       true|    2455 Corry Alley|      North Carolina|        17|a799fcafe47d7fb19...|
|    ydudeniei| 19|       Ted|    Knowlys|       true|      31 South Drive|                Ohio|        19|a799fcafe47d7fb19...|
|     fmyttonm| 23|  Annmarie|     Lafond|       true|     35 Oriole Place|             Georgia|        23|a799fcafe47d7fb19...|
|   bfletcherr| 28|      Toma|     Sokell|       true|39641 Eggendart Hill|            Maryland|        28|a799fcafe47d7fb19...|
|     gturleyt| 30|       Ram|    Lefever|       true|   9969 Laurel Alley|               Texas|        30|a799fcafe47d7fb19...|
|    calyukinu| 31|    Raddie|    Heindle|       true|   811 Talmadge Road|                Ohio|        31|a799fcafe47d7fb19...|
| ckleinlererw| 33|    Wallie|       Caws|       true|   9999 Kenwood Pass|              Oregon|        33|a799fcafe47d7fb19...|
|  pshanklandx| 34|    Derril|Varfolomeev|       true|     4 Jenifer Court|             Florida|        34|a799fcafe47d7fb19...|
|    enelane12| 39|     Kelcy|     Wheway|       true|93207 Morningstar...|             Florida|        39|a799fcafe47d7fb19...|
|    sfollet13| 40|    Dorree|    Rookeby|       true|       2 Troy Circle|          California|        40|a799fcafe47d7fb19...|
|      mtesh14| 41|    Martyn|       Tott|       true|       728 Muir Lane|             Florida|        41|a799fcafe47d7fb19...|
|   tseyfart16| 43|     Cally|      Thody|       true|   1 Graceland Plaza|             Florida|        43|a799fcafe47d7fb19...|
|   hfarrier18| 45|       Ted|   Pittaway|       true|767 Little Fleur ...|      North Carolina|        45|a799fcafe47d7fb19...|
|     nabbie1b| 48|      Fifi|    Lidgley|       true|6744 Sutherland Road|      South Carolina|        48|a799fcafe47d7fb19...|
|  ystadding1d| 50|    Ashely|     O'Hern|       true|   929 Scoville Park|             Florida|        50|a799fcafe47d7fb19...|
|hhallgalley1g| 53|   Diannne|Osbaldeston|       true|        0 Mesta Pass|           Tennessee|        53|a799fcafe47d7fb19...|
|   ageaveny1n| 60|     Sonny|     Jeskin|       true| 50 Sutherland Drive|       Massachusetts|        60|a799fcafe47d7fb19...|
+-------------+---+----------+-----------+-----------+--------------------+--------------------+----------+--------------------+
only showing top 20 rows

# Create user dataframe to match active_user table
clean_user_df = cleaned_df.select(["id", "first_name", "last_name", "username"])
clean_user_df.show()+---+----------+-----------+-------------+
| id|first_name|  last_name|     username|
+---+----------+-----------+-------------+
|  6|    Fraser|    Korneev|   fstappard5|
|  7|    Demott|     Rapson|   lhambling6|
| 15|   Sadella|      Jaram|    wheinerte|
| 17|    Hewitt|    Trammel| droughsedgeg|
| 19|       Ted|    Knowlys|    ydudeniei|
| 23|  Annmarie|     Lafond|     fmyttonm|
| 28|      Toma|     Sokell|   bfletcherr|
| 30|       Ram|    Lefever|     gturleyt|
| 31|    Raddie|    Heindle|    calyukinu|
| 33|    Wallie|       Caws| ckleinlererw|
| 34|    Derril|Varfolomeev|  pshanklandx|
| 39|     Kelcy|     Wheway|    enelane12|
| 40|    Dorree|    Rookeby|    sfollet13|
| 41|    Martyn|       Tott|      mtesh14|
| 43|     Cally|      Thody|   tseyfart16|
| 45|       Ted|   Pittaway|   hfarrier18|
| 48|      Fifi|    Lidgley|     nabbie1b|
| 50|    Ashely|     O'Hern|  ystadding1d|
| 53|   Diannne|Osbaldeston|hhallgalley1g|
| 60|     Sonny|     Jeskin|   ageaveny1n|
+---+----------+-----------+-------------+
only showing top 20 rows

# Create user dataframe to match billing_info table
clean_billing_df = cleaned_df.select(["billing_id", "street_address", "state", "username"])
clean_billing_df.show()+----------+--------------------+--------------------+-------------+
|billing_id|      street_address|               state|     username|
+----------+--------------------+--------------------+-------------+
|         6|  76084 Novick Court|           Minnesota|   fstappard5|
|         7|    86320 Dahle Park|District of Columbia|   lhambling6|
|        15|7528 Waxwing Terrace|         Connecticut|    wheinerte|
|        17|    2455 Corry Alley|      North Carolina| droughsedgeg|
|        19|      31 South Drive|                Ohio|    ydudeniei|
|        23|     35 Oriole Place|             Georgia|     fmyttonm|
|        28|39641 Eggendart Hill|            Maryland|   bfletcherr|
|        30|   9969 Laurel Alley|               Texas|     gturleyt|
|        31|   811 Talmadge Road|                Ohio|    calyukinu|
|        33|   9999 Kenwood Pass|              Oregon| ckleinlererw|
|        34|     4 Jenifer Court|             Florida|  pshanklandx|
|        39|93207 Morningstar...|             Florida|    enelane12|
|        40|       2 Troy Circle|          California|    sfollet13|
|        41|       728 Muir Lane|             Florida|      mtesh14|
|        43|   1 Graceland Plaza|             Florida|   tseyfart16|
|        45|767 Little Fleur ...|      North Carolina|   hfarrier18|
|        48|6744 Sutherland Road|      South Carolina|     nabbie1b|
|        50|   929 Scoville Park|             Florida|  ystadding1d|
|        53|        0 Mesta Pass|           Tennessee|hhallgalley1g|
|        60| 50 Sutherland Drive|       Massachusetts|   ageaveny1n|
+----------+--------------------+--------------------+-------------+
only showing top 20 rows

# Create user dataframe to match payment_info table
clean_payment_df = cleaned_df.select(["billing_id", "cc_encrypted"])
clean_payment_df.show()+----------+--------------------+
|billing_id|        cc_encrypted|
+----------+--------------------+
|         6|a799fcafe47d7fb19...|
|         7|a799fcafe47d7fb19...|
|        15|a799fcafe47d7fb19...|
|        17|a799fcafe47d7fb19...|
|        19|a799fcafe47d7fb19...|
|        23|a799fcafe47d7fb19...|
|        28|a799fcafe47d7fb19...|
|        30|a799fcafe47d7fb19...|
|        31|a799fcafe47d7fb19...|
|        33|a799fcafe47d7fb19...|
|        34|a799fcafe47d7fb19...|
|        39|a799fcafe47d7fb19...|
|        40|a799fcafe47d7fb19...|
|        41|a799fcafe47d7fb19...|
|        43|a799fcafe47d7fb19...|
|        45|a799fcafe47d7fb19...|
|        48|a799fcafe47d7fb19...|
|        50|a799fcafe47d7fb19...|
|        53|a799fcafe47d7fb19...|
|        60|a799fcafe47d7fb19...|
+----------+--------------------+
only showing top 20 rows

Postgres Setup_____no_output_____

# Configure settings for RDS
mode = "append"
jdbc_url="jdbc:postgresql://<connection string>:5432/<database-name>"
config = {"user":"postgres",
          "password": "<password>",
          "driver":"org.postgresql.Driver"}
_____no_output_____# Write DataFrame to active_user table in RDS

clean_user_df.write.jdbc(url=jdbc_url, table='active_user', mode=mode, properties=config)_____no_output_____# Write dataframe to billing_info table in RDS

clean_billing_df.write.jdbc(url=jdbc_url, table='billing_info', mode=mode, properties=config)_____no_output_____# Write dataframe to payment_info table in RDS

clean_payment_df.write.jdbc(url=jdbc_url, table='payment_info', mode=mode, properties=config)_____no_output_____


# Notebook from CylonScion/python-causality-handbook
Path: causal-inference-for-the-brave-and-true/07-Beyond-Confounders.ipynb

# 07 - Beyond Confounders

## Good Controls

We've seen how adding additional controls to our regression model can help identify causal effect. If the control is a confounder, adding it to the model is not just nice to have, but is a requirement. When the unwary see this, a natural response is to throw whatever he can measure into the model. In today's world of big data, this could easily be more than 1000 variables. As it turns out, this is not only unnecessary, but can be detrimental to causal identification. We will now turn our attention to controls that are not confounders. First, let's take a look at the good ones. Then, we will delve into harmful controls.

As a motivating example, let's suppose you are a data scientist in the collections team of a fintech. Your next task is to figure out the impact of sending an email asking people to negotiate their debt. Your response variable is the amount of payments from the late customers.

To answer this question, your team selects 5000 random customers from your late customers base to do a random test. For every customer, you flip a coin, if its heads, the customer receives the email; otherwise, it is left as a control. With this test, you hope to find out how much extra money the email generates._____no_output_____

import warnings
warnings.filterwarnings('ignore')

import pandas as pd
import numpy as np
from scipy import stats
from matplotlib import style
import seaborn as sns
from matplotlib import pyplot as plt
import statsmodels.formula.api as smf
import graphviz as gr

style.use("fivethirtyeight")_____no_output_____data = pd.read_csv("./data/collections_email.csv")
data.head()_____no_output_____

Since the data is random, you know that a simple difference in means estimates the Average Treatment Effect. In other words, nothing can have caused the treatment but the randomisation, so the potential outcomes are independent of the treatment: \\((Y_0, Y_1)\perp T\\).

$
ATE = E[Y_i|T_i=1] - E[Y_i|T_i=0]
$

Since you are smart and want to place a confidence interval around your estimate, you use a linear regression._____no_output_____

print("Difference in means:",
      data.query("email==1")["payments"].mean() - data.query("email==0")["payments"].mean())

model = smf.ols('payments ~ email', data=data).fit()
model.summary().tables[1]Difference in means: -0.6202804021329484

Sadly, the estimated ATE is -0.62, which is pretty weird. How can sending an email make late customers pay less than average? Still, the P-value is so high that this probably doesn't mean anything. What you should do now? Go back to your team with a tail between your legs and say that the test is inconclusive and you need more data? Not so fast.

Notice how your data has some other interesting columns. For example, `credit_limit` represents the customer's credit line prior to he or she getting late. `risk_score` corresponds to the estimated risk of the customer prior to the delivery of the email. It makes sense to think that credit limit and risk are probably very good predictors of payments. But how can that be useful?

First, let's understand why we can fail to find statistical significance in a treatment even when it is there. It could be that, like in this case, the treatment has very little impact on the outcome. If you think about it, what makes people pay their debt is, by and large, factors outside the control of the collections department. People pay their debt because they find a new job, manage their finances, income and so on. In statistical terms, we can say that **the variability of payments is explained much more by other factors other than by the email**.

To get a visual understanding of it, we can plot the payments against the treatment variable email. I've also plotted the fitted line of the model above in red. To help visualization, I've added a little bit of noise to the email variable so that it doesn't get smashed at the zero or one. _____no_output_____

sns.scatterplot("email", "payments",
                alpha=0.8,
                data=data.assign(email=data["email"] + np.random.normal(0, 0.01, size=len(data["email"]))))
plt.plot(np.linspace(-0.2, 1.2), model.params[0] + np.linspace(-1, 2) * model.params[1], c="C1")
plt.xlabel("Email")
plt.ylabel("Payments");_____no_output_____

We can see how wildly payments vary in a single treatment group. Visually, it looks like it is going from a little bit under 400 to 1000 in both groups. If the impact of the email is in the order of say 5.00 or 10.00 R$, it is no wonder it will be hard to find it inside all the variability.

Fortunately, regression can help us lower this variability. The trick is to use additional controls. **If a variable is a good predictor of the outcome, it will explain away a lot of its variance**. If risk and credit limit are good predictors of payment, we can control them to make it easier to find the impact of the email on payments. If we remember how regression works, this has an intuitive explanation. Adding extra variables to a regression means keeping them constant while looking at the treatment. So, the reasoning goes, if we look at similar levels of risk and credit limit, the variance of the response variable `payments` should be smaller. Or, in other words, if risk and credit line predicts payments very well, customers with a similar risk and credit line should also have similar payment levels, hence with less variance.

![img](./data/img/beyond-conf/y-pred.png)

To demonstrate this, let's resort to the partialling out way of breaking regression into 2 steps. First, we will regress the treatment, email, and the outcome, payments, on the additional controls, credit limit and risk score. Second, we will regress the residual of the treatment on the residuals of payments, both obtained in step 1. (This is purely pedagogical, in practice you won't need to go through all the hassle)._____no_output_____

model_email = smf.ols('email ~ credit_limit + risk_score', data=data).fit()
model_payments = smf.ols('payments ~ credit_limit + risk_score', data=data).fit()

residuals = pd.DataFrame(dict(res_payments=model_payments.resid, res_email=model_email.resid))

model_treatment = smf.ols('res_payments ~ res_email', data=residuals).fit()_____no_output_____

This lowers the variance of the dependent variable. By regressing payments on credit limit and risk and obtaining the residuals for this model, we are creating a new dependent variable with much less variability than the original one. The last model also uncovers the `ATE` with valid standard error estimate.

Just out of curiosity, we can also check that the model that predicts the treatment should not be able to lower the variance of it. That's because email is, by design, random, so nothing can predict it._____no_output_____

print("Payments Variance", np.var(data["payments"]))
print("Payments Residual Variance", np.var(residuals["res_payments"]))

print("Email Variance", np.var(data["email"]))
print("Email Residual Variance", np.var(residuals["res_email"]))

model_treatment.summary().tables[1]Payments Variance 10807.612416
Payments Residual Variance 5652.453558466207
Email Variance 0.24991536
Email Residual Variance 0.24918421069820032

Notice how the variance of payments went from 10807 to 5652. We've decreased it by almost half once we control for risk and credit limits. Also notice that we didn't manage to reduce the variability of the treatment email. This makes sense, since risk and credit line does not predict email (nothing does, by definition of randomness).

Now, we see something much more reasonable. This new estimate tells us that we should expect customers that received the email to pay, on average, 4.4 reais more than those in the control group. This estimate is now statistically different from zero. We can also visualize how the variance is now lower within each control group._____no_output_____

sns.scatterplot("res_email", "res_payments", data=residuals)
plt.plot(np.linspace(-0.7, 1), model_treatment.params[0] + np.linspace(-1, 2) * model_treatment.params[1], c="C1")
plt.xlabel("Email Residuals")
plt.ylabel("Payments Residuals");_____no_output_____

As I've said, we did this for pedagogical reasons. In practice, you can simply add the controls to the regression model together with the treatment and the estimates will be exactly the same._____no_output_____

model_2 = smf.ols('payments ~ email + credit_limit + risk_score', data=data).fit()
model_2.summary().tables[1]_____no_output_____

To wrap it up, anytime we have a control that is a good predictor of the outcome, even if it is not a confounder, adding it to our model is a good idea. It helps lowering the variance of our treatment effect estimates. Here is a picture of what this situation looks like with causal graphs._____no_output_____

g = gr.Digraph()
g.edge("X", "Y"), g.edge("T", "Y")
g.node("T", color="gold")

g.node("email", color="gold")
g.edge("credit_limit", "payments")
g.edge("risk_score", "payments")
g.edge("email", "payments")

g_____no_output_____

## Mostly Harmful Controls

As a second motivating example, let's consider a drug test scenario with 2 hospitals. Both of them are conducting randomised trials on a new drug to treat a certain illness. The outcome of interest is days hospitalised. If the treatment is effective, it will lower the amount of days the patient stays in the hospital. For one of the hospitals, the policy regarding the random treatment is to give it to 90% of its patients while 10% get a placebo. The other hospital has a different policy: it gives the drug to a random 10% of its patients and 90% get a placebo. You are also told that the hospital that gives 90% of the true drug and 10% of placebo usually gets more severe cases of the illness to treat. _____no_output_____

hospital = pd.read_csv("./data/hospital_treatment.csv")
hospital.head()_____no_output_____

Since you are dealing with randomized data, your first instinct is to simply run a regression of the outcome on the treatment._____no_output_____

hosp_1 = smf.ols('days ~ treatment', data=hospital).fit()
hosp_1.summary().tables[1]_____no_output_____

But you find some counterintuitive results. How can the treatment be increasing the number of days in the hospital? The answer lies in the fact that we are running 2 different experiments. Severity is positively linked with more days at the hospital and since the hospital with more severe cases also gives more of the drug, the drug becomes positively correlated with more days at the hospital. When we look at both hospital together, we have that \\(E[Y_0|T=0]>E[Y_0|T=1]\\), that is, the potential outcome of the untreated is, on average, higher than that of the treated because there are more untreated in the hospital with less severe cases. In other words, severity acts as a confounder, determining the hospital the patient goes and, hence, the probability of receiving the drug.

There are 2 ways of fixing that. The first one, which defeats the purpose of using data from both hospitals, is to simply look at the ATE in each hospital individually._____no_output_____

hosp_2 = smf.ols('days ~ treatment', data=hospital.query("hospital==0")).fit()
hosp_2.summary().tables[1]_____no_output_____hosp_3 = smf.ols('days ~ treatment', data=hospital.query("hospital==1")).fit()
hosp_3.summary().tables[1]_____no_output_____

In this case, we did get an intuitive result of the ATE. It looks like now the drug is in fact lowering the amount of days at the hospital. However, since we are looking at each hospital individually, there are not enough data points. As a consequence, we are unable to find statistically significant results.

The other approach, which leverages the power of regression, is to control for severity by including it in the model._____no_output_____

hosp_4 = smf.ols('days ~ treatment + severity', data=hospital).fit()
hosp_4.summary().tables[1]_____no_output_____

The question that arises next is, should we also include hospital in the model? After all, we know that hospitals cause the treatment right? Well, that is true, but once we've controlled for severity, hospital is no longer correlated with the outcome number of days hospitalised. And we know that to be a confounder a variable has to cause both the treatment and the outcome. In this case, we have a variable that only causes the treatment.

But maybe controlling for it lowers the variance, right? Well, not true again. In order for a control to lower the variance, it has to be a good predictor of the outcome, not of the treatment, which is the case here.

Still, we might want to control it right? It can't hurt... Or can it?_____no_output_____

hosp_5 = smf.ols('days ~ treatment + severity + hospital', data=hospital).fit()
hosp_5.summary().tables[1]_____no_output_____

Surprisingly, it can hurt!

![img](./data/img/beyond-conf/shocked.png)

Adding hospital on top of severity as a control introduced MORE variance to our ATE estimator. How can that be? The answer lies in the formula for the standard error of the regression coefficient.

$
\hat{\sigma}^2 = \dfrac{1}{n-2} \sum( y_i - \hat{y}_i )^2
$

$
\text{Var}(\hat{\beta}_2) = \dfrac{\sigma^2}{\sum(x_i - \bar{x})^2}
$

From this formula, we can see that the standard error is inversely proportional to the variance of the variable \\(X\\). This means that, if \\(X\\) doesn't change much, it will be hard to estimate its effect on the outcome. This also makes intuitive sense. Take it to the extreme and pretend you want to estimate the effect of a drug, so you conduct a test with 10000 individuals but only 1 of them get the treatment. This will make finding the ATE very hard, we will have to rely on comparing a single individual with everyone else. Another way to say this is that we need lots of variability in the treatment to make it easier to find its impact.

As to why including hospitals in the model increases the error of our estimate, it is because it is a good predictor of the treatment and not of the outcome (once we control for severity). So, by predicting the treatment, it effectively makes it so that it's variance is lower! Once again, we can resort to partitioning our regression above into it's 2 steps to see this._____no_output_____

model_treatment = smf.ols('treatment ~ severity + hospital', data=hospital).fit()
model_days = smf.ols('days ~ severity + hospital', data=hospital).fit()

residuals = pd.DataFrame(dict(res_days=model_days.resid, res_treatment=model_treatment.resid))

model_treatment = smf.ols('res_days ~ res_treatment', data=residuals).fit()

model_treatment.summary().tables[1]_____no_output_____print("Treatment Variance", np.var(hospital["treatment"]))
print("Treatment Residual Variance", np.var(residuals["res_treatment"]))Treatment Variance 0.234375
Treatment Residual Variance 0.057529091872119076

Also, don't take my word for it! You can check that the SE formula above is true:_____no_output_____

sigma_hat = sum(model_treatment.resid**2)/(len(model_treatment.resid)-2)
var = sigma_hat/sum((residuals["res_treatment"] - residuals["res_treatment"].mean())**2)
print("SE of the Coeficient:", np.sqrt(var))SE of the Coeficient: 3.446973767486902

So the bottom line is that we should add controls that are both correlated with the treatment and the outcome (confounder), like the severity in the model above. We should also add controls that are good predictors of the outcome, even if they are not confounders, because they lower the variance of our estimates. However, we should **NOT** add controls that are just good predictors of the treatment, because they will increase the variance of our estimates.

Here is a picture of what this situation looks like with causal graphs._____no_output_____

g = gr.Digraph()

g.edge("X", "T"), g.edge("T", "Y")
g.node("T", color="gold")

g.node("treatment", color="gold")
g.edge("severity", "hospital")
g.edge("severity", "days")
g.edge("hospital", "treatment")
g.edge("treatment", "days")

g_____no_output_____

## Bad Controls - Selection Bias

Let's go back to the collections email example. Remember that the email was randomly assigned to customers. We've already explained what `credit_limit` and `risk_score` is. Now, let's look at the remaining variables. `opened` is a dummy variable for the customer opening the email or not. `agreement` is another dummy marking if the customers contacted the collections department to negotiate their debt, after having received the email. Which of the following models do you think is more appropriate? The first is a model with the treatment variable plus `credit_limit` and `risk_score`; the second adds `opened` and `agreement` dummies._____no_output_____

email_1 = smf.ols('payments ~ email + credit_limit + risk_score', data=data).fit()
email_1.summary().tables[1]_____no_output_____email_2 = smf.ols('payments ~ email + credit_limit + risk_score + opened + agreement', data=data).fit()
email_2.summary().tables[1]_____no_output_____

While the first model finds statistically significant results for the email, the second one does not. But maybe the second one is the right model and there's no effect for the email. After all, this model controls for more factors, so it should be more robust right? By now you probably know that this is not the case. What is left is to figure out what is.

We know that we MUST add confounding variables. Variables that cause both the treatment and the outcome. We also know that it is a good idea to add controls that predict the outcome very well. This is not required, but it's nice to have. We also know that it is a bad idea to add controls that predict only the treatment. Again, this is not a deadly sin, but is nice to avoid. So what kind of controls are `opened` and `agreement`? Turns out, they are neither of the above.

If you think about it, `opened` and `agreement` are surely correlated with the email. After all, you can't open the email if you didn't receive it and we've also said that the agreement only considers renegotiation that happened after the email has been sent. But **they don't cause email! Instead, they are caused by it!**

Whenever I need to understand what kind of variables I'm dealing with, I always like to think about their causal graph. Let's do these here_____no_output_____

g = gr.Digraph()

g.edge("email", "payments")
g.edge("email", "opened")
g.edge("email", "agreement")
g.edge("opened", "payments")
g.edge("opened", "agreement")
g.edge("agreement", "payments")

g.edge("credit_limit", "payments")
g.edge("credit_limit", "opened")
g.edge("credit_limit", "agreement")
g.edge("risk_score", "payments")
g.edge("risk_score", "opened")
g.edge("risk_score", "agreement")

g_____no_output_____

We know nothing causes email, because it's random by design. And we know (or at least we have strong reasons to believe) that credit limit and risk cause payments. We also think that email causes payments. As for `opened`, we think that it does cause payments. Intuitively, people that opened the collection email are more willing to negotiate and pay their debt. We also think that `opened` causes agreements for the same reasons as it causes payments. Moreover, we know `opened` is caused by email and we have reasons to believe people with different risk and credit limits have different open rates for the emails, so credit limit and risk also causes opened. As for agreement, we also think that it is caused by `opened`. If we think about the payments response variable, we can think of is as the result of a funnel:

$
email -> opened -> agreement -> payment
$

We also think that different levels of risk and line have different propensity of doing an agreement, so we will mark them as also causing agreement. As for email and agreement, we could make an argument that some people just read the subject of the email and that makes them more likely to make an agreement. The point is that email could also cause agreement without passing through open.

What we notice with this graph is that opened and agreement are both in the causal path from email to payments. So, if we control for them with regression, we would be saying "this is the effect of email while keeping `opened` and `agreement` fixed". However, both are part of the causal effect of the email, so we don't want to hold them fixed. Instead, we could argue that email increases payments precisely because it boosts the agreement rate. If we fix those variables, we are removing some of the true effect from the email variable.

With potential outcome notation, we can say that, due to randomization \\(E[Y_0|T=0] = E[Y_0|T=1]\\). However, even with randomization, when we control for agreement, treatment and control are no longer comparable. In fact, with some intuitive thinking, we can even guess how they are different:

$
E[Y_0|T=0, Agreement=0] > E[Y_0|T=1, Agreement=0]
$

$
E[Y_0|T=0, Agreement=1] > E[Y_0|T=1, Agreement=1]
$

The first equation makes it explicit that we think those without the email and the agreement are better than those with the email and without the agreement. That is because, if the treatment has a positive effect, those that didn't make an agreement **even after having received the email** are probably worse in terms of payments compared to those that also didn't do the agreement but also didn't get the extra incentive of the email. As for the second equation, those that did the agreement even without having received the treatment are probably better than those that did the agreement but had the extra incentive of the email.

This might be very confusing the first time you read it (it was for me), but make sure you understand it. Read it again if necessary. Then, a similar kind of reasoning can be done with the opened variable. Try to make it yourself.

This sort of bias is so pervasive it has its own name. While confounding is the bias from failing to control for a common cause, **selection bias is when we control for a common effect or a variable in between the path from cause to effect.** As a rule of thumb, always include confounders and variables that are good predictors of \\(Y\\) in your model. Always exclude variables that are good predictors of only \\(T\\), mediators between the treatment and outcome or common effect of the treatment and outcome.

![img](./data/img/beyond-conf/selection.png)

Selection bias is so pervasive that not even randomization can fix it. Better yet, it is often introduced by the ill advised, even in random data! Spotting and avoiding selection bias requires more practice than skill. Often, they appear underneath some supposedly clever idea, making it even harder to uncover. Here are some examples of selection biased I've encountered:

    1. Adding a dummy for paying the entire debt when trying to estimate the effect of a collections strategy on payments.
    2. Controlling for white vs blue collar jobs when trying to estimate the effect of schooling on earnings
    3. Controlling for conversion when estimating the impact of interest rates on loan duration
    4. Controlling for marital happiness when estimating the impact of children on extramarital affairs
    5. Breaking up payments modeling E[Payments] into one binary model that predict if payment will happen and another model that predict how much payment will happen given that some will: E[Payments|Payments>0]*P(Payments>0)

What is notable about all these ideas is how reasonable they sound. Selection bias often does. Let this be a warning. As a matter of fact, I myself have fallen into the traps above many many times before I learned how bad they were. One in particular, the last one, deserves further explanation because it looks so clever and catches lots of data scientists off guard. It's so pervasive that it has its own name: **The Bad COP**!

### Bad COP

The situation goes like this. You have a continuous variable that you want to predict but its distribution is overrepresented at zero. For instance, if you want to model consumer spending, you will have something like a gamma distribution, but with lots of zeros._____no_output_____

plt.hist(np.concatenate([
    np.random.gamma(5, 50, 1000),
    np.zeros(700)
]), bins=20)
plt.xlabel("Customer Spend")
plt.title("Distribution Customer Spend");_____no_output_____

When a data scientist sees this, the first idea that pops into his head is to break up modeling into 2 steps. The first is the participation, that is, the probability that \\(Y > 0\\). In our spend example, this would be modeling if the customer decided to spend or not. The second part models \\(Y\\) for those that decided to participate. It is the Conditional-on-Positives effect. In our case, this would be how much the customer spends after he or she decided they would spend anything. If we would like to estimate the effect of the treatment \\(T\\) on expenditures, it would look something this:

$
E[Y_i|T_i] = E[Y_i|Y_i>0, T_i]P(Y_i>0|T_i)
$

There is nothing wrong with the participation model \\(P(Y_i>0|T_i)\\). In fact, if \\(T\\) is randomly assigned, it will capture the increase in probability of spending due to the treatment. Also, there is nothing wrong with the decomposition above. It's mathematically true, by the law of total probabilities.

The issue is in estimating the COP part. **It will be biased even under random assignment**. On an intuitive level, it's not crazy to think that some units are zeros only because they didn't get the treatment. The treatment would cause them to **not** be zeros. On the other hand, some units are never zeros. The treatment might increase their outcome, but even without it, they would not be zeros. Now, the key point is understanding that these two types of units are not comparable. Those that are never zeros have high \\(Y_0\\) compared to those that are zeros if they didn't get the treatment. In fact, for the latter, \\(Y_0=0\\).

Knowing this, if we remove the zeros, we will keep the never zeros on both the treated and the control group. But we would remove from the control those that flip from zero to not zero with the treatment. This would make the treatment and control no longer comparable, because the control would only contain those that are never zeros and that have higher \\(Y_0\\), while the treated would contain both types of units.

Now that we have an intuitive understanding of the problem, let's check it from a mathematical angle. To see it, let's break up the treatment effect. Under random assignment, it is equal to the difference in means

$$
\begin{align*}
&E[Y_i|T_i=1] - E[Y_i|T_i=0]\\
&=E[Y_i|Y_i>0, T_i=1]P(Y_i>0|T_i=1) - E[Y_i|Y_i>0, T_i=0]P(Y_i>0|T_i=0)\\
&=\underbrace{\{P(Y_i>0|T_i=1) - P(Y_i>0|T_i=0)\}}_{Participation \ Effect} * E[Y_i|Y_i>0, T_i=1]\\
&+\underbrace{\{E[Y_i|Y_i>0, T_i=1] - E[Y_i|Y_i>0, T_i=0]\}}_{COP \ Effect} * P(Y_i>0|T_i=0)
\end{align*}
$$

Where the last equality comes from adding and subtracting \\(E[Y_i|Y_i>0, T_i=1]P(Y_i>0|T_i=0)\\) and rearranging the terms. This means that the difference in averages is composed of two parts: first, it's the difference in the probability that the outcome \\(y\\) is positive. This is called the participation effect because it measures the increase in the probability that customers will participate in spending. Second, it's the difference in the outcome conditioned on participation, the COP effect. So far so good. There is nothing wrong with this. It is a mathematical truth. The problem comes when we try to estimate each part separately

This becomes more evident if we analyse the COP effect even further.

$$
\begin{align*}
E[Y_i|Y_i>0, T_i=1] - E[Y_i|Y_i>0, T_i=0]&=E[Y_{i1}|Y_{i1}>0]-E[Y_{i0}|Y_{i0}>0] \\
&=\underbrace{E[Y_{i1} - Y_{i0}|Y_{i1}>0]}_{Causal \ Effect} + \underbrace{\{ E[Y_{i0}|Y_{i1}>0] - E[Y_{i0}|Y_{i0}>0] \}}_{Selection \ Bias}
\end{align*}
$$

where the second equality comes after we add and subtract \\(E[Y_{i0}|Y_{i1}>0]\\). When we break up the COP effect, we get first the causal effect on the participant subpopulation. In our example, this would be the causal effect on those that decide to spend something. Second, we get a bias term which is the difference in \\(Y_0\\) for those that decide to participate when assigned to the treatment (\\(E[Y_{i0}|Y_{i1}>0]\\)) and those that that participate even without the treatment (\\(E[Y_{i0}|Y_{i0}>0]\\)). In our case, this bias is probably negative, since those that spend when assigned to the treatment, had they not received the treatment, would probably spend less than those that spend even without the treatment \\(E[Y_{i0}|Y_{i1}>0] < E[Y_{i0}|Y_{i0}>0]\\).

![img](./data/img/beyond-conf/cop.png)

Now, I know that COP bias is super counterintuitive at first, so I think it is worth going through a visual example. Let's say that we want to estimate how a marketing campaign increases how much people spend on our product. This marketing campaign has been randomized, so we don't need to worry about confounding. In this example, we can break up the customers into two segments. First, there are those that will only buy our products if they see a marketing campaign. Let's call these customers the frugal ones. They don't spend unless we give them an extra push. Then there are the customers that will spend even without the campaign. The campaign makes them spend more, but they would already spend without seeing it anyway. Let's call them the rich customers. In the figure, I've displayed the counterfactuals in light colors and dashed lines.

![img](./data/img/beyond-conf/cop-ex1.png)

To estimate the ATE of the campaign, since we have randomization, all we need to do is compare the treated with the untreated. But, suppose we use the COP formulation where we break up estimation into two models, a participation model that estimates \\(P(Y_i>0|T_i)\\) and the COP, which estimates \\(E[Y_i|Y_i>0]\\). This removes everyone that didn't spend from the analysis.

![img](./data/img/beyond-conf/cop-ex2.png)

When we do that, the treated and control are no longer comparable. As we can see, the untreated is now only composed of the segment of customers that will spend even without the campaign. Also notice that we can even know the direction of the bias here. It will be \\(E[Y_{i0}|Y_{i1}>0] - E[Y_{i0}|Y_{i0}>0]\\) or \\(E[Y_{i0}|\text{Frugal and Rich}] - E[Y_{i0}|Rich]\\). This is obviously negative, as the rich spend more than the frugal customers. As a result, once we filter only the participant population, our estimate of the ATE becomes biased, even if there was no bias at first due to randomization. I sincerely hope this convinces you to avoid COP like the plague. I see too many Data Scientists doing this separate estimation, unaware of the problems that it brings. _____no_output_____To wrap up selection bias, we need to always remind ourselves to never control for a variable that is either in between the treatment and the outcome or is a common effect of the outcome and the treated. In graphical language, here is what bad control looks like:_____no_output_____

g = gr.Digraph()

g.edge("T", "X_1"), g.node("T", color="gold"), g.edge("X_1", "Y"), g.node("X_1", color="red")
g.edge("T", "X_2"), g.edge("Y", "X_2"), g.node("X_2", color="red")

g_____no_output_____

## Key Ideas

In this section, we've looked at variables that are not confounders and if we should add them or not to our model for causal identification. We've seen that variables that are good predictors of the outcome \\(y\\) should be added to the model even if they don't predict \\(T\\) (are not confounders). This is because predicting \\(Y\\) lowers its variance and makes it more likely that we will see statistically significant results when estimating the causal effect. Next, we've seen that it is a bad idea to add variables that predict the treatment but not the outcome. Those variables reduce the variability of the treatment, making it harder for us to find the causal effect. Finally, we've looked at selection bias. This is bias that arises when we control for variables in the causal path from the treatment to the outcome or variables that are common effects of the treatment and the outcome.

## References

I like to think of this entire book as a tribute to Joshua Angrist, Alberto Abadie and Christopher Walters for their amazing Econometrics class. Most of the ideas here are taken from their classes at the American Economic Association. Watching them is what is keeping me sane during this tough year of 2020.
* [Cross-Section Econometrics](https://www.aeaweb.org/conference/cont-ed/2017-webcasts)
* [Mastering Mostly Harmless Econometrics](https://www.aeaweb.org/conference/cont-ed/2020-webcasts)

I'll also like to reference the amazing books from Angrist. They have shown me that Econometrics, or 'Metrics as they call it, is not only extremely useful but also profoundly fun.

* [Mostly Harmless Econometrics](https://www.mostlyharmlesseconometrics.com/)
* [Mastering 'Metrics](https://www.masteringmetrics.com/)

My final reference is Miguel Hernan and Jamie Robins' book. It has been my trustworthy companion in the most thorny causal questions I had to answer.

* [Causal Inference Book](https://www.hsph.harvard.edu/miguel-hernan/causal-inference-book/)

![img](./data/img/poetry.png)

## Contribute

Causal Inference for the Brave and True is an open-source material on causal inference, the statistics of science. It uses only free software, based in Python. Its goal is to be accessible monetarily and intellectually.
If you found this book valuable and you want to support it, please go to [Patreon](https://www.patreon.com/causal_inference_for_the_brave_and_true). If you are not ready to contribute financially, you can also help by fixing typos, suggesting edits or giving feedback on passages you didn't understand. Just go to the book's repository and [open an issue](https://github.com/matheusfacure/python-causality-handbook/issues). Finally, if you liked this content, please share it with others who might find it useful and give it a [star on GitHub](https://github.com/matheusfacure/python-causality-handbook/stargazers)._____no_output_____


# Notebook from spsrc/sostat2021
Path: tutorials/daniela_machine_learning/SOSTAT2021_ML_DataPrep.ipynb

# Data Preparation for the SOSTAT 2021 Tutorials

## Imports_____no_output_____

# make plots interactive and import plotting functionality
%matplotlib notebook
import matplotlib.pyplot as plt

# pretty plotting
import seaborn as sns

# my standard styles for plots
sns.set_style("whitegrid")
sns.set_context("talk")

# Always need numpy
import numpy as np

# data array operations
import pandas as pd_____no_output_____from astroquery.sdss import SDSS_____no_output_____stars = SDSS.query_sql("""SELECT * from (SELECT top 50000
objid, ra, dec, psfmag_u-extinction_u AS mag_u,
psfmag_g-extinction_g AS mag_g, psfmag_r-extinction_r AS mag_r, psfmag_i-extinction_i AS mag_i,psfmag_z-extinction_z AS mag_z,z AS spec_z,dered_u - dered_g AS u_g_color,
dered_g - dered_r AS g_r_color,dered_r - dered_i AS r_i_color,dered_i - dered_z AS i_z_color,class
FROM SpecPhoto
WHERE
 (class = 'STAR')
 ) as sp
""")_____no_output_____galaxies = SDSS.query_sql("""SELECT * from (SELECT top 50000
objid, ra, dec, psfmag_u-extinction_u AS mag_u,
psfmag_g-extinction_g AS mag_g, psfmag_r-extinction_r AS mag_r, psfmag_i-extinction_i AS mag_i,psfmag_z-extinction_z AS mag_z,z AS spec_z,dered_u - dered_g AS u_g_color,
dered_g - dered_r AS g_r_color,dered_r - dered_i AS r_i_color,dered_i - dered_z AS i_z_color,class
FROM SpecPhoto
WHERE
 (class = 'GALAXY')
 ) as sp
""")_____no_output_____stars = stars.to_pandas()
stars["class"] = np.array(stars["class"], dtype=str)
_____no_output_____galaxies = galaxies.to_pandas()
galaxies["class"] = np.array(galaxies["class"], dtype=str)
_____no_output_____comb = pd.concat([stars[:10000], galaxies[:10000]])_____no_output_____comb.head()_____no_output_____comb.to_csv("sdss_dataset1.csv", index=False)_____no_output_____

Let's also generate a shared test set:_____no_output_____

test_set = pd.concat([stars[-1000:], galaxies[-1000:]])_____no_output_____test_set = test_set.sample(frac=1, replace=False).reset_index(drop=True)_____no_output_____test_set.head()_____no_output_____test_set.to_csv("sdss_testset1.csv")_____no_output_____

## Neural Network Data_____no_output_____

from astroML.datasets import fetch_sdss_galaxy_colors_____no_output_____NOBJECTS = 50000_____no_output_____query_text = ('\n'.join(("SELECT TOP %i" % NOBJECTS,
                         "  p.u, p.g, p.r, p.i, p.z, s.class, s.z AS redshift, s.zerr AS redshift_err",
                         "FROM PhotoObj AS p",
                         "  JOIN SpecObj AS s ON s.bestobjid = p.objid",
                         "WHERE ",
                         "  p.u BETWEEN 0 AND 19.6",
                         "  AND p.g BETWEEN 0 AND 20",
                         "  AND s.class <> 'UNKNOWN'",
                         "  AND s.class <> 'STAR'",
                         "  AND s.class <> 'SKY'",
                         "  AND s.class <> 'STAR_LATE'")))

_____no_output_____data = SDSS.query_sql(query_text)/astro/users/dhuppenk/.conda/envs/pytorch/lib/python3.8/site-packages/astroquery/sdss/core.py:862: VisibleDeprecationWarning: Reading unicode strings without specifying the encoding argument is deprecated. Set the encoding, use None for the system default.
  arr = np.atleast_1d(np.genfromtxt(io.BytesIO(response.content),
data = data.to_pandas()
data["class"] = np.array(data["class"], dtype=str)

_____no_output_____data.shape_____no_output_____data.head()_____no_output_____training_data = data[:20000]

test_data = data[-4000:]_____no_output_____training_data.shape_____no_output_____test_data.shape_____no_output_____training_data.to_csv("sdss_dataset2.csv", index=False)

test_data.to_csv("sdss_testset2.csv", index=False)_____no_output_____


# Notebook from crazyhottommy/scripts-general-use
Path: Python/pybedtools_intro.ipynb

### This notebook is to get myself to be familair with the pybedtools
import the pybedtools module _____no_output_____

import pybedtools
import sys
import os_____no_output_____

get the working directory and you can change to the directory you want by os.chdir(path)
list all the files in the directory os.listdir(path)_____no_output_____

os.getcwd()
# use a pre-shipped bed file as an example
a = pybedtools.example_bedtool('a.bed')_____no_output_____

a is a bedtool object, one can acess the object by index_____no_output_____

print a[0]
print a[1]
feature = a[1]chr1	1	100	feature1	0	+

chr1	100	200	feature2	0	+

see what type of the interval is by Interval.file_type
All features, no matter what the file type, have chrom, start, stop, name, score, and strand attributes. Note that start and stop are long integers, while everything else (including score) is a string._____no_output_____

print feature.file_type
print feature
print feature.chrom
print feature.start
print feature.stop
print feature.name
print feature.score
print feature.strand
print feature.fieldsbed
chr1	100	200	feature2	0	+

chr1
100
200
feature2
0
+
['chr1', '100', '200', 'feature2', '0', '+']

interval can also be accessed by index or like a dictionary_____no_output_____

print feature[0]
print feature["chrom"]
print feature[1]
print feature["start"]chr1
chr1
100
100
print a[1:4]<itertools.islice object at 0x10d42b418>

slice get an itertools object that can be interated _____no_output_____

for interval in a[1:4]:
    print intervalchr1	100	200	feature2	0	+

chr1	150	500	feature3	0	-

chr1	900	950	feature4	0	+

for each interval, one can access the chr,star, end by _____no_output_____

for interval in a[1:4]:
    print interval.chromchr1
chr1
chr1

Let's do some intersection for 2 bed files_____no_output_____

a = pybedtools.example_bedtool('a.bed')
b = pybedtools.example_bedtool('b.bed')_____no_output_____print a.head() # print out only the first 10 lines if you have big bed file
print b.head()chr1	1	100	feature1	0	+
 chr1	100	200	feature2	0	+
 chr1	150	500	feature3	0	-
 chr1	900	950	feature4	0	+
 None
chr1	155	200	feature5	0	-
 chr1	800	901	feature6	0	+
 None

use the Bedtools.intersect() method _____no_output_____

a_and_b = a.intersect(b)_____no_output_____a_and_b.head()chr1	155	200	feature2	0	+
 chr1	155	200	feature3	0	-
 chr1	900	901	feature4	0	+

one can add flags to the intersect call just as the command line intersectbed

-wa Write the original entry in A for each overlap. may have duplicated entries from A

-u Write original A entry **once** if any overlaps found in B. In other words, just report the fact at least one overlap was found in B.
The following toy example returns the same result for -u and -wa flag. _____no_output_____a.intersect(b, wa=True).head()_____no_output_____

a.intersect(b, u=True).head()chr1	100	200	feature2	0	+
 chr1	150	500	feature3	0	-
 chr1	900	950	feature4	0	+

### saving files
save the Bedtool object to a file, you can add a trackline._____no_output_____

c = a_and_b.saveas('intersection-of-a-and-b.bed', trackline='track name="a and b"')_____no_output_____os.listdir(".")
print ctrack name="a and b"
chr1	155	200	feature2	0	+
chr1	155	200	feature3	0	-
chr1	900	901	feature4	0	+

one can chain the methods of pybedtools just like the pipe in the command line.
The following intersect a with b first, and save the intersection in a file.
because the intersect() method returns a Bedtool object, it can be chained using .merge()
method and finally saved the mearged bed file
_____no_output_____

 x4 = a\
    .intersect(b, u=True)\
    .saveas('a-with-b.bed')\
    .merge()\
    .saveas('a-with-b-merged.bed')_____no_output_____

### demonstrate the filter method
grep out only the intervals with length bigger than 100_____no_output_____

print a
for interval in a:
    print len(interval)chr1	1	100	feature1	0	+
chr1	100	200	feature2	0	+
chr1	150	500	feature3	0	-
chr1	900	950	feature4	0	+

99
100
350
50

Let's use filter to extract the intervals with length >100_____no_output_____

print a.filter(lambda x: len(x) >100)chr1	150	500	feature3	0	-

Or, use a more generic function _____no_output_____

def len_filter(feature, L):
    return len(feature) > L_____no_output_____

Then call this function inside the filter method:_____no_output_____

print a.filter(len_filter, 100)chr1	150	500	feature3	0	-

we got the same results as using the lambda. Using len_filter function is more flexiabl, as you can supply any length that you want to filter._____no_output_____### demonstrate the each() method
each() method can apply a function for each interval in the BedTool object.
It is similar to the apply functions in R
Let's add counts of how many hits in b intersect a_____no_output_____

with_count = a.intersect(b, c=True)
print with_countchr1	1	100	feature1	0	+	0
chr1	100	200	feature2	0	+	1
chr1	150	500	feature3	0	-	1
chr1	900	950	feature4	0	+	1

Normalize the counts by dividing the length of the interval.use a scalar of 0.001 to normalize it to
counts per 1kb_____no_output_____

def normalize_count(feature, scalar=0.001):
    count = float(feature[-1])
    normalized_count = count/len(feature) * scalar
    ## write the score back, need to turn it to a string first
    feature.score = str(normalized_count)
    return feature_____no_output_____print with_count.each(normalize_count)chr1	1	100	feature1	0.0	+	0
chr1	100	200	feature2	1e-05	+	1
chr1	150	500	feature3	2.85714285714e-06	-	1
chr1	900	950	feature4	2e-05	+	1


# Notebook from SalishSeaCast/analysis-keegan
Path: notebooks/Evaluations/CTD_Individual_year_evaluations/2018_Puget_CTD_Evaluation.ipynb

A notebook which contains an evaluation of the SalishSeaCast model within PugetSound. More specifically, it compares Observed data on temperature, salinity, and chlorophyll concentrations  collected from CTD data and compares it to various model phytoplankon outputs. This evaluation includes scatter plots and 2d histograms visualizing the similarity of observed and model data as well as tables of evaluation statistics. _____no_output_____

import sys
sys.path.append('/ocean/kflanaga/MEOPAR/analysis-keegan/notebooks/Tools')_____no_output_____import numpy as np
import matplotlib.pyplot as plt
import os
import pandas as pd
import netCDF4 as nc
import datetime as dt
from salishsea_tools import evaltools as et, viz_tools
import gsw
import matplotlib.gridspec as gridspec
import matplotlib as mpl
import matplotlib.dates as mdates
import cmocean as cmo
import scipy.interpolate as sinterp
import pickle
import cmocean
import json
import f90nml
import Keegan_eval_tools as ket
from collections import OrderedDict
from matplotlib.colors import LogNorm

fs=16
mpl.rc('xtick', labelsize=fs)
mpl.rc('ytick', labelsize=fs)
mpl.rc('legend', fontsize=fs)
mpl.rc('axes', titlesize=fs)
mpl.rc('axes', labelsize=fs)
mpl.rc('figure', titlesize=fs)
mpl.rc('font', size=fs)
mpl.rc('font', family='sans-serif', weight='normal', style='normal')

import warnings
#warnings.filterwarnings('ignore')
from IPython.display import Markdown, display

%matplotlib inline_____no_output_____

years=[2009,2010]
year=2010
modelversion='nowcast-green.201905'
PATH= '/results2/SalishSea/nowcast-green.201905/'
datadir='/ocean/eolson/MEOPAR/obs/WADE/ptools_data/ecology'_____no_output_____# Parameters
years = [2017, 2018]
year = 2018
modver = "nowcast-green.201905"
PATH = "/results2/SalishSea/nowcast-green.201905/"
saveloc = "/ocean/kflanaga/MEOPAR/savedData"
_____no_output_____display(Markdown('''## Year: '''+ str(year)))_____no_output_____display(Markdown('''### Model output: '''+ PATH))_____no_output_____

## Yearly model-data comparisons of chlorophyll, temperature and salinity between 201905 runs and CTD observations_____no_output_____

##### Loading in pickle file data
saveloc='/ocean/kflanaga/MEOPAR/savedData'
with open(os.path.join(saveloc,f'data_CTD_{modelversion}_{year}.pkl'),'rb') as hh:
    data_all=pickle.load(hh)_____no_output_____# Creating seperate dataframes for each season (includes only dataabove 15 metre depth.)
iz=(data_all.Z<15)
DJF_CTD=data_all.loc[iz&(data_all.dtUTC>=dt.datetime(years[0],12,1))&(data_all.dtUTC<=dt.datetime(years[1],3,1)),:]
MAM_CTD=data_all.loc[iz&(data_all.dtUTC<=dt.datetime(years[1],6,1))&(data_all.dtUTC>dt.datetime(years[1],3,1)),:]
JJA_CTD=data_all.loc[iz&(data_all.dtUTC<=dt.datetime(years[1],9,1))&(data_all.dtUTC>dt.datetime(years[1],6,1)),:]
SON_CTD=data_all.loc[iz&(data_all.dtUTC>dt.datetime(years[1],9,1))&(data_all.dtUTC<=dt.datetime(years[1],11,30)),:]_____no_output_____data_CTD=data_all.loc[(data_all.dtUTC>=dt.datetime(years[0],12,1))&(data_all.dtUTC<=dt.datetime(years[1],11,30)),:]_____no_output_____# defning a dictionary that brakes the data down by individual stations
datstat_CTD=dict()
for ind, istation in enumerate(data_CTD.Station.unique()):
    datstat_CTD[istation]=data_CTD.loc[data_CTD.Station == istation]

datreg_CTD=dict()
for iregion in data_CTD.Basin.unique():
    datreg_CTD[iregion] = data_CTD.loc[data_CTD.Basin==iregion]_____no_output_____### These groupings will be used to calculate statistics. The keys are labels and
### the values are corresponding dataframe views
statsubs_CTD=OrderedDict({'z < 15 m':data_CTD.loc[data_CTD.Z<15],
                      '15 m < z < 22 m':data_CTD.loc[(data_CTD.Z>=15)&(data_CTD.Z<22)],
                      'z >= 22 m':data_CTD.loc[data_CTD.Z>=22],
                      'z > 50 m':data_CTD.loc[data_CTD.Z>50],
                      'all':data_CTD,
                      'z < 15 m, DJF':DJF_CTD,
                      'z < 15 m, MAM':MAM_CTD,
                      'z < 15 m, JJA':JJA_CTD,
                      'z < 15 m, SON': SON_CTD})
for iregion in data_CTD.Basin.unique():
    statsubs_CTD[iregion]=datreg_CTD[iregion]
statsubs_CTD.keys()_____no_output_____

## Map of observation locations in each basin_____no_output_____

fig, ax = plt.subplots(1,1,figsize = (6,6))
with nc.Dataset('/data/vdo/MEOPAR/NEMO-forcing/grid/bathymetry_201702.nc') as grid:
    viz_tools.plot_coastline(ax, grid, coords = 'map',isobath=.1)
colors=('blue','green','firebrick','darkorange','darkviolet','fuchsia',
        'royalblue','darkgoldenrod','mediumspringgreen','deepskyblue')
for ind, iregion in enumerate(data_CTD.Basin.unique()):
    ax.plot(datreg_CTD[iregion]['Lon'], datreg_CTD[iregion]['Lat'],'.',
            color = colors[ind], label=iregion)
ax.set_ylim(47, 49)
ax.legend(bbox_to_anchor=[1,.6,0,0])
ax.set_xlim(-124, -122);
ax.set_title('Observation Locations');_____no_output_____

# Absolute Salinity (g/kg)_____no_output_____### Table containing evaluation statistics for relevant subsets_____no_output_____

# Setting Variable for Temperature.
obsvar='SA'
modvar='mod_vosaline'
regions=data_CTD.Basin.unique()_____no_output_____statsDict_CTSA={year:dict()}
statsDict_CTSA[year]['SA']=OrderedDict()
for isub in statsubs_CTD:
    print(isub)
    statsDict_CTSA[year]['SA'][isub]=dict()
    var=statsDict_CTSA[year]['SA'][isub]
    var['N'],mmean,omean,var['Bias'],var['RMSE'],var['WSS']=et.stats(statsubs_CTD[isub].loc[:,[obsvar]],
                                                                     statsubs_CTD[isub].loc[:,[modvar]])
tbl,tdf=et.displayStats(statsDict_CTSA[year]['SA'],level='Subset',suborder=list(statsubs_CTD.keys()))
tblz < 15 m
15 m < z < 22 m
z >= 22 m
z > 50 m
all
z < 15 m, DJF
z < 15 m, MAM
z < 15 m, JJA
z < 15 m, SON
Whidbey Basin
Main Basin
South Basin
Hood Canal Basin
Strait of Georgia
Strait of Juan de Fuca
Admiralty Inlet

### Model vs Observed comparisons by depth and region_____no_output_____

lims=(0,40)

fig, ax = plt.subplots(1,2,figsize = (16,7))
ps,l=ket.byDepth(ax[0],data_CTD,obsvar,modvar,lims)
ax[0].set_title('S$_A$ (g kg$^{-1}$) By Depth')

ps,l=ket.byRegion(ax[1],data_CTD,datreg_CTD,obsvar,modvar,lims)
ax[1].set_title('S$_A$ (g kg$^{-1}$) By Region');_____no_output_____

### A closer look at Salinity in South Basin_____no_output_____### 2D Histogram of model vs observed_____no_output_____

fig, ax = plt.subplots(1,1,figsize= (12,8))
ps = ket.hist2d(ax,fig,data_CTD,obsvar,modvar,lims,fontsize=18)
ax.set_title('S$_A$ (g kg$^{-1}$)',fontsize=18)_____no_output_____

### 2d histograms of individual regions_____no_output_____

ax=ket.byRegion_hist2d(datreg_CTD,regions,obsvar,modvar,lims)_____no_output_____

### Model vs observed comparisons for multiple seasons_____no_output_____

seasons=[DJF_CTD,MAM_CTD,JJA_CTD,SON_CTD]
season_titles=['Dec-Feb','Mar-May','Jun-Aug','Sep-Nov']

fig, ax = plt.subplots(1,4,figsize = (16,3.3))
ket.bySeason(ax,seasons,obsvar,modvar,lims,season_titles)
fig,ax=plt.subplots(1,1,figsize=(20,.3))
ax.plot(data_CTD.dtUTC,np.ones(np.shape(data_CTD.dtUTC)),'k.')
ax.set_xlim((dt.datetime(year,1,1),dt.datetime(year,12,31)))
ax.set_title('data_CTD Timing')
ax.yaxis.set_visible(False)_____no_output_____

# Conservative Temperature_____no_output_____

#Setting variables for temperature.
obsvar='CT'
modvar='mod_votemper'_____no_output_____

### Table containing evaluation statistics for relevant subsets_____no_output_____

statsDict_CTSA[year]['CT']=OrderedDict()
for isub in statsubs_CTD:
    statsDict_CTSA[year]['CT'][isub]=dict()
    var=statsDict_CTSA[year]['CT'][isub]
    var['N'],mmean,omean,var['Bias'],var['RMSE'],var['WSS']=et.stats(statsubs_CTD[isub].loc[:,[obsvar]],
                                                                     statsubs_CTD[isub].loc[:,[modvar]])
tbl,tdf=et.displayStats(statsDict_CTSA[year]['CT'],level='Subset',suborder=list(statsubs_CTD.keys()))
tbl_____no_output_____

### Model vs Observed comparisons by depth and region_____no_output_____

lims=(0,40)
fig, ax = plt.subplots(1,2,figsize = (16,7))
ps,l=ket.byDepth(ax[0],data_CTD,obsvar,modvar,lims)
ax[0].set_title('$\Theta$ ($^{\circ}$C) By Depth')

ps,l=ket.byRegion(ax[1],data_CTD,datreg_CTD,obsvar,modvar,lims)
ax[1].set_title('$\Theta$ ($^{\circ}$C) By Region');_____no_output_____

### 2D Histogram of model vs observed_____no_output_____

fig, ax = plt.subplots(1,1,figsize= (12,8))
ps = ket.hist2d(ax,fig,data_CTD,obsvar,modvar,lims,fontsize=18)
ax.set_title('$\Theta$ ($^{\circ}$C)',fontsize=18)_____no_output_____

### 2d histograms of individual regions_____no_output_____

ax=ket.byRegion_hist2d(datreg_CTD,regions,obsvar,modvar,lims)_____no_output_____

### Model vs observed comparisons for multiple seasons_____no_output_____

seasons=[DJF_CTD,MAM_CTD,JJA_CTD,SON_CTD]
season_titles=['Dec-Feb','Mar-May','Jun-Aug','Sep-Nov']

fig, ax = plt.subplots(1,4,figsize = (16,3.3))
ket.bySeason(ax,seasons,obsvar,modvar,lims,season_titles)
fig,ax=plt.subplots(1,1,figsize=(20,.3))
ax.plot(data_CTD.dtUTC,np.ones(np.shape(data_CTD.dtUTC)),'k.')
ax.set_xlim((dt.datetime(year,1,1),dt.datetime(year,12,31)))
ax.set_title('data_CTD Timing')
ax.yaxis.set_visible(False)_____no_output_____

### Temperature-Salinity by Region_____no_output_____

def tsplot(ax,svar,tvar):
    limsS=(0,36)
    limsT=(5,20)
    ss,tt=np.meshgrid(np.linspace(limsS[0],limsS[1],20),np.linspace(limsT[0],limsT[1],20))
    rho=gsw.rho(ss,tt,np.zeros(np.shape(ss)))
    r=ax.contour(ss,tt,rho,colors='k')
    ps=list()
    for ind, iregion in enumerate(data_CTD.Basin.unique()):
        p=ax.plot(datreg_CTD[iregion][svar], datreg_CTD[iregion][tvar],'.',
                color = colors[ind], label=iregion)
        ps.append(p[0])
    l=ax.legend(handles=ps,bbox_to_anchor=(1.01,1))
    ax.set_ylim(limsT)
    ax.set_xlim(limsS)
    ax.set_ylabel('$\Theta$ ($^{\circ}$C)')
    ax.set_xlabel('S$_A$ (g kg$^{-1}$)')
    ax.set_aspect((limsS[1]-limsS[0])/(limsT[1]-limsT[0]))
    return_____no_output_____fig,ax=plt.subplots(1,2,figsize=(16,3.5))
tsplot(ax[0],'SA','CT')
ax[0].set_title('Observed')
tsplot(ax[1],'mod_vosaline','mod_votemper')
ax[1].set_title('Modelled')_____no_output_____

### Summary table of temperature and salinity evaluation statistics_____no_output_____

#No longer entirely accurate, should create seperate ones for each of the chlorophyll
#comparisons and the CT SA analysis. Also should probably not log transform for the table.
tbl,tdf=et.displayStats(statsDict_CTSA[year],level='Variable',suborder=list(statsubs_CTD.keys()))
tbl_____no_output_____


# Notebook from sachinpr0001/data_science
Path: ml_repo/Data Visualisation/Data Visualisation.ipynb

# Data Visualisation

#### Install Matplotlib

`pip install matplotlib`_____no_output_____Data Science Pipeline
-----------------------------
- Understand the Problem Statement
- Acquire Data & Cleaning
- Data Visualisation / Exploratory Analysis
- Machine Learning Modelling
- Share Results/ Interesting Visualisations about findings_____no_output_____

import matplotlib.pyplot as plt
import numpy as np
_____no_output_____themes = plt.style.available
print(themes)['bmh', 'classic', 'dark_background', 'fast', 'fivethirtyeight', 'ggplot', 'grayscale', 'seaborn-bright', 'seaborn-colorblind', 'seaborn-dark-palette', 'seaborn-dark', 'seaborn-darkgrid', 'seaborn-deep', 'seaborn-muted', 'seaborn-notebook', 'seaborn-paper', 'seaborn-pastel', 'seaborn-poster', 'seaborn-talk', 'seaborn-ticks', 'seaborn-white', 'seaborn-whitegrid', 'seaborn', 'Solarize_Light2', 'tableau-colorblind10', '_classic_test']
plt.style.use("seaborn")_____no_output_____# Line Plot
x = np.arange(10)
y1 = x**2
y2 = 2*x + 3
print(x)
print(y1)
print(y2)[0 1 2 3 4 5 6 7 8 9]
[ 0  1  4  9 16 25 36 49 64 81]
[ 3  5  7  9 11 13 15 17 19 21]
# Line Plots
plt.plot(x,y1,color='red',label="Apple",marker='o')
plt.plot(x,y2,color='green',label="Kiwi",linestyle="dashed",marker='*')
plt.xlabel("Time")
plt.ylabel("Price")
plt.title("Prices of Fruits over Time")
plt.legend()
plt.show()
_____no_output_____

_____no_output_____

prices = np.array([1,2,3,4])**3
print(prices)[ 1  8 27 64]
plt.plot(prices)
plt.show()_____no_output_____

### Scatter Plots_____no_output_____

# Adjust the Size of Any Plot
plt.figure(figsize=(6,6))
plt.scatter(x,y1,color='red',label="Apple",marker='^')
plt.scatter(x,y2,color='green',label="Kiwi",linestyle="dashed")
plt.xlabel("Time")
plt.ylabel("Price")
plt.title("Scatter Plots - Prices Over Time")
plt.legend()
plt.show()_____no_output_____

## Bar-graphs_____no_output_____

plt.style.use("seaborn-white")
x_coordinates = np.array([0,1,2])*2
plt.bar(x_coordinates-0.25,[10,20,15],width=0.5,label="Current Year",tick_label=["Gold","Platinum","Silver"]) # Current Year
plt.bar(x_coordinates+0.25,[20,10,12],width=0.5,label="Next Year") # Next Year
plt.title("Metal Price Comparision")
plt.xlabel("Metal")
plt.ylabel("Price")
plt.ylim(0,40)
plt.xlim(-2,5)
plt.legend()
plt.show()_____no_output_____plt.style.use("dark_background")

plt.figure(figsize=(5,5))
x_coordinates = np.array([0,1,2])*2
plt.bar(x_coordinates-0.25,[10,20,15],width=0.5,label="Current Year",tick_label=["Gold","Platinum","Silver"],color="yellow") # Current Year
plt.bar(x_coordinates+0.25,[20,10,12],width=0.5,label="Next Year",color="orange") # Next Year
plt.title("Metal Price Comparision")
plt.xlabel("Metal")
plt.ylabel("Price")
plt.ylim(0,40)
plt.legend()
plt.show()_____no_output_____

#### Pie Chart
_____no_output_____

plt.style.use("seaborn")_____no_output_____plt.figure(figsize=(3,3))
subjects = "Maths","Chem","Physics","English"
weightage = [20,10,15,5]

plt.pie(weightage,labels=subjects,explode=(0,0,0.1,0),autopct='%1.1f%%',shadow=True)
plt.title("Subjects ")
plt.show()_____no_output_____

### Histogram_____no_output_____

Xsn = np.random.randn(100)
sigma = 8
u = 70
X = np.round(Xsn*sigma + u)
X2 = np.round(Xsn*7 + 40)

print(X,X2)[58. 69. 73. 78. 65. 74. 65. 81. 63. 84. 68. 66. 66. 73. 74. 71. 77. 70.
 74. 81. 62. 57. 68. 75. 74. 68. 76. 58. 88. 61. 72. 64. 72. 67. 74. 57.
 71. 66. 77. 71. 63. 66. 68. 70. 56. 81. 79. 63. 70. 59. 67. 64. 64. 81.
 71. 81. 74. 71. 71. 71. 53. 74. 55. 77. 68. 60. 62. 68. 47. 81. 70. 69.
 74. 64. 73. 67. 70. 70. 87. 81. 79. 70. 81. 72. 82. 76. 63. 73. 83. 69.
 80. 75. 65. 65. 72. 55. 71. 67. 67. 77.] [29. 39. 42. 47. 35. 43. 35. 49. 33. 53. 38. 36. 37. 43. 44. 41. 46. 40.
 44. 49. 33. 29. 38. 44. 44. 38. 45. 30. 55. 32. 42. 34. 42. 37. 44. 29.
 41. 37. 46. 41. 34. 36. 38. 40. 28. 50. 48. 34. 40. 30. 37. 34. 35. 50.
 41. 50. 43. 41. 41. 41. 25. 44. 27. 46. 38. 31. 33. 39. 20. 50. 40. 39.
 44. 35. 43. 38. 40. 40. 55. 50. 48. 40. 49. 41. 51. 46. 34. 43. 51. 39.
 49. 45. 35. 36. 42. 27. 41. 38. 37. 46.]
plt.style.use("seaborn")

plt.hist(X2,alpha=0.8,label="Physics")
plt.hist(X,alpha=0.9,label="Maths")
plt.ylabel("Prob/FreqCount of Students")
plt.xlabel("Marks Range")
plt.title("Histogram")
plt.show()_____no_output_____

## Data Visualisation Challenge

Given a Movies Dataset, your taskis is to make a visualisation that -
- Plots the Length of Movie Title Names on X-Axis
- Frequency Counts on Y-axis ie Number of Movies having 'x' characters in their title
_____no_output_____

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt_____no_output_____# Read a CSV File
df = pd.read_csv("../2. Working with Libraries/movie_metadata.csv")_____no_output_____df.head(n=10)_____no_output_____df.columns_____no_output_____titles = list(df.get('movie_title'))_____no_output_____print(type(titles))<class 'list'>
print(titles[:5])['Avatar\xa0', "Pirates of the Caribbean: At World's End\xa0", 'Spectre\xa0', 'The Dark Knight Rises\xa0', 'Star Wars: Episode VII - The Force Awakens\xa0            ']
freq_titles = {}

for title in titles:
    length = len(title)

    if freq_titles.get(length) is None:
        freq_titles[length] = 1
    else:
        freq_titles[length] += 1_____no_output_____print(freq_titles){7: 138, 41: 17, 8: 226, 22: 138, 55: 3, 12: 330, 13: 306, 24: 85, 39: 13, 35: 30, 17: 239, 18: 219, 43: 8, 16: 264, 44: 9, 15: 284, 42: 14, 23: 109, 11: 328, 36: 21, 19: 213, 10: 294, 27: 69, 20: 176, 32: 43, 26: 74, 25: 82, 14: 274, 21: 138, 31: 47, 51: 5, 6: 139, 5: 91, 63: 1, 3: 8, 9: 267, 38: 24, 28: 51, 29: 65, 34: 31, 57: 1, 47: 4, 4: 35, 37: 18, 48: 4, 46: 7, 40: 14, 45: 4, 49: 4, 50: 3, 30: 36, 33: 21, 2: 5, 59: 2, 60: 1, 56: 1, 84: 1, 54: 2, 87: 1, 53: 2, 72: 1, 69: 1, 80: 1, 67: 1}
X = np.array(list(freq_titles.keys()))
Y = np.array(list(freq_titles.values()))
#print(X,Y)_____no_output_____plt.scatter(X,Y)
plt.xlabel("Length of Movie Title")
plt.ylabel("Freq Count")
plt.title("Movie Data Visualiation Problem")
plt.show()_____no_output_____

#### Mini Challenge
- Generate Random 100 Datapoints and Scatter Them!
_____no_output_____