# Notebook from bigfooted/covid19-seir
Path: presentation_seir.ipynb

from IPython.display import Image,display
from IPython.core.display import HTML
import ipywidgets as widgets
import matplotlib.pyplot as plt_____no_output_____%%html
_____no_output_____#!python
# from: https://scipy-cookbook.readthedocs.io/items/LoktaVolterraTutorial.html
from numpy import *
#import pylab as p
import matplotlib.pyplot as p
# Definition of parameters
a = 1.
b = 0.1
c = 1.5
d = 0.75
#@interact(beta=widgets.FloatSlider(min=0.1, max=2, step=0.1, value=0.4),gamma=widgets.FloatSlider(min=0.1, max=1, step=0.1, value=0.2))

def dX_dt(X, t=0):
    """ Return the growth rate of fox and rabbit populations. """
    return array([ a*X[0] -   b*X[0]*X[1] ,
                  -c*X[1] + d*b*X[0]*X[1] ])
#!python
X_f0 = array([     0. ,  0.])
X_f1 = array([ c/(d*b), a/b])
all(dX_dt(X_f0) == zeros(2) ) and all(dX_dt(X_f1) == zeros(2)) # => True
#!python
def d2X_dt2(X, t=0):
    """ Return the Jacobian matrix evaluated in X. """
    return array([[a -b*X[1],   -b*X[0]     ],
                  [b*d*X[1] ,   -c +b*d*X[0]] ])
#! python
A_f0 = d2X_dt2(X_f0)                    # >>> array([[ 1. , -0. ],
                                        #            [ 0. , -1.5]])
#!python
A_f1 = d2X_dt2(X_f1)                    # >>> array([[ 0.  , -2.  ],
                                        #            [ 0.75,  0.  ]])
# whose eigenvalues are +/- sqrt(c*a).j:
lambda1, lambda2 = linalg.eigvals(A_f1) # >>> (1.22474j, -1.22474j)
# They are imaginary numbers. The fox and rabbit populations are periodic as follows from further
# analysis. Their period is given by:
T_f1 = 2*pi/abs(lambda1)                # >>> 5.130199
#!python
from scipy import integrate
t = linspace(0, 15,  1000)              # time
X0 = array([10, 5])                     # initials conditions: 10 rabbits and 5 foxes
X, infodict = integrate.odeint(dX_dt, X0, t, full_output=True)
infodict['message']                     # >>> 'Integration successful.'
#!python
rabbits, foxes = X.T
#f1 = p.figure()
p.plot(t, rabbits, 'r-', label='Rabbits')
p.plot(t, foxes  , 'b-', label='Foxes')
p.grid()
p.legend(loc='best')
p.xlabel('time')
p.ylabel('population')
p.title('Evolution of fox and rabbit populations')
p.show()
#f1.savefig('rabbits_and_foxes_1.png')
_____no_output_____values  = linspace(0.3, 0.9, 5)                          # position of X0 between X_f0 and X_f1
vcolors = p.cm.autumn_r(linspace(0.3, 1., len(values)))  # colors for each trajectory

#f2 = p.figure()

#-------------------------------------------------------
# plot trajectories
for v, col in zip(values, vcolors):
    X0 = v * X_f1                               # starting point
    X = integrate.odeint( dX_dt, X0, t)         # we don't need infodict here
    #p.plot( X[:,0], X[:,1], lw=5.0*v, color=col, label='X0=(%.f, %.f)' % ( X0[0], X0[1]) )
    p.plot( X[:,0], X[:,1], lw=5.0, color=col, label='X0=(%.f, %.f)' % ( X0[0], X0[1]) )

#-------------------------------------------------------
# define a grid and compute direction at each point
ymax = p.ylim(ymin=0)[1]                        # get axis limits
xmax = p.xlim(xmin=0)[1]
nb_points   = 20

x = linspace(0, xmax, nb_points)
y = linspace(0, ymax, nb_points)

X1 , Y1  = meshgrid(x, y)                       # create a grid
DX1, DY1 = dX_dt([X1, Y1])                      # compute growth rate on the gridt
M = (hypot(DX1, DY1))                           # Norm of the growth rate
M[ M == 0] = 1.                                 # Avoid zero division errors
DX1 /= M                                        # Normalize each arrows
DY1 /= M

#-------------------------------------------------------
# Drow direction fields, using matplotlib 's quiver function
# I choose to plot normalized arrows and to use colors to give information on
# the growth speed
p.title('Trajectories and direction fields')
Q = p.quiver(X1, Y1, DX1, DY1, M, pivot='mid', cmap=p.cm.jet)
#Q = p.quiver(X1, Y1, DX1, DY1, M, pivot='mid')
p.xlabel('Number of rabbits')
p.ylabel('Number of foxes')
p.legend()
p.grid()
p.xlim(0, xmax)
p.ylim(0, ymax)
#f2.savefig('rabbits_and_foxes_2.png')
p.show()
_____no_output_____fig, axes = plt.subplots(nrows=1, ncols=2, figsize=(5, 3))
axes[0].plot(x1, y1)
axes[1].plot(x2, y2)
fig.tight_layout()_____no_output_____#!python
def IF(X):
    u, v = X
    return u**(c/a) * v * exp( -(b/a)*(d*u+v) )
# We will verify that IF remains constant for different trajectories
for v in values:
    X0 = v * X_f1                               # starting point
    X = integrate.odeint( dX_dt, X0, t)
    I = IF(X.T)                                 # compute IF along the trajectory
    I_mean = I.mean()
    delta = 100 * (I.max()-I.min())/I_mean
    #print("X0=(%2.f,%2.f) => I ~ %.1f |delta = %.3G",X0[0], X0[1], I_mean, delta)
# >>> X0=( 6, 3) => I ~ 20.8 |delta = 6.19E-05 %
#     X0=( 9, 4) => I ~ 39.4 |delta = 2.67E-05 %
#     X0=(12, 6) => I ~ 55.7 |delta = 1.82E-05 %
#     X0=(15, 8) => I ~ 66.8 |delta = 1.12E-05 %
#     X0=(18, 9) => I ~ 72.4 |delta = 4.68E-06 %

#!python
#-------------------------------------------------------
# plot iso contours
nb_points = 80                              # grid size
x = linspace(0, xmax, nb_points)
y = linspace(0, ymax, nb_points)
X2 , Y2  = meshgrid(x, y)                   # create the grid
Z2 = IF([X2, Y2])                           # compute IF on each point
f3 = p.figure()
CS = p.contourf(X2, Y2, Z2, cmap=p.cm.Purples_r, alpha=0.5)
CS2 = p.contour(X2, Y2, Z2, colors='black', linewidths=2. )
p.clabel(CS2, inline=1, fontsize=16, fmt='%.f')
p.grid()
p.xlabel('Number of rabbits')
p.ylabel('Number of foxes')
p.ylim(1, ymax)
p.xlim(1, xmax)
p.title('IF contours')
f3.savefig('rabbits_and_foxes_3.png')
p.show()_____no_output_____from IPython.display import clear_output
def create_multipleChoice_widget(description, options, correct_answer):
    if correct_answer not in options:
        options.append(correct_answer)

    correct_answer_index = options.index(correct_answer)

    radio_options = [(words, i) for i, words in enumerate(options)]
    alternativ = widgets.RadioButtons(
        options = radio_options,
        description = '',
        disabled = False
    )

    description_out = widgets.Output()
    with description_out:
        print(description)

    feedback_out = widgets.Output()

    def check_selection(b):
        a = int(alternativ.value)
        if a==correct_answer_index:
            s = '\x1b[6;30;42m' + "Correct!" + '\x1b[0m' +"\n" #green color
        else:
            s = '\x1b[5;30;41m' + "No! Please try again!" + '\x1b[0m' +"\n" #red color
        with feedback_out:
            clear_output()
            print(s)
        return

    check = widgets.Button(description="submit")
    check.on_click(check_selection)

    return widgets.VBox([description_out, alternativ, check, feedback_out])
    _____no_output_____Q1 = create_multipleChoice_widget('What kind of figure is figure b?',['A phase portrait','A Poincare map','A time series plot'],'A phase portrait')
Q2 = create_multipleChoice_widget('What happens if more foxes are introduced?',['Rabbits and foxes go extinct','Occasional rabbit plagues','The foxes all die'],'Occasional rabbit plagues')
#Q2 = create_multipleChoice_widget('What happens if more foxes are introduced?',['Rabbits and foxes go extinct','Rabbit plagues ','Foxes will die'],'Rabbit plagues')
Q3 = create_multipleChoice_widget('What type of point is the origin?',['A limit cycle','A saddle point','An equilibrium point'],'A saddle point')_____no_output_____display(Q1)
display(Q2)
display(Q3)_____no_output_____

        Jupyter labs in the classroom

 written in python, with easy online notebooks

Online lecture notes and slides

interactive data manipulation and visualization

tools for data analysis, programming, GUI creation, etc...

_____no_output_____

        Nonlinear dynamics and control: systems of first order ODEs

        Background:

Ordinary differential equations $\alpha$

Numerical methods

Transport Phenomena

_____no_output_____

HTML("""

    """)
_____no_output_____

          SIR model of a virus outbreak

            The outbreak and spread of a contagious disease can be modeled by the SIR model:

  $S(t)$ : number of Susceptible persons (persons that are not infected or immune)

  $I(t)$ : number of Infected persons

  $R(t)$ : number of Recovered persons (immune)

  $N = S+I+R$ : total population

  $\beta$ : infection rate

  $\alpha$ : recovery rate

            In this model, the total population size remains constant: nobody dies. Note that R can be also interpreted as the number of persons that are not ill anymore, either alive or dead.

_____no_output_____

       SIR model of a virus outbreak

            SIR model of a virus outbreak

            Classic SIR model:
            Kermack-McKendrick:

                $\frac{dS}{dt}=-\beta S I$

                $\frac{dI}{dt}=\beta S I - \gamma I $

                $\frac{dR}{dt}=\gamma I$

  $S(t)$ : number of Susceptible persons (persons that are not infected or immune)

  $I(t)$ : number of Infected persons

  $R(t)$ : number of Recovered persons (immune)

  $N = S+I+R$ : total population

  $\beta$ : infection rate

  $\alpha$ : recovery rate

            In this model, the total population size remains constant: nobody dies. Note that R can be also interpreted as the number of persons that are not ill anymore, either alive or dead.

_____no_output_____

%%html

       SIR model of a virus outbreak

            SIR model of a virus outbreak

            Classic SIR model:
            Kermack-McKendrick:

                $\frac{dS}{dt}=-\beta S I$

                $\frac{dI}{dt}=\beta S I - \gamma I $

                $\frac{dR}{dt}=\gamma I$

  $S(t)$ : number of Susceptible persons (persons that are not infected or immune)

  $I(t)$ : number of Infected persons

  $R(t)$ : number of Recovered persons (immune)

  $N = S+I+R$ : total population

  $\beta$ : infection rate

  $\alpha$ : recovery rate

            In this model, the total population size remains constant: nobody dies. Note that R can be also interpreted as the number of persons that are not ill anymore, either alive or dead.

_____no_output_____%%html

       SIR model of a virus outbreak

            SIR model of a virus outbreak

            Classic SIR model:
            Kermack-McKendrick:

                $\frac{dS}{dt}=-\beta S I$

                $\frac{dI}{dt}=\beta S I - \gamma I $

                $\frac{dR}{dt}=\gamma I$

  $S(t)$ : number of Susceptible persons (persons that are not infected or immune)

  $I(t)$ : number of Infected persons

  $R(t)$ : number of Recovered persons (immune)

  $N = S+I+R$ : total population

  $\beta$ : infection rate

  $\alpha$ : recovery rate

            In this model, the total population size remains constant: nobody dies. Note that R can be also interpreted as the number of persons that are not ill anymore, either alive or dead.

   _____no_output_____#%matplotlib inline

import matplotlib.pyplot as plt
from ipywidgets import interact

import ipywidgets as widgets

import numpy as np
import pandas as pd
from scipy.integrate import solve_ivp
from scipy.optimize import minimize
import matplotlib as mpl
#import matplotlib.pyplot as plt
from datetime import timedelta, datetime
# for nan
#import math

#S_0 = 15000
#population size
N = 1000000
I_0 = 10
R_0 = 0
S_0 = N - I_0 - R_0

# nr of points
size=72

# beta = transmission rate [number per day]
beta = 0.4

# gamma = 1/infectuous period = 1/T, T=5 days
gamma = 0.2

#R = beta/gamma

#def f(x):
#    return x
#
##interact(f, x=widgets.IntSlider(min=-10, max=30, step=1, value=10));
#interact(f, x=widgets.FloatSlider(min=0.4,max=0.6,step=0.01,value=0.5))

# @ means its a decorator, functions are called implicitly
@interact(beta=widgets.FloatSlider(min=0.1, max=2, step=0.1, value=0.4),gamma=widgets.FloatSlider(min=0.1, max=1, step=0.1, value=0.2))
def sirsol(beta,gamma):

    def SIR(t, y):
            S = y[0]
            I = y[1]
            R = y[2]
            return [-beta*S*I/N, beta*S*I/N - gamma*I,gamma*I]

    #beta=n
    SIR_sol = solve_ivp(SIR, [0, size], [S_0,I_0,R_0], t_eval=np.arange(0, size, 0.5))

    #index = confirmed.index
    #print("index = ",index)
    #print
    #x=np.arange(len(new_index))

    fig = plt.figure()
    ax = plt.subplot(111)

    plt.rcParams["figure.figsize"] = [12,12]
    plt.rcParams["axes.linewidth"] = 2.0
    plt.rcParams['lines.linewidth'] = 5
    plt.rcParams['lines.markersize'] = 10
    plt.rcParams['lines.markeredgecolor'] = 'black'

    plt.rcParams['font.size'] = 16
    plt.rcParams['axes.labelsize'] = 16
    plt.rcParams['axes.labelweight'] = 'bold'
    plt.rcParams['xtick.labelsize'] = 14
    plt.rcParams['ytick.labelsize'] = 14
    plt.rcParams['legend.fontsize'] = 12
    plt.rcParams['figure.titlesize'] = 12

    plt.xticks([0,7,14,21,28,35,42,49,56,63,70],['start outbreak','week 1','week 2','week 3','week 4','week 5','week 6','week 7','week 8','week 9','week 10'],rotation=45)

    plt.plot(SIR_sol.t,SIR_sol.y[0],label='Susceptible S')
    #plt.plot(prediction.y[1],label='predicted exposed E')
    plt.plot(SIR_sol.t,SIR_sol.y[1],label='Infected I')
    plt.plot(SIR_sol.t,0.96*SIR_sol.y[2],label='Recovered R')
    plt.plot(SIR_sol.t,0.04*SIR_sol.y[2],label='Deaths D')
    ax.legend(loc='upper left', frameon=False)

    plt.show()

_____no_output_____import matplotlib.pyplot as plt
from ipywidgets import interact
import ipywidgets as widgets
import numpy as np
import pandas as pd
from scipy.integrate import solve_ivp
from scipy.optimize import minimize
import matplotlib as mpl
from datetime import timedelta, datetime

F_0 = 5
R_0 = 5

# nr of points
size=16

#a = 0.4
#b = 0.2
#c=1
#d=1

# @ means its a decorator, functions are called implicitly
@interact(a=widgets.FloatSlider(min=0.1, max=2, step=0.05, value=1.0),
          b=widgets.FloatSlider(min=0.1, max=1, step=0.05, value=0.1),
          c=widgets.FloatSlider(min=0.1, max=2, step=0.05, value=1.5),
          d=widgets.FloatSlider(min=0.1, max=1, step=0.05, value=0.75))
def sirsol(a,b,c,d):
    def lotka_volterra(t, y):
            F = y[0]
            R = y[1]
            return [a*F - b*F*R, -c*R + d*b*F*R]

    #beta=n
    SIR_sol = solve_ivp(lotka_volterra, [0, size], [F_0,R_0], t_eval=np.arange(0, size, 0.1))

    fig = plt.figure()
    ax = plt.subplot(111)

    plt.rcParams["figure.figsize"] = [12,12]
    plt.rcParams["axes.linewidth"] = 2.0
    plt.rcParams['lines.linewidth'] = 5
    plt.rcParams['lines.markersize'] = 10
    plt.rcParams['lines.markeredgecolor'] = 'black'

    plt.rcParams['font.size'] = 16
    plt.rcParams['axes.labelsize'] = 16
    plt.rcParams['axes.labelweight'] = 'bold'
    plt.rcParams['xtick.labelsize'] = 14
    plt.rcParams['ytick.labelsize'] = 14
    plt.rcParams['legend.fontsize'] = 12
    plt.rcParams['figure.titlesize'] = 12

    plt.xlabel('time [weeks]')
    plt.ylabel('population')
    plt.grid(True)
#    plt.xticks([0,7,14,21,28,35,42,49,56,63,70],['start outbreak','week 1','week 2','week 3','week 4','week 5','week 6','week 7','week 8','week 9','week 10'],rotation=45)

    plt.plot(SIR_sol.t,SIR_sol.y[0],label='Foxes')
    plt.plot(SIR_sol.t,SIR_sol.y[1],label='Rabbits')

    ax.legend(loc='upper left', frameon=False)

    plt.show()
    _____no_output_____

next slide: comparing with data: pulldown list of all country data data. We find that a fit might be difficult_____no_output_____next slide: fit of coefficients, realization that coefficients are changing in time. Fits take time to compute... do we want this realtime?_____no_output_____next slide: _____no_output_____


# Notebook from ljmartin/estimating_sem_timeseries
Path: scrapbook_code/autocorrelation.ipynb

# Error of the estimated mean for time-series data

A common analysis task is determining the mean	value of a metric in a population - here we consider the metric to be the arithmetic mean. Because we don't have access to the entire population, this is almost always done by taking some sample data from the broader population, measuring the mean in the sample, then taking that measurement as an estimation of the true value (hence 'estimator'). Because the estimator doesn't return the exact true value, it should be accompanied by error bars. For sample data that is independent, calculating the standard error of the estimated mean is trivial using a formula. Some other data, for example time series, are highly correlated and thus need some adjustments to determine the error.

Despite the broad range of fields that measure time series data - ecology, econometrics, physical simulations, astronomy, etc... - there's no single, accepted approach to calculating the standard error of the mean, motiviating this article.

This article demonstrates four different techniques for determing the standard error of the estimated mean:
- Block averaging
- Using a correction factor
- Autoregressive (AR(1)) maximum likelihood fit
- Autoregressive (AR(1)) bayesian fit

------------

First, to demonstrate the problem, below shows examples of independent data (top panel) and highly correlated data time series data (bottom panel):
_____no_output_____

import matplotlib.pyplot as plt
import numpy as np
#np.random.seed(2572367)
np.random.seed(257367)

def gen_correlated_curve(ac, num=100, innovation=1):
    y = np.zeros((num,))
    for i in range(1,num):
        y[i] = ac * y[i-1] + np.random.normal()*innovation
    return y

uncorrelated = np.random.randn(100)
correlated = gen_correlated_curve(0.90)

fig, ax = plt.subplots(ncols=1, nrows=2); fig.set_figheight(10); fig.set_figwidth(10)
ax[0].plot(uncorrelated, '-o', mfc='white'); ax[0].set_title('Independent sample'); ax[0].axhline(0, c='k', linestyle='--')
ax[1].plot(correlated, '-o', mfc='white'); ax[1].set_title('Highly correlated sample'); ax[1].axhline(0, c='k', linestyle='--')
_____no_output_____

The key difference is that in the case of highly correlated data, each new point is very similar to the previous one. This is known as autocorrelation. For independent data this is not the case - each new point has no memory of the previous points even though all points are still centred on a common mean value.

This causes difficulty when calculating error bars. Normally, for independent data, when estimating the mean value you would calculate the standard error of that estimate using:

\begin{align}
SEM = \frac{s}{\sqrt{n}}
\end{align}

which in english says "the standard error of the (estimated) mean, called $SEM$, is equal to the standard deviation ($s$) of the sample divided by the square root of the number of samples". The formula isn't so special, it can be found on the wikipedia entry for [standard error of the mean](https://en.wikipedia.org/wiki/Standard_error).

The problem with using this formula for highly correlated data is the $\sqrt{n}$ in the denominator - it can become very large for time series data, especially if you take samples at high frequency. Consider an example: measure your body temperature every hour for four weeks. It fluctuates over the day/night cycle, but doesn't change too much hour by hour. In that case, $n=672$. Now imagine measuring it every minute for a month, so $n=40320$. The $n$ has multiplied by $60$, meaning the $SEM$ becomes smaller by $\sqrt{60}$. Yet in both cases you sampled for the same total time. Your actual body temperature would have varied by about the same amount, meaning you would expect the error bars to be about the same.

Put another way, high frequency sampling of a time-varying process gives you lots of unnecessary data points because each point is similar to its neighbours, and this makes the sample look very precise when it isn't.

_____no_output_____# Autocorrelation

This correlation between adjacent points can be visualized using an `autocorrelation plot`. Autocorrelation plots range from `-1 to 1` and measure how correlated adjacent points are, with `1` being total positive correlation, `0` being no correlation at all, and `-1` being total negative correlation. Autocorrelation plots measure how correlated a sample is to another point at some time lag. When the time lag is zero, autocorrelation must be `1` because each point must be completely positively correlated with itself!

The below shows an example of three time series with different levels of positive correlation. When the autocorrelation plot reaches zero, it's safe to assume that the time lag is big enough that any data point has no memory of another data point separated by that lag. This value of the time lag is characteristic to an autocorrelated process, and is called the 'autocorrelation time' ($\tau$, tau). In practice, there's a lot of noise around the autocorrelation of zero which appears even using independent data.

`??? See Detecting Signals from Data with Noise: Theory and Applications` for significance testing of autocorrelation vs. white noise
_____no_output_____

plt.figure(figsize=(10, 5))
for ac, label in zip([0, 0.8, 0.99], ['Not autocorrelated at all', 'A bit autocorrelated', 'Very autocorrelated']):
    curve = gen_correlated_curve(ac, num=500)
    #calculate autocorrelation:
    co = np.correlate(curve-curve.mean(),curve-curve.mean(), mode='same')
    co = co/co.max()
    co = co[int(co.shape[0]/2):]

    plt.plot(co,label=label)

plt.legend()
plt.xlabel('Lag time', fontsize=16)
_=plt.ylabel('Autocorrelation', fontsize=16)
_____no_output_____

# Approach 1: Block averaging
'Undoing' the autocorrelation allows us to calculate $SEM$ as if the sample was independent. One way to do this is block averaging. First, a baseline value for the $SEM$ is calculated using all points. To do one round of block averaging, every second point and it's adjacent point forward in time are averaged, creating 'blocks' of size 2 and resulting in a dataset with $ \frac{n}{2} $ points.

Here's an example:_____no_output_____

correlated = gen_correlated_curve(0.99, num=25)
x = np.arange(len(correlated))

plt.figure(figsize=(10,5))
plt.title('One round of block averaging')
plt.plot(x,correlated,mew=1, linewidth=0.5, label='Autocorrelated data')

#plot pairs:
for i in range(0,13):
    plt.plot(x[i*2:i*2+2],correlated[i*2:i*2+2],'-o', c='C0', mfc='white', mew=2, linewidth=3, zorder=0)

#do one round of block averaging, blocksize of 2
x_ = x[:len(x)-(len(x)%2)].reshape(-1,2).mean(1)
y_ = correlated[:len(x)-(len(x)%2)].reshape(-1,2).mean(1)

plt.scatter(x_, y_, facecolor='white', edgecolor='red', linewidth=2,zorder=2, label='Block size=2')
plt.xlabel('Time',fontsize=16)
plt.ylabel('Value',fontsize=16)
_=plt.legend()_____no_output_____

By increasing the size of the blocks from 2,3,4..., the hope is that eventually adjacent blocks are far enough apart in time that they are totally uncorrelated and the data resemble an independent sample. This occurs when the blocks are bigger than the autocorrelation time $\tau$. It appears as an asymptote in the $SEM$, and would also correspond to reaching zero in the autocorrelation plot.

To demonstrate, the below shows blocks up to `size=200` using 5000 points from an autocorrelated time series: _____no_output_____

from scipy.stats import sem

def do_block_averaging(corr):
    sems = list()
    x=np.arange(len(corr))
    #for blocksize in range(1, int(len(corr)//25)+1):
    for blocksize in range(1, int(len(corr)**(2/3))+1):
        #this is the blocking bit:
        x_ = x[:len(x)-(len(x)%blocksize)].reshape(-1,blocksize).mean(1)
        y_ = corr[:len(x)-(len(x)%blocksize)].reshape(-1,blocksize).mean(1)
        sems.append(sem(y_))
    return sems

correlated = gen_correlated_curve(0.6, num=5000)
x = np.arange(correlated.shape[0])

blocked_sems = do_block_averaging(correlated)_____no_output_____#plot:
fig, ax = plt.subplots()
fig.set_figwidth(10)
fig.set_figheight(5)

ax.plot(blocked_sems, label='Blocked SEM')
ax.set_xlabel('Block Size', fontsize=14)
ax.set_ylabel('SEM',fontsize=14)
_=ax.legend()_____no_output_____

There are a few problems with this approach:

- **Wasting data**. In order to maintain identically distributed blocks, each block should be exactly the same size. So, what if the block size doesn't evenly divide the total number of measurements? The overhanging data has to be discarded, which is a waste of perfectly good measurements
- **Noise** - as seen above, the $SEM$ values become noisy after the asymptote is reached, rather than converging towards the true value. This is a consequence of cutting data - as the block size grows, larger and larger overhanging chunks get removed until the blocksize is an even divisor of the number of samples. The changes in the amount of data being used leads to jumps in the calculated $SEM$.
- **Arbitrary choice of asymptote**. By eye it seems pretty obvious what region the $SEM$ lies in, but how do you choose a consensus value? One way to remedy this is to automate the choice by fitting some curve that also grows to an asymptote like the $SEM$. There doesn't appear to be a known curve to use for the fit, but curves based on `y=arctan(x)` work. However, on the edge cases where the optimum block size is only 2 or 3, or it's nearly as many samples as you have, the fit can be (wildly) inaccurate.

Because we have a good smooth curve above, the below demonstrates fitting the curve `y= a*arctan(b*(x-c))` to find the asymptote of the $SEM$.
_____no_output_____

# fitting to arctan
from scipy import optimize

def block_avg_fit(x, a, b, c):
    return a * np.arctan(b*(x-c))

popt, pcov = optimize.curve_fit(block_avg_fit, np.arange(len(blocked_sems)), np.array(blocked_sems))
_____no_output_____#plot:
ax.plot(np.arange(len(blocked_sems)),
        block_avg_fit(np.arange(len(blocked_sems)), popt[0], popt[1],popt[2]),
        label='Line of best fit')
ax.axhline(popt[0] *np.pi/2, label='SEM from block avg. asymptote', linestyle='--', c='C1')
ax.legend()
fig_____no_output_____

So block bootstrapping works well if you have lots of data and a smooth curve. But what to do in other cases? Below demonstrates some alternatives that use all the available data and give definite answers without requiring a good block averaging plot._____no_output_____# Approach 2: Estimating $n_{eff}$ from the autocorrelation function

Block averaging works by reducing the number of data points $n$ until the $SEM$ converges. At this point, we expect adjacent points to look independent. The value of $n$ that achieves this is known as the 'effective $n$', or $n_{eff}$, and it reflects how many independently sampled data points you actually have in the sample. There are other ways to calculate $n_{eff}$ that are more direct than block averaging, and interestingly they can result in fractional values.

Adjusting the wikipedia equation for $SEM$, we now have:

\begin{align}
SEM = \frac{s}{\sqrt{n_{eff}}}
\end{align}

Calculating $n_{eff}$ normally happens by first calculating the autocorrelation time $\tau$. The autocorrelation time is the time it takes for the process to forget a past value - meaning  the autocorrelation plot should be close to zero. This time, of course, doesn't need to be an integer value, which is how $n_{eff}$ can be fractional.

\begin{align}
n_{eff} = \frac{n}{\tau}
\end{align}

So how to find $\tau$? The two techniques given below, called here Sokal and Chodera (see refs below), estimate $\tau$ from the autocorrelation function and have reasonable agreement with block averaging if you sampled for many multiples of $\tau$.

>Chodera: see https://github.com/choderalab/pymbar for code, and `Use of the Weighted Histogram Analysis Method for the Analysis of Simulated and Parallel TemperingSimulations`
>
>Sokal: see https://dfm.io/posts/autocorr/ for code, [here](https://pdfs.semanticscholar.org/0bfe/9e3db30605fe2d4d26e1a288a5e2997e7225.pdf) for Sokal notes
_____no_output_____

import statsmodels.api as sm
def next_pow_two(n):
    #lew: complicated one-liner: int(2**(np.ceil(np.log2(number))))
    i = 1
    while i < n:
        i = i << 1
    return i

def autocorr_func_1d(x):
    n = next_pow_two(len(x))
    # Compute the FFT and then (from that) the auto-correlation function
    f = np.fft.fft(x - np.mean(x), n=2*n)
    acf = np.fft.ifft(f * np.conjugate(f))[:len(x)].real
    acf /= 4*n
    acf /= acf[0]
    return acf

# Automated windowing procedure following Sokal (1989)
def auto_window(taus, c):
    m = np.arange(len(taus)) < c * taus
    if np.any(m):
        return np.argmin(m)
    return len(taus) - 1

def sokal_autocorr_time(corr):
    f = autocorr_func_1d(corr)
    taus = 2.0*np.cumsum(f)-1.0
    c=5
    window = auto_window(taus, c)
    return taus[window]

def statineff(corr, mintime=3):
    N = corr.size
    C_t = sm.tsa.stattools.acf(corr, fft=True, unbiased=True, nlags=N)
    t_grid = np.arange(N).astype('float')
    g_t = 2.0 * C_t * (1.0 - t_grid / float(N))
    ind = np.where((C_t <= 0) & (t_grid > mintime))[0][0]
    g = 1.0 + g_t[1:ind].sum()
    return max(1.0, g)

#chodera:
chodera_autoc_time = statineff(correlated)

#sokal:
sokal_autoc_time = sokal_autocorr_time(correlated)

n=len(correlated)
chodera_sem = np.std(correlated) / np.sqrt(n/chodera_autoc_time)
sokal_sem = np.std(correlated) / np.sqrt(n/sokal_autoc_time)_____no_output_____#plot:
ax.axhline(chodera_sem, label='SEM from Chodera', c='green')
ax.axhline(sokal_sem, label='SEM from Sokal', c='red')
ax.legend(fontsize=14)
fig_____no_output_____

# Approach 3: Autoregressive processes
A third way to estimate the $SEM$ is by treating it as an autoregressive process. Here, it helps to understand what it is that makes time series correlated with themselves. The simplest model of autocorrelation is called an `autoregressive model` where we only consider the time lag at lag=1, commonly just called `AR(1)`. [Wikipedia](https://en.wikipedia.org/wiki/Autoregressive_model) has some more details on these, but here's the short version. In an AR(1) process, each new point is equal to the previous point multiplied by a number $\rho$ (rho), plus or minus some white noise (i.e. random noise).

In maths, using $X_t$ to mean 'point $X$ at time $t$' and $\epsilon$ (epsilon) to mean 'random noise':

\begin{align}
X_t = \rho X_{t-1} + \epsilon
\end{align}

When $\rho$ is less than 1, each step pulls the point back towards zero (hence auto*regressive*), meaning the time series will maintain a mean of zero. Moving average models are also commonly used, but a stationary mean makes things a bit easier. If your time series isn't centred on zero, the AR(1) model can just add a constant that shifts the whole sample back to zero, and then still use the model above.

First, let's estimate the $\rho$ parameter. This can be done using linear regression, where you calculate the slope of the line that goes between any point and its adjacent neighbour. Remember, the true $\rho$ in this case was 0.6.
_____no_output_____

from statsmodels.tsa.ar_model import AutoReg

#find estimated rho using ordinary least squares
result = AutoReg(correlated-correlated.mean(), lags = [1]).fit()
estimated_rho = result.params[1]
print('###################################')
print(f'Estimated rho is {estimated_rho}, with 95% CI of {result.conf_int()[1]}')
print('###################################')###################################
Estimated rho is 0.6111243087797814, with 95% CI of [0.58917639 0.63307223]
###################################

Not a bad estimate for $\rho$. Next, we can use a handy formula, which is a consequence of assuming an autoregressive AR(1) process, to get an estimate for the $SEM$. First the $SEM$ gets calculated naively, giving the wrong answer. Then it's multiplied by a correction factor $k$, which is:

\begin{align}
k = \sqrt{\frac{1+\rho}{1-\rho}}
\end{align}

So now we have:

\begin{align}
SEM = \frac{ks}{\sqrt{n}}
\end{align}_____no_output_____

correction = np.sqrt((1+estimated_rho)/(1-estimated_rho))

naive_sem = sem(correlated)
corrected_sem = naive_sem*correction

ax.axhline(corrected_sem, label='SEM from correction', c='purple')
ax.legend()
fig_____no_output_____

Before moving on, it's also neat to see how big this correction factor can get. At $\rho$ close to 0, i.e. less than about 0.2, $k$ is barely greater than 1, meaning the naive $SEM$ doesn't change much. But as $\rho$ approaches 1, the $k$ can grow very large. Because you might be multiplying small numbers by big numbers here, you would probably need very large sample sizes to get precise estimates if the process is highly autocorrelated.  _____no_output_____

def correction_factor(rho, n):
    d = ((n-1)*rho - n*rho**2 + rho**(n+1)) / (1-rho)**2
    k = np.sqrt( (1 + (2*d)/n) / ( 1 - (2*d)/(n*(n-1))  ) )
    return k

x = np.linspace(0,0.9,100)
plt.figure(figsize=(10,5))
plt.ylabel('Correction factor, $k$',fontsize=14); plt.xlabel('Autocorrelation at lag 1, $\\rho$',fontsize=16)
_ = plt.plot(x, correction_factor(x, 100))
_____no_output_____

So, to take stock:
Block averaging clumps the data together in order to reduce $n$, but doesn't make use of all data. Other approaches estimate $n_{eff}$ from the autocorrelation plot. Alternatively you can assume an AR(1) process, fit with ordinary least squares to find $\rho$, then use a correction factor to fix the naively-calculated $SEM$.

The final technique here also uses an AR(1) model, but instead of using a correction equation, it simulates how probable a range of different mean values are,then reports the region of highest probability._____no_output_____
# Approach 4: AR(1) via Bayesian estimation
The above approaches report what is known as the `maximum likelihood` estimate of the $SEM$. It's called maximum likelihood simply because it's the single most likely value after you factor in any model assumptions that were made (like the AR(1) assumption, or normally-distributed noise).

An alternative is Bayesian estimation. There are, of course, still model assumptions - this still uses an AR(1) model. The trick here is that we try and find the $\rho$ parameter by also estimating where the mean value is. Afterwards, we can throw away the $\rho$ estimate and just look at the most probable values for the mean. These most probable values are encompassed in a 'highest posterior density' which can be directly interpreted as a probability (unlike confidence intervals).

The below demonstrates a Bayesian fit of an AR(1) model, using the `PyMC3` library, which jointly estimates the value of the mean and the $\rho$, and can then report a full spectrum of likely values for the mean (or for $\rho$ if you want). The 95% highest posterior density is used instead of the $SEM$._____no_output_____

import pymc3 as pm
import logging
logger = logging.getLogger('pymc3')
logger.setLevel(logging.ERROR)

def do_bayes_sem(corr):
    with pm.Model() as ar1:
        k_ = pm.Uniform('k',-1,1) #we assume process is stationary, so -1<k_<1
        tau_ = pm.Gamma('tau',mu=1,sd=1)
        center = pm.Normal('center', mu=corr.mean(), sigma=5) #set the prior for the mean to be centred onthe population mean
        likelihood = pm.AR1('likelihood', k=k_, tau_e=tau_, observed=corr-center)
        trace = pm.sample()
    return trace
#slower:
#     with pm.Model() as ar1:
#         rho = pm.Uniform('rho', -1,1)
#         center = pm.Normal('center', mu=corr.mean(), sigma=5)
#         likelihood = pm.AR('likelihood', rho, sigma=1.0, observed=corr-center)
#         trace = pm.sample(500, chains=1, progressbar=True)
#     return trace

trace = do_bayes_sem(correlated)
Sampling 2 chains, 0 divergences: 100%|██████████| 2000/2000 [00:01<00:00, 1488.18draws/s]
import pymc3 as pm
with pm.Model() as ar1:
    k_ = pm.Uniform('k',-1,1) #we assume process is stationary, so -1<k_<1
    tau_ = pm.Gamma('tau',mu=1,sd=1)
    center = pm.Normal('center', mu=correlated.mean(), sigma=5) #set the prior for the mean to be centred onthe population mean
    likelihood = pm.AR1('likelihood', k=k_, tau_e=tau_, observed=correlated-center)

with ar1:
    trace = pm.sample()

kdeplot(trace['center'])
Auto-assigning NUTS sampler...
Initializing NUTS using jitter+adapt_diag...
Multiprocess sampling (2 chains in 2 jobs)
NUTS: [center, tau, k]
Sampling 2 chains, 0 divergences: 100%|██████████| 2000/2000 [00:02<00:00, 917.89draws/s]
hpd = pm.stats.hpd(trace['center'])
hpd_sem = (hpd[1]-hpd[0]/2)/1.96

ax.axhline(hpd_sem, c='black', linestyle='--', linewidth=3,
            label='SEM from Bayesian estimation')

ax.legend()
fig

_____no_output_____

There's a lot more analysis one might do with the results here. As a taste, you can plot the posterior density of the mean value, along with the 95% probability bounds (equivalent to the $SEM$). In lay terms, this means it can tell you what the 95% limits are, as well as how probable all the intermediate values are for the mean, which arguably is much more useful than just the error bounds.

For example:
_____no_output_____

from seaborn import kdeplot
plt.figure(figsize=(10,5))
kdeplot(trace['center'], shade=True, label='Posterior density of mean value')

hpd = pm.stats.hpd(trace['center'])
plt.errorbar([hpd.mean()], [0], xerr = np.abs(hpd.mean()-hpd[:,None]),c='C3',
             linewidth=4,fmt='o', mfc='white', label='95% highest posterior density')
plt.axvline(hpd.mean(), c='C3', linestyle='--')
plt.legend()_____no_output_____rpts = 100
bavg_results = np.zeros([rpts,5])
chodera_results = np.zeros([rpts,5])
sokal_results = np.zeros([rpts,5])
correction_results = np.zeros([rpts,5])
bayes_results = np.zeros([rpts,5])
estimated_means = np.zeros([rpts,5])_____no_output_____import tqdm
for row in tqdm.tqdm_notebook(range(rpts)):
    for col, datasize in enumerate([30,100,300,1000,3000]):
        cor = gen_correlated_curve(0.9, datasize)
        estimated_means[row,col]=cor.mean()

        #block avg:
        s = do_block_averaging(cor)

        popt, pcov = optimize.curve_fit(block_avg_fit, np.arange(1, int(datasize**(2/3))+1), np.array(s), maxfev=25000)

        block_avg_sem = popt[0] *np.pi/2
        bavg_results[row,col]=block_avg_sem

        #chodera, sokal:
        chodera_autoc_time = statineff(cor)
        sokal_autoc_time = sokal_autocorr_time(cor)
        n=len(cor)
        chodera_sem = np.std(cor) / np.sqrt(n/chodera_autoc_time)
        chodera_results[row,col] = chodera_sem
        sokal_sem = np.std(cor) / np.sqrt(n/sokal_autoc_time)
        sokal_results[row,col] = sokal_sem

        #correction factor:
        result = AutoReg(cor-cor.mean(), lags = [1]).fit()
        estimated_rho = result.params[1]
        correction = np.sqrt((1+estimated_rho)/(1-estimated_rho))
        naive_sem = sem(cor)
        corrected_sem = naive_sem*correction
        correction_results[row,col]=corrected_sem

        #bayesian:
        trace = do_bayes_sem(cor)
        hpd = pm.stats.hpd(trace['center'])
        hpd_sem = (hpd[1]-hpd[0]/2)/1.96
        bayes_results[row,col]=hpd_sem
        print(block_avg_sem, chodera_sem, sokal_sem, corrected_sem, hpd_sem)

        _____no_output_____for res in [bavg_results, chodera_results, sokal_results,
            correction_results, bayes_results]:
    ci_95 = 1.96*res[:135]
    print(((ci_95-np.abs(estimated_means[:135]))>0).sum(0)/100)
[0.86 0.88 0.92 0.92 0.95]
[0.71 0.82 0.88 0.91 0.96]
[0.49 0.75 0.85 0.87 0.95]
[0.79 0.87 0.92 0.95 0.94]
[0.95 0.95 0.96 0.99 0.95]
my_results = list()
for res in [bavg_results, chodera_results, sokal_results,
            correction_results, bayes_results]:
    ci_95 = 1.96*res
    print(((ci_95-np.abs(estimated_means))>0).sum(0))
    my_results.append(((ci_95-np.abs(estimated_means))>0).sum(0))

z=1.96
for col, datasize, label in zip(range(6), [30, 100, 300,1000,3000], ['Block avg', 'Chodera' ,'Sokal', 'Correction', 'Bayes']):
    for color, item in enumerate(np.array(my_results)[:,col]):
        it = item/100
        confint = z* np.sqrt((it*(1-it))/100)
        plt.errorbar(datasize+np.random.uniform(-datasize*0.1, datasize*0.1),
                     it, mfc='white' , elinewidth=2, capsize=2,yerr=confint,c='C'+str(color), label=label)

plt.xscale('log')
plt.axhline(0.95)
plt.legend()[86 88 92 92 95]
[71 82 88 91 96]
[49 75 85 87 95]
[79 87 92 95 94]
[95 95 96 99 95]
np.random.uniform(-0.1, 0.1, len(datasizes))_____no_output_____items = np.array(my_results)/100
confint = z*((items*(1-items))/100)**0.5
datasizes = [30, 100, 300,1000,3000]
for b, lab, ci in zip(items, ['Block avg', 'Chodera' ,'Sokal', 'Correction', 'Bayes'], confint):
    plt.errorbar(np.log10(datasizes)+np.random.uniform(-0.025, 0.025, len(datasizes)),
                 b, yerr=ci, label=lab)

#plt.xscale('log')
plt.axhline(0.95, c='k')
plt.legend()
_____no_output_____ci_____no_output_____rpts = 1000
bavg_results2 = np.zeros([rpts,6])
chodera_results2 = np.zeros([rpts,6])
sokal_results2 = np.zeros([rpts,6])
correction_results2 = np.zeros([rpts,6])
estimated_means2 = np.zeros([rpts,6])_____no_output_____rpts = 1000
bavg_results2 = np.zeros([rpts,7])
chodera_results2 = np.zeros([rpts,7])
sokal_results2 = np.zeros([rpts,7])
correction_results2 = np.zeros([rpts,7])
estimated_means2 = np.zeros([rpts,7])
for row in tqdm.tqdm_notebook(range(rpts)):
    for col, datasize in enumerate([30, 100, 300,1000,3000]):
        cor = gen_correlated_curve(0.9, datasize)
        estimated_means2[row,col]=cor.mean()

        #block avg:
        s = do_block_averaging(cor)

        popt, pcov = optimize.curve_fit(block_avg_fit, np.arange(1, int(datasize**(2/3))+1), np.array(s), maxfev=25000)

        block_avg_sem = popt[0] *np.pi/2
        bavg_results2[row,col]=block_avg_sem

        #chodera, sokal:
        chodera_autoc_time = statineff(cor)
        sokal_autoc_time = sokal_autocorr_time(cor)
        n=len(cor)
        chodera_sem = np.std(cor) / np.sqrt(n/chodera_autoc_time)
        chodera_results2[row,col] = chodera_sem
        sokal_sem = np.std(cor) / np.sqrt(n/sokal_autoc_time)
        sokal_results2[row,col] = sokal_sem

        #correction factor:
        result = AutoReg(cor-cor.mean(), lags = [1]).fit()
        estimated_rho = result.params[1]
        correction = np.sqrt((1+estimated_rho)/(1-estimated_rho))
        naive_sem = sem(cor)
        corrected_sem = naive_sem*correction
        correction_results2[row,col]=corrected_sem

        _____no_output_____cor = gen_correlated_curve(0.6, 30)
print(np.mean(cor))
#block avg:
s = do_block_averaging(cor)
popt, pcov = optimize.curve_fit(block_avg_fit, np.arange(1, int(datasize**(2/3))+1), np.array(s), maxfev=25000)
#plt.plot(s)0.4719414507883181
my_results = list()
for res in [bavg_results2, chodera_results2, sokal_results2,
            correction_results2]:
    ci_95 = 1.96*res
    print(((ci_95-np.abs(estimated_means2))>0).sum(0)/10)
    my_results.append(((ci_95-np.abs(estimated_means2))>0).sum(0)/10)

z=1.96
for col, datasize in zip(range(6), [30, 100, 300,1000,3000,10000]):
    for color, item in enumerate(np.array(my_results)[:,col]):
        it = item/100
        confint = z* np.sqrt((it*(1-it))/1000)
        plt.errorbar(datasize+np.random.uniform(-datasize*0.1, datasize*0.1),
                     it, mfc='white' , elinewidth=2, capsize=2,yerr=confint,c='C'+str(color))

plt.xscale('log')
plt.axhline(0.95)[78.  87.  91.4 95.1 94.2 94.8  0. ]
[58.2 80.9 89.4 94.8 94.5 95.1  0. ]
[39.7 70.2 83.1 93.1 93.5 94.3  0. ]
[70.6 84.9 90.4 95.3 94.5 94.7  0. ]
z=1.96
for col, datasize in zip(range(6), [30, 100, 300,1000,3000,10000]):
    for color, item in enumerate(np.array(my_results)[:,col]):
        it = item/100
        confint = z* np.sqrt((it*(1-it))/1000)
        plt.errorbar(datasize+np.random.uniform(-datasize*0.1, datasize*0.1),
                     it, mfc='white' , elinewidth=2, capsize=2,yerr=confint,c='C'+str(color))

plt.xscale('log')
plt.axhline(0.95)_____no_output_____confint_____no_output_____690000+750000-65000_____no_output_____

See:
*Confidence intervals in stationary autocorrelated time series*. Halkos, George and Kevork, Ilias,
at: https://mpra.ub.uni-muenchen.de/31840/1/MPRA_paper_31840.pdf , who reference:
`Fishman, G., 1978. Principles of Discrete Event Simulation. Wiley, New York.`_____no_output_____the below is from a cooky approach - fitting the np.log(periodogram) by linear regression to estimate the value at 0.
`A Spectral Method for Confidence Interval Generation and Run Length Control in Simulations`

I found it in Thomson:
`A Comparison of Methods for Computing Autocorrelation Time`_____no_output_____

from scipy.signal import periodogram
#out = periodogram(correlated)
#out= welch(correlated)
peri=np.abs(np.fft.fft(correlated)**2)/len(correlated)
out = peri[:peri.shape[0]//2]
from statsmodels.tools import add_constant
from statsmodels.regression.linear_model import OLS
x_ = np.arange(len(out[1:200]))
y_ = np.log(out[1:200])

res = OLS(y_, add_constant(x_)).fit()

I = np.e**res.params[0]
print(I, I/np.var(correlated))_____no_output_____

When you don't have enough data, the above can be a bad approximation. The full version is a bit messier:

\begin{align}
k = \sqrt{\frac{1+2\delta / n}{1 - 2\delta/(n(n-1))}}
\end{align}

where

\begin{align}
\delta = \frac{(n-1)\rho - n\rho^2 + \rho^{n+1}}{(1-\rho)^2}
\end{align}_____no_output_____


# Notebook from frazer-lab/cardips-ipsc-eqtl
Path: notebooks/RNA-Seq Analysis.ipynb

# RNA-Seq Analysis

I'd like to explore some different aspects of the expression profiles for
the different samples. There are some additional samples besides those
used for the eQTL study because for some people we sequenced multiple clones
etc.

I'll generally be looking at the all of the data but I'll also look
at a subset of European individuals from different families to remove
some of the genetic and ancestry signal. While it's true that some family
members are genetically unrelated, I will just take one person per family
to avoid similarities from shared familial environment._____no_output_____

import copy
import cPickle
import os
import subprocess

import cdpybio as cpb
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from scipy.linalg import svd
import scipy.stats as stats
import seaborn as sns
import statsmodels.formula.api as smf

import cardipspy as cpy
import ciepy

%matplotlib inline
%load_ext rpy2.ipython

outdir = os.path.join(ciepy.root, 'output',
                      'rna_seq_analysis')
cpy.makedir(outdir)

private_outdir = os.path.join(ciepy.root, 'private_output',
                              'rna_seq_analysis')
cpy.makedir(private_outdir)_____no_output_____sns.set_context('notebook')
sns.set_style('whitegrid')_____no_output_____fn = os.path.join(ciepy.root, 'output', 'input_data', 'wgs_metadata.tsv')
wgs_meta = pd.read_table(fn, index_col=0, squeeze=True)
fn = os.path.join(ciepy.root, 'output', 'input_data', 'rnaseq_metadata.tsv')
rna_meta = pd.read_table(fn, index_col=0)
fn = os.path.join(ciepy.root, 'output', 'input_data', 'subject_metadata.tsv')
subject_meta = pd.read_table(fn, index_col=0)

ext_meta = pd.read_table(os.path.join(ciepy.root, 'output', 'input_data', 'GSE73211.tsv'), index_col=0,
                         low_memory=False)

tpm = pd.read_table(os.path.join(ciepy.root, 'output', 'input_data', 'rsem_tpm.tsv'), index_col=0,
                                 low_memory=False)
ext_tpm = pd.read_table(os.path.join(ciepy.root, 'output', 'input_data', 'GSE73211_tpm.tsv'), index_col=0,
                        low_memory=False)

gene_info = pd.read_table(cpy.gencode_gene_info, index_col=0)

fn = os.path.join(ciepy.root, 'output', 'input_data', 'star_logs.tsv')
star_logs = pd.read_table(fn, index_col=0)

fn = os.path.join(ciepy.root, 'output', 'input_data', 'picard_metrics.tsv')
picard = pd.read_table(fn, index_col=0)_____no_output_____meta = rna_meta.merge(subject_meta, left_on='subject_id',
                      right_index=True, how='inner')_____no_output_____

## STAR Logs_____no_output_____

fig, axs = plt.subplots(2, 2)
axs = axs.flatten()
ax = axs[0]
k = 'Number of input reads'
star_logs[k].hist(ax=ax)
ax.set_ylabel('Number of samples')
ax.set_xlabel(k)
ax = axs[1]
k = 'Uniquely mapped reads number'
star_logs[k].hist(ax=ax)
ax.set_ylabel('Number of samples')
ax.set_xlabel(k)
ax = axs[2]
k = 'Uniquely mapped reads %'
star_logs[k].hist(ax=ax)
ax.set_ylabel('Number of samples')
ax.set_xlabel(k)
ax = axs[3]
k = 'Average mapped length'
star_logs[k].hist(ax=ax)
ax.set_ylabel('Number of samples')
ax.set_xlabel(k)
plt.tight_layout()_____no_output_____n = star_logs['Number of input reads'].mean()
print('Average of {:.2f} million read pairs per sample'.format(n / 10**6))Average of 21.59 million read pairs per sample
star_logs['Uniquely mapped reads %'].min()_____no_output_____star_logs['Uniquely mapped reads %'].median()_____no_output_____wgs_meta.cell.value_counts()_____no_output_____

## Picard Metrics_____no_output_____

picard[['MEDIAN_INSERT_SIZE', 'PERCENT_DUPLICATION', u'MEDIAN_5PRIME_TO_3PRIME_BIAS']].hist();_____no_output_____picard.PERCENT_DUPLICATION.median()_____no_output_____picard[['MEDIAN_INSERT_SIZE', 'PERCENT_DUPLICATION', u'MEDIAN_5PRIME_TO_3PRIME_BIAS']].describe()_____no_output_____

## Expression Distribution_____no_output_____

s = (tpm == 0).sum(axis=1)
print(s.value_counts().head(2))
s.hist(bins=100)
plt.title('Number of samples with zero TPM for each gene')
plt.ylabel('Number of genes')
plt.xlabel('Number of samples');250    19020
0      15313
dtype: int64

We can see overal that there are a fair number of genes that are not expressed
in any samples._____no_output_____

s = (tpm[gene_info.ix[tpm.index, 'gene_type'] == 'protein_coding'] == 0).sum(axis=1)
print(s.value_counts().head(2))
s.hist(bins=100)
plt.title('Number of samples with zero TPM for each protein coding gene')
plt.ylabel('Number of genes')
plt.xlabel('Number of samples');0      13524
250      906
dtype: int64

Protein coding genes are highly enriched for being expressed in all samples.

## Pluripotency gene expression_____no_output_____

plur_markers = ['POU5F1', 'SOX2', 'NANOG', 'ZFP42', 'LIN28A']
diff_markers = ['T', 'EOMES', 'SOX17', 'FOXA2', 'GATA4', 'HAND1',
                'CDX2', 'PAX6', 'SOX1', 'EN1']
def get_gene_id(x):
    return gene_info[gene_info.gene_name == x].index[0]
plur_markers = pd.Series(plur_markers, index=[get_gene_id(x) for x in plur_markers])
diff_markers = pd.Series(diff_markers, index=[get_gene_id(x) for x in diff_markers])

url = 'http://www.nature.com/nbt/journal/v33/n11/extref/nbt.3387-S5.xlsx'
scorecard = pd.read_excel(url)
scorecard = scorecard.drop(scorecard.columns[2:], axis=1)
scorecard = scorecard[scorecard.gene.apply(lambda x: x in gene_info.gene_name.values)]
scorecard.index = [get_gene_id(x) for x in scorecard.gene]
scorecard = scorecard[scorecard['class'].apply(lambda x: x in ['Mesoderm', 'Pluri'])]

tpm_all = pd.concat([tpm, ext_tpm], axis=1)_____no_output_____tdf = np.log10(tpm_all.ix[scorecard.index].T + 1)
tdf = tdf - tdf.mean()
tdf = tdf / tdf.std()_____no_output_____cs = pd.Series(dict(zip(list(set(ext_meta.cell_type)), sns.color_palette('colorblind')[3:6])))
rc = ([sns.color_palette('colorblind')[2]] * tpm.shape[1]) + list(cs[ext_meta.cell_type])
cs = pd.Series(dict(zip(list(set(scorecard['class'])), sns.color_palette("Set2", 7))))
cc = cs[scorecard['class']]

sns.clustermap(tdf, xticklabels=[], yticklabels=[], col_colors=cc, row_colors=rc);#, col_cluster=False);_____no_output_____

## All samples

### Variable Genes

I'm interested in what genes are variable among all samples_____no_output_____

# Filter for robustly expressed genes and take log.
tpm_f = tpm[(tpm == 0).sum(axis=1) == 0]
log_tpm = np.log10(tpm_f + 1)
# Mean center.
log_tpm_c = (log_tpm.T - log_tpm.mean(axis=1)).T
# Variance normalize.
log_tpm_n = (log_tpm_c.T / log_tpm_c.std(axis=1)).T_____no_output_____cov = log_tpm.std(axis=1) / log_tpm.mean(axis=1)
cov.sort_values(ascending=False, inplace=True)
cov.hist()
plt.title('Coefficient of variation histogram')
plt.xlabel('Coefficient of variation')
plt.ylabel('Number of genes');_____no_output_____

### Clustering_____no_output_____

## SVD_____no_output_____

res = cpb.analysis.SVD(log_tpm, scale_variance=True)_____no_output_____res.plot_variance_explained(xtick_start=10, xtick_spacing=20, cumulative=False)_____no_output_____res.plot_variance_explained(num_pc=30)_____no_output_____pc_anova = res.pc_anova(meta[['sex', 'ethnicity_group', 'sequence_id']])
pc_anova.pvalue_____no_output_____fig, ax = res.plot_pc_scatter(
    'PC1', 'PC2',
    color=meta.ix[res.v.index, 'sequence_id'],
    color_name='Flowcell')/frazer01/home/cdeboever/software/anaconda/envs/cie/lib/python2.7/site-packages/matplotlib/axes/_axes.py:492: UserWarning: You have mixed positional and keyword arguments, some input will be discarded.
  warnings.warn("You have mixed positional and keyword "
fig, ax = res.plot_pc_scatter(
    'PC2', 'PC3',
    color=meta.ix[res.v.index, 'sequence_id'],
    color_name='Flowcell')_____no_output_____fig, ax = res.plot_pc_scatter(
    'PC3', 'PC4',
    color=meta.ix[res.v.index, 'sequence_id'],
    color_name='Flowcell')_____no_output_____

The ANOVA and PC plots show that the different batches on the flowcells
definitely have an effect on the expression values. The flowcell with ID
23 was done a month or so after the other three flowcells (23 had the
samples that failed QC from the first three flowcells) so it's not surprising
that it's more different.

## Subset of Samples_____no_output_____

a = meta.dropna(subset=['family_id']).drop_duplicates(subset=['family_id'])
b = meta[meta.family_id.isnull()]
euro_no_fam = pd.concat([a, b])_____no_output_____# Filter for robustly expressed genes and take log.
subset_tpm_f = tpm.ix[(tpm == 0).sum(axis=1) == 0, euro_no_fam.index]
subset_log_tpm = np.log10(subset_tpm_f + 1)
# Mean center.
subset_log_tpm_c = (subset_log_tpm.T - subset_log_tpm.mean(axis=1)).T
# Variance normalize.
subset_log_tpm_n = (subset_log_tpm_c.T / subset_log_tpm_c.std(axis=1)).T_____no_output_____subset_cov = subset_log_tpm.std(axis=1) / subset_log_tpm.mean(axis=1)
subset_cov.sort_values(ascending=False, inplace=True)
subset_cov.hist()
plt.title('Coefficient of variation histogram')
plt.xlabel('Coefficient of variation')
plt.ylabel('Number of genes');_____no_output_____subset_svd = cpb.analysis.SVD(subset_log_tpm, scale_variance=True)_____no_output_____subset_svd.plot_variance_explained(xtick_start=10, xtick_spacing=10, cumulative=False)_____no_output_____subset_svd.plot_variance_explained(num_pc=30)_____no_output_____fig, ax = subset_svd.plot_pc_scatter(
    'PC1', 'PC2',
    color=meta.ix[subset_svd.v.index, 'sequence_id'],
    marker=meta.ix[subset_svd.v.index, 'sex'],
    color_name='Flowcell',
    marker_name='Sex')_____no_output_____subset_svd_500 = cpb.analysis.SVD(subset_log_tpm_n.ix[subset_cov.head(500).index])_____no_output_____fig, ax = subset_svd_500.plot_pc_scatter(
    'PC1', 'PC2',
    color=meta.ix[subset_svd.v.index, 'sequence_id'],
    marker=meta.ix[subset_svd.v.index, 'sex'],
    color_name='Flowcell',
    marker_name='Sex')_____no_output_____

TODO: Working here. Update stuff below._____no_output_____

t = subset_log_tpm_n.ix[subset_cov.head(500).index]
ct = t.corr(method='spearman')
cg = sns.clustermap(
    ct, xticklabels=False, yticklabels=False,
    row_colors=color_meta.ix[ct.index, ['sex']].T.values,
    col_colors=color_meta.ix[ct.index, ['sequence_id']].T.values)_____no_output_____

tdf = subset_log_tpm_n.ix[subset_cov.head(500).index]
fn = os.path.join(outdir, 'subset_sparse_pca.tsv')
p = os.path.join(outdir, 'subset_sparse_pca.pickle')
if not os.path.exists(fn):
    from sklearn import decomposition
    sparse_pca = decomposition.SparsePCA(n_components=5, n_jobs=30)
    sparse_pca.fit(tdf.T)

    ind = ['PC{}'.format(x) for x in
            range(1, sparse_pca.components_.shape[0] + 1)]
    sparse_components = pd.DataFrame(sparse_pca.components_,
                                      columns=tdf.index,
                                      index=ind).T
    sparse_components.to_csv(fn, sep='\t')
    cPickle.dump(sparse_pca, open(p, 'wb'))
else:
    sparse_componenets = pd.read_table(fn, index_col=0, header=0)
    sparse_pca = cPickle.load(open(p, 'rb'))_____no_output_____#plt.scatter(subset_svd_500.u['PC1'], sparse_pca.components_[0], lw=0, alpha=0.8)
ax = sns.jointplot(subset_svd_500.u['PC1'], sparse_pca.components_[0],
                   stat_func=None, alpha=0.25)
plt.xlabel('PCA loading')
plt.ylabel('Sparse PCA loading')
ax.set_axis_labels(xlabel='Distance in kb',
                   ylabel='$-\log_{10}$ $p$-value')
plt.tight_layout()
#plt.title('PCA vs. sparse PCA first eigenvector');_____no_output_____sparse_pca.components_[0].max()_____no_output_____ax = sns.jointplot(x=subset_svd_500.u['PC1'].values, y=sparse_pca.components_[0],
                   stat_func=None, alpha=0.5)
ax.set_axis_labels(xlabel='PCA eigenvector',
                   ylabel='Sparse PCA eigenvector')
plt.tight_layout()_____no_output_____genes = [x.split('.')[0] for x in subset_cov.index]
sig = np.array([False] * len(genes))
sig[0:500] = True
subset_var_go_results = cpb.analysis.goseq_gene_enrichment(genes, sig)_____no_output_____genes = [x.split('.')[0] for x in subset_cov.index]
sig = pd.Series(np.array([False] * len(genes)), index=subset_cov.index)
se = pd.Series(sparse_pca.components_[0], index=tdf.index)
sig[se[se != 0].index] = True
spc1_var_go_results = cpb.analysis.goseq_gene_enrichment(genes, sig.values)_____no_output_____spc1_var_go_results.head()_____no_output_____se = pd.Series(sparse_pca.components_[0], index=tdf.index)
t = subset_log_tpm_n.ix[se[se != 0].index]
cg = sns.clustermap(
    t, xticklabels=False, yticklabels=False,
    col_colors=color_meta.ix[ct.index, ['sex', 'sequence_id']].T.values)_____no_output_____