# Notebook from MIDOSS/analysis-ashutosh
Path: climatology_analysis_notebooks/River_plume.ipynb

# Freshet analysis_____no_output_____

import numpy as np
import numpy.ma as ma
import matplotlib.pyplot as plt
import xarray as xr
import cmocean
import pickle
import time
import datetime
from salishsea_tools import viz_tools
from salishsea_tools import geo_tools
from salishsea_tools import grid_tools
from salishsea_tools import utilities
%matplotlib inline_____no_output_____

## Loading Datasets_____no_output_____#### Load datasets -> Save locally as numpy arrays_____no_output_____

salinity = xr.open_dataset("https://salishsea.eos.ubc.ca/erddap/griddap/ubcSSg3DTracerFields1hV18-06")
#ssh = xr.open_dataset("https://salishsea.eos.ubc.ca/erddap/griddap/ubcSSgSurfaceTracerFields1hV18-06")_____no_output_____

#### Load arrays_____no_output_____

times_salinity = salinity.time.sel(time = salinity.time.dt.year.isin([2015,2016,2017,2018])).values.astype('datetime64[s]').astype(datetime.datetime)
Y = np.arange(540); X = np.arange(355)_____no_output_____#salmon_lat = lat.values[56][258]
#turn_lat = lat.values[143][240]
#sog_lat = lat.values[274][242]
Xcoord = 258; Ycoord = 56_____no_output_____

##### Creating local file for SoG timeseries_____no_output_____

for year in [2015, 2016, 2017,2018]:
    filename= ("/home/abhudia/Desktop/salinity/Raw/{}salinity.pickle").format(str(year))
    unpickle = open(filename, "rb")
    locals()["salinity" + str(year)] = pickle.load(unpickle)
    times = []
    for time in times_salinity:
        if time.year == year:
            times.append(time)
    locals()["salinity" + str(year)] = xr.DataArray(locals()["salinity" + str(year)], coords = [times, Y, X], dims = ['time', 'Y' , 'X'])
    locals()["salmon" + str(year)] = locals()["salinity" + str(year)].isel(X = Xcoord, Y = Ycoord).values
    locals()["salinity" + str(year)] = False
    fname = "salmon{}.pickle".format(str(year))
    picklefile = open(fname, 'wb')
    pickle.dump(locals()["salmon" + str(year)], picklefile)
    picklefile.close()
    locals()["salmon" + str(year)] = False_____no_output_____

##### Loading SoG time series and corresponding time lists_____no_output_____

for year in [2015, 2016, 2017, 2018]:
    fname = "/home/abhudia/Desktop/salinity/3points/sog{}.pickle".format(str(year))
    file = open(fname, "rb")
    locals()["sog" + str(year)] = pickle.load(file)
    locals()["tsog" + str(year)] = []
    for time in times_salinity:
        if time.year == year:
            locals()["tsog" + str(year)].append(time)

for year in [2015, 2016, 2017, 2018]:
    fname = "/home/abhudia/Desktop/salinity/3points/salmon{}.pickle".format(str(year))
    file = open(fname, "rb")
    locals()["salmon" + str(year)] = pickle.load(file)
    locals()["tsalmon" + str(year)] = []
    for time in times_salinity:
        if time.year == year:
            locals()["tsalmon" + str(year)].append(time)

for year in [2015, 2016, 2017, 2018]:
    fname = "/home/abhudia/Desktop/salinity/3points/turn{}.pickle".format(str(year))
    file = open(fname, "rb")
    locals()["turn" + str(year)] = pickle.load(file)
    locals()["tturn" + str(year)] = []
    for time in times_salinity:
        if time.year == year:
            locals()["tturn" + str(year)].append(time)   _____no_output_____

# Freshet Analysis_____no_output_____

years = [2015,2016,2017,2018]
freshet_salinity = []_____no_output_____def running_mean(x, N):
    cumsum = np.cumsum(np.insert(x, 0, 0))
    return (cumsum[N:] - cumsum[:-N]) / float(N)_____no_output_____

## Freshet 2015_____no_output_____

freshet2015 = np.min(sog2015)
freshet_salinity.append(freshet2015)
text = "the lowest salinity was {} g kg-1, occuring at point {} \n".format(freshet2015, str(np.where(sog2015 == freshet2015)[0]))
print(text)

fig = plt.figure(figsize = (20,10))
plt.subplot(211)
plt.plot(tsog2015, sog2015)
plt.title('Salinity time series at Strait of Georgia point 2015')
plt.xlabel('Time')
plt.ylabel('Salinity (g kg-1)')

plt.subplot(212)
plt.plot(tsog2015[3700:3740], sog2015[3700:3740])
plt.title('Salinity time series near Freshet at Strait of Georgia point 2015')
plt.xlabel('Time')
plt.ylabel('Salinity (g kg-1)')the lowest salinity was 7.072141170501709 g kg-1, occuring at point [3729]

## Freshet 2016_____no_output_____

freshet2016 = np.min(sog2016)
freshet_salinity.append(freshet2016)
text = "the lowest salinity was {} g kg-1, occuring at point {} \n".format(freshet2016, str(np.where(sog2016 == freshet2016)[0]))
print(text)

fig = plt.figure(figsize = (20,10))
plt.subplot(211)
plt.plot(tsog2016, sog2016)
plt.title('Salinity time series at Strait of Georgia point 2016')
plt.xlabel('Time')
plt.ylabel('Salinity (g kg-1)')

plt.subplot(212)
plt.plot(tsog2016[4010:4040], sog2016[4010:4040])
plt.title('Salinity time series near Freshet at Strait of Georgia point 2016')
plt.xlabel('Time')
plt.ylabel('Salinity (g kg-1)')the lowest salinity was 10.337932586669922 g kg-1, occuring at point [4024]

## Freshet 2017_____no_output_____

freshet2017 = np.min(sog2017)
freshet_salinity.append(freshet2017)
text = "the lowest salinity was {} g kg-1, occuring at point {} \n".format(freshet2017, str(np.where(sog2017 == freshet2017)[0]))
print(text)

fig = plt.figure(figsize = (20,10))
plt.subplot(211)
plt.plot(tsog2017, sog2017)
plt.title('Salinity time series at Strait of Georgia point 2017')
plt.xlabel('Time')
plt.ylabel('Salinity (g kg-1)')

plt.subplot(212)
plt.plot(tsog2017[3700:3770], sog2017[3700:3770])
plt.title('Salinity time series near Freshet at Strait of Georgia point 2017')
plt.xlabel('Time')
plt.ylabel('Salinity (g kg-1)')the lowest salinity was 3.0763986110687256 g kg-1, occuring at point [3728]

## Freshet 2018_____no_output_____

freshet2018 = np.min(sog2018)
freshet_salinity.append(freshet2018)
text = "the lowest salinity was {} g kg-1, occuring at point {} \n".format(freshet2018, str(np.where(sog2018 == freshet2018)[0]))
print(text)

fig = plt.figure(figsize = (20,10))
plt.subplot(211)
plt.plot(tsog2018, sog2018)
plt.title('Salinity time series at Strait of Georgia point 2018')
plt.xlabel('Time')
plt.ylabel('Salinity (g kg-1)')

plt.subplot(212)
plt.plot(tsog2018[3400:3450], sog2018[3400:3450])
plt.title('Salinity time series near Freshet at Strait of Georgia point 2018')
plt.xlabel('Time')
plt.ylabel('Salinity (g kg-1)')the lowest salinity was 2.1126484870910645 g kg-1, occuring at point [3439]

## Overall_____no_output_____

freshmin = years[np.argmin(freshet_salinity)]; freshmax = years[np.argmax(freshet_salinity)]
freshest = min(freshet_salinity); staleest = max(freshet_salinity)
text = "The year with the freshest freshet was {} (min salinity {} g kg-1)\nThe year with the least fresh freshet was {} (min salinity {} g kg-1)".format(freshmin, freshest, freshmax, staleest)
print(text)The year with the freshest freshet was 2018 (min salinity 2.1126484870910645 g kg-1)
The year with the least fresh freshet was 2017 (min salinity 10.337932586669922 g kg-1)


# Notebook from alabatie/moments-dnns
Path: notebooks/Complements on fully-connected networks.ipynb

# Fully-Connected Networks

This notebook considers experiments equivalent to Fig. $2$, $3$, $4$, $5$ for fully-connected networks. All parameters are left unchanged apart from the parameters `batch_size` and `kernel_size`. The fully-connected behaviour is simply enforced by setting `kernel_size = 1` with the effect of flattening the original images and reducing the spatial extent to $n=1$.

For the experiments of Fig. $3$, $4$, $5$, the number of realizations is reduced to `num_realizations = 200`. This has the effect of making the curves slightly more noisy, but it is already enough to gain insights._____no_output_____## Moments of Vanilla Nets

The results here are equivalent to Fig. $2$ with the only difference that $\log \mu_2(\mathrm{d}\mathbf{x}^l)$ diffuses faster. _____no_output_____

from moments_dnns.run_experiment import run_experiment
from moments_dnns.manage_experiments import prune_experiment

# this experiment ran in the cloud, but it is left here to show the function call
run_experiment(architecture='vanilla', total_depth=200, kernel_size=1, num_channels=128,
               dataset='cifar10', # boundary conditions are not relevant for fully-connected networks
               batch_size=1024, num_realizations=10000, name_experiment='vanilla_histo_fc',
               num_computations=4, # only compute 4 moments, at depth 50, 100, 150, 200
               compute_reff_signal=False, compute_reff_noise=False, verbose=True)

# in the npy/ folder, only the moments relevant for the plots are kept in order to limit disk space
prune_experiment(type_plot='vanilla_histo', name_experiment='vanilla_histo_fc')_____no_output_____from moments_dnns.manage_experiments import load_experiment
from moments_dnns.plots import plot_vanilla_histo
%matplotlib inline

moments = load_experiment('vanilla_histo_fc')
plot_vanilla_histo(moments, use_tex=True, name_fig=None)_____no_output_____

## Evolution of Vanilla Nets

The evolution of fully-connected vanilla nets is equivalent to the evolution of Fig. $3$ with the convergence to $\delta \chi^l \to 1$ and the convergence to the pathology of one-dimensional signal: $r_\text{eff}(\mathbf{x}^l) \to 1$. _____no_output_____

from moments_dnns.run_experiment import run_experiment
from moments_dnns.manage_experiments import prune_experiment

# this experiment ran in the cloud, but it is left here to show the function call
run_experiment(architecture='vanilla', total_depth=200, kernel_size=1, num_channels=512,
               dataset='cifar10',  # boundary conditions are not relevant for fully-connected networks
               batch_size=64, num_realizations=200, name_experiment='vanilla_fc',
               compute_reff_signal=True, compute_reff_noise=False, verbose=True)

# in the npy/ folder, only the moments relevant for the plots are kept in order to limit disk space
prune_experiment(type_plot='vanilla', name_experiment='vanilla_fc')_____no_output_____from moments_dnns.manage_experiments import load_experiment
from moments_dnns.plots import plot_vanilla
%matplotlib inline

moments_vanilla = load_experiment('vanilla_fc')
plot_vanilla(moments_vanilla, use_tex=True, name_fig=None)_____no_output_____

## Evolution of Batch-Normalized Feedforward Nets

*As in Fig. $4$, fully-connected batch-normalized feedforward nets are subject to the pathology of exploding sensitivity. However, there is an earlier saturation of the pathologies in the signal, with relatively large $r_\text{eff}(\mathbf{x}^l)$ and low $\mu_4(\mathbf{z}^l)$.*

The reason is that rare events cannot be arbitrarily rare for a distribution comprised of a finite number of point masses, $\frac{1}{M} \sum^M_{i=1} \delta_{p_i}$. Indeed it is shown in Section A of the paper that the kurtosis of $M$ point masses is roughly bounded by $M$. For convolutional networks, $M$ is larger than the batch size since the data distribution depends on the input $\mathbf{x}$ and the spatial position $\alpha$

Smaller batch size leads to larger $\delta^{}_\phi \chi^l$ and smaller $\delta^{}_\text{BN} \chi^l$. Interestingly, this also leads to the noise geting ill-conditioned with lower $r_\text{eff}(\mathrm{d}\mathbf{x}^l)$._____no_output_____

from moments_dnns.run_experiment import run_experiment
from moments_dnns.manage_experiments import prune_experiment

# these experiments ran in the cloud, but they are left here to show the function calls
run_experiment(architecture='bn_ff', total_depth=200, kernel_size=1, num_channels=512,
               dataset='cifar10',  # boundary conditions are not relevant for fully-connected networks
               batch_size=64, num_realizations=200, name_experiment='bn_ff_fc_64',
               compute_reff_signal=True, compute_reff_noise=True, verbose=True)

run_experiment(architecture='bn_ff', total_depth=200, kernel_size=1, num_channels=512,
               dataset='cifar10',  # boundary conditions are not relevant for fully-connected networks
               batch_size=128, num_realizations=200, name_experiment='bn_ff_fc_128',
               compute_reff_signal=True, compute_reff_noise=True, verbose=True)

run_experiment(architecture='bn_ff', total_depth=200, kernel_size=1, num_channels=512,
               dataset='cifar10',  # boundary conditions are not relevant for fully-connected networks
               batch_size=256, num_realizations=200, name_experiment='bn_ff_fc_256',
               compute_reff_signal=True, compute_reff_noise=True, verbose=True)

run_experiment(architecture='bn_ff', total_depth=200, kernel_size=1, num_channels=512,
               dataset='cifar10',  # boundary conditions are not relevant for fully-connected networks
               batch_size=512, num_realizations=200, name_experiment='bn_ff_fc_512',
               compute_reff_signal=True, compute_reff_noise=True, verbose=True)

# in the npy/ folder, only the moments relevant for the plots are kept in order to limit disk space
prune_experiment(type_plot='bn_ff', name_experiment='bn_ff_fc_64')
prune_experiment(type_plot='bn_ff', name_experiment='bn_ff_fc_128')
prune_experiment(type_plot='bn_ff', name_experiment='bn_ff_fc_256')
prune_experiment(type_plot='bn_ff', name_experiment='bn_ff_fc_512')_____no_output_____from moments_dnns.manage_experiments import load_experiment
from moments_dnns.plots import plot_bn_ff
%matplotlib inline

moments_bn_ff = load_experiment('bn_ff_fc_64')
plot_bn_ff(moments_bn_ff, use_tex=True, name_fig=None)_____no_output_____from moments_dnns.manage_experiments import load_experiment
from moments_dnns.plots import plot_bn_ff
%matplotlib inline

moments_bn_ff = load_experiment('bn_ff_fc_128')
plot_bn_ff(moments_bn_ff, use_tex=True, name_fig=None)_____no_output_____from moments_dnns.manage_experiments import load_experiment
from moments_dnns.plots import plot_bn_ff
%matplotlib inline

moments_bn_ff = load_experiment('bn_ff_fc_256')
plot_bn_ff(moments_bn_ff, use_tex=True, name_fig=None)_____no_output_____from moments_dnns.manage_experiments import load_experiment
from moments_dnns.plots import plot_bn_ff
%matplotlib inline

moments_bn_ff = load_experiment('bn_ff_fc_512')
plot_bn_ff(moments_bn_ff, use_tex=True, name_fig=None)_____no_output_____

## Evolution of Batch-Normalized ResNets

*The evolution of batch-normalized resnets is the slowed down version of the evolution of batch-normalized feedforward nets*, with perfect power-law fit of $\chi^l$ and the subplots (a), (c), (d) very similar to the subplots (a), (c), (d) for batch-normalized feedforward nets._____no_output_____

from moments_dnns.run_experiment import run_experiment
from moments_dnns.manage_experiments import prune_experiment

# this experiment ran in the cloud, but it is left here to show the function call
run_experiment(architecture='bn_res', total_depth=500, kernel_size=1, num_channels=512,
               dataset='cifar10', res_depth=2,  # boundary conditions are not relevant for fully-connected networks
               batch_size=64, num_realizations=200, name_experiment='bn_res_fc',
               compute_reff_signal=True, compute_reff_noise=True, verbose=True)

# in the npy/ folder, only the moments relevant for the plots are kept in order to limit disk space
prune_experiment(type_plot='bn_res', name_experiment='bn_res_fc')_____no_output_____from moments_dnns.manage_experiments import load_experiment
from moments_dnns.plots import plot_bn_res
%matplotlib inline

moments_bn_res = load_experiment('bn_res_fc')
plot_bn_res(moments_bn_res, use_tex=True , name_fig=None)_____no_output_____


# Notebook from pmezap/computational-macroeconomics
Path: Lecture Notebooks/Econ126_Class_12_blank.ipynb

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
plt.style.use('classic')
%matplotlib inline_____no_output_____

# Class 12: Introduction to the `linearsolve` Module

In general, dynamic stochastic general equilibrium (DSGE) models do not admit analytic (i.e., pencil-and-paper) solutions and they are time-consuming to work with. The `linearsolve` module approximates, solves, and simulates DSGE models and therefore makes DSGE models easier to use.

## Installing `linearsolve`

`linearsolve` is not included in the Anaconda Python installation and so before you can import it, you need to download and install the `linearsolve` package from PyPI, the Python Package Index. In Windows, open the Anaconda Prompt and in Mac, open the Terminal and run the following commmand:

    pip install linearsolve

You only have to install the package once._____no_output_____

# Import the linearsolve under the 'ls' namespace
_____no_output_____

## Example: A One-Equation Model of TFP

Consider the following AR(1) specification for $\log$ TFP:

\begin{align}
\log A_{t+1} & = \rho \log A_t + \epsilon_{t+1}
\end{align}

where $\epsilon_{t+1} \sim \mathcal{N}(0,\sigma^2)$. Let's simulate the model with `linearsolve`. To do this we need to do several things:

1. Create a Pandas series that stores the names of the parameters of the model.
2. Define a function that returns the equilibrium conditions of the model solved for zero.
3. Initialize an instance of the `linearsolve.model` class
4. Compute and input the steady state of the model.
5. Approximate and solve the model.
6. Compute simulations of the model._____no_output_____**Step 1:** Create a variable called `parameters` that stores parameter values as a Pandas Series._____no_output_____

# Create a variable called 'parameters' that stores the model parameter values in a Pandas Series

# Print stored parameter values
_____no_output_____# Create variable called 'varNames' that stores the variable names in a list with state variables ordered first

# Create variable called 'shockNames' that stores an exogenous shock name for each state variable.
_____no_output_____

**Step 2:** Define a function called `equilibrium_equations` that evaluates the equilibrium
equations of model solved for zero. The function should accept three arguments:

1. `variables_forward`: Values of $t+1$-dated variables
2. `variables_current`: Values of $t-1$-dated variables
3. `parameters`: Pandas Series with the parameters for the model

The function should return a NumPy array of the model's equilibrium conditions solved for zero._____no_output_____

# Define a function that evaluates the equilibrium conditions of the model solved for zero. PROVIDED
def equilibrium_equations(variables_forward,variables_current,parameters):

    # Create a variable called 'p' that storres the model's parameters. PROVIDED.
    p = parameters

    # Create variable called 'cur' that stores the values of current (date t) variables. PROVIDED.
    cur = variables_current

    # Create variable called 'fwd' that stores the values of one-period-ahead (date t+1) variables. PROVIDED.
    fwd = variables_forward

    # Create variable called 'tfp_proc' that returns the law of motion for TFP solved for zero (exogenous shock excluded)

    # Return equilibrium conditions stacked in a numpy array. PROVIDED.
    return np.array([

_____no_output_____

**Step 3:** Initialize a model instance using the `ls.model()` function. The

1. `equations`: Name of function that stores the equilibrium conditions
2. `nstates`: Number of *state* variables (i.e., variables that are *predetermined*)
3. `varNames`: List of the names of the endogenous variables
4. `shockNames`: List of the names of the exogenous shocks
5. `parameters`: Pandas Series of parameter values_____no_output_____

# Initialize the model into a variable named 'rbc_model' using the ls.model() function

_____no_output_____

**Step 4:** Set the steady state of the model. Either use the `.compute_ss()` method which requires an initial guess of what the steady state is. Or set the steady state `.ss` attribute directly._____no_output_____

# Compute the steady state numerically using .compute_ss() method of model

_____no_output_____# Set the .ss attribute of model directly
_____no_output_____

**Step 5:** Compute a transform the model into a log-linear approximation around the nonstochastic steady state. Then rewrite the equilibrium conditions so that all endogenous variables are expressed as linear functions of state variables and exogenous shocks._____no_output_____

# Appproximate and solve using the .approximate_and_solve() method of model
_____no_output_____

**Step 6:** Simulate the model using one of the following methods:

1. `.impulse()`: Compute impulse responses to a one-time shock to each exogenous variables. Results stored in `.irs` attribute.
2. `.stoch_sim()`: Compute stochastic simulation. Results stored in `.simulated` attribute.

First, we'll compute an impulse response simulation. Let's consider the effect of a one time shock to $\epsilon$ of 0.01 in period 5. Simulate 41 periods._____no_output_____

# Compute impulse response of a to a one-time shock to e_a
_____no_output_____

The impulse response simulations are stored in the `.irs` attribute as a dictionary with keys equal to the names of the exogenous shocks. _____no_output_____

# Print model.irs
_____no_output_____

Let's look at the first 10 rows of the `'e_a'` element of `model.irs`._____no_output_____

# Print first 10 rows of the element in model.irs that corresponds to the shock to TFP
_____no_output_____# Plot simulated impulse response to e_a
_____no_output_____

Next, we'll use the `.stoch_sim()` method to compute a stochastic simulation. The method takes arguments:

1. `seed`: Seed of NumPy RNG. (Optional)
2. `T`: Number of periods to simulate
3. `covMat`: Covariance matrix for exogenous shock process

The simulation should be for 201 periods._____no_output_____

# Compute stochastic simulation

# Print the first 10 rows of `model.simulated`
_____no_output_____

The stochastic simulations are stored in the `.stoch_sim` attribute as a Pandas `DataFrame`._____no_output_____

# Plot the stochastic simulation
_____no_output_____

## Example 2: The Stochastic Solow Growth Model Revisited

Now consider the following system of equations:

\begin{align}
Y_t & = A_t K_t^{\alpha} \\
K_{t+1} & = sY_t + (1-\delta) K_t\\
\log A_{t+1} & = \rho \log A_t + \epsilon_{t+1}
\end{align}

where $\epsilon_{t+1} \sim \mathcal{N}(0,\sigma^2)$. Let's simulate the model with `linearsolve`. Before proceding, let's also go ahead and rewrite the model with all variables moved to the lefthand side of the equations:

\begin{align}
0 & = A_t K_t^{\alpha} - Y_t \\
0 & = sY_t + (1-\delta) K_t - K_{t+1}\\
0 & = \rho \log A_t + \epsilon_{t+1} - \log A_{t+1}
\end{align}

Capital and TFP are called *state variables* because they're $t+1$ values are predetermined. Output is called a *costate* or *control* variable. Note that the model as 3 endogenous variables with 2 state variables.

Use the following values for the simulation:

| $\rho$ | $\sigma$ | $s$ | $\alpha$ | $\delta $ | $T$  |
|--------|----------|-----|----------|-----------|------|
| 0.75   | 0.006    | 0.1 | 0.35     |  0.025    | 201  |_____no_output_____### Initialization, Approximation, and Solution

The next several cells initialize the model in `linearsolve` and then approximate and solve it._____no_output_____

# Create a variable called 'parameters' that stores the model parameter values in a Pandas Series

# Print the model's parameters
_____no_output_____# Create variable called 'varNames' that stores the variable names in a list with state variables ordered first

# Create variable called 'shockNames' that stores an exogenous shock name for each state variable.
_____no_output_____# Define a function that evaluates the equilibrium conditions of the model solved for zero. PROVIDED
def equilibrium_equations(variables_forward,variables_current,parameters):

    # Parameters. PROVIDED
    p = parameters

    # Current variables. PROVIDED
    cur = variables_current

    # Forward variables. PROVIDED
    fwd = variables_forward

    # Production function

    # Capital evolution

    # Exogenous tfp

    # Stack equilibrium conditions into a numpy array

_____no_output_____

Next, initialize the model._____no_output_____

# Initialize the model into a variable named 'rbc_model'

_____no_output_____# Compute the steady state numerically using .compute_ss() method of model

# Print the computed steady state
_____no_output_____# Find the log-linear approximation around the non-stochastic steady state and solve using .approximate_and_solve() method of model
_____no_output_____

### A Few Details About the Approximation (Optional)

The previous step constructs a log-linear approximation of the model and then solves for the endogenous variables as functions of the state variables and exogenous shocks only.

View the approximated model by calling the `.approximated()` method._____no_output_____

# Print the log-linear approximation to the models's equilibrium conditions
_____no_output_____

Each variable represents the log-deviation from the steady state of the respective variable in our model. For example, the variable `y[t]` means $\log(Y_t) - \log(Y)$ in terms of the stochastic Solow model. But how do these equations relate to the original model?

The first equation appears to be:

\begin{align}
0 &= -2.1095\cdot a_t-0.7383\cdot k_t + 2.1095\cdot y_t
\end{align}

Note that dividing by 2.1095 and solving for $y_t$ yields:

\begin{align}
y_t &= a_t +0.3499\cdot k_t,
\end{align}

so the coefficient on $k_t$ appears to be close to $\alpha=0.35$. We can derive this linear equation directly. First, start with the production function:

\begin{align}
Y_t &= A_t K_t^{\alpha}.
\end{align}

Then divide both sides by steady state output:

\begin{align}
\frac{Y_t}{Y} &= \frac{A_t K_t^{\alpha}}{AK^{\alpha}} \, = \, \frac{A_t}{A}\frac{K_t^{\alpha}}{K^{\alpha}}.
\end{align}

Then, take the log of both sides:
\begin{align}
\log\left(\frac{Y_t}{Y} \right)&= \log\left(\frac{A_t}{A}\right) + \alpha\log\left(\frac{K_t}{K}\right).
\end{align}

finally, letting $y_t = \log(Y_t/Y)$, k_t = $\log(K_t/K)$, and $a_t = \log(A_t/A)$, we have:

\begin{align}
y_t &= a_t + \alpha k_t.
\end{align}

However, understanding this process isn't as important as being able to interpret the graphs and statistics that compute using the output of `linearsolve`.

### A Few Details About the Solution (Optional)

It's also worth seeing what it means for a model to be *solved*. After `linearsolve` computes the log-linear approximation to the model, it solves for each endogenous variable as a function of state variables only. View the solved model by calling the `.solved()` method._____no_output_____

# Print the solved model
_____no_output_____

### Impulse Responses

Compute a 41 period impulse responses of the model's variables to a 0.01 unit shock to TFP in period 5._____no_output_____

# Compute impulse responses

# Print the first 10 rows of the computed impulse responses.
_____no_output_____# Plot the computed impulse responses to a TFP shock

_____no_output_____

### Stochastic Simulation

Compute a 201 period stochastic simulation of the model's variables. Set the variance of $\epsilon_t$ to $\sigma^2$ and the variance of the shock to capital to 0 so that the covariance matrix for the shock process is:

\begin{align}
\text{Covariance matrix} & = \left[\begin{array}{cc} \sigma^2 & 0\\ 0 & 0\end{array} \right]
\end{align}_____no_output_____

# Compute stochastic simulation and print the simulated values.

# Print first 10 rows of model.simulated
_____no_output_____# Plot the computed stochastic simulation

_____no_output_____# Compute standard deviations of simulated TFP, output, and capital
_____no_output_____# Compute correlation coefficients of simulated TFP, output, and capital
_____no_output_____


# Notebook from instamatic-dev/pyserialem
Path: demos/montage_processing_serialem.ipynb

%load_ext autoreload
%autoreload 2_____no_output_____

# PySerialEM - montaging

https://github.com/instamatic-dev/pyserialem

This notebook shows how to process a grid montage acquired using `SerialEM`. The data for this demo were collected on a zeolite sample (2020-02-12), using a JEOL JEM-1400 @ 120 kV in combination with a TVIPS F-416 camera.

The data are available from zenodo: https://doi.org/10.5281/zenodo.3923718

These data were chosen, because the stitching from SerialEM was particularly bad. We will show an example of how `PySerialEM` can be used to try to get a better montage.

For this demo to work, change the `work` directory below to point at the right location._____no_output_____

from pyserialem import Montage
import numpy as np
from pathlib import Path
np.set_printoptions(suppress=True)

# work directory
work = Path(r"C:\s\2020-02-12\serialem_montage")
work_____no_output_____

## Setting up the montage

Load the `gm.mrc` file and the associated images. For SerialEM data, the gridshape must be specified, because it cannot be obtained from the data or `.mdoc` direction. There are also several parameters to tune the orientation of the images to match them with the input of the stagematrix (if needed). First we must get the coordinate settings to match those of SerialEM. These variables ap_____no_output_____

m = Montage.from_serialem_mrc(
    work / 'gm.mrc',
    gridshape=(5,5),
    direction='updown',
    zigzag=True,
    flip=False,
    image_rot90 = 3,
    image_flipud = False,
    image_fliplr = True,
)

m.gridspec_____no_output_____

First, we can check what the data actually look like. To do so, we can simply `stitch` and `plot` the data using a `binning=4` to conserve a bit of memory. This naively plots the data at the expected positions. Although the stitching is not that great, it's enough to get a feeling for the data.

Note that SerialEM includes the pixel coordinates in the `.mdoc` file, so it is not necessary to calculate these again. Instead, the `PieceCoordinates` are mapped to `m.coords`._____no_output_____

# Use `optimized = False` to prevent using the aligned piece coordinates
m.stitch(binning=4, optimized=False)
m.plot()_____no_output_____

SerialEM has also already calculated the aligned image coordinates (`AlignedPieceCoordsVS`/`AlignedPieceCoords`). These can be accessed via the `.optimized_coords` attribute. To plot, you can do the following:_____no_output_____

# optimized = True is the default, so it can be left out
m.stitch(binning=4, optimized=True)
m.plot()

montage_serialem = m.stitched  # store reference for later_____no_output_____

The stitching from SerialEM is particularly bad, so we can try to optimize it using the algorithm in `pyserialem`.
First, we must ensure that we entered the gridspec correctly. If the layout of the tiles below does not look right (i.e. similar to above), go back to loading the `Montage` and fiddle with the rotation of the images. The operation below just places the tiles at the positions calculated by `pyserialem`._____no_output_____

m.calculate_montage_coords()
m.stitch(binning=4, optimized=False)
m.plot()_____no_output_____

It is still possible to try to get better stitching using the algorithm in `pyserialem`:

 1. Better estimate the difference vectors between each tile using cross correlation
 2. Optimize the coordinates of the difference vectors using least-squares minimization

This approach is based on *Globally optimal stitching of tiled 3D microscopic image acquisitions* by Preibish et al., Bioinformatics 25 (2009), 1463–1465 (https://doi.org/10.1093/bioinformatics/btp184).

Some metrics, such as the obtained shifts and FFT scores are plotted to evaluate the stitching._____no_output_____

# Use cross correlation to get difference vectors
m.calculate_difference_vectors(
    threshold=0.08,
#     method='skimage',
    plot=False
)

# plot the fft_scores
m.plot_fft_scores()

# plot the pixel shifts
m.plot_shifts()

# get coords optimized using cross correlation
m.optimize_montage_coords(plot=True)

# stitch image, use binning 4 for speed-up and memory conservation
m.stitch(binning=4)

# plot the stitched image
m.plot()

montage_pyserialem = m.stitched  # store reference for later_____no_output_____

It's not a perfect stitching, but much better than what SerialEM produced! I believe the reason is that SerialEM does some adjustments to the stageposition as it is moving. Below is an example of the same grid map collected with [instamatic](http://github.com/instamatic-dev/instamatic), using the same coordinates and imaging conditions, reconstructed with the same algorithm._____no_output_____

import tifffile
import matplotlib.pyplot as plt

with tifffile.TiffFile(work / 'stitched_instamatic.tiff') as f:
    montage_instamatic = f.asarray()_____no_output_____fig, (ax0, ax1, ax2) = plt.subplots(ncols=3, figsize=(20,10))

ax0.imshow(montage_serialem)
ax0.set_title('Data: SerialEM\n'
              'Stitching: SerialEM')

ax1.imshow(montage_pyserialem)
ax1.set_title('Data: SerialEM\n'
              'Stitching: PySerialEM')

ax2.imshow(montage_instamatic)
ax2.set_title('Data: Instamatic\n'
              'Stitching: PySerialEM');_____no_output_____

When the image has been stitched (with or without optimization), we can look for the positions of the grid squares/squircles.

First, we should tell `pyserialem` about the imaging conditions by setting the stagematrix to relate the pixelpositions back to stage coordinates.

The easiest way to do it is to pass the `StageToCameraMatrix` directly. It can be found in `SerialEMcalibrations.txt`. Look for the last number, which gives the magnification.

(They can also be set directly via `.set_stagematrix` and `.set_pixelsize`)_____no_output_____

StageToCameraMatrix = "StageToCameraMatrix 10 0 8.797544 0.052175 0.239726 8.460119   0.741238   100"
m.set_stagematrix_from_serialem_calib(StageToCameraMatrix)

m.stagematrix_____no_output_____

This also sets the pixelsize._____no_output_____

m.pixelsize_____no_output_____

To find the holes, call the method `.find_holes`. The grid squares are identified as objects roughly sized `diameter` with a tolerance of 10%. The median as well as 5/95 percentiles are printed to evaluate the hole size distribution. By default the `otsu` method is used to define the threshold, but the threshold can be changed if the segmentation looks poor._____no_output_____

stagecoords, imagecoords = m.find_holes(
    plot=True,
    tolerance=0.2)_____no_output_____

IF a `.nav` file was saved, the stage coordinates can be added and read back into `SerialEM`._____no_output_____

from pyserialem import read_nav_file, write_nav_file

nav = read_nav_file(work / "nav.nav")
map_item = nav[0]
items = map_item.add_marker_group(coords=stagecoords/1000, kind='stage')
write_nav_file("nav_new.nav", map_item, *items)_____no_output_____

It is possible to optimize the stage coordinates for more efficient navigation. In this example, the total stage movement can be reduced by about 75%, which will save a lot of time. The function uses the _two-opt_ algorithm for finding the shortest path: https://en.wikipedia.org/wiki/2-opt._____no_output_____

from pyserialem.navigation import sort_nav_items_by_shortest_path

stagecoords = sort_nav_items_by_shortest_path(
    stagecoords,
    plot=True
)_____no_output_____

We can re-run the command (or set the `threshold` to something like `0.01`) to try to get a better path. In this case it's possible to improve it a little bit more._____no_output_____

stagecoords = sort_nav_items_by_shortest_path(
    stagecoords,
    threshold=0.01,
    plot=True
)_____no_output_____


# Notebook from ucsd-ccbb/crispr_network_analysis
Path: network_analysis_CRISPR_essential_genes_18-04-25.ipynb

# Network analysis of CRISPR essential genes in pancreatic cancer

------------

Author: Brin Rosenthal (sbrosenthal@ucsd.edu)

-----------_____no_output_____

import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
import networkx as nx
import seaborn as sns

import community

import mygene
mg = mygene.MyGeneInfo()

# latex rendering of text in graphs
import matplotlib as mpl
mpl.rc('text', usetex = False)
mpl.rc('font', family = 'serif')

% matplotlib inline

import visJS2jupyter.visJS_module
import visJS2jupyter.visualizations
_____no_output_____

# Load the CRISPR data and RNAseq data

Update 12/07/17: Use intersection of RNAseq fdr significant genes and CRISPR significant genes as seeds

3DGvs3DV --> selects genes sensitive to drug
3DVvsT21 --> selects genes essential for 'stemness'_____no_output_____

# ---- Now updated file from Nikki ------

rna_df = pd.read_excel('../data/rnaseq_crispr_171207.xlsx',sheetname='RNA-seq',index_col='symbol')
rna_df.head()
# crispr_T21_df = pd.read_csv('../data/RS_MAGeCK_comparison/RS_T0_T21.gene_summary.txt',sep='\t',index_col='Unnamed: 0')
# crispr_T21_df.head()

# crispr_3DV_df = pd.read_csv('../data/RS_MAGeCK_comparison/RS_3DV.gene_summary.txt',sep='\t',index_col='Unnamed: 0')
# crispr_3DV_df.head(10)_____no_output_____#crispr_3DV_df = pd.read_excel('../data/rnaseq_crispr_171207.xlsx',sheetname='Crispr 3DV',skiprows=1,
#                             index_col='Unnamed: 0')

# load new crispr data
crispr_3DV_df = pd.read_csv('../data/3DV_enhancers_p_fdr_180119.txt',sep='\t',index_col='Unnamed: 0')

crispr_3DV_df.head()_____no_output_____crispr_3DV_df.loc['Rorc']_____no_output_____print(len(crispr_3DV_df[crispr_3DV_df['permutation FDR']<0.5]))94
# intersection of crispr-significant and rna-expressed

rna_expressed = rna_df.index.tolist()

crispr_rna_expressed = crispr_3DV_df[crispr_3DV_df['permutation FDR']<0.5]
crispr_rna_expressed = crispr_rna_expressed.loc[rna_expressed].dropna()
crispr_rna_expressed = crispr_rna_expressed.sort_values('permutation FDR',ascending=True)

print(len(crispr_rna_expressed))
crispr_rna_expressed.head()
crispr_rna_expressed = crispr_rna_expressed.index.tolist() # cast to gene names57
# intersection dataset filtered by FC

rna_FC2 = rna_df[rna_df['log2 GFP/NEG'].abs()>2]
crispr_rna_fc =  crispr_3DV_df[crispr_3DV_df['permutation FDR']<0.5] # filter by FDR
crispr_rna_fc = crispr_rna_fc.loc[rna_FC2.index.tolist()].dropna() # filter by rna FC
crispr_rna_fc = crispr_rna_fc.sort_values('permutation FDR',ascending=True)

print(len(crispr_rna_fc))
crispr_rna_fc
crispr_rna_fc = crispr_rna_fc.index.tolist() # cast to gene names10
crispr_rna_fc_____no_output_____

# Load Interactome (for now- mouse STRING)

Download from here:
https://string-db.org/cgi/download.pl?sessionId=Df4zRlbs4NEr&species_text=Mus+musculus_____no_output_____

def load_STRING(datafile = '/Users/brin/Documents/CCBB_tickets_data/STRING/mouse/10090.protein.links.v10.5.txt',conf_thresh=700,species='mouse',
               cap_choice='lower'):
    '''
    Helper function to parse and load STRING network

    '''

    string_df = pd.read_csv(datafile,sep=' ')
    string_df = string_df.loc[string_df['combined_score']>conf_thresh]

    # create the network
    G_str = nx.Graph()
    G_str.add_weighted_edges_from(zip(string_df['protein1'],string_df['protein2'],string_df['combined_score']))

    # convert from ensembleID to gene symbol
    string_genes_temp = [n[n.find('.')+1:] for n in G_str.nodes()]

    mg_temp = mg.querymany(string_genes_temp,scopes='ensemblprotein',fields='symbol',species=species)
    ensembl_list = [x['query'] for x in mg_temp]
    symbol_list = [x['symbol'] if 'symbol' in x.keys() else 'None' for x in mg_temp]
    ensembl_to_symbol = dict(zip(ensembl_list,symbol_list))
    ensembl_to_symbol = pd.Series(ensembl_to_symbol)

    # relabel nodes with symbols
    G_str = nx.relabel_nodes(G_str,dict(zip(G_str.nodes(),string_genes_temp)))
    G_str = nx.relabel_nodes(G_str,dict(ensembl_to_symbol[ensembl_to_symbol!='None']))  # only keep the proteins that have associated genes

    if cap_choice=='lower':
        genes = [g[0]+g[1:].lower() for g in G_str.nodes()]
        G_str = nx.relabel_nodes(G_str,dict(zip(G_str.nodes(),genes)))
    elif cap_choice=='upper':
        genes = [g.upper() for g in G_str.nodes()]
        G_str = nx.relabel_nodes(G_str,dict(zip(G_str.nodes(),genes)))

    return G_str_____no_output_____# HIGH CONFIDENCE STRING
#G_str = load_STRING(datafile='/Users/brin/Documents/CCBB_tickets_data/STRING/mouse/10090.protein.links.v10.5.txt',
#                                     conf_thresh=700,species='mouse',cap_choice='lower')

G_str = nx.read_gpickle('/Users/brin/Documents/CCBB_tickets_data/STRING/mouse/G_str.gpickle')

# MEDIUM CONFIDENCE STRING
# G_str = load_STRING(datafile='/Users/brin/Documents/CCBB_tickets_data/STRING/mouse/10090.protein.links.v10.5.txt',
#                                      conf_thresh=400,species='mouse',cap_choice='lower')_____no_output_____# remove UBC
if 'Ubc' in G_str.nodes():
    G_str.remove_node('Ubc')

_____no_output_____'Lipt2' in G_str.nodes()_____no_output_____np.intersect1d(crispr_rna_fc,G_str.nodes())_____no_output_____len(G_str.edges())_____no_output_____

# Select which interactome to use_____no_output_____

Gint = G_str

if 'None' in Gint.nodes():
    Gint.remove_node('None')
_____no_output_____# development versions of return_node_to_color and return_edge_to_color

def return_node_to_color(G,field_to_map='degree',cmap=mpl.cm.jet,alpha = 1.0, color_vals_transform = None,ceil_val=10,
                        color_max_frac = 1.0,color_min_frac = 0.0,vmin=None,vmax=None):

    '''
    Function to return a dictionary mapping nodes (keys) to colors (values), based on the selected field_to_map.
        - field_to_map must be a node attribute
        - cmap must be a valid matplotlib colormap
        - color_max_frac and color_min_frac allow user to set lower and upper ranges for colormap

    '''

    nodes_with_data = [(n[0],n[1][field_to_map]) for n in G.nodes(data=True)]

    if color_vals_transform == 'log':
        nodes,data = zip(*nodes_with_data)
        min_dn0 = np.nanmin([d for d in data if d>0])
        data = [np.log(np.max([d,min_dn0])) for d in data]  # set the zero d values to minimum non0 value
        data = [(d-np.nanmin(data)) for d in data] # shift so we don't have any negative values
        nodes_with_data = zip(nodes,data)

    elif color_vals_transform == 'sqrt':
        nodes,data = zip(*nodes_with_data)
        data = [np.sqrt(d) for d in data]
        nodes_with_data = zip(nodes,data)

    elif color_vals_transform == 'ceil':
        nodes,data = zip(*nodes_with_data)
        data = [min(d,ceil_val) for d in data]
        nodes_with_data = zip(nodes,data)
    else:
        nodes,data = zip(*nodes_with_data)

    # if vmin and vmax aren't set, set them to min and max of the data
    if vmin == None:
        vmin = np.nanmin(data)
    if vmax == None:
        vmax = np.nanmax(data)

    node_to_mapField = dict(nodes_with_data)

    color_to_mult = 256*(color_max_frac-color_min_frac)
    color_to_add = 256*color_min_frac
    print(color_to_mult)
    print(color_to_add)
    print(np.nanmax(list(node_to_mapField.values())))

    color_list = [np.multiply(cmap(int(float(node_to_mapField[d]-vmin)/(vmax-vmin)*color_to_mult+color_to_add)),256)
                  if ~np.isnan(node_to_mapField[d])
                  else [np.nan]
                  for d in G.nodes()]

    color_list = [(int(c[0]),int(c[1]),int(c[2]),alpha)
                  if ~np.isnan(c[0])
                  else (200,200,200,alpha)
                  for c in color_list]

    node_to_color = dict(zip(G.nodes(),['rgba'+str(c) for c in color_list]))

    return node_to_color

def return_edge_to_color(G,field_to_map='degree',cmap=mpl.cm.jet,alpha = 1.0, color_vals_transform = None,ceil_val=10,
                        vmin=None,vmax=None):

    '''
    Function to return a dictionary mapping edges (keys) to colors (values), based on the selected field_to_map.
        - field_to_map must be an edge attribute
        - cmap must be a valid matplotlib colormap

    '''

    edges_with_data = [(e[0],e[1],e[2][field_to_map]) for e in G.edges(data=True)]

    edges1,edges2,data = zip(*edges_with_data)

    if color_vals_transform == 'log':
        data = [np.log(d) for d in data]
        data = [(d-np.min(data)) for d in data] # shift so we don't have any negative values
        edges_with_data = zip(zip(edges1,edges2),data)

    elif color_vals_transform == 'sqrt':
        data = [np.sqrt(d) for d in data]
        edges_with_data = zip(zip(edges1,edges2),data)

    elif color_vals_transform == 'ceil':
        data = [max(d,ceil_val) for d in data]
        edges_with_data = zip(zip(edges1,edges2),data)
    else:

        edges_with_data = zip(zip(edges1,edges2),data)

    # if vmin and vmax aren't set, set them to min and max of the data
    if vmin == None:
        vmin = np.nanmin(data)
    if vmax == None:
        vmax = np.nanmax(data)

    edge_to_mapField = dict(edges_with_data)

    color_list = [np.multiply(cmap(int(float(edge_to_mapField[d]-vmin)/(vmax-vmin)*256)),256) for d in G.edges()]

    color_list = [(int(c[0]),int(c[1]),int(c[2]),alpha) for c in color_list]

    edge_to_color = dict(zip(G.edges(),['rgba'+str(c) for c in color_list]))

    return edge_to_color

def bias_position_by_partition(pos,partition,r=1.0,focal_partitions=None,scale_by_groupsize=False,
                              use_orig_angle=False):
    '''
    Bias the positions by partition membership, to group them together

    '''

    partition = pd.Series(partition)

    # if focal_partitions haven't been defined, use all of them
    if focal_partitions==None:
        focal_partitions = list(np.unique(partition))

    counter=-1
    for p in focal_partitions:
        counter+=1

        focal_nodes = partition[partition==p].index.tolist()

        # if use_orig_angle==True, move groups apart by their original angle
        if use_orig_angle:
            xlist = [pos[n][0] for n in focal_nodes]
            ylist = [pos[n][1] for n in focal_nodes]
            xmean = np.mean(xlist)
            ymean = np.mean(ylist)

            rho,phi = cart2pol(xmean,ymean)
        else:
            phi = float(counter)/(len(focal_partitions))*2*np.pi

        # if scale_by_groupsize==True, move smaller groups further towards periphery
        if scale_by_groupsize:
            rtemp=r*1.0/np.sqrt(len(focal_nodes))
        else:
            rtemp=r

        for n in focal_nodes:
            pos[n][0]+=pol2cart(rtemp,phi)[0]
            pos[n][1]+=pol2cart(rtemp,phi)[1]

    return pos

def cart2pol(x, y):
    rho = np.sqrt(x**2 + y**2)
    phi = np.arctan2(y, x)
    return(rho, phi)

def pol2cart(rho, phi):
    x = rho * np.cos(phi)
    y = rho * np.sin(phi)
    return(x, y)_____no_output_____

# Bottom-up approach: use network propagation to explore network neighborhood near genes of interest

- Start propagation from Roman's essential genes
_____no_output_____

# make a table of filtering steps, for MS
crispr_sig = crispr_3DV_df[crispr_3DV_df['permutation FDR']<0.5].index.tolist()
seed_filters_df = pd.DataFrame(np.zeros((len(crispr_sig),6)),index=crispr_sig)
seed_filters_df.columns=['CRISPR fdr','RNAseq logFC', 'CRISPR-essential','RNA-expressed','RNA-DE','in_STRING']
seed_filters_df['CRISPR fdr'].loc[crispr_sig]=crispr_3DV_df.loc[crispr_sig]['permutation FDR']
seed_filters_df['RNAseq logFC'].loc[crispr_sig]=rna_df.loc[crispr_sig]['log2 GFP/NEG']
seed_filters_df['CRISPR-essential'].loc[crispr_sig] = True

seed_filters_df['RNA-expressed'].loc[crispr_sig] = False
seed_filters_df['RNA-expressed'].loc[crispr_rna_expressed] = True

seed_filters_df['RNA-DE'].loc[crispr_sig] = False
seed_filters_df['RNA-DE'].loc[crispr_rna_fc] = True

seed_filters_df['in_STRING'].loc[crispr_sig]=False
crispr_sig_string = list(np.intersect1d(crispr_sig,G_str.nodes()))
seed_filters_df['in_STRING'].loc[crispr_sig_string] = True

#seed_filters_df.to_csv('network_prop_seed_filters.csv')

seed_filters_df.head(15)/Users/brin/anaconda/lib/python2.7/site-packages/pandas/core/indexing.py:179: SettingWithCopyWarning:
A value is trying to be set on a copy of a slice from a DataFrame

See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy
  self._setitem_with_indexer(indexer, value)
crispr_rna_expressed
_____no_output_____# seed genes for network propagation

#seed_genes = list(np.intersect1d(crispr_rna_expressed,Gint.nodes()))

seed_genes = list(np.intersect1d(crispr_rna_fc,Gint.nodes()))
print(len(seed_genes))5
# calculate the normalized adjacency matrix
Wprime = visJS2jupyter.visualizations.normalized_adj_matrix(Gint)_____no_output_____# run the network propagation simulation
Fnew = visJS2jupyter.visualizations.network_propagation(Gint,Wprime,seed_genes)_____no_output_____Fnew.sort_values(ascending=False,inplace=True)
Fnew.head(10)_____no_output_____Fnew_rank = pd.DataFrame(Fnew)
Fnew_rank['rank']=range(len(Fnew_rank))
Fnew_rank.loc[['Celsr1','Celsr2','Pear1','Megf11']]_____no_output_____# select the top 500 genes proximal to the seed set

hot_genes = Fnew.head(500).index.tolist()

# ---------- Just hot genes --------------------
G_hot = nx.subgraph(Gint,hot_genes)

G_hot = max(nx.connected_component_subgraphs(G_hot), key=len) # take the Largest connected component
len(G_hot.nodes())_____no_output_____'Wnt1' in G_hot.nodes()_____no_output_____# plot the hot subnetwork
# add pval and fc as node attributes
#nx.set_node_attributes(G_sub,'pval',dict(crispr_df.loc[G_sub.nodes()]['Pval']))
nx.set_node_attributes(G_hot,'3DV_FDR',
                       dict(crispr_3DV_df.loc[G_hot.nodes()]['permutation FDR']))
#nx.set_node_attributes(G_hot,'Statistic',
#                       dict(crispr_3DV_df.loc[G_hot.nodes()]['s']))

# add rna fc as node attributes
nx.set_node_attributes(G_hot,'rna_logFC',dict(rna_df.loc[G_hot.nodes()]['log2 GFP/NEG']))
nx.set_node_attributes(G_hot,'rna_lfdr',dict(rna_df.loc[G_hot.nodes()]['lfdr GFP-NEG']))

# cluster the network
partition = community.best_partition(G_hot)
nx.set_node_attributes(G_hot,'partition',partition)
partition= pd.Series(partition)_____no_output_____G_hot.nodes(data=True)[0]_____no_output_____from networkx.drawing.nx_agraph import graphviz_layout_____no_output_____#pos = nx.spring_layout(G_hot,weight=None)
pos = graphviz_layout(G_hot,prog='neato')
for n in pos.keys():
    pos[n]=np.array([pos[n][0]/200.0,pos[n][1]/200.0])

#pos = bias_position_by_partition(pos,partition,r=1.9,use_orig_angle=False,focal_partitions = [0,1,2,6,3,8,4,5,7])
#pos = bias_position_by_partition(pos,partition,r=1.9,use_orig_angle=False,focal_partitions = [0,1,2,4,3,5])
#pos = bias_position_by_partition(pos,partition,r=1.7,use_orig_angle=False, focal_partitions = [4,1,2,5,6,0,3])
pos = bias_position_by_partition(pos,partition,r=1.7,use_orig_angle=True)_____no_output_____len(G_hot.edges())_____no_output_____# draw the network and save for export to cytoscape

nodes = G_hot.nodes()
nodes_data = G_hot.nodes(data=True)
edges = G_hot.edges()
edges_data = G_hot.edges(data=True)
numnodes = len(nodes)
numedges = len(edges)

node_to_title = {}
node_to_size = {}
node_to_HL = {}
node_to_shape={}
node_to_label={}
for node in nodes_data:
    node_to_title[node[0]]=node[0]+': crispr 3DV FDR = '+str(node[1]['3DV_FDR'])
    node_to_title[node[0]]+='
group = ' + str(partition.loc[node[0]])
    node_to_title[node[0]]+='
rna log2FC = ' + str(node[1]['rna_logFC'])
    node_to_title[node[0]]+='
rna lfdr = ' + str(node[1]['rna_lfdr'])

    # only label seed nodes or significant nodes
    if node[0] in crispr_3DV_df.index:
        if (node[0] in seed_genes) or (np.abs(node[1]['rna_logFC'])>3.0):
            node_to_label[node[0]]=str(node[0])
        else:
            node_to_label[node[0]]=''
    else:
        node_to_label[node[0]]=''

    #node_to_size[node[0]]=-np.log(node[1]['lfdr'])-1.2 # subtract log(0.3)
    #node_to_size[node[0]]=np.abs(node[1]['Statistic'])-5 # map size to statistic
    #node_to_size[node[0]]=np.abs(node[1]['rna_logFC'])
    #node_to_size[node[0]]=-np.log(node[1]['rna_lfdr'])*3+2
    node_to_size[node[0]]=-np.log(node[1]['3DV_FDR'])*5+.1

    if node[0] in seed_genes:
        node_to_HL[node[0]]=1
    else:
        node_to_HL[node[0]]=0

#     if node[0] in seeds_crispr:
#         node_to_shape[node[0]]='triangle'
#     elif node[0] in seeds_rna:
#         node_to_shape[node[0]]='square'
    if node[0] in seed_genes:
        node_to_shape[node[0]]='triangle'
        node_to_size[node[0]]=25
    else:
        node_to_shape[node[0]]='dot'
        node_to_size[node[0]]=5

#node_to_color = return_node_to_color(G_hot,'partition',mpl.cm.Set1,color_max_frac=1)
#node_to_color = return_node_to_color(G_hot,'logFC_avg',mpl.cm.Blues_r,vmax=0,vmin=-3)
node_to_color = return_node_to_color(G_hot,'rna_logFC',mpl.cm.coolwarm,vmax=5,vmin=-5)
#node_to_color = return_node_to_color(G_hot,'Statistic',mpl.cm.autumn_r,vmax=5,vmin=0)

nodes_rna_logFC = nx.get_node_attributes(G_hot,'rna_logFC')
nodes_rna_lfdr = nx.get_node_attributes(G_hot,'rna_lfdr')
nodes_3DV_FDR = nx.get_node_attributes(G_hot,'3DV_FDR')
nodes_partition = nx.get_node_attributes(G_hot,'partition')
nodes_dict = [{"id":str(n),"color":node_to_color[n],
               "node_size":node_to_size[n],
               "border_width":node_to_HL[n],
               "node_shape":node_to_shape[n],
               "node_label":node_to_label[n],
                "rna_logFC":[nodes_rna_logFC[n] if not np.isnan(nodes_rna_logFC[n]) else 0.0][0],
               "rna_lfdr":[nodes_rna_lfdr[n] if not np.isnan(nodes_rna_lfdr[n]) else 0.0][0],
               "3DV_FDR":[nodes_3DV_FDR[n] if not np.isnan(nodes_3DV_FDR[n]) else 0.0][0],
               "partition":nodes_partition[n],
              "x":pos[n][0]*1000,
              "y":pos[n][1]*1000} for n in nodes
              ] # NOTE rna_logFC/lfdr = 0 means not expressed!

#                "title":node_to_title[n],

node_map = dict(zip(nodes,range(len(nodes))))  # map to indices for source/target in edges

# make the within-group edges darker
edge_to_color={}
for e in edges:
    p1 = partition[e[0]]
    p2 = partition[e[1]]
    if p1==p2:
        edge_to_color[e]="rgba(200, 250, 200,.3)"
    else:
        edge_to_color[e]="rgba(181, 227, 217,.2)"
#edge_to_color = return_edge_to_color(G_hot,field_to_map='weight',cmap=mpl.cm.BuGn,vmin=200,alpha=.2)

# only draw the strongest edges because we can't render all of them
edges_dict = [{"source":node_map[edges[i][0]], "target":node_map[edges[i][1]],
              "color":edge_to_color[edges[i]],"title":'test'} for i in range(len(edges)) if edges_data[i][2]['weight']>700]

# set some network-wide styles
visJS2jupyter.visJS_module.visjs_network(nodes_dict,edges_dict,
                          node_size_multiplier=10,
                          node_size_field='node_size',
                          node_size_transform = 'Math.sqrt',
                          node_color_highlight_border='red',
                          node_color_highlight_background='#D3918B',
                          node_color_hover_border='blue',
                          node_color_border='white',
                          node_color_hover_background='#8BADD3',
                          node_label_field='node_label',
                          node_font_size=120,
                          node_font_color='rgba(240,240,240,1.0)',
                          physics_enabled=False,
                          edge_color_highlight='#8A324E',
                          edge_color_hover='#8BADD3',
                          edge_width=3,
                          edge_smooth_enabled=False,
                          edge_smooth_type='cubicBezier',
                          edge_smooth_roundness=0.5,
                          scaling_factor=1,
                          node_scaling_label_draw_threshold=0,
                          graph_id=1,
                          graph_title='',export_network=False,export_file='crispr_p5_rnaFC_medium_string.json')

256.0
0.0
5.262947675
500
500
# save hot subnetwork genes
print(len(seed_genes))
pd.Series(G_hot.nodes()).to_csv('G_hot.txt',index=False)8
# print out genes in individual clusters
for n in partition[partition==4].index.tolist():
    print(n)Aars
Abcc8
Aff1
Ambp
Apc
Avpi1
Chd4
Comp
Ctse
Ddx5
ENSMUSP00000127885
Eed
Eml1
Eno2
Epor
Ereg
Etv5
Fmnl2
Fth1
Gan
Gm128
Gpr182
Grap
Hopx
Ide
Ighg2c
Lat2
Mbnl1
Mst1r
Naa50
Nagpa
Ndp
Nin
Nrip3
Nts
Omp
Opn3
Ptprf
Ran
Rasl2-9
Rit2
Rorc
Sh3gl1
Slc5a5
Slc6a3
Son
Tap2
Tcf23
Trim37
Ubr5
Ust
Zbtb32