# Notebook from jkombe/Evolutions
Path: examples/Exact Diagonalisation - Isinglattice.ipynb

<code>
# import all relevant libraries dependencies and set up the backend
import time
import IPython
import numpy as np
import matplotlib
import math
import pandas
import pickle
import subprocess
import matplotlib.pyplot as plt
%matplotlib inline

import scipy
import scipy.special
import scipy.optimize
import scipy.sparse.linalg as LA
import scipy.sparse as sparse
from scipy.io import loadmat
from scipy.optimize import curve_fit
from scipy.signal import argrelextrema

# add the ED.py functions to the notebook
import sys
#import hubbard as ED

from matplotlib import animation, rc
# rc('font',**{'family':'sans-serif','sans-serif':['Helvetica'],'size':12})
# rc('font',**{'family':'serif','serif':['FreeSerif'],'size':12})
rc('text', usetex=True)
from cycler import cycler

c1_a = '#003AF0'
c2_a = '#008F24'
c3_a = '#FE7B22'
c4_a = '#FE000C'
c5_a = '#FB30F2'
c6_a = '#82E0AA' #'#FFE135'
c7_a = '#9966CC'
c8_a = '#7FFFD4'
c9_a = '#66FF00'
c10_a = '#8B008B'
c11_a = '#000000'
colours = [c1_a,c2_a,c3_a,c4_a,c5_a,c6_a,c7_a,c8_a,c9_a,c10_a,c11_a]

matplotlib.rcParams['backend']='MacOSX'
matplotlib.rcParams['savefig.dpi']=250
matplotlib.rcParams['text.usetex']=True
# matplotlib.rcParams['text.latex.unicode']=True
# matplotlib.rcParams['axes.color_cycle'] = [c1_ac2_ac3_ac4_ac5_a]
matplotlib.rcParams['axes.prop_cycle'] = cycler('color',colours)
# matplotlib.rcParams['axes.'] = \in\
# matplotlib.rcParams['legend.fancybox']=True
matplotlib.rcParams['legend.frameon']=False
# matplotlib.rcParams['legend.fontsize']=10
matplotlib.rcParams['figure.figsize'] = (10,8)
# matplotlib.rcParams['axes.color_cycle'] = colors
# matplotlib.rcParams['axes.prop_cycle'] = colors_____no_output_____# define system parameters
L = 2
W = 4
couplings = [0., 0.5, 0.1]
hx = [0.5 for _ in range(L * W)]
hz = [0. for _ in range(L * W)]
dt = 0.01
total_time = 5.0
ave = 50
J = [[0. for _ in range(L * W)] for _ in range(L * W)]
for i in range(L * W):
    for j in range(L * W):
        s_r, s_c = i // W, i % W
        t_r, t_c = j // W, j % W
        if s_r == t_r:
            h_dist = min([abs(s_c - t_c), W - abs(s_c - t_c)])
            J[i][j] = couplings[h_dist]
        elif s_c == t_c:
            v_dist = min([abs(s_r - t_r), L - abs(s_r - t_r)])
            J[i][j] = couplings[v_dist]

times = np.linspace(0, total_time, num=10)
initial_state = [[0, 1, 0, 0],
                 [0, 0, 0, 0]]

p = {}
p['L'] = L # system length
p['W'] = W # system width
p['N'] = p['L'] * p['W'] # number of system in the system
p['hx'] = np.asarray(hx) # transverse Ising field for each site
p['hz'] = np.asarray(hz) # longitudinal Ising field for each site
p['J'] = np.asarray(J) # Ising interaction for all sites (i,j)
p['t_initial'] = times[0] # begin of time evolution
p['dt'] = (times[1] - times[0]) # time-step
p['t_final'] = times[-1] # end of time evolution_____no_output_____# do exact evolution
tic = time.time()
flat_initial_state = [item for sub in initial_state for item in sub]
# I think the definition of 0 and 1 is reversed (for me Z|0> = |0>) so I flip the initial state
# flat_initial_state = [1 - f for f in flat_initial_state]
sim, state_table = evolve(p, state_table, flat_initial_state, kind="list", trotterised=True)
toc = time.time()
print("Exact evolution took ", toc-tic, "s")/Users/Johannes/anaconda3/lib/python3.7/site-packages/scipy/sparse/_index.py:84: SparseEfficiencyWarning: Changing the sparsity structure of a csc_matrix is expensive. lil_matrix is more efficient.
  self._set_intXint(row, col, x.flat[0])
# plot
fig = plt.figure()
qubits = [(0, 0), (0, 1), (0, 2), (1, 3)]
for i, q in enumerate(qubits):
    site = p['W']*q[0]+q[1]
    # plt.plot(times, result[:, q[0], q[1]], label='{}'.format(q), color=colors[i])
    # Z measurement
    plt.plot(sim['Time'], sim['Zi Site {}'.format(site)], label='{}'.format(q), color=colours[i])
    # computational basis measurement
    # plt.plot(sim['Time'], 0.5*(1 - sim['Zi Site {}'.format(site)]), label='{}'.format(q), color=colours[i])
plt.xlabel('time')
plt.ylabel(r'$\langle Z_{i} \rangle$')
plt.legend();_____no_output_____
</code>


# Notebook from ouyang-lab/KARR-seq
Path: example/NB_visualize.ipynb

import matplotlib.pyplot as plt

from analysis.visualize.utils_contactmap import plot_contactmap, plot_diffmap, plot_contactmap_w_arcbands_____no_output_____

### Plotting Contactmaps
Using these functions, contact maps can be constructed from the pairs files to visualize your transcripts/isoforms of interest. First, we visualize the differences among the following methods (KARR-seq, PARIS and RIC-seq) by looking at the Telomerase RNA component, TERC, transcript. By proving a dot-bracket notation file, we can overlap secondary structures onto the contact map. Here, we use the known secondary structure obtained from [Rfam](https://rfam.org/family/RF00024) to illustrate._____no_output_____

data_KARRseq = ["data/inner/hrefseq/MAPQ1_SPAN0/G1_kethoxal-HEK293T_M13_B01.dedup.pairs.gz"]
data_PARIS = ["data/inner/hrefseq/MAPQ1_SPAN0/PARIS-HEK293T_B97_B01.nodedup.pairs.gz"]
data_RIC = ["data/inner/hrefseq/MAPQ1_SPAN0/RIC-HeLa-rRNA-_M94_B01.dedup.pairs.gz"]

fig, ax = plt.subplots(1, 3, figsize=(18, 8))

# KARR-seq (Left)
plot_contactmap(data_KARRseq, p_chrom1="NR_001566", p_resolution=10, p_max=3, p_figsize=(6, 6),
                p_structure=False, p_genome="hg19", p_display="full", p_balance=False,
                p_dot_size=0.5, f_dot="data/sec_struct/TERC.dot",
                f_size="data/reference/hg19/hg19_refseq_wo_version.sizes", p_ax=(fig, ax[0]), p_verbose=False)

# PARIS (Center)
plot_contactmap(data_PARIS, p_chrom1="NR_001566", p_resolution=10, p_max=3, p_figsize=(6, 6),
                p_structure=False, p_genome="hg19", p_display="full", p_balance=False,
                p_dot_size=0.5, f_dot="/mnt/chenga/projects/rnarna/analysis/TechComparison/TERC.dot",
                f_size="data/reference/hg19/hg19_refseq_wo_version.sizes", p_ax=(fig, ax[1]), p_verbose=False)

# RIC-seq (Right)
plot_contactmap(data_RIC, p_chrom1="NR_001566", p_resolution=10, p_max=3, p_figsize=(6, 6),
                p_structure=False, p_genome="hg19", p_display="full", p_balance=False,
                p_dot_size=0.5, f_dot="/mnt/chenga/projects/rnarna/analysis/TechComparison/TERC.dot",
                f_size="data/reference/hg19/hg19_refseq_wo_version.sizes", p_ax=(fig, ax[2]), p_verbose=False)_____no_output_____

### Clustering chimeric reads and classifying them into loops, left stripes and right stripes
To cluster the chimeric groups, we first define a similarity function by counting the amount of overlapd between two chimeric pairs. We can then construct the graph by building the similarity matrix, and then cluster the chimeric reads into chimeric groups by using a greedy modularity algorithm. We can then classify the chimeric groups into loops, left stripes and right stripes.  _____no_output_____

from analysis.clustering.cluster_pairs import cluster_chimeric_pairs
data_txt = "data/pairs/hrefseq/MAPQ1_SPAN0/G1_kethoxal-HepG2-refold_M13_R01.txt.gz"

dict_kwargs = {
    "p_iid": "NM_001404", "p_genome": None,
    "p_long": 200, "p_span": 10, "p_overlap": 10, "p_min_reads": 3,
    "p_min_interactions": 200, "p_max_interactions": 20000,
    "f_sizes": None, "f_tsne": None, "f_pairs": data_txt, "f_output": None, "f_bed": None
}
results = cluster_chimeric_pairs(**dict_kwargs)NM_001404 None 200 10 10 3 200 20000 None None /mnt/chenga/projects/rnarna/data/star/pairs/se/hrefseq/MAPQ1_SPAN0/G1_kethoxal-HepG2-refold_M13_R01.txt.gz None None
13
[['NM_001404', 471, 540, 'NM_001404', 823, 890], ['NM_001404', 816, 898, 'NM_001404', 986, 1074], ['NM_001404', 275, 336, 'NM_001404', 478, 566], ['NM_001404', 1198, 1287, 'NM_001404', 1365, 1467], ['NM_001404', 458, 541, 'NM_001404', 1367, 1444], ['NM_001404', 926, 998, 'NM_001404', 1401, 1470], ['NM_001404', 847, 906, 'NM_001404', 1260, 1407], ['NM_001404', 409, 495, 'NM_001404', 1093, 1195], ['NM_001404', 171, 222, 'NM_001404', 942, 1071], ['NM_001404', 614, 691, 'NM_001404', 1187, 1236], ['NM_001404', 937, 976, 'NM_001404', 1176, 1296], ['NM_001404', 1017, 1195, 'NM_001404', 1185, 1219], ['NM_001404', 194, 287, 'NM_001404', 1373, 1417]]

### Plotting Contactmaps with ArcBands
To visualize the chimeric groups, we can plot them as arcbands and align it above the heatmap. Here, we will compare an orthologue pair from mouse (Eef1g, mESC) and human (EEF1G, HepG2). Using a BLAST analysis, even though they have a 91.4% precentage identity, their general topologies are similar, suggesting that structure are conserved and plausibly functional. If the secondary structure is not available, you can use RNAfold to predict the ensemble base-pairing matrix by setting
`p_structure=True`.
Here we plot the mouse Eef1g ..._____no_output_____

data_mouse = ["data/pairs/mrefseq/MAPQ1_SPAN0/G1_kethoxal-mESC-refold_M13_R01.dedup.pairs.gz",
              "data/pairs/mrefseq/MAPQ1_SPAN0/G1_kethoxal-mESC-refold_M13_R02.dedup.pairs.gz"]

f_cluster_mouse = "data/clustering/output/mrefseq/G1_kethoxal-mESC-refold_M13_C01.dedup.classification.txt"

plot_contactmap_w_arcbands(data_mouse, f_cluster_mouse, p_chrom1="NM_026007.4", p_resolution=25, p_max=3, p_figsize=(10, 10),
                           p_structure=True, p_genome=None, p_display="upper", p_balance=False,
                           f_fasta = "data/reference/mm10/mm10_refseq.fa",
                           f_size = "data/reference/mm10/mm10_refseq.sizes",
                           p_dot_size=1.0, p_multi=False, p_verbose=False)_____no_output_____

... and then for human EEF1G_____no_output_____

data_human = ["data/pairs/hrefseq/MAPQ1_SPAN0/G1_kethoxal-HepG2-refold_M13_R01.dedup.pairs.gz",
              "data/pairs/hrefseq/MAPQ1_SPAN0/G1_kethoxal-HepG2-refold_M13_R02.dedup.pairs.gz"]
f_cluster_human = "data/clustering/output/hrefseq/G1_kethoxal-HepG2-refold_M13_C01.dedup.classification.txt"

plot_contactmap_w_arcbands(data_human, f_cluster_human, p_chrom1="NM_001404", p_resolution=25, p_max=3, p_figsize=(10, 10),
                           p_structure=True, p_genome="hg19", p_display="upper", p_balance=False,
                           f_fasta = "data/reference/hg19/hg19_refseq_wo_version.fasta",
                           f_size = "data/reference/hg19/hg19_refseq_wo_version.sizes",
                           p_dot_size=1.0, p_multi=False, p_verbose=False)_____no_output_____

### Differential Maps
To identify the features separating the in-vitro and in-vivo contact maps, we can use the plot_diffmap function._____no_output_____

data_vitro = ["data/pairs/hrefseq/MAPQ1_SPAN0/G1_kethoxal-HepG2-refold_M13_R01.dedup.pairs.gz"]
data_vivo = ["data/pairs/hrefseq/MAPQ1_SPAN0/G1_kethoxal-HepG2_M13_R01.dedup.pairs.gz"]

fig, ax = plt.subplots(1, 2, figsize=(18, 8))

plot_contactmap(data_vitro, p_chrom1="NM_001085", p_resolution=25, p_max=5, p_figsize=(6, 6),
             p_structure=True, p_genome="hg19", p_display="full", p_balance=False,
             p_dot_size=1.0,
             f_size="data/reference/hg19/hg19_refseq_wo_version.sizes", p_ax=(fig, ax[0]), p_verbose=False)

plot_contactmap(data_vivo, p_chrom1="NM_001085", p_resolution=25, p_max=5, p_figsize=(6, 6),
             p_structure=True, p_genome="hg19", p_display="full", p_balance=False,
             p_dot_size=1.0,
             f_size="data/reference/hg19/hg19_refseq_wo_version.sizes", p_ax=(fig, ax[1]), p_verbose=False)_____no_output_____data_pairs = ["data/pairs/hrefseq/MAPQ1_SPAN0/G1_kethoxal-HepG2-refold_M13_R01.dedup.pairs.gz",
              "data/pairs/hrefseq/MAPQ1_SPAN0/G1_kethoxal-HepG2_M13_R01.dedup.pairs.gz"]

plot_diffmap(data_pairs, p_chrom1="NM_001085", p_resolution=25, p_max=5, p_figsize=(6, 6),
             p_structure=True, p_genome="hg19", p_display="upper", p_balance=False,
             p_dot_size=1.0, p_diff="norm_diff",
             f_size="data/reference/hg19/hg19_refseq_wo_version.sizes", p_verbose=True)['NM_001085:0-1628|NM_001085:0-1628']
Dim: (66, 66) | Total Counts: 2165
Dim: (66, 66) | Total Counts: 1635
1 1
(66, 66)
(66, 66)


# Notebook from chaudha4/introtodeeplearning
Path: lab1/Part1_TensorFlow.ipynb

_____no_output_____# Lab 1: Intro to TensorFlow and Music Generation with RNNs

# Part 1: Intro to TensorFlow

## 0.1 Install TensorFlow

TensorFlow is a software library extensively used in machine learning. Here we'll learn how computations are represented and how to define a simple neural network in TensorFlow. For all the labs in 6.S191 2020, we'll be using the latest version of TensorFlow, TensorFlow 2, which affords great flexibility and the ability to imperatively execute operations, just like in Python. You'll notice that TensorFlow 2 is quite similar to Python in its syntax and imperative execution. Let's install TensorFlow and a couple of dependencies.
_____no_output_____

%tensorflow_version 2.x
import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt_____no_output_____# Download and import the MIT 6.S191 package
!pip install mitdeeplearning
import mitdeeplearning as mdlCollecting mitdeeplearning
  Downloading https://files.pythonhosted.org/packages/8b/3b/b9174b68dc10832356d02a2d83a64b43a24f1762c172754407d22fc8f960/mitdeeplearning-0.1.2.tar.gz (2.1MB)
     |████████████████████████████████| 2.1MB 7.7MB/s
Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from mitdeeplearning) (1.18.5)
Requirement already satisfied: regex in /usr/local/lib/python3.6/dist-packages (from mitdeeplearning) (2019.12.20)
Requirement already satisfied: tqdm in /usr/local/lib/python3.6/dist-packages (from mitdeeplearning) (4.41.1)
Requirement already satisfied: gym in /usr/local/lib/python3.6/dist-packages (from mitdeeplearning) (0.17.3)
Requirement already satisfied: scipy in /usr/local/lib/python3.6/dist-packages (from gym->mitdeeplearning) (1.4.1)
Requirement already satisfied: pyglet<=1.5.0,>=1.4.0 in /usr/local/lib/python3.6/dist-packages (from gym->mitdeeplearning) (1.5.0)
Requirement already satisfied: cloudpickle<1.7.0,>=1.2.0 in /usr/local/lib/python3.6/dist-packages (from gym->mitdeeplearning) (1.3.0)
Requirement already satisfied: future in /usr/local/lib/python3.6/dist-packages (from pyglet<=1.5.0,>=1.4.0->gym->mitdeeplearning) (0.16.0)
Building wheels for collected packages: mitdeeplearning
  Building wheel for mitdeeplearning (setup.py) ... done
  Created wheel for mitdeeplearning: filename=mitdeeplearning-0.1.2-cp36-none-any.whl size=2114585 sha256=46cd9649a74ddd2407ca4e035811c6908c362db14024371fbdb43b1225365c5c
  Stored in directory: /root/.cache/pip/wheels/27/e1/73/5f01c787621d8a3c857f59876c79e304b9b64db9ff5bd61b74
Successfully built mitdeeplearning
Installing collected packages: mitdeeplearning
Successfully installed mitdeeplearning-0.1.2

## 1.1 Why is TensorFlow called TensorFlow?

TensorFlow is called 'TensorFlow' because it handles the flow (node/mathematical operation) of Tensors, which are data structures that you can think of as multi-dimensional arrays. Tensors are represented as n-dimensional arrays of base dataypes such as a string or integer -- they provide a way to generalize vectors and matrices to higher dimensions.

The ```shape``` of a Tensor defines its number of dimensions and the size of each dimension. The ```rank``` of a Tensor provides the number of dimensions (n-dimensions) -- you can also think of this as the Tensor's order or degree.

Let's first look at 0-d Tensors, of which a scalar is an example:_____no_output_____

sport = tf.constant("Tennis", tf.string)
number = tf.constant(1.41421356237, tf.float64)

print("`sport` is a {}-d Tensor".format(tf.rank(sport).numpy()))
print("`number` is a {}-d Tensor".format(tf.rank(number).numpy()))`sport` is a 0-d Tensor
`number` is a 0-d Tensor

Vectors and lists can be used to create 1-d Tensors:_____no_output_____

sports = tf.constant(["Tennis", "Basketball"], tf.string)
numbers = tf.constant([3.141592, 1.414213, 2.71821], tf.float64)

print("`sports` is a {}-d Tensor with shape: {}".format(tf.rank(sports).numpy(), tf.shape(sports)))
print("`numbers` is a {}-d Tensor with shape: {}".format(tf.rank(numbers).numpy(), tf.shape(numbers)))

print(tf.shape(sports))
print(tf.shape(numbers))`sports` is a 1-d Tensor with shape: [2]
`numbers` is a 1-d Tensor with shape: [3]
tf.Tensor([2], shape=(1,), dtype=int32)
tf.Tensor([3], shape=(1,), dtype=int32)

Next we consider creating 2-d (i.e., matrices) and higher-rank Tensors. For examples, in future labs involving image processing and computer vision, we will use 4-d Tensors. Here the dimensions correspond to the number of example images in our batch, image height, image width, and the number of color channels._____no_output_____

### Defining higher-order Tensors ###

'''TODO: Define a 2-d Tensor'''
matrix = tf.constant([[1,1,1],[2,2,2]], tf.float64)

assert isinstance(matrix, tf.Tensor), "matrix must be a tf Tensor object"
assert tf.rank(matrix).numpy() == 2_____no_output_____'''TODO: Define a 4-d Tensor.'''
# Use tf.zeros to initialize a 4-d Tensor of zeros with size 10 x 256 x 256 x 3.
#   You can think of this as 10 images where each image is RGB 256 x 256.
images = tf.zeros([10,256,256,3])

assert isinstance(images, tf.Tensor), "matrix must be a tf Tensor object"
assert tf.rank(images).numpy() == 4, "matrix must be of rank 4"
assert tf.shape(images).numpy().tolist() == [10, 256, 256, 3], "matrix is incorrect shape"_____no_output_____tf.zeros([2,3,4,5])_____no_output_____

As you have seen, the ```shape``` of a Tensor provides the number of elements in each Tensor dimension. The ```shape``` is quite useful, and we'll use it often. You can also use slicing to access subtensors within a higher-rank Tensor:_____no_output_____

row_vector = matrix[1]
column_vector = matrix[:,2]
scalar = matrix[1, 2]

print("`row_vector`: {}".format(row_vector.numpy()))
print("`column_vector`: {}".format(column_vector.numpy()))
print("`scalar`: {}".format(scalar.numpy()))`row_vector`: [2. 2. 2.]
`column_vector`: [1. 2.]
`scalar`: 2.0

## 1.2 Computations on Tensors

A convenient way to think about and visualize computations in TensorFlow is in terms of graphs. We can define this graph in terms of Tensors, which hold data, and the mathematical operations that act on these Tensors in some order. Let's look at a simple example, and define this computation using TensorFlow:

![alt text](https://raw.githubusercontent.com/aamini/introtodeeplearning/master/lab1/img/add-graph.png)_____no_output_____

# Create the nodes in the graph, and initialize values
a = tf.constant(15)
b = tf.constant(61)

# Add them!
c1 = tf.add(a,b)
c2 = a + b # TensorFlow overrides the "+" operation so that it is able to act on Tensors
print(c1)
print(c2)tf.Tensor(76, shape=(), dtype=int32)
tf.Tensor(76, shape=(), dtype=int32)

Notice how we've created a computation graph consisting of TensorFlow operations, and how  the output is a Tensor with value 76 -- we've just created a computation graph consisting of operations, and it's executed them and given us back the result.

Now let's consider a slightly more complicated example:

![alt text](https://raw.githubusercontent.com/aamini/introtodeeplearning/master/lab1/img/computation-graph.png)

Here, we take two inputs, `a, b`, and compute an output `e`. Each node in the graph represents an operation that takes some input, does some computation, and passes its output to another node.

Let's define a simple function in TensorFlow to construct this computation function:_____no_output_____

### Defining Tensor computations ###

# Construct a simple computation function
def func(a,b):
  '''TODO: Define the operation for c, d, e (use tf.add, tf.subtract, tf.multiply).'''
  c = tf.add(a,b)
  d = tf.subtract(a,b)
  e = tf.multiply(a,b)
  return e_____no_output_____

Now, we can call this function to execute the computation graph given some inputs `a,b`:_____no_output_____

# Consider example values for a,b
a, b = 1.5, 2.5
# Execute the computation
e_out = func(a,b)
print(e_out)tf.Tensor(3.75, shape=(), dtype=float32)

Notice how our output is a Tensor with value defined by the output of the computation, and that the output has no shape as it is a single scalar value._____no_output_____

a = tf.constant([[1,2,1],[3,4,3]])
b = tf.constant([5,6,7])
tf.multiply(a,b)_____no_output_____

## 1.3 Neural networks in TensorFlow
We can also define neural networks in TensorFlow. TensorFlow uses a high-level API called [Keras](https://www.tensorflow.org/guide/keras) that provides a powerful, intuitive framework for building and training deep learning models.

Let's first consider the example of a simple perceptron defined by just one dense layer: $ y = \sigma(Wx + b)$, where $W$ represents a matrix of weights, $b$ is a bias, $x$ is the input, $\sigma$ is the sigmoid activation function, and $y$ is the output. We can also visualize this operation using a graph:

![alt text](https://raw.githubusercontent.com/aamini/introtodeeplearning/master/lab1/img/computation-graph-2.png)

Tensors can flow through abstract types called [```Layers```](https://www.tensorflow.org/api_docs/python/tf/keras/layers/Layer) -- the building blocks of neural networks. ```Layers``` implement common neural networks operations, and are used to update weights, compute losses, and define inter-layer connectivity. We will first define a ```Layer``` to implement the simple perceptron defined above._____no_output_____

### Defining a network Layer ###

# n_output_nodes: number of output nodes
# input_shape: shape of the input
# x: input to the layer

class OurDenseLayer(tf.keras.layers.Layer):
  def __init__(self, n_output_nodes):
    # Abhi - super.__init__() would do the same but for multiple
    # inheritance, this can allow us more fine grained control.
    # See https://realpython.com/python-super/
    super(OurDenseLayer, self).__init__()
    self.n_output_nodes = n_output_nodes

  def build(self, input_shape):
    d = int(input_shape[-1])
    # Define and initialize parameters: a weight matrix W and bias b
    # Note that parameter initialization is random!
    self.W = self.add_weight("weight", shape=[d, self.n_output_nodes]) # note the dimensionality
    self.b = self.add_weight("bias", shape=[1, self.n_output_nodes]) # note the dimensionality

  def call(self, x):
    '''TODO: define the operation for z (hint: use tf.matmul)'''
    z = tf.add(self.b, tf.matmul(x, self.W))

    '''TODO: define the operation for out (hint: use tf.sigmoid)'''
    y = tf.sigmoid(z)
    return y

# Since layer parameters are initialized randomly, we will set a random seed for reproducibility
tf.random.set_seed(1)
layer = OurDenseLayer(3)
layer.build((1,2))
x_input = tf.constant([[1,2.]], shape=(1,2))
y = layer.call(x_input)

# test the output!
print(y.numpy())
mdl.lab1.test_custom_dense_layer_output(y)[[0.2697859  0.45750412 0.66536945]]
[PASS] test_custom_dense_layer_output

Conveniently, TensorFlow has defined a number of ```Layers``` that are commonly used in neural networks, for example a [```Dense```](https://www.tensorflow.org/api_docs/python/tf/keras/layers/Dense?version=stable). Now, instead of using a single ```Layer``` to define our simple neural network, we'll use the  [`Sequential`](https://www.tensorflow.org/versions/r2.0/api_docs/python/tf/keras/Sequential) model from Keras and a single [`Dense` ](https://www.tensorflow.org/versions/r2.0/api_docs/python/tf/keras/layers/Dense) layer to define our network. With the `Sequential` API, you can readily create neural networks by stacking together layers like building blocks. _____no_output_____

### Defining a neural network using the Sequential API ###

# Import relevant packages
from tensorflow.keras import Sequential
from tensorflow.keras.layers import Dense

# Define the number of outputs
n_output_nodes = 3

# First define the model
model = Sequential()

'''TODO: Define a dense (fully connected) layer to compute z'''
# Remember: dense layers are defined by the parameters W and b!
# You can read more about the initialization of W and b in the TF documentation :)
# https://www.tensorflow.org/api_docs/python/tf/keras/layers/Dense?version=stable
dense_layer = tf.keras.layers.Dense(units=n_output_nodes)

# Add the dense layer to the model
model.add(dense_layer)
_____no_output_____

That's it! We've defined our model using the Sequential API. Now, we can test it out using an example input:_____no_output_____

# Test model with example input
x_input = tf.constant([[1,2.],[3,4],[3,4]], shape=(3,2))
print("The rank of a tensor is the number of indices required to uniquely select each element of the tensor.", tf.rank(x_input))
print(tf.shape(x_input))
'''TODO: feed input into the model and predict the output!'''
model_output = model.predict(x_input)
print(model_output)The rank of a tensor is the number of indices required to uniquely select each element of the tensor. tf.Tensor(2, shape=(), dtype=int32)
tf.Tensor([3 2], shape=(2,), dtype=int32)
[[ 0.62118787 -0.08692831  1.6387595 ]
 [ 0.6649711  -0.7461755   3.2854748 ]
 [ 0.6649711  -0.7461755   3.2854748 ]]
 model.predict( [[1,2],[1,2]] )_____no_output_____

In addition to defining models using the `Sequential` API, we can also define neural networks by directly subclassing the [`Model`](https://www.tensorflow.org/api_docs/python/tf/keras/Model?version=stable) class, which groups layers together to enable model training and inference. The `Model` class captures what we refer to as a "model" or as a "network". Using Subclassing, we can create a class for our model, and then define the forward pass through the network using the `call` function. Subclassing affords the flexibility to define custom layers, custom training loops, custom activation functions, and custom models. Let's define the same neural network as above now using Subclassing rather than the `Sequential` model._____no_output_____

### Defining a model using subclassing ###

from tensorflow.keras import Model
from tensorflow.keras.layers import Dense

class SubclassModel(tf.keras.Model):

  # In __init__, we define the Model's layers
  def __init__(self, n_output_nodes):
    super(SubclassModel, self).__init__()
    '''TODO: Our model consists of a single Dense layer. Define this layer.'''
    self.dense_layer = tf.keras.layers.Dense(units=n_output_nodes)

  # In the call function, we define the Model's forward pass.
  def call(self, inputs):
    return self.dense_layer(inputs)_____no_output_____

Just like the model we built using the `Sequential` API, let's test out our `SubclassModel` using an example input.

_____no_output_____

n_output_nodes = 3
model = SubclassModel(n_output_nodes)

x_input = tf.constant([[1,2.]], shape=(1,2))

print(model.call(x_input))tf.Tensor([[-0.84747624  0.52264667 -0.06161666]], shape=(1, 3), dtype=float32)

Importantly, Subclassing affords us a lot of flexibility to define custom models. For example, we can use boolean arguments in the `call` function to specify different network behaviors, for example different behaviors during training and inference. Let's suppose under some instances we want our network to simply output the input, without any perturbation. We define a boolean argument `isidentity` to control this behavior:_____no_output_____

### Defining a model using subclassing and specifying custom behavior ###

from tensorflow.keras import Model
from tensorflow.keras.layers import Dense

class IdentityModel(tf.keras.Model):

  # As before, in __init__ we define the Model's layers
  # Since our desired behavior involves the forward pass, this part is unchanged
  def __init__(self, n_output_nodes):
    super(IdentityModel, self).__init__()
    self.dense_layer = tf.keras.layers.Dense(n_output_nodes, activation='sigmoid')

  '''TODO: Implement the behavior where the network outputs the input, unchanged,
      under control of the isidentity argument.'''
  def call(self, inputs, isidentity=False):

    if isidentity:
      return inputs

    return self.dense_layer(inputs)

    '''TODO: Implement identity behavior'''_____no_output_____

Let's test this behavior:_____no_output_____

n_output_nodes = 3
model = IdentityModel(n_output_nodes)

x_input = tf.constant([[1,2.]], shape=(1,2))
'''TODO: pass the input into the model and call with and without the input identity option.'''
out_activate = model.call(x_input, True)
out_identity = model.call(x_input, False)

print("Network output with activation: {}; network identity output: {}".format(out_activate.numpy(), out_identity.numpy()))Network output with activation: [[1. 2.]]; network identity output: [[0.19695838 0.6330006  0.7668015 ]]

Now that we have learned how to define `Layers` as well as neural networks in TensorFlow using both the `Sequential` and Subclassing APIs, we're ready to turn our attention to how to actually implement network training with backpropagation._____no_output_____## 1.4 Automatic differentiation in TensorFlow

[Automatic differentiation](https://en.wikipedia.org/wiki/Automatic_differentiation)
is one of the most important parts of TensorFlow and is the backbone of training with
[backpropagation](https://en.wikipedia.org/wiki/Backpropagation). We will use the TensorFlow GradientTape [`tf.GradientTape`](https://www.tensorflow.org/api_docs/python/tf/GradientTape?version=stable) to trace operations for computing gradients later.

When a forward pass is made through the network, all forward-pass operations get recorded to a "tape"; then, to compute the gradient, the tape is played backwards. By default, the tape is discarded after it is played backwards; this means that a particular `tf.GradientTape` can only
compute one gradient, and subsequent calls throw a runtime error. However, we can compute multiple gradients over the same computation by creating a ```persistent``` gradient tape.

First, we will look at how we can compute gradients using GradientTape and access them for computation. We define the simple function $ y = x^2$ and compute the gradient:_____no_output_____

### Gradient computation with GradientTape ###

# y = x^2
# Example: x = 3.0
x = tf.Variable(3.0)

# Initiate the gradient tape
with tf.GradientTape() as tape:
  # Define the function
  y = x * x
# Access the gradient -- derivative of y with respect to x
dy_dx = tape.gradient(y, x)

assert dy_dx.numpy() == 6.0_____no_output__________no_output_____

In training neural networks, we use differentiation and stochastic gradient descent (SGD) to optimize a loss function. Now that we have a sense of how `GradientTape` can be used to compute and access derivatives, we will look at an example where we use automatic differentiation and SGD to find the minimum of $L=(x-x_f)^2$. Here $x_f$ is a variable for a desired value we are trying to optimize for; $L$ represents a loss that we are trying to  minimize. While we can clearly solve this problem analytically ($x_{min}=x_f$), considering how we can compute this using `GradientTape` sets us up nicely for future labs where we use gradient descent to optimize entire neural network losses._____no_output_____

### Function minimization with automatic differentiation and SGD ###

# Initialize a random value for our initial x
x = tf.Variable([tf.random.normal([1])])
print("Initializing x={}".format(x.numpy()))

learning_rate = 5.8e-2 # learning rate for SGD
history = []
# Define the target value
x_f = 4

# We will run SGD for a number of iterations. At each iteration, we compute the loss,
#   compute the derivative of the loss with respect to x, and perform the SGD update.
for i in range(50):
  with tf.GradientTape() as tape:
    '''TODO: define the loss as described above'''
    loss = (x - x_f) * (x - x_f)

  # loss minimization using gradient tape
  grad = tape.gradient(loss, x) # compute the derivative of the loss with respect to x
  new_x = x - learning_rate*grad # sgd update
  x.assign(new_x) # update the value of x
  history.append(x.numpy()[0])

# Plot the evolution of x as we optimize towards x_f!
plt.plot(history)
plt.plot([0, 500],[x_f,x_f])
plt.legend(('Predicted', 'True'))
plt.xlabel('Iteration')
plt.ylabel('x value')Initializing x=[[-1.87371]]

`GradientTape` provides an extremely flexible framework for automatic differentiation. In order to back propagate errors through a neural network, we track forward passes on the Tape, use this information to determine the gradients, and then use these gradients for optimization using SGD._____no_output_____


# Notebook from cschrumm/TreeDeepSpelling
Path: .ipynb_checkpoints/training_algorithm-checkpoint.ipynb

<code>
import math
import random
import torch
import matplotlib.pyplot as plt
from scipy.spatial import distance
import itertools
import jellyfish
import numpy as np
import os, shutil_____no_output_____# import the words I'm using
%run partial_words.py
%run core.py
%run imgCls.py_____no_output_____def delete_files_dir(folder):
    for filename in os.listdir(folder):
        file_path = os.path.join(folder, filename)
        try:
            if os.path.isfile(file_path) or os.path.islink(file_path):
                os.unlink(file_path)
            elif os.path.isdir(file_path):
                shutil.rmtree(file_path)
        except Exception as e:
            print('Failed to delete %s. Reason: %s' % (file_path, e))_____no_output_____delete_files_dir("./alternate_training")_____no_output_____class DeepSpellingChecker:
    #ky_dct = load_keys(dr_lc)
    def __init__(self,encode_dir,net_dir):
        self.encode_dir = encode_dir
        self.net_dir = net_dir
        self.ky_dct = load_keys(encode_dir)
        self.mapp = None
        return
    def encode_image(self,wrd):
        return encode_word(wrd,self.ky_dct)
    def create_word_mistake(self,wrd):
        return word_mix(wrd)
    def can_use_gpu(self):
        return torch.cuda.is_available()
    def train_words(self,wrd_list):
        rt_name = "mdl_spelling"
        dcMapping = {}
        maxDepth = 30
        wrl = WordManage()
        wrl.set_words(wrd_list,0)
        # clean out the network directory
        delete_files_dir(self.net_dir)
        #
        train_and_choose(rt_name,wrl,dcMapping,maxDepth,self.net_dir)

        file_path = self.mapping_file()

        save_json(dcMapping,file_path)

        self.mapp = dcMapping
        return

    def mapping_file(self):
        return  os.path.join(self.net_dir, "word_tree_mapping.json")

    def best_word_match(self,wrd,dbg=False):

        file_path = self.mapping_file()

        if self.mapp == None:
            self.mapp = load_json(file_path)

        rt_name = "mdl_spelling"

        wd  = spell_word(wrd,rt_name,self.mapp,self.net_dir, dbg)

        if len(wd) > 1:
            srt = sorted(wd,key=lambda ky : jellyfish.damerau_levenshtein_distance(wrd,ky))
            print("matches... ",srt)
            return srt[0]
        else:
            return wd[0]

    def print_network_layout(self):
        print(ImgNet(5))
        return
_____no_output_____def show_image(img):
    plt.figure(figsize=(10,2))
    #plt.imshow(img,aspect="auto",extent=(0,5,1e0,1e5),interpolation='nearest')
    plt.imshow(img,aspect="auto",interpolation='nearest')
    plt.show()
    return_____no_output_____deep_sp = DeepSpellingChecker("","./alternate_training")_____no_output_____deep_sp.print_network_layout()ImgNet(
  (conv1): Conv2d(1, 32, kernel_size=(2, 2), stride=(1, 1))
  (conv2): Conv2d(32, 16, kernel_size=(2, 2), stride=(1, 1), padding=(1, 1))
  (dropout1): Dropout2d(p=0.25, inplace=False)
  (dropout2): Dropout2d(p=0.25, inplace=False)
  (fc1): Linear(in_features=2048, out_features=64, bias=True)
  (fc2): Linear(in_features=64, out_features=5, bias=True)
)
deep_sp.can_use_gpu()_____no_output_____show_image(deep_sp.encode_image("This is a phrase encoded for demo to get a rough idea of what they look like"))_____no_output_____for i in range(10):
    print(deep_sp.create_word_mistake("conditional"))conditional
conditimnal
coenidtional
conditional
conditional
conditio al
codnitional
conditiohnal
condig ional
jonditional
word_to_train = ["cat","dog","fish","shark","lark","child","log","chicken","ostrich","zebra","fox","kitten","saint bernard","salmon","cobra","rattlesnake"]
deep_sp.train_words(top_255)train tree..
category  0  total word count  45
recursing on  mdl_spelling_0  total words..  45
train tree..
category  0  total word count  5
recursing on  mdl_spelling_0_0  total words..  5
train tree..
category  0  total word count  0
category  1  total word count  3
recursing on  mdl_spelling_0_0_1  total words..  3
Words.. ['context', 'collection', 'connection']
train tree..
category  2  total word count  0
category  3  total word count  1
category  4  total word count  1
category  1  total word count  4
recursing on  mdl_spelling_0_1  total words..  4
Words.. ['teacher', 'president', 'statement', 'interaction']
train tree..
category  0  total word count  1
category  1  total word count  1
category  2  total word count  0
category  3  total word count  2
recursing on  mdl_spelling_0_1_3  total words..  2
Words.. ['statement', 'interaction']
train tree..
category  4  total word count  0
category  2  total word count  19
recursing on  mdl_spelling_0_2  total words..  19
train tree..
category  0  total word count  1
category  1  total word count  16
recursing on  mdl_spelling_0_2_1  total words..  16
train tree..
category  0  total word count  0
category  1  total word count  0
category  2  total word count  6
recursing on  mdl_spelling_0_2_1_2  total words..  6
train tree..
category  0  total word count  4
recursing on  mdl_spelling_0_2_1_2_0  total words..  4
Words.. ['disk', 'road', 'city', 'steak']
train tree..
category  1  total word count  2
recursing on  mdl_spelling_0_2_1_2_1  total words..  2
Words.. ['concept', 'aspect']
train tree..
category  2  total word count  0
category  3  total word count  0
category  4  total word count  0
category  3  total word count  0
category  4  total word count  10
recursing on  mdl_spelling_0_2_1_4  total words..  10
train tree..
category  2  total word count  1
category  3  total word count  1
category  4  total word count  0
category  3  total word count  11
recursing on  mdl_spelling_0_3  total words..  11
train tree..
category  0  total word count  0
category  1  total word count  8
recursing on  mdl_spelling_0_3_1  total words..  8
train tree..
category  2  total word count  0
category  3  total word count  1
category  4  total word count  2
recursing on  mdl_spelling_0_3_4  total words..  2
Words.. ['way', 'map']
train tree..
category  4  total word count  6
recursing on  mdl_spelling_0_4  total words..  6
train tree..
category  0  total word count  1
category  1  total word count  2
recursing on  mdl_spelling_0_4_1  total words..  2
Words.. ['love', 'lake']
train tree..
category  2  total word count  1
category  3  total word count  1
category  4  total word count  1
category  1  total word count  44
recursing on  mdl_spelling_1  total words..  44
train tree..
category  0  total word count  6
recursing on  mdl_spelling_1_0  total words..  6
train tree..
category  0  total word count  1
category  1  total word count  2
recursing on  mdl_spelling_1_0_1  total words..  2
Words.. ['passion', 'version']
train tree..
category  2  total word count  1
category  3  total word count  1
category  4  total word count  1
category  1  total word count  7
recursing on  mdl_spelling_1_1  total words..  7
train tree..
category  0  total word count  2
recursing on  mdl_spelling_1_1_0  total words..  2
Words.. ['development', 'department']
train tree..
category  1  total word count  1
category  2  total word count  2
recursing on  mdl_spelling_1_1_2  total words..  2
Words.. ['equipment', 'apartment']
train tree..
category  3  total word count  1
category  4  total word count  1
category  2  total word count  12
recursing on  mdl_spelling_1_2  total words..  12
train tree..
category  0  total word count  1
category  1  total word count  6
recursing on  mdl_spelling_1_2_1  total words..  6
train tree..
category  0  total word count  1
category  1  total word count  1
category  2  total word count  3
recursing on  mdl_spelling_1_2_1_2  total words..  3
Words.. ['attention', 'opinion', 'agreement']
train tree..
category  3  total word count  1
category  4  total word count  0
category  2  total word count  2
recursing on  mdl_spelling_1_2_2  total words..  2
Words.. ['investment', 'expression']
train tree..
category  3  total word count  2
recursing on  mdl_spelling_1_2_3  total words..  2
Words.. ['information', 'environment']
train tree..
category  4  total word count  1
category  3  total word count  7
recursing on  mdl_spelling_1_3  total words..  7
train tree..
category  0  total word count  1
category  1  total word count  2
recursing on  mdl_spelling_1_3_1  total words..  2
Words.. ['addition', 'situation']
train tree..
category  2  total word count  1
category  3  total word count  2
recursing on  mdl_spelling_1_3_3  total words..  2
Words.. ['solution', 'emphasis']
train tree..
category  4  total word count  1
category  4  total word count  12
recursing on  mdl_spelling_1_4  total words..  12
train tree..
category  0  total word count  1
category  1  total word count  7
recursing on  mdl_spelling_1_4_1  total words..  7
train tree..
category  0  total word count  1
category  1  total word count  2
recursing on  mdl_spelling_1_4_1_1  total words..  2
Words.. ['combination', 'competition']
train tree..
category  2  total word count  2
recursing on  mdl_spelling_1_4_1_2  total words..  2
Words.. ['population', 'foundation']
train tree..
category  3  total word count  1
category  4  total word count  1
category  2  total word count  1
category  3  total word count  1
category  4  total word count  2
recursing on  mdl_spelling_1_4_4  total words..  2
Words.. ['communication', 'application']
train tree..
category  2  total word count  3
recursing on  mdl_spelling_2  total words..  3
Words.. ['opportunity', 'relationship', 'responsibility']
train tree..
category  0  total word count  1
category  1  total word count  1
category  2  total word count  1
category  3  total word count  0
category  4  total word count  0
category  3  total word count  45
recursing on  mdl_spelling_3  total words..  45
train tree..
category  0  total word count  33
recursing on  mdl_spelling_3_0  total words..  33
train tree..
category  0  total word count  0
category  1  total word count  8
recursing on  mdl_spelling_3_0_1  total words..  8
train tree..
category  0  total word count  1
category  1  total word count  0
category  2  total word count  3
recursing on  mdl_spelling_3_0_1_2  total words..  3
Words.. ['power', 'oven', 'the']
train tree..
category  3  total word count  4
recursing on  mdl_spelling_3_0_1_3  total words..  4
Words.. ['camera', 'paper', 'user', 'dealer']
train tree..
category  4  total word count  0
category  2  total word count  25
recursing on  mdl_spelling_3_0_2  total words..  25
train tree..
category  0  total word count  4
recursing on  mdl_spelling_3_0_2_0  total words..  4
Words.. ['person', 'thing', 'heart', 'union']
train tree..
category  1  total word count  9
recursing on  mdl_spelling_3_0_2_1  total words..  9
train tree..
category  2  total word count  0
category  3  total word count  4
recursing on  mdl_spelling_3_0_2_3  total words..  4
Words.. ['idea', 'area', 'series', 'nation']
train tree..
category  4  total word count  8
recursing on  mdl_spelling_3_0_2_4  total words..  8
train tree..
category  0  total word count  5
recursing on  mdl_spelling_3_0_2_4_0  total words..  5
train tree..
category  0  total word count  0
category  1  total word count  1
category  2  total word count  0
category  3  total word count  3
recursing on  mdl_spelling_3_0_2_4_0_3  total words..  3
Words.. ['marketing', 'location', 'tradition']
train tree..
category  4  total word count  1
category  1  total word count  0
category  2  total word count  0
category  3  total word count  3
recursing on  mdl_spelling_3_0_2_4_3  total words..  3
Words.. ['media', 'phone', 'scene']
train tree..
category  4  total word count  0
category  3  total word count  0
category  4  total word count  0
category  1  total word count  0
category  2  total word count  12
recursing on  mdl_spelling_3_2  total words..  12
train tree..
category  0  total word count  2
recursing on  mdl_spelling_3_2_0  total words..  2
Words.. ['photo', 'but']
train tree..
category  1  total word count  2
recursing on  mdl_spelling_3_2_1  total words..  2
Words.. ['effort', 'payment']
train tree..
category  2  total word count  2
recursing on  mdl_spelling_3_2_2  total words..  2
Words.. ['success', 'student']
train tree..
category  3  total word count  5
recursing on  mdl_spelling_3_2_3  total words..  5
train tree..
category  4  total word count  1
category  3  total word count  0
category  4  total word count  0
category  4  total word count  119
recursing on  mdl_spelling_4  total words..  119
train tree..
category  0  total word count  28
recursing on  mdl_spelling_4_0  total words..  28
train tree..
category  0  total word count  3
recursing on  mdl_spelling_4_0_0  total words..  3
Words.. ['television', 'analysis', 'response']
train tree..
category  1  total word count  9
recursing on  mdl_spelling_4_0_1  total words..  9
train tree..
category  0  total word count  2
recursing on  mdl_spelling_4_0_1_0  total words..  2
Words.. ['language', 'resource']
train tree..
category  1  total word count  2
recursing on  mdl_spelling_4_0_1_1  total words..  2
Words.. ['nature', 'article']
train tree..
category  2  total word count  2
recursing on  mdl_spelling_4_0_1_2  total words..  2
Words.. ['medicine', 'magazine']
train tree..
category  3  total word count  1
category  4  total word count  2
recursing on  mdl_spelling_4_0_1_4  total words..  2
Words.. ['income', 'storage']
train tree..
category  2  total word count  7
recursing on  mdl_spelling_4_0_2  total words..  7
train tree..
category  0  total word count  1
category  1  total word count  3
recursing on  mdl_spelling_4_0_2_1  total words..  3
Words.. ['office', 'advice', 'mixture']
train tree..
category  2  total word count  1
category  3  total word count  1
category  4  total word count  1
category  3  total word count  3
recursing on  mdl_spelling_4_0_3  total words..  3
Words.. ['product', 'thought', 'alcohol']
train tree..
category  4  total word count  6
recursing on  mdl_spelling_4_0_4  total words..  6
train tree..
category  0  total word count  1
category  1  total word count  1
category  2  total word count  1
category  3  total word count  1
category  4  total word count  2
recursing on  mdl_spelling_4_0_4_4  total words..  2
Words.. ['industry', 'security']
train tree..
category  1  total word count  8
recursing on  mdl_spelling_4_1  total words..  8
train tree..
category  0  total word count  3
recursing on  mdl_spelling_4_1_0  total words..  3
Words.. ['difference', 'appearance', 'importance']
train tree..
category  1  total word count  1
category  2  total word count  1
category  3  total word count  2
recursing on  mdl_spelling_4_1_3  total words..  2
Words.. ['performance', 'perspective']
train tree..
category  4  total word count  1
category  2  total word count  4
recursing on  mdl_spelling_4_2  total words..  4
Words.. ['boyfriend', 'technology', 'philosophy', 'psychology']
train tree..
category  0  total word count  1
category  1  total word count  1
category  2  total word count  1
category  3  total word count  1
category  4  total word count  0
category  3  total word count  17
recursing on  mdl_spelling_4_3  total words..  17
train tree..
category  0  total word count  2
recursing on  mdl_spelling_4_3_0  total words..  2
Words.. ['understanding', 'percentage']
train tree..
category  1  total word count  2
recursing on  mdl_spelling_4_3_1  total words..  2
Words.. ['grandmother', 'attitude']
train tree..
category  2  total word count  6
recursing on  mdl_spelling_4_3_2  total words..  6
train tree..
category  0  total word count  1
category  1  total word count  1
category  2  total word count  2
recursing on  mdl_spelling_4_3_2_2  total words..  2
Words.. ['internet', 'instance']
train tree..
category  3  total word count  1
category  4  total word count  1
category  3  total word count  5
recursing on  mdl_spelling_4_3_3  total words..  5
train tree..
category  0  total word count  0
category  1  total word count  1
category  2  total word count  1
category  3  total word count  1
category  4  total word count  2
recursing on  mdl_spelling_4_3_3_4  total words..  2
Words.. ['cigarette', 'patience']
train tree..
category  4  total word count  2
recursing on  mdl_spelling_4_3_4  total words..  2
Words.. ['knowledge', 'membership']
train tree..
category  4  total word count  62
recursing on  mdl_spelling_4_4  total words..  62
train tree..
category  0  total word count  36
recursing on  mdl_spelling_4_4_0  total words..  36
train tree..
category  0  total word count  8
recursing on  mdl_spelling_4_4_0_0  total words..  8
train tree..
category  0  total word count  6
recursing on  mdl_spelling_4_4_0_0_0  total words..  6
train tree..
category  0  total word count  1
category  1  total word count  0
category  2  total word count  4
recursing on  mdl_spelling_4_4_0_0_0_2  total words..  4
Words.. ['history', 'control', 'variety', 'contract']
train tree..
category  3  total word count  0
category  4  total word count  1
category  1  total word count  1
category  2  total word count  0
category  3  total word count  0
category  4  total word count  1
category  1  total word count  8
recursing on  mdl_spelling_4_4_0_1  total words..  8
train tree..
category  0  total word count  1
category  1  total word count  0
category  2  total word count  1
category  3  total word count  4
recursing on  mdl_spelling_4_4_0_1_3  total words..  4
Words.. ['science', 'disease', 'politics', 'college']
train tree..
category  4  total word count  2
recursing on  mdl_spelling_4_4_0_1_4  total words..  2
Words.. ['month', 'death']
train tree..
category  2  total word count  5
recursing on  mdl_spelling_4_4_0_2  total words..  5
train tree..
category  0  total word count  1
category  1  total word count  1
category  2  total word count  1
category  3  total word count  1
category  4  total word count  1
category  3  total word count  2
recursing on  mdl_spelling_4_4_0_3  total words..  2
Words.. ['physics', 'blood']
train tree..
category  4  total word count  13
recursing on  mdl_spelling_4_4_0_4  total words..  13
train tree..
category  0  total word count  5
recursing on  mdl_spelling_4_4_0_4_0  total words..  5
train tree..
category  1  total word count  2
recursing on  mdl_spelling_4_4_0_4_1  total words..  2
Words.. ['problem', 'property']
train tree..
category  2  total word count  2
recursing on  mdl_spelling_4_4_0_4_2  total words..  2
Words.. ['system', 'library']
train tree..
category  3  total word count  2
recursing on  mdl_spelling_4_4_0_4_3  total words..  2
Words.. ['quality', 'currency']
train tree..
category  4  total word count  2
recursing on  mdl_spelling_4_4_0_4_4  total words..  2
Words.. ['ability', 'policy']
train tree..
category  1  total word count  1
category  2  total word count  4
recursing on  mdl_spelling_4_4_2  total words..  4
Words.. ['reading', 'safety', 'reality', 'setting']
train tree..
category  3  total word count  11
recursing on  mdl_spelling_4_4_3  total words..  11
train tree..
category  0  total word count  1
category  1  total word count  0
category  2  total word count  2
recursing on  mdl_spelling_4_4_3_2  total words..  2
Words.. ['depth', 'memory']
train tree..
category  3  total word count  4
recursing on  mdl_spelling_4_4_3_3  total words..  4
Words.. ['world', 'family', 'health', 'wealth']
train tree..
category  4  total word count  4
recursing on  mdl_spelling_4_4_3_4  total words..  4
Words.. ['fishing', 'meaning', 'finding', 'housing']
train tree..
category  4  total word count  10
recursing on  mdl_spelling_4_4_4  total words..  10
train tree..
category  0  total word count  2
recursing on  mdl_spelling_4_4_4_0  total words..  2
Words.. ['people', 'recipe']
train tree..
category  1  total word count  0
category  2  total word count  4
recursing on  mdl_spelling_4_4_4_2  total words..  4
Words.. ['method', 'freedom', 'flight', 'length']
train tree..
category  3  total word count  1
category  4  total word count  3
recursing on  mdl_spelling_4_4_4_4  total words..  3
Words.. ['activity', 'growth', 'category']
train tree..
arr = deep_sp.best_word_match("kitten")_____no_output_____arr_____no_output_____for d in arr:
    print(jellyfish.damerau_levenshtein_distance("c4t",d))1
3
3
3
def test_words(wrds):
    wrl = WordManage()
    wrl.set_words(wrds,0)
    mdl = build_choose_and_train(wrl,dbg=True,out_cat=5)
    wrds = wrl.all_words()
    cats = get_category(mdl,wrds,dbg=True)
    print("categories... ",cats)
    return_____no_output_____im = ImgNet(5)
im.apply(custom_init_weights)
im = best_move(im)
get_category(im,['Chris','Schrumm','Programmer','Test','Nice','Some','Zero','Whatever','Mouse','Other'])_____no_output_____ww = word_batch(['Chris','Schrumm'])_____no_output_____def display_im(im):
    plt.figure(figsize=(25, 150),dpi=80)
    plt.imshow(im)_____no_output_____display_im(ww[0][0])_____no_output_____tt = numpy_to_tensor(np.array([3,4,5]))
tt.to(best_device())
tensor_info(tt)_____no_output_____word_batch(["chris","schrumm"]).shape_____no_output_____wrl = WordManage()
wrl.set_words(sbst,0)
ntwrk_name = "mdl_spelling"
dr = "./net_data"
fl = dr + "/" + ntwrk_name + ".bin"
mdl = best_move(ImgNet(5))
load_file_into_model(mdl,fl)
all_wrds = wrl.all_words()
cat_w = get_category(mdl,all_wrds)
out_cat = 5_____no_output_____build_choose_and_train(wrl,dbg=True)_____no_output_____cat_list = [[all_wrds[v] for v,ct in enumerate(cat_w) if ct == i ] for i in range(out_cat)]_____no_output_____targ_arr = []
in_arr = []

for _ in range(15):
    for i in range(out_cat):
        #print("lenght list ",len(cat_list[i]))
        lst = random.sample(cat_list[i], k=min(len(cat_list[i]),15))
        #print(len(lst))

        for w in lst:
            targ_arr.append(i)
            wd = word_mix(w)
            wrd = encode_word(wd)
            wrd = np.expand_dims(wrd,axis=0)
            in_arr.append(wrd)

tn_in, tn_trg  = np.stack(in_arr),np.array(targ_arr,dtype=np.long)
_____no_output_____l = tn_trg.shape[0]_____no_output_____l_____no_output_____def train_loop(epoch,tn_in,tn_arg, model, optimizer, scheduler,dbg):

    example_size = tn_arg.shape[0]
    example_indexes = [x for x in range(example_size)]
    for i in range(epoch):
        for b in batch(example_indexes,256):
            #print(b[0],b[-1])

            optimizer.zero_grad()

            data = tn_in[b[0]:b[-1]]
            target = tn_arg[b[0]:b[-1]]

            #print("training data ",data.shape)

            data = numpy_to_tensor(data)
            target = numpy_to_tensor(target)

            output = model(data)

            loss = F.nll_loss(output,target)
            loss.backward()

            optimizer.step()

        scheduler.step()
        if dbg:
            print("lr.. ",scheduler.get_lr())_____no_output_____'''
def build_choose_and_train(wrl,dbg=False,out_cat=5):
    out_num = 5
    targ_arr = []
    in_arr = []
    wrds = choose_spread_combo(out_num, wrl)

    for en,v in enumerate(wrds):
        targ_arr.append(en)
        wrd = encode_word(v)
        wrd = np.expand_dims(wrd,axis=0)
        in_arr.append(wrd)
        if dbg:
            print("word ",v," category ",en)

    for i in range(1000):
        for en,v in enumerate(wrds):
            targ_arr.append(en)
            wd = word_mix(v)
            #if dbg:
            #    print("word ",v," mix ",wd," category ",en)
            wrd = encode_word(wd)
            wrd = np.expand_dims(wrd,axis=0)
            in_arr.append(wrd)

    if dbg:
        print("length of input.. ",len(in_arr))

    tn_in, tn_trg  = np.stack(in_arr),np.array(targ_arr,dtype=np.long)

    epoch = 6

    # example_size = len(targ_arr)
    # example_indexes = [x for x in range(example_size)]

    model, optimizer, scheduler = build_net_opt_schedule(out_cat)

    train_loop(epoch,tn_in,tn_trg, model, optimizer, scheduler,dbg)

    if dbg:
        print("second half training..")

    # choose category based on the partially trained model..

    all_wrds = wrl.all_words()
    cat_w = get_category(model,all_wrds)

    #for i,ct in enumerate(cat_w):
    cat_list = [[all_wrds[v] for v,ct in enumerate(cat_w) if ct == i ] for i in range(out_cat)]

    in_arr = []
    targ_arr = []

    epoch = 40

    for _ in range(30):
        for i in range(out_cat):
            #print("lenght list ",len(cat_list[i]))
            lst = random.sample(cat_list[i], k=min(len(cat_list[i]),15))
            #print(len(lst))

            for w in lst:
                targ_arr.append(i)
                wd = word_mix(w)
                wrd = encode_word(wd)
                wrd = np.expand_dims(wrd,axis=0)
                in_arr.append(wrd)

    tn_in, tn_trg  = np.stack(in_arr),np.array(targ_arr,dtype=np.long)

    train_loop(epoch,tn_in,tn_trg, model, optimizer, scheduler,dbg)

    if dbg:
        model.eval()
        with torch.no_grad():
            example_size = tn_trg.shape[0]
            example_indexes = [x for x in range(example_size)]

            for b in batch(example_indexes,256):
                data = tn_in[b[0]:b[-1]]
                target = tn_trg[b[0]:b[-1]]

                data = numpy_to_tensor(data)
                target = numpy_to_tensor(target)
                output = model(data)

                # test_loss += F.nll_loss(output, target, reduction='sum').item()  # sum up batch loss
                pred = output.argmax(dim=1, keepdim=True)  # get the index of the max log-probability
                correct = pred.eq(target.view_as(pred)).sum().item()

                print("correct on batch.. ", (100.0 * correct) / len(b))
    return model
'''_____no_output_____data_ky = load_json("word_tree_mapping.json")_____no_output_____ntwrk_name = "mdl_spelling"
wrd = "marketing"
dr = "./net_data"

mdl = best_move(ImgNet(5))

fl = dr + "/" + ntwrk_name + ".bin"

nt = torch.load(fl)

mdl.load_state_dict(nt)
mdl.eval()

cats = get_category(mdl,[wrd])_____no_output_____cats_____no_output_____mmp = data_ky
sb_ntwrk = ntwrk_name + "_" +  str(cats[0])

word = [k for k,v in mmp.items() if v == sb_ntwrk]_____no_output_____sb_ntwrk_____no_output_____# pell_word(wrd,ntwrk_name,mmp,dr, dbg=False):
spell_word("dibate","mdl_spelling",data_ky,"./net_data",dbg=True)category..  [0]
traverse down..  ./net_data/mdl_spelling_0.bin
category..  [1]
traverse down..  ./net_data/mdl_spelling_0_1.bin
category..  [4]
traverse down..  ./net_data/mdl_spelling_0_1_4.bin
category..  [2]
traverse down..  ./net_data/mdl_spelling_0_1_4_2.bin
category..  [4]
traverse down..  ./net_data/mdl_spelling_0_1_4_2_4.bin
category..  [0]
key  debate  value  mdl_spelling_0_1_4_2_4_0
rt_name = "mdl_spelling"
dcMapping = {}
maxDepth = 30
btch_size = 128
wrl = WordManage()
wrl.set_words(sbst,0)
train_and_choose(rt_name,wrl,dcMapping,maxDepth,btch_size)
save_json(dcMapping,"word_tree_mapping.json")_____no_output_____wrl.word_count()_____no_output_____vl = "nice"
wrl = WordManage()
wrl.set_words(["cat","dog","fish","shark","lark","child","log","chicken","ostrich","zebra","fox","kitten","saint bernard","salmon","cobra","rattlesnake"],0)_____no_output_____rtTreeName = "mdl_spelling"
avWords = wrl
wdDic = {}_____no_output_____mdl = build_choose_and_train(avWords,True)word  fish  category  0
word  fox  category  1
word  rattlesnake  category  2
word  log  category  3
word  saint bernard  category  4
length of input..  1255
lr..  [0.768]
lr..  [0.73728]
lr..  [0.7077887999999999]
lr..  [0.679477248]
second half training..
input shape..  (1360, 1, 3, 256)
lr..  [0.6522981580799999]
lr..  [0.6262062317567999]
lr..  [0.6011579824865279]
lr..  [0.5771116631870667]
input shape..  (1360, 1, 3, 256)
lr..  [0.5540271966595841]
lr..  [0.5318661087932006]
lr..  [0.5105914644414726]
lr..  [0.4901678058638137]
input shape..  (1360, 1, 3, 256)
lr..  [0.4705610936292612]
lr..  [0.4517386498840907]
lr..  [0.43366910388872704]
lr..  [0.41632233973317795]
correct on batch..  81.640625
correct on batch..  88.28125
correct on batch..  88.671875
correct on batch..  87.5
correct on batch..  83.203125
correct on batch..  90.0
wdDic_____no_output_____save_network(mdl,"test")_____no_output_____mdl = best_move(ImgNet(5))
load_data_into(mdl,"test")
#mdl = load_network("test")_____no_output_____wrl.set_category('bike',2)
wrl.set_category('cause',2)
wrl.set_category('clerk',2)_____no_output_____l = wrl.filter_list(2)_____no_output_____# ['tension', 'session']
wd_e = encode_word("food")_____no_output_____def display_im(im):
    plt.figure(figsize=(25, 150),dpi=80)
    plt.imshow(im)_____no_output_____
</code>


# Notebook from himahuja/course-content
Path: tutorials/W3D1_RealNeurons/student/W3D1_Tutorial2.ipynb

_____no_output_____# Neuromatch Academy: Week 3, Day 1, Tutorial 2
# Real Neurons: Effects of Input Correlation
__Content creators:__ Qinglong Gu, Songtin Li, John Murray, Richard Naud, Arvind Kumar

__Content reviewers:__ Maryam Vaziri-Pashkam, Ella Batty, Lorenzo Fontolan, Richard Gao, Matthew Krause, Spiros Chavlis, Michael Waskom_____no_output_____---
# Tutorial Objectives
In this tutorial, we will use the leaky integrate-and-fire (LIF) neuron model (see Tutorial 1) to study how they transform input correlations to output properties (transfer of correlations). In particular, we are going to write a few lines of code to:

- inject correlated GWN in a pair of neurons

- measure correlations between the spiking activity of the two neurons

- study how the transfer of correlation depends on the statistics of the input, i.e. mean and standard deviation._____no_output_____---
# Setup_____no_output_____

# Import libraries
import matplotlib.pyplot as plt
import numpy as np
import time_____no_output_____# @title Figure Settings
import ipywidgets as widgets       # interactive display
%config InlineBackend.figure_format = 'retina'
# use NMA plot style
plt.style.use("https://raw.githubusercontent.com/NeuromatchAcademy/course-content/master/nma.mplstyle")
my_layout = widgets.Layout()_____no_output_____# @title Helper functions
def default_pars(**kwargs):
  pars = {}

  ### typical neuron parameters###
  pars['V_th'] = -55.     # spike threshold [mV]
  pars['V_reset'] = -75.  # reset potential [mV]
  pars['tau_m'] = 10.     # membrane time constant [ms]
  pars['g_L'] = 10.       # leak conductance [nS]
  pars['V_init'] = -75.   # initial potential [mV]
  pars['V_L'] = -75.      # leak reversal potential [mV]
  pars['tref'] = 2.       # refractory time (ms)

  ### simulation parameters ###
  pars['T'] = 400. # Total duration of simulation [ms]
  pars['dt'] = .1  # Simulation time step [ms]

  ### external parameters if any ###
  for k in kwargs:
    pars[k] = kwargs[k]

  pars['range_t'] = np.arange(0, pars['T'], pars['dt'])  # Vector of discretized
                                                         # time points [ms]
  return pars

def run_LIF(pars, Iinj):
  """
  Simulate the LIF dynamics with external input current

  Args:
    pars       : parameter dictionary
    Iinj       : input current [pA]. The injected current here can be a value or an array

  Returns:
    rec_spikes : spike times
    rec_v      : mebrane potential
  """

  # Set parameters
  V_th, V_reset = pars['V_th'], pars['V_reset']
  tau_m, g_L = pars['tau_m'], pars['g_L']
  V_init, V_L = pars['V_init'], pars['V_L']
  dt, range_t = pars['dt'], pars['range_t']
  Lt = range_t.size
  tref = pars['tref']

  # Initialize voltage and current
  v = np.zeros(Lt)
  v[0] = V_init
  Iinj = Iinj * np.ones(Lt)
  tr = 0.

  # simulate the LIF dynamics
  rec_spikes = []   # record spike times
  for it in range(Lt - 1):
    if tr > 0:
      v[it] = V_reset
      tr = tr - 1
    elif v[it] >= V_th:  # reset voltage and record spike event
      rec_spikes.append(it)
      v[it] = V_reset
      tr = tref / dt

    # calculate the increment of the membrane potential
    dv = (-(v[it] - V_L) + Iinj[it] / g_L) * (dt / tau_m)

    # update the membrane potential
    v[it + 1] = v[it] + dv

  rec_spikes = np.array(rec_spikes) * dt

  return v, rec_spikes

def my_GWN(pars, sig, myseed=False):
  """
  Function that calculates Gaussian white noise inputs

  Args:
    pars       : parameter dictionary
    mu         : noise baseline (mean)
    sig        : noise amplitute (standard deviation)
    myseed     : random seed. int or boolean
                 the same seed will give the same random number sequence

  Returns:
    I          : Gaussian white noise input
  """

  # Retrieve simulation parameters
  dt, range_t = pars['dt'], pars['range_t']
  Lt = range_t.size

  # Set random seed. You can fix the seed of the random number generator so
  # that the results are reliable however, when you want to generate multiple
  # realization make sure that you change the seed for each new realization
  if myseed:
      np.random.seed(seed=myseed)
  else:
      np.random.seed()

  # generate GWN
  # we divide here by 1000 to convert units to sec.
  I_GWN = sig * np.random.randn(Lt) * np.sqrt(pars['tau_m'] / dt)

  return I_GWN

def Poisson_generator(pars, rate, n, myseed=False):
  """
  Generates poisson trains

  Args:
    pars       : parameter dictionary
    rate       : noise amplitute [Hz]
    n          : number of Poisson trains
    myseed     : random seed. int or boolean

  Returns:
    pre_spike_train : spike train matrix, ith row represents whether
                      there is a spike in ith spike train over time
                      (1 if spike, 0 otherwise)
  """

  # Retrieve simulation parameters
  dt, range_t = pars['dt'], pars['range_t']
  Lt = range_t.size

  # set random seed
  if myseed:
      np.random.seed(seed=myseed)
  else:
      np.random.seed()

  # generate uniformly distributed random variables
  u_rand = np.random.rand(n, Lt)

  # generate Poisson train
  poisson_train = 1. * (u_rand < rate * (dt / 1000.))

  return poisson_train

def example_plot_myCC():
  pars = default_pars(T=50000, dt=.1)

  c = np.arange(10) * 0.1
  r12 = np.zeros(10)
  for i in range(10):
    I1gL, I2gL = correlate_input(pars, mu=20.0, sig=7.5, c=c[i])
    r12[i] = my_CC(I1gL, I2gL)

  plt.figure()
  plt.plot(c, r12, 'bo', alpha=0.7, label='Simulation', zorder=2)
  plt.plot([-0.05, 0.95], [-0.05, 0.95], 'k--', label='y=x',
           dashes=(2, 2), zorder=1)
  plt.xlabel('True CC')
  plt.ylabel('Sample CC')
  plt.legend(loc='best')

def LIF_output_cc(pars, mu, sig, c, bin_size, n_trials=20):
  """ Simulates two LIF neurons with correlated input and computes output correlation

  Args:
  pars       : parameter dictionary
  mu         : noise baseline (mean)
  sig        : noise amplitute (standard deviation)
  c          : correlation coefficient ~[0, 1]
  bin_size   : bin size used for time series
  n_trials   : total simulation trials

  Returns:
  r          : output corr. coe.
  sp_rate    : spike rate
  sp1        : spike times of neuron 1 in the last trial
  sp2        : spike times of neuron 2 in the last trial
  """

  r12 = np.zeros(n_trials)
  sp_rate = np.zeros(n_trials)
  for i_trial in range(n_trials):
    I1gL, I2gL = correlate_input(pars, mu, sig, c)
    _, sp1 = run_LIF(pars, pars['g_L'] * I1gL)
    _, sp2 = run_LIF(pars, pars['g_L'] * I2gL)

    my_bin = np.arange(0, pars['T'], bin_size)

    sp1_count, _ = np.histogram(sp1, bins=my_bin)
    sp2_count, _ = np.histogram(sp2, bins=my_bin)

    r12[i_trial] = my_CC(sp1_count[::20], sp2_count[::20])
    sp_rate[i_trial] = len(sp1) / pars['T'] * 1000.

  return r12.mean(), sp_rate.mean(), sp1, sp2

def plot_c_r_LIF(c, r, mycolor, mylabel):
  z = np.polyfit(c, r, deg=1)
  c_range = np.array([c.min() - 0.05, c.max() + 0.05])
  plt.plot(c, r, 'o', color=mycolor, alpha=0.7, label=mylabel, zorder=2)
  plt.plot(c_range, z[0] * c_range + z[1], color=mycolor, zorder=1)

_____no_output_____

The helper function contains the:

- Parameter dictionary: `default_pars( **kwargs)`
- LIF simulator: `run_LIF`
- Gaussian white noise generator: `my_GWN(pars, sig, myseed=False)`
- Poisson type spike train generator: `Poisson_generator(pars, rate, n, myseed=False)`
- Two LIF neurons with correlated inputs simulator: `LIF_output_cc(pars, mu, sig, c, bin_size, n_trials=20)`
- Some additional plotting utilities_____no_output_____---
# Section 1: Correlations (Synchrony)
Correlation or synchrony in neuronal activity can be described for any readout of brain activity. Here, we are concerned with the spiking activity of neurons.

In the simplest way, correlation/synchrony refers to coincident spiking of neurons, i.e., when two neurons spike together, they are firing in **synchrony** or are **correlated**. Neurons can be synchronous in their instantaneous activity, i.e., they spike together with some probability. However, it is also possible that spiking of a neuron at time $t$ is correlated with the spikes of another neuron with a delay (time-delayed synchrony).

## Origin of synchronous neuronal activity:
- Common inputs, i.e., two neurons are receiving input from the same sources. The degree of correlation of the shared inputs is proportional to their output correlation.
- Pooling from the same sources. Neurons do not share the same input neurons but are receiving inputs from neurons which themselves are correlated.
- Neurons are connected to each other (uni- or bi-directionally): This will only give rise to time-delayed synchrony. Neurons could also be connected via gap-junctions.
- Neurons have similar parameters and initial conditions.

## Implications of synchrony
When neurons spike together, they can have a stronger impact on downstream neurons. Synapses in the brain are sensitive to the temporal correlations (i.e., delay) between pre- and postsynaptic activity, and this, in turn, can lead to the formation of functional neuronal networks - the basis of unsupervised learning (we will study some of these concepts in a forthcoming tutorial).

Synchrony implies a reduction in the dimensionality of the system. In addition, correlations, in many cases, can impair the decoding of neuronal activity._____no_output_____

# @title Video 1: Input & output correlations
from IPython.display import YouTubeVideo
video = YouTubeVideo(id="nsAYFBcAkes", width=854, height=480, fs=1)
print("Video available at https://youtube.com/watch?v=" + video.id)
videoVideo available at https://youtube.com/watch?v=nsAYFBcAkes

## How to study the emergence of correlations
_____no_output_____A simple model to study the emergence of correlations is to inject common inputs to a pair of neurons and measure the output correlation as a function of the fraction of common inputs.

Here, we are going to investigate the transfer of correlations by computing the correlation coefficient of spike trains recorded from two unconnected LIF neurons, which received correlated inputs.

The input current to LIF neuron $i$ $(i=1,2)$ is:

\begin{equation}
\frac{I_i}{g_L} =\mu_i + \sigma_i (\sqrt{1-c}\xi_i + \sqrt{c}\xi_c) \quad (1)
\end{equation}

where $\mu_i$ is the temporal average of the current. The Gaussian white noise $\xi_i$ is independent for each neuron, while $\xi_c$ is common to all neurons. The variable $c$ ($0\le c\le1$) controls the fraction of common and independent inputs. $\sigma_i$ shows the variance of the total input.

So, first, we will generate correlated inputs._____no_output_____

# @title
#@markdown Execute this cell to get a function for generating correlated GWN inputs
def correlate_input(pars, mu=20., sig=7.5, c=0.3):
  """
  Args:
    pars       : parameter dictionary
    mu         : noise baseline (mean)
    sig        : noise amplitute (standard deviation)
    c.         : correlation coefficient ~[0, 1]

  Returns:
    I1gL, I2gL : two correlated inputs with corr. coe. c
  """

  # generate Gaussian whute noise xi_1, xi_2, xi_c
  xi_1 = my_GWN(pars, sig)
  xi_2 = my_GWN(pars, sig)
  xi_c = my_GWN(pars, sig)

  # Generate two correlated inputs by Equation. (1)
  I1gL = mu + np.sqrt(1. - c) * xi_1 + np.sqrt(c) * xi_c
  I2gL = mu + np.sqrt(1. - c) * xi_2 + np.sqrt(c) * xi_c

  return I1gL, I2gL

print(help(correlate_input))Help on function correlate_input in module __main__:

correlate_input(pars, mu=20.0, sig=7.5, c=0.3)
    Args:
      pars       : parameter dictionary
      mu         : noise baseline (mean)
      sig        : noise amplitute (standard deviation)
      c.         : correlation coefficient ~[0, 1]

    Returns:
      I1gL, I2gL : two correlated inputs with corr. coe. c

None

### Exercise 1: Compute the correlation

The _sample correlation coefficient_ between two input currents $I_i$ and $I_j$ is defined as the sample covariance of $I_i$ and $I_j$ divided by the square root of the sample variance of $I_i$ multiplied with the square root of the sample variance of $I_j$. In equation form:

\begin{align}
r_{ij} &= \frac{cov(I_i, I_j)}{\sqrt{var(I_i)} \sqrt{var(I_j)}}\\
cov(I_i, I_j) &= \sum_{k=1}^L (I_i^k -\bar{I}_i)(I_j^k -\bar{I}_j) \\
var(I_i) &= \sum_{k=1}^L (I_i^k -\bar{I}_i)^2
\end{align}

where $\bar{I}_i$ is the sample mean, k is the time bin, and L is the length of $I$.  This means that $I_i^k$ is current i at time $k\cdot dt$. Note that the equations above are not accurate for sample covariances and variances as they should be additionally divided by L-1 - we have dropped this term because it cancels out in the sample correlation coefficient formula.

The _sample correlation coefficient_ may also be referred to as the _sample Pearson correlation coefficient_. Here, is a beautiful paper that explains multiple ways to calculate and understand correlations [Rodgers and Nicewander 1988](https://www.stat.berkeley.edu/~rabbee/correlation.pdf).

In this exercise, we will create a function, `my_CC` to compute the sample correlation coefficient between two time series. Note that while we introduced this computation here in the context of input currents, the sample correlation coefficient is used to compute the correlation between any two time series - we will use it later on binned spike trains._____no_output_____

def my_CC(i, j):
  """
  Args:
    i, j  : two time series with the same length

  Returns:
    rij   : correlation coefficient
  """
  ########################################################################
  ## TODO for students: compute rxy, then remove the NotImplementedError #
  # Tip1: array([a1, a2, a3])*array([b1, b2, b3]) = array([a1*b1, a2*b2, a3*b3])
  # Tip2: np.sum(array([a1, a2, a3])) = a1+a2+a3
  # Tip3: square root, np.sqrt()
  # Fill out function and remove
  raise NotImplementedError("Student exercise: compute the sample correlation coefficient")
  ########################################################################
  # Calculate the covariance of i and j
  cov = ...
  # Calculate the variance of i
  var_i = ...
  # Calculate the variance of j
  var_j = ...
  # Calculate the correlation coefficient
  rij = ...

  return rij

# Uncomment the line after completing the my_CC function
# example_plot_myCC()_____no_output_____

[*Click for solution*](https://github.com/NeuromatchAcademy/course-content/tree/master//tutorials/W3D1_RealNeurons/solutions/W3D1_Tutorial2_Solution_03e44bdc.py)

*Example output:*

_____no_output_____### Exercise 2: Measure the correlation between spike trains

After recording the spike times of the two neurons, how can we estimate their correlation coefficient?

In order to find this, we need to bin the spike times and obtain two time series. Each data point in the time series is the number of spikes in the corresponding time bin. You can use `np.histogram()` to bin the spike times.

Complete the code below to bin the spike times and calculate the correlation coefficient for two Poisson spike trains. Note that `c` here is the ground-truth correlation coefficient that we define._____no_output_____

# @title

# @markdown Execute this cell to get a function for generating correlated Poisson inputs (generate_corr_Poisson)

def generate_corr_Poisson(pars, poi_rate, c, myseed=False):
  """
  function to generate correlated Poisson type spike trains
  Args:
    pars       : parameter dictionary
    poi_rate   : rate of the Poisson train
    c.         : correlation coefficient ~[0, 1]

  Returns:
    sp1, sp2   : two correlated spike time trains with corr. coe. c
  """

  range_t = pars['range_t']

  mother_rate = poi_rate / c
  mother_spike_train = Poisson_generator(pars, rate=mother_rate,
                                         n=1, myseed=myseed)[0]
  sp_mother = range_t[mother_spike_train > 0]

  L_sp_mother = len(sp_mother)
  sp_mother_id = np.arange(L_sp_mother)
  L_sp_corr = int(L_sp_mother * c)

  np.random.shuffle(sp_mother_id)
  sp1 = np.sort(sp_mother[sp_mother_id[:L_sp_corr]])

  np.random.shuffle(sp_mother_id)
  sp2 = np.sort(sp_mother[sp_mother_id[:L_sp_corr]])

  return sp1, sp2

print(help(generate_corr_Poisson))Help on function generate_corr_Poisson in module __main__:

generate_corr_Poisson(pars, poi_rate, c, myseed=False)
    function to generate correlated Poisson type spike trains
    Args:
      pars       : parameter dictionary
      poi_rate   : rate of the Poisson train
      c.         : correlation coefficient ~[0, 1]

    Returns:
      sp1, sp2   : two correlated spike time trains with corr. coe. c

None
def corr_coeff_pairs(pars, rate, c, trials, bins):
  """
  Calculate the correlation coefficient of two spike trains, for different
  realizations

  Args:
      pars   : parameter dictionary
      rate   : rate of poisson inputs
      c      : correlation coefficient ~ [0, 1]
      trials  : number of realizations
      bins   : vector with bins for time discretization

  Returns:
    r12      : correlation coefficient of a pair of inputs
  """

  r12 = np.zeros(n_trials)

  for i in range(n_trials):
    ##############################################################
    ## TODO for students: Use np.histogram to bin the spike time #
    ##   e.g., sp1_count, _= np.histogram(...)
    # Use my_CC() compute corr coe, compare with c
    # Note that you can run multiple realizations and compute their r_12(diff_trials)
    # with the defined function above. The average r_12 over trials can get close to c.
    # Note: change seed to generate different input per trial
    # Fill out function and remove
    raise NotImplementedError("Student exercise: compute the correlation coefficient")
    ##############################################################

    # Generate correlated Poisson inputs
    sp1, sp2 = generate_corr_Poisson(pars, ..., ..., myseed=2020+i)

    # Bin the spike times of the first input
    sp1_count, _ = np.histogram(..., bins=...)

    # Bin the spike times of the second input
    sp2_count, _ = np.histogram(..., bins=...)

    # Calculate the correlation coefficient
    r12[i] = my_CC(..., ...)

  return r12

poi_rate = 20.
c = 0.2  # set true correlation
pars = default_pars(T=10000)

# bin the spike time
bin_size = 20  # [ms]
my_bin = np.arange(0, pars['T'], bin_size)
n_trials = 100  # 100 realizations

# Uncomment to test your function
# r12 = corr_coeff_pairs(pars, rate=poi_rate, c=c, trials=n_trials, bins=my_bin)
# print(f'True corr coe = {c:.3f}')
# print(f'Simu corr coe = {r12.mean():.3f}')_____no_output_____

Sample output

```
True corr coe = 0.200
Simu corr coe = 0.197
```_____no_output_____[*Click for solution*](https://github.com/NeuromatchAcademy/course-content/tree/master//tutorials/W3D1_RealNeurons/solutions/W3D1_Tutorial2_Solution_c79ab02d.py)

_____no_output_____---
# Section 2: Investigate the effect of input correlation on the output correlation

Now let's combine the aforementioned two procedures. We first generate the correlated inputs by Equation (1). Then we inject the correlated inputs $I_1, I_2$ into a pair of neurons and record their output spike times. We continue measuring the correlation between the output and
investigate the relationship between the input correlation and the output correlation._____no_output_____## Drive a neuron with correlated inputs and visualize its output
In the following, you will inject correlated GWN in two neurons. You need to define the mean (`gwn_mean`), standard deviation (`gwn_std`), and input correlations (`c_in`).

We will simulate $10$ trials to get a better estimate of the output correlation. Change the values in the following cell for the above variables (and then run the next cell) to explore how they impact the output correlation._____no_output_____

# Play around with these parameters

pars = default_pars(T=80000, dt=1.)  # get the parameters
c_in = 0.3  # set input correlation value
gwn_mean = 10.
gwn_std = 10.
_____no_output_____# @title

# @markdown Do not forget to execute this cell to simulate the LIF

bin_size = 10.  # ms

starttime = time.perf_counter()  # time clock

r12_ss, sp_ss, sp1, sp2 = LIF_output_cc(pars, mu=gwn_mean, sig=gwn_std, c=c_in,
                                        bin_size=bin_size, n_trials=10)

# just the time counter
endtime = time.perf_counter()
timecost = (endtime - starttime) / 60.
print(f"Simulation time = {timecost:.2f} min")

print(f"Input correlation = {c_in}")
print(f"Output correlation = {r12_ss}")

plt.figure(figsize=(12, 6))
plt.plot(sp1, np.ones(len(sp1)) * 1, '|', ms=20, label='neuron 1')
plt.plot(sp2, np.ones(len(sp2)) * 1.1, '|', ms=20, label='neuron 2')
plt.xlabel('time (ms)')
plt.ylabel('neuron id.')
plt.xlim(1000, 8000)
plt.ylim(0.9, 1.2)
plt.legend()
plt.show()Simulation time = 0.05 min
Input correlation = 0.3
Output correlation = 0.07493360325177058

## Think!
- Is the output correlation always smaller than the input correlation? If yes, why?
- Should there be a systematic relationship between input and output correlations?

You will explore these questions in the next figure but try to develop your own intuitions first!_____no_output_____Lets vary `c_in` and plot the relationship between the `c_in` and output correlation. This might take some time depending on the number of trials. _____no_output_____

#@title

#@markdown Don't forget to execute this cell!

pars = default_pars(T=80000, dt=1.)  # get the parameters
bin_size = 10.
c_in = np.arange(0, 1.0, 0.1)  # set the range for input CC
r12_ss = np.zeros(len(c_in))  # small mu, small sigma

starttime = time.perf_counter() # time clock
for ic in range(len(c_in)):
  r12_ss[ic], sp_ss, sp1, sp2 = LIF_output_cc(pars, mu=10.0, sig=10.,
                                              c=c_in[ic], bin_size=bin_size,
                                              n_trials=10)

endtime = time.perf_counter()
timecost = (endtime - starttime) / 60.
print(f"Simulation time = {timecost:.2f} min")

plt.figure(figsize=(7, 6))
plot_c_r_LIF(c_in, r12_ss, mycolor='b', mylabel='Output CC')
plt.plot([c_in.min() - 0.05, c_in.max() + 0.05],
         [c_in.min() - 0.05, c_in.max() + 0.05],
         'k--', dashes=(2, 2), label='y=x')

plt.xlabel('Input CC')
plt.ylabel('Output CC')
plt.legend(loc='best', fontsize=16)
plt.show()Simulation time = 0.51 min

[*Click for solution*](https://github.com/NeuromatchAcademy/course-content/tree/master//tutorials/W3D1_RealNeurons/solutions/W3D1_Tutorial2_Solution_ecc2167a.py)

_____no_output_____---
# Section 3: Correlation transfer function
The above plot of input correlation vs. output correlation is called the __correlation transfer function__ of the neurons. _____no_output_____## Section 3.1: How do the mean and standard deviation of the GWN affect the correlation transfer function?

The correlations transfer function appears to be linear. The above can be taken as the input/output transfer function of LIF neurons for correlations, instead of the transfer function for input/output firing rates as we had discussed in the previous tutorial (i.e., F-I curve).

What would you expect to happen to the slope of the correlation transfer function if you vary the mean and/or the standard deviation of the GWN?_____no_output_____

#@markdown Execute this cell to visualize correlation transfer functions

pars = default_pars(T=80000, dt=1.) # get the parameters
no_trial = 10
bin_size = 10.
c_in = np.arange(0., 1., 0.2)  # set the range for input CC
r12_ss = np.zeros(len(c_in))   # small mu, small sigma
r12_ls = np.zeros(len(c_in))   # large mu, small sigma
r12_sl = np.zeros(len(c_in))   # small mu, large sigma

starttime = time.perf_counter()  # time clock
for ic in range(len(c_in)):
  r12_ss[ic], sp_ss, sp1, sp2 = LIF_output_cc(pars, mu=10.0, sig=10.,
                                              c=c_in[ic], bin_size=bin_size,
                                              n_trials=no_trial)
  r12_ls[ic], sp_ls, sp1, sp2 = LIF_output_cc(pars, mu=18.0, sig=10.,
                                              c=c_in[ic], bin_size=bin_size,
                                              n_trials=no_trial)
  r12_sl[ic], sp_sl, sp1, sp2 = LIF_output_cc(pars, mu=10.0, sig=20.,
                                              c=c_in[ic], bin_size=bin_size,
                                              n_trials=no_trial)
endtime = time.perf_counter()
timecost = (endtime - starttime) / 60.
print(f"Simulation time = {timecost:.2f} min")

plt.figure(figsize=(7, 6))
plot_c_r_LIF(c_in, r12_ss, mycolor='b', mylabel=r'Small $\mu$, small $\sigma$')
plot_c_r_LIF(c_in, r12_ls, mycolor='y', mylabel=r'Large $\mu$, small $\sigma$')
plot_c_r_LIF(c_in, r12_sl, mycolor='r', mylabel=r'Small $\mu$, large $\sigma$')
plt.plot([c_in.min() - 0.05, c_in.max() + 0.05],
         [c_in.min() - 0.05, c_in.max() + 0.05],
         'k--', dashes=(2, 2), label='y=x')
plt.xlabel('Input CC')
plt.ylabel('Output CC')
plt.legend(loc='best', fontsize=14)
plt.show()Simulation time = 0.77 min

### Think!
Why do both the mean and the standard deviation of the GWN affect the slope of the correlation transfer function? _____no_output_____[*Click for solution*](https://github.com/NeuromatchAcademy/course-content/tree/master//tutorials/W3D1_RealNeurons/solutions/W3D1_Tutorial2_Solution_bb194a8f.py)

_____no_output_____## Section 3.2: What is the rationale behind varying $\mu$ and $\sigma$?
The mean and the variance of the synaptic current depends on the spike rate of a Poisson process. We can use [Campbell's theorem](https://en.wikipedia.org/wiki/Campbell%27s_theorem_(probability)) to estimate the mean and the variance of the synaptic current:

\begin{align}
\mu_{\rm syn} = \lambda J \int P(t) \\
\sigma_{\rm syn} = \lambda J \int P(t)^2 dt\\
\end{align}

where $\lambda$ is the firing rate of the Poisson input, $J$ the amplitude of the postsynaptic current and $P(t)$ is the shape of the postsynaptic current as a function of time.

Therefore, when we varied $\mu$ and/or $\sigma$ of the GWN, we mimicked a change in the input firing rate. Note that, if we change the firing rate, both $\mu$ and $\sigma$ will change simultaneously, not independently.

Here, since we observe an effect of $\mu$ and $\sigma$ on correlation transfer, this implies that the input rate has an impact on the correlation transfer function.
_____no_output_____### Think!

- What are the factors that would make output correlations smaller than input correlations? (Notice that the colored lines are below the black dashed line)
- What does it mean for the correlation in the network?
- Here we have studied the transfer of correlations by injecting GWN. But in the previous tutorial, we mentioned that GWN is unphysiological. Indeed, neurons receive colored noise (i.e., Shot noise or OU process). How do these results obtained from injection of GWN apply to the case where correlated spiking inputs are injected in the two LIFs? Will the results be the same or different?

Reference
- De La Rocha, Jaime, et al. "Correlation between neural spike trains increases with firing rate." Nature (2007) (https://www.nature.com/articles/nature06028/)

- Bujan AF, Aertsen A, Kumar A. Role of input correlations in shaping the variability and noise correlations of evoked activity in the neocortex. Journal of Neuroscience. 2015 Jun 3;35(22):8611-25. (https://www.jneurosci.org/content/35/22/8611)_____no_output_____[*Click for solution*](https://github.com/NeuromatchAcademy/course-content/tree/master//tutorials/W3D1_RealNeurons/solutions/W3D1_Tutorial2_Solution_4575a736.py)

_____no_output_____---
# Summary

In this tutorial, we studied how the input correlation of two LIF neurons is mapped to their output correlation. Specifically, we:

- injected correlated GWN in a pair of neurons,

- measured correlations between the spiking activity of the two neurons, and

- studied how the transfer of correlation depends on the statistics of the input, i.e., mean and standard deviation.

Here, we were concerned with zero time lag correlation. For this reason, we restricted estimation of correlation to instantaneous correlations. If you are interested in time-lagged correlation, then we should estimate the cross-correlogram of the spike trains and find out the dominant peak and area under the peak to get an estimate of output correlations.

We leave this as a future to-do for you if you are interested._____no_output_____---
# Bonus 1: Example of a conductance-based LIF model
Above, we have written code to generate correlated Poisson spike trains. You can write code to stimulate the LIF neuron with such correlated spike trains and study the correlation transfer function for spiking input and compare it to the correlation transfer function obtained by injecting correlated GWNs._____no_output_____

# @title Function to simulate conductance-based LIF

def run_LIF_cond(pars, I_inj, pre_spike_train_ex, pre_spike_train_in):
  """
  conductance-based LIF dynamics

  Args:
    pars               : parameter dictionary
    I_inj              : injected current [pA]. The injected current here can
                         be a value or an array
    pre_spike_train_ex : spike train input from presynaptic excitatory neuron
    pre_spike_train_in : spike train input from presynaptic inhibitory neuron

  Returns:
    rec_spikes         : spike times
    rec_v              : mebrane potential
    gE                 : postsynaptic excitatory conductance
    gI                 : postsynaptic inhibitory conductance
  """

  # Retrieve parameters
  V_th, V_reset = pars['V_th'], pars['V_reset']
  tau_m, g_L = pars['tau_m'], pars['g_L']
  V_init, E_L = pars['V_init'], pars['E_L']
  gE_bar, gI_bar = pars['gE_bar'], pars['gI_bar']
  VE, VI = pars['VE'], pars['VI']
  tau_syn_E, tau_syn_I = pars['tau_syn_E'], pars['tau_syn_I']
  tref = pars['tref']
  dt, range_t = pars['dt'], pars['range_t']
  Lt = range_t.size

  # Initialize
  tr = 0.
  v = np.zeros(Lt)
  v[0] = V_init
  gE = np.zeros(Lt)
  gI = np.zeros(Lt)
  Iinj = I_inj * np.ones(Lt)  # ensure I has length Lt

  if pre_spike_train_ex.max() == 0:
    pre_spike_train_ex_total = np.zeros(Lt)
  else:
    pre_spike_train_ex_total = pre_spike_train_ex * np.ones(Lt)

  if pre_spike_train_in.max() == 0:
    pre_spike_train_in_total = np.zeros(Lt)
  else:
    pre_spike_train_in_total = pre_spike_train_in * np.ones(Lt)

  # simulation
  rec_spikes = []  # recording spike times
  for it in range(Lt - 1):
    if tr > 0:
      v[it] = V_reset
      tr = tr - 1
    elif v[it] >= V_th:   # reset voltage and record spike event
      rec_spikes.append(it)
      v[it] = V_reset
      tr = tref / dt
    # update the synaptic conductance
    gE[it+1] = gE[it] - (dt / tau_syn_E) * gE[it] + gE_bar * pre_spike_train_ex_total[it + 1]
    gI[it+1] = gI[it] - (dt / tau_syn_I) * gI[it] + gI_bar * pre_spike_train_in_total[it + 1]

    # calculate the increment of the membrane potential
    dv = (-(v[it] - E_L) - (gE[it + 1] / g_L) * (v[it] - VE) - \
            (gI[it + 1] / g_L) * (v[it] - VI) + Iinj[it] / g_L) * (dt / tau_m)

    # update membrane potential
    v[it + 1] = v[it] + dv

  rec_spikes = np.array(rec_spikes) * dt

  return v, rec_spikes, gE, gI

print(help(run_LIF_cond))Help on function run_LIF_cond in module __main__:

run_LIF_cond(pars, I_inj, pre_spike_train_ex, pre_spike_train_in)
    conductance-based LIF dynamics

    Args:
      pars               : parameter dictionary
      I_inj              : injected current [pA]. The injected current here can
                           be a value or an array
      pre_spike_train_ex : spike train input from presynaptic excitatory neuron
      pre_spike_train_in : spike train input from presynaptic inhibitory neuron

    Returns:
      rec_spikes         : spike times
      rec_v              : mebrane potential
      gE                 : postsynaptic excitatory conductance
      gI                 : postsynaptic inhibitory conductance

None

## Interactive Demo: Correlated spike input to an LIF neuron

In the following you can explore what happens when the neurons receive correlated spiking input.

You can vary the correlation between excitatory input spike trains. For simplicity, the correlation between inhibitory spike trains is set to 0.01.

Vary both excitatory rate and correlation and see how the output correlation changes. Check if the results are qualitatively similar to what you observed previously when you varied the $\mu$ and $\sigma$.
_____no_output_____

# @title

# @markdown Make sure you execute this cell to enable the widget!

my_layout.width = '450px'
@widgets.interact(
  pwc_ee=widgets.FloatSlider(0.3, min=0.05, max=0.99, step=0.01,
                             layout=my_layout),
  exc_rate=widgets.FloatSlider(1e3, min=500., max=5e3, step=50.,
                               layout=my_layout),
  inh_rate=widgets.FloatSlider(500., min=300., max=5e3, step=5.,
                               layout=my_layout),
)

def EI_isi_regularity(pwc_ee, exc_rate, inh_rate):
  pars = default_pars(T=1000.)
  # Add parameters
  pars['V_th'] = -55.     # spike threshold [mV]
  pars['V_reset'] = -75.  # reset potential [mV]
  pars['tau_m'] = 10.     # membrane time constant [ms]
  pars['g_L'] = 10.       # leak conductance [nS]
  pars['V_init'] = -65.   # initial potential [mV]
  pars['E_L'] = -75.      # leak reversal potential [mV]
  pars['tref'] = 2.       # refractory time (ms)

  pars['gE_bar'] = 4.0    # [nS]
  pars['VE'] = 0.         # [mV] excitatory reversal potential
  pars['tau_syn_E'] = 2.  # [ms]
  pars['gI_bar'] = 2.4    # [nS]
  pars['VI'] = -80.       # [mV] inhibitory reversal potential
  pars['tau_syn_I'] = 5.  # [ms]

  my_bin = np.arange(0, pars['T']+pars['dt'], .1)  # 20 [ms] bin-size

  # exc_rate = 1e3
  # inh_rate = 0.4e3
  # pwc_ee = 0.3
  pwc_ii = 0.01
  # generate two correlated spike trains for excitatory input
  sp1e, sp2e = generate_corr_Poisson(pars, exc_rate, pwc_ee)

  sp1_spike_train_ex, _ = np.histogram(sp1e, bins=my_bin)
  sp2_spike_train_ex, _ = np.histogram(sp2e, bins=my_bin)

  # generate two uncorrelated spike trains for inhibitory input
  sp1i, sp2i = generate_corr_Poisson(pars, inh_rate, pwc_ii)
  sp1_spike_train_in, _ = np.histogram(sp1i, bins=my_bin)
  sp2_spike_train_in, _ = np.histogram(sp2i, bins=my_bin)

  v1, rec_spikes1, gE, gI = run_LIF_cond(pars, 0, sp1_spike_train_ex, sp1_spike_train_in)
  v2, rec_spikes2, gE, gI = run_LIF_cond(pars, 0, sp2_spike_train_ex, sp2_spike_train_in)

  # bin the spike time
  bin_size = 20  # [ms]
  my_bin = np.arange(0, pars['T'], bin_size)
  spk_1, _ = np.histogram(rec_spikes1, bins=my_bin)
  spk_2, _ = np.histogram(rec_spikes2, bins=my_bin)

  r12 = my_CC(spk_1, spk_2)
  print(f"Input correlation = {pwc_ee}")
  print(f"Output correlation = {r12}")

  plt.figure(figsize=(14, 7))
  plt.subplot(211)
  plt.plot(sp1e, np.ones(len(sp1e)) * 1, '|', ms=20,
           label='Exc. input 1')
  plt.plot(sp2e, np.ones(len(sp2e)) * 1.1, '|', ms=20,
           label='Exc. input 2')
  plt.plot(sp1i, np.ones(len(sp1i)) * 1.3, '|k', ms=20,
           label='Inh. input 1')
  plt.plot(sp2i, np.ones(len(sp2i)) * 1.4, '|k', ms=20,
           label='Inh. input 2')

  plt.ylim(0.9, 1.5)
  plt.legend()
  plt.ylabel('neuron id.')

  plt.subplot(212)
  plt.plot(pars['range_t'], v1, label='neuron 1')
  plt.plot(pars['range_t'], v2, label='neuron 2')
  plt.xlabel('time (ms)')
  plt.ylabel('membrane voltage $V_{m}$')
  plt.tight_layout()
  plt.show()Input correlation = 0.3
Output correlation = 0.5240669646233612

Above, we are estimating the output correlation for one trial. You can modify the code to get a trial average of output correlations.

_____no_output_____---
# Bonus 2: Ensemble Response

Finally, there is a short BONUS lecture video on the firing response of an ensemble of neurons to time-varying input. There are no associated coding exercises - just enjoy._____no_output_____

#@title Video 2 (Bonus): Response of ensemble of neurons to time-varying input
from IPython.display import YouTubeVideo
video = YouTubeVideo(id="78_dWa4VOIo", width=854, height=480, fs=1)
print("Video available at https://youtube.com/watch?v=" + video.id)
videoVideo available at https://youtube.com/watch?v=78_dWa4VOIo