# Notebook from aude-forcione-lambert/jupyter-test-lab
Path: .ipynb_checkpoints/analysis-checkpoint.ipynb

**analysis.ipynb**

**Description:** Interactive auxillary support for my summer internship repport. This is the analysis of three recurrent neural networks using mainly primary component analysis (PCA) and slow point analysis.

**Author:** Aude Forcione-lambert, University of Montreal, Faculty of Arts and Sciences

**Email:** aude.forcione-lambert@umontreal.ca_____no_output_____# Importations_____no_output_____

import numpy as np
import scipy
from sklearn.decomposition import PCA

import matplotlib as mpl
import matplotlib.pyplot as plt
from matplotlib.lines import Line2D
from mpl_toolkits.mplot3d import Axes3D_____no_output_____# Uncomment for interactive graphics. Be aware that this will slow down the program.
#%matplotlib notebook_____no_output_____

# Useful variables_____no_output_____

# Number of neurons
nb_neurons = 100
# Identity matrix matching the main matrix's size
I = np.diag(np.ones(nb_neurons))
# Possible switches by index
# Possible outputs are either represented by tensions (3 channels of real numbers roughly between -1 and 1) or by an
# index between 0 and 8. Index and tension of each three channels are related by the binary representation of the
# index. The unit digit corresponds to channel one, a 0 digit corresponds to a tension of -1 and a 1 to a tension of 1.
switch = np.array([[0,1],[0,2],[0,4],[1,3],[1,5],[2,3],[2,6],[3,7],[4,5],[4,6],[5,7],[6,7]])
# Colors used in graphics for each index
colors = ('indianred','darkorange','gold','lime','turquoise','royalblue','blueviolet','magenta')_____no_output_____

# Useful functions_____no_output_____

#_Converts channel tensions to the corresponding index... Honestly it's pretty self-explanatory............................._Do_I_really_have_to_do_this_everywhere?_I'm_not_using_three_meters_long_variable_names_for_fun..............................................................Fine, I'll do it. This is for you, all my past programming teachers........................ Shoutout to Sonia Gounar, you are one great teacher, even if you did always remove some of my points for non-sufficient comments... I might have deserved it.............................................................................................................................................................................................................................I need coffee
def tension_to_index(tension):
    tension = np.round(tension)
    return int((tension[0]+1)/2+(tension[1]+1)+2*(tension[2]+1))

# Same but the other way around
def index_to_tension(index):
    tension = np.zeros((3))
    tension[0] = (index%2)*2-1
    # These >> are bitshifts. Bitshifts are cool
    tension[1] = ((index >> 1)%2)*2-1
    tension[2] = ((index >> 2)%2)*2-1
    return tension

# Calculates next network state in the absence of input. Useful for q function
def F(state,w):
    return np.tanh(np.matmul(state,w))

# Energy function used for finding slow points
def q(state,w):
    return 1/2.0*np.linalg.norm(F(state,w)-state)**2

# Find all instances of a short pattern in a long array
def matchPattern(base_array,pattern):
    indexes = []
    for i in range(len(base_array)-len(pattern)):
        if np.all(base_array[i:i+len(pattern)] == pattern):
            indexes.append(i)
    return np.array(indexes)_____no_output_____

# Data importation_____no_output_____

# Input array  (3 channels)
x_test_array = np.loadtxt('Data/x_test_array.gz')
# Expected output array (3 channels)
y_test_array = np.loadtxt('Data/y_test_array.gz')

# Loss over time for random_seed = 7 and time_delay = 0
#s7d0_loss = np.loadtxt('Data/s7d0_loss.gz')
# Output during testing
s7d0_output = np.loadtxt('Data/s7d0_output.gz')
# States of neurons during testing
s7d0_states = np.loadtxt('Data/s7d0_states.gz')
# W matrix (or strengths of neurons connections)
s7d0_w = np.loadtxt('Data/s7d0_w.gz')
# Wout matrix
s7d0_wout = np.loadtxt('Data/s7d0_wout.gz')

# Same for random_seed = 7 and time_delay = 4
s7d4_loss = np.loadtxt('Data/s7d4_loss.gz')
s7d4_output = np.loadtxt('Data/s7d4_output.gz')
s7d4_states = np.loadtxt('Data/s7d4_states.gz')
s7d4_w = np.loadtxt('Data/s7d4_w.gz')
s7d4_wout = np.loadtxt('Data/s7d4_wout.gz')

# Same for random_seed = 7 and time_delay = 8
#s7d8_loss = np.loadtxt('Data/s7d8_loss.gz')
s7d8_output = np.loadtxt('Data/s7d8_output.gz')
s7d8_states = np.loadtxt('Data/s7d8_states.gz')
s7d8_w = np.loadtxt('Data/s7d8_w.gz')
s7d8_wout = np.loadtxt('Data/s7d8_wout.gz')

# Win matrix for random_seed = 7 and all time delays
#s7_win = np.loadtxt('Data/s7_win.gz')_____no_output_____

# Task Definition_____no_output_____

fig,ax_array = plt.subplots(3,3,sharex=True,figsize=(8,4))
fig.subplots_adjust(wspace=0)

# Channel 1
# Expected output
ax_array[0,0].plot(y_test_array[10000:10150,0],'salmon')
# Input
ax_array[0,0].plot(x_test_array[10000:10150,0],'--',color='darkred',linewidth=1)
# Style
ax_array[0,0].set_ylim(-1.2,1.2)
ax_array[0,0].set_title('Channel 1',fontsize=10)

# Channel 2
# Expected output
ax_array[0,1].plot(y_test_array[10000:10150,1],'limegreen')
# Input
ax_array[0,1].plot(x_test_array[10000:10150,1],'--',color='darkgreen',linewidth=1)
# Style
ax_array[0,1].set_ylim(-1.2,1.2)
ax_array[0,1].set_title('Channel 2',fontsize=10)
ax_array[0,1].get_yaxis().set_ticks([])

# Channel 3
# Expected output
ax_array[0,2].plot(y_test_array[10000:10150,2],'dodgerblue')
# Input
ax_array[0,2].plot(x_test_array[10000:10150,2],'--',color='darkblue',linewidth=1)
# Style
ax_array[0,2].set_ylim(-1.2,1.2)
ax_array[0,2].set_title('Channel 3',fontsize=10)
ax_array[0,2].get_yaxis().set_ticks([])

ax_array[0,0].set_ylabel('Without\ndelay')

for j in [1,2]:
    delay = 4*j

    # Channel 1
    # Visualisation of delay
    for i in np.where(np.abs(np.round((y_test_array[10000-delay:10150-delay,0]-x_test_array[10000:10150,0])))==2)[0] :
        ax_array[j,0].axvspan(i,i+delay,color='lightgray')
    # Expected output
    ax_array[j,0].plot(y_test_array[10000-delay:10150-delay,0],'salmon')
    # Input
    ax_array[j,0].plot(x_test_array[10000:10150,0],'--',color='darkred',linewidth=1)
    # Style
    ax_array[j,0].set_ylim(-1.2,1.2)

    # Channel 2
    # Visualisation of delay
    for i in np.where(np.abs(np.round((y_test_array[10000-delay:10150-delay,1]-x_test_array[10000:10150,1])))==2)[0] :
        ax_array[j,1].axvspan(i,i+delay,color='lightgray')
    # Expected output
    ax_array[j,1].plot(y_test_array[10000-delay:10150-delay,1],'limegreen')
    # Input
    ax_array[j,1].plot(x_test_array[10000:10150,1],'--',color='darkgreen',linewidth=1)
    # Make pwetty
    ax_array[j,1].set_ylim(-1.2,1.2)
    ax_array[j,1].get_yaxis().set_ticks([])

    # Channel 3
    # Visualisation of delay
    for i in np.where(np.abs(np.round((y_test_array[10000-delay:10150-delay,2]-x_test_array[10000:10150,2])))==2)[0] :
        ax_array[j,2].axvspan(i,i+delay,color='lightgray')
    # Expected output
    ax_array[j,2].plot(y_test_array[10000-delay:10150-delay,2],'dodgerblue')
    # Input
    ax_array[j,2].plot(x_test_array[10000:10150,2],'--',color='darkblue',linewidth=1)
    # Style
    ax_array[j,2].set_ylim(-1.2,1.2)
    ax_array[j,2].get_yaxis().set_ticks([])

    ax_array[j,0].set_ylabel('Delayed\n'+str(delay)+' time-steps')

ax_array[2,1].set_xlabel('Elapsed time steps')

custom_legend = [
    Line2D([0], [0], color='k', linestyle='--', linewidth=1),
    Line2D([0], [0], color='dimgray',linewidth=1.5),
    mpl.patches.Patch(facecolor='lightgray', edgecolor='lightgray')
]
ax_array[0,1].legend(custom_legend, ['Input','Expected\noutput','Duration\nof delay'], loc='upper center', bbox_to_anchor=(2.3,-0.05), ncol=1)_____no_output_____

---
Task definition. The neural network is expected to match and maintain the last input for each channel while preventing cross-talk. For the delayed task definition, the neural network must in addition delay its response (and therefore remember the input) during a certain number of time steps.

---_____no_output_____# Training_____no_output_____

plt.figure(figsize=(8,5))
plt.subplot(1,1,1)
plt.xlabel("Epoch index")
plt.ylabel("Mean squared error")
plt.plot(np.arange(3000),s7d4_loss[:3000])_____no_output_____

---
Sample loss over training epoch. This graphic was generated during training of the RNN with a random seed of 7 and a delay of 4 time-steps.

---_____no_output_____# Elementary case_____no_output_____

# Simple pca analysis of the test for random_seed=7 and time_delay=0

s7d0_pca = PCA(n_components=3)
s7d0_pca.fit(s7d0_states)
s7d0_pc1, s7d0_pc2, s7d0_pc3 = s7d0_pca.transform(s7d0_states).T

# Type of output for the neurons states
s7d0_out_type = np.array([tension_to_index(out) for out in s7d0_output])_____no_output_____# Drawing of pca analysis

fig = plt.figure(figsize=(8,6))
ax = fig.add_subplot(111, projection='3d')

#
for i in range(3000):
    ax.scatter(s7d0_pc1[i], s7d0_pc2[i], s7d0_pc3[i], c=colors[s7d0_out_type[i]], alpha=0.05)

ax.plot(s7d0_pc1[:3000], s7d0_pc2[:3000], s7d0_pc3[:3000], '-k', linewidth=0.5, alpha=0.5)

custom_legend = [
    Line2D([0], [0], color = 'w', markersize = 15, markerfacecolor = colors[tension_to_index([1,1,1])], marker = '.', label = 'Out (1,1,1)'),
    Line2D([0], [0], color = 'w', markersize = 15, markerfacecolor = colors[tension_to_index([-1,1,1])], marker = '.', label = 'Out (-1,1,1)'),
    Line2D([0], [0], color = 'w', markersize = 15, markerfacecolor = colors[tension_to_index([1,-1,1])], marker = '.', label = 'Out (1,-1,1)'),
    Line2D([0], [0], color = 'w', markersize = 15, markerfacecolor = colors[tension_to_index([-1,-1,1])], marker = '.', label = 'Out (-1,-1,1)'),
    Line2D([0], [0], color = 'w', markersize = 15, markerfacecolor = colors[tension_to_index([1,1,-1])], marker = '.', label = 'Out (1,1,-1)'),
    Line2D([0], [0], color = 'w', markersize = 15, markerfacecolor = colors[tension_to_index([-1,1,-1])], marker = '.', label = 'Out (-1,1,-1)'),
    Line2D([0], [0], color = 'w', markersize = 15, markerfacecolor = colors[tension_to_index([1,-1,-1])], marker = '.', label = 'Out (1,-1,-1)'),
    Line2D([0], [0], color = 'w', markersize = 15, markerfacecolor = colors[tension_to_index([-1,-1,-1])], marker = '.', label = 'Out (-1,-1,-1)'),
    Line2D([0], [0], color = 'gray', label = 'States trajectory')
]
ax.legend(handles = custom_legend, loc=9, ncol=3)
ax.set_xlabel('PC1')
ax.set_ylabel('PC2')
ax.set_zlabel('PC3')
ax.view_init(39,-126)_____no_output_____

---
Dynamics during 3000 time-steps of the test for the network without delay. The states of the neurons are plotted according to the first three principal components. Each point represents the network's state at a different time-step and is color-coded according to the effective output value at that time. The trajectory is plotted in gray. In the absence of input, the network will maintain its position in phase-space. Injecting an input in the network induces a switch of the network towards the new output.

---_____no_output_____# Adding delay_____no_output_____Delay = 4_____no_output_____

# Simple pca analysis of the test for random_seed=7 and time_delay=4

s7d4_pca = PCA(n_components=3)
s7d4_pca.fit(s7d4_states)
s7d4_pc1, s7d4_pc2, s7d4_pc3 = s7d4_pca.transform(s7d4_states).T

# Type of output for the neurons states
s7d4_out_type = np.array([tension_to_index(out) for out in s7d4_output])_____no_output_____# Drawing of pca analysis

fig = plt.figure(figsize=(8,6))
ax = fig.add_subplot(111, projection='3d')

#
for i in range(3000):
    ax.scatter(s7d4_pc1[i], s7d4_pc2[i], s7d4_pc3[i], c=colors[s7d4_out_type[i]], alpha=0.05)

ax.plot(s7d4_pc1[:3000], s7d4_pc2[:3000], s7d4_pc3[:3000], '-k', linewidth=0.5, alpha=0.5)

custom_legend = [
    Line2D([0], [0], color = 'w', markersize = 15, markerfacecolor = colors[tension_to_index([1,1,1])], marker = '.', label = 'Out (1,1,1)'),
    Line2D([0], [0], color = 'w', markersize = 15, markerfacecolor = colors[tension_to_index([-1,1,1])], marker = '.', label = 'Out (-1,1,1)'),
    Line2D([0], [0], color = 'w', markersize = 15, markerfacecolor = colors[tension_to_index([1,-1,1])], marker = '.', label = 'Out (1,-1,1)'),
    Line2D([0], [0], color = 'w', markersize = 15, markerfacecolor = colors[tension_to_index([-1,-1,1])], marker = '.', label = 'Out (-1,-1,1)'),
    Line2D([0], [0], color = 'w', markersize = 15, markerfacecolor = colors[tension_to_index([1,1,-1])], marker = '.', label = 'Out (1,1,-1)'),
    Line2D([0], [0], color = 'w', markersize = 15, markerfacecolor = colors[tension_to_index([-1,1,-1])], marker = '.', label = 'Out (-1,1,-1)'),
    Line2D([0], [0], color = 'w', markersize = 15, markerfacecolor = colors[tension_to_index([1,-1,-1])], marker = '.', label = 'Out (1,-1,-1)'),
    Line2D([0], [0], color = 'w', markersize = 15, markerfacecolor = colors[tension_to_index([-1,-1,-1])], marker = '.', label = 'Out (-1,-1,-1)'),
    Line2D([0], [0], color = 'gray', label = 'States trajectory')
]
ax.legend(handles = custom_legend, loc=9, ncol=3)
ax.set_xlabel('PC1')
ax.set_ylabel('PC2')
ax.set_zlabel('PC3')
ax.view_init(8,-53)_____no_output_____

---
Dynamics of the delayed network. The network must generate the same output as before but delayed by 4 time-steps.

---_____no_output_____Delay = 8_____no_output_____

# Simple pca analysis of the test for random_seed=7 and time_delay=8

s7d8_pca = PCA(n_components=3)
s7d8_pca.fit(s7d8_states)
s7d8_pc1, s7d8_pc2, s7d8_pc3 = s7d8_pca.transform(s7d8_states).T

# Type of output for the neurons states
s7d8_out_type = np.array([tension_to_index(out) for out in s7d8_output])_____no_output_____# Drawing of pca analysis

fig = plt.figure(figsize=(8,6))
ax = fig.add_subplot(111, projection='3d')

# Color coded points for neurons' states at each time step
for i in range(3000):
    ax.scatter(s7d8_pc1[i], s7d8_pc2[i], s7d8_pc3[i], c=colors[s7d8_out_type[i]], alpha=0.05)
# Path over time of neurons' states
ax.plot(s7d8_pc1[:3000], s7d8_pc2[:3000], s7d8_pc3[:3000], '-k', linewidth=0.5, alpha=0.5)

custom_legend = [
    Line2D([0], [0], color = 'w', markersize = 15, markerfacecolor = colors[tension_to_index([1,1,1])], marker = '.', label = 'Out (1,1,1)'),
    Line2D([0], [0], color = 'w', markersize = 15, markerfacecolor = colors[tension_to_index([-1,1,1])], marker = '.', label = 'Out (-1,1,1)'),
    Line2D([0], [0], color = 'w', markersize = 15, markerfacecolor = colors[tension_to_index([1,-1,1])], marker = '.', label = 'Out (1,-1,1)'),
    Line2D([0], [0], color = 'w', markersize = 15, markerfacecolor = colors[tension_to_index([-1,-1,1])], marker = '.', label = 'Out (-1,-1,1)'),
    Line2D([0], [0], color = 'w', markersize = 15, markerfacecolor = colors[tension_to_index([1,1,-1])], marker = '.', label = 'Out (1,1,-1)'),
    Line2D([0], [0], color = 'w', markersize = 15, markerfacecolor = colors[tension_to_index([-1,1,-1])], marker = '.', label = 'Out (-1,1,-1)'),
    Line2D([0], [0], color = 'w', markersize = 15, markerfacecolor = colors[tension_to_index([1,-1,-1])], marker = '.', label = 'Out (1,-1,-1)'),
    Line2D([0], [0], color = 'w', markersize = 15, markerfacecolor = colors[tension_to_index([-1,-1,-1])], marker = '.', label = 'Out (-1,-1,-1)'),
    Line2D([0], [0], color = 'gray', label = 'States trajectory')
]
ax.legend(handles = custom_legend, loc=9, ncol=3)
ax.set_xlabel('PC1')
ax.set_ylabel('PC2')
ax.set_zlabel('PC3')
ax.view_init(15,-128)_____no_output_____

---
Dynamics of the network delayed by 8 time-steps.

---_____no_output_____# Slow point analysis_____no_output_____0 delay_____no_output_____

# Calculating approximate cube
s7d0_center_array = []
for i in range(8):
    indexes = np.where(np.equal(s7d0_out_type,i*np.ones(len(s7d0_out_type))))[0]
    s7d0_center= np.mean(s7d0_states[indexes],axis=0)
    s7d0_center_array.append(s7d0_center)
s7d0_center_array = np.array(s7d0_center_array)

s7d0_cube = s7d0_center_array[switch]_____no_output_____# Finding slow points
s7d0_min_q = []
for i in range(300):
    opt_result = scipy.optimize.minimize(q,(np.random.rand(nb_neurons)-0.5)*10,args=(s7d0_w))
    # if {optimization was a success} and ( {this is the first slow point} or {slow point does not correspond to one already found} )
    if opt_result['success'] and (len(s7d0_min_q)==0 or np.all([np.linalg.norm(np.abs(min_q_i-opt_result['x']))>1 for min_q_i in s7d0_min_q])):
        s7d0_min_q.append(opt_result['x'])
s7d0_min_q = np.array(s7d0_min_q)

# Calculating the number of free dimensions for each slow point
s7d0_q_type = []
for min_q_i in s7d0_min_q:
    J = np.empty((nb_neurons,nb_neurons))
    for i in range(nb_neurons):
        J[i,:] = np.dot(s7d0_w[i,:],(1/np.cosh(min_q_i[i]))**2)-I[i,:]
    s7d0_q_type.append(sum(np.linalg.eig(J)[0]>0))_____no_output_____fig = plt.figure(figsize=(8,6))
ax = fig.add_subplot(111, projection='3d')

for edge in s7d0_cube:
    edge_pc1, edge_pc2, edge_pc3 = s7d0_pca.transform(edge).T
    ax.plot(edge_pc1, edge_pc2, edge_pc3, '--', c='darkgray', lw=1)

min_q_pc1, min_q_pc2, min_q_pc3 = s7d0_pca.transform(s7d0_min_q).T
for i in range(s7d0_min_q.shape[0]):
    if(s7d0_q_type[i]<=3):
        ax.scatter(min_q_pc1[i], min_q_pc2[i], min_q_pc3[i], c=['k','b','g','y'][s7d0_q_type[i]], marker='x', alpha=0.7)
    else:
        ax.scatter(min_q_pc1[0,i], min_q_pc2[1,i], min_q_pc3[2,i], c='r', marker='^', alpha=0.7)
        ax.text(min_q_pc1[i], min_q_pc2[i], min_q_pc3[i], str(s7d0_q_type[i]))

custom_legend = [
    Line2D([0], [0], color = 'w', markersize = 10, markerfacecolor = 'k', marker = 'X', label = 'Fixed point'),
    Line2D([0], [0], color = 'w', markersize = 10, markerfacecolor = 'b', marker = 'X', label = 'Saddle point (1 unst dim)'),
    Line2D([0], [0], color = 'w', markersize = 10, markerfacecolor = 'g', marker = 'X', label = 'Saddle point (2 unst dims)'),
    Line2D([0], [0], color = 'w', markersize = 10, markerfacecolor = 'y', marker = 'X', label = 'Saddle point (3 unst dims)'),
    Line2D([0], [0], color = 'darkgray', linestyle='--', label = 'Approx. test trajectory')
]
ax.legend(handles = custom_legend, loc=9, ncol=2)
ax.set_xlabel('PC1')
ax.set_ylabel('PC2')
ax.set_zlabel('PC3')
ax.view_init(8,-21)_____no_output_____

---
Slow point analysis of RNN without delay. For clarity, a cube is used to mark the approximate trajectory of neurons states during the test. The slow points are color coded with respect to their number of unstable dimensions. This is consistent with the findings of Susslio and Banach.

---_____no_output_____4 delay_____no_output_____

# Calculating approximate cube
s7d4_center_array = []
for i in range(8):
    indexes = np.where(np.equal(s7d4_out_type,i*np.ones(len(s7d4_out_type))))[0]
    s7d4_center= np.mean(s7d4_states[indexes],axis=0)
    s7d4_center_array.append(s7d4_center)
s7d4_center_array = np.array(s7d4_center_array)

s7d4_cube = s7d4_center_array[switch]_____no_output_____# Finding slow points
s7d4_min_q = []
for i in range(300):
    opt_result = scipy.optimize.minimize(q,(np.random.rand(nb_neurons)-0.5)*10,args=(s7d4_w))
    # if {optimization was a success} and ( {this is the first slow point} or {slow point does not correspond to one already found} )
    if opt_result['success'] and (len(s7d4_min_q)==0 or np.all([np.linalg.norm(np.abs(min_q_i-opt_result['x']))>1 for min_q_i in s7d4_min_q])):
        s7d4_min_q.append(opt_result['x'])
s7d4_min_q = np.array(s7d4_min_q)

# Calculating the number of free dimensions for each slow point
s7d4_q_type = []
for min_q_i in s7d4_min_q:
    J = np.empty((nb_neurons,nb_neurons))
    for i in range(nb_neurons):
        J[i,:] = np.dot(s7d4_w[i,:],(1/np.cosh(min_q_i[i]))**2)-I[i,:]
    s7d4_q_type.append(sum(np.linalg.eig(J)[0]>0))_____no_output_____fig = plt.figure(figsize=(8,6))
ax = fig.add_subplot(111, projection='3d')

for edge in s7d4_cube:
    edge_pc1, edge_pc2, edge_pc3 = s7d4_pca.transform(edge).T
    ax.plot(edge_pc1, edge_pc2, edge_pc3, '--', c='darkgray', lw=1)

min_q_pc1, min_q_pc2, min_q_pc3 = s7d4_pca.transform(s7d4_min_q).T
for i in range(s7d4_min_q.shape[0]):
    if(s7d4_q_type[i]<=3):
        ax.scatter(min_q_pc1[i], min_q_pc2[i], min_q_pc3[i], c=['k','b','g','y'][s7d4_q_type[i]], marker='x', alpha=0.7)
    else:
        ax.scatter(min_q_pc1[i], min_q_pc2[i], min_q_pc3[i], c='r', marker='^', alpha=0.7)
        ax.text(min_q_pc1[i], min_q_pc2[i], min_q_pc3[i], str(s7d4_q_type[i]))

custom_legend = [
    Line2D([0], [0], color = 'w', markersize = 10, markerfacecolor = 'k', marker = 'X', label = 'Fixed point'),
    Line2D([0], [0], color = 'w', markersize = 10, markerfacecolor = 'b', marker = 'X', label = 'Saddle point (1 unst dim)'),
    Line2D([0], [0], color = 'w', markersize = 10, markerfacecolor = 'g', marker = 'X', label = 'Saddle point (2 unst dims)'),
    Line2D([0], [0], color = 'w', markersize = 10, markerfacecolor = 'y', marker = 'X', label = 'Saddle point (3 unst dims)'),
    Line2D([0], [0], color = 'darkgray', linestyle='--', label = 'Approx. test trajectory')
]
ax.legend(handles = custom_legend, loc=9, ncol=2)
ax.set_xlabel('PC1')
ax.set_ylabel('PC2')
ax.set_zlabel('PC3')
ax.view_init(31,141)_____no_output_____

---
Slow point analysis of RNN delayed by 4 time-steps. There are no discernable difference with the slow point analysis of the network without delay. The delay is rather implemented by changing the direction of the input vectors in phase space.

---_____no_output_____8 delay_____no_output_____

# Calculating approximate cube
s7d8_center_array = []
for i in range(8):
    indexes = np.where(np.equal(s7d8_out_type,i*np.ones(len(s7d8_out_type))))[0]
    s7d8_center= np.mean(s7d8_states[indexes],axis=0)
    s7d8_center_array.append(s7d8_center)
s7d8_center_array = np.array(s7d8_center_array)

s7d8_cube = s7d8_center_array[switch]_____no_output_____# Finding slow points
s7d8_min_q = []
for i in range(500):
    opt_result = scipy.optimize.minimize(q,(np.random.rand(nb_neurons)-0.5)*10,args=(s7d8_w))
    # if {optimization was a success} and ( {this is the first slow point} or {slow point does not correspond to one already found} )
    if opt_result['success'] and (len(s7d8_min_q)==0 or np.all([np.linalg.norm(np.abs(min_q_i-opt_result['x']))>0.1 for min_q_i in s7d8_min_q])):
        s7d8_min_q.append(opt_result['x'])
s7d8_min_q = np.array(s7d8_min_q)

# Calculating the number of free dimensions for each slow point
s7d8_q_type = []
for min_q_i in s7d8_min_q:
    J = np.empty((nb_neurons,nb_neurons))
    for i in range(nb_neurons):
        J[i,:] = np.dot(s7d8_w[i,:],(1/np.cosh(min_q_i[i]))**2)-I[i,:]
    s7d8_q_type.append(sum(np.linalg.eig(J)[0]>0))

# This might take a while.................................................................................................................................................................................................................................. So... how was your day?.......................................................................................................still going...................................................................the weather sure is weathery today, huh?..................................................................................zzzzzzzzzzzzzz...........zzzzzzzz.........zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz AH! I'm awake, I'm awake......................................................................................................................................Alright, try to guess what number I'm thinking about...............................................................Come on, guess.........................................................................Wrong! It was 239.062... Unless you guessed 239.062, in that case, good job!..........................................................................................................................Yep, still not done............................................_____no_output_____fig = plt.figure(figsize=(8,6))
ax = fig.add_subplot(111, projection='3d')

for edge in s7d8_cube:
    edge_pc1, edge_pc2, edge_pc3 = s7d8_pca.transform(edge).T
    ax.plot(edge_pc1, edge_pc2, edge_pc3, '--', c='darkgray', lw=1)

min_q_pc1, min_q_pc2, min_q_pc3 = s7d8_pca.transform(s7d8_min_q).T
for i in range(s7d8_min_q.shape[0]):
    if(s7d8_q_type[i]<=3):
        ax.scatter(min_q_pc1[i], min_q_pc2[i], min_q_pc3[i], c=['k','b','g','y'][s7d8_q_type[i]], marker='x', alpha=0.7)
    else:
        ax.scatter(min_q_pc1[i], min_q_pc2[i], min_q_pc3[i], c='r', marker='^', alpha=0.7)
        ax.text(min_q_pc1[i], min_q_pc2[i], min_q_pc3[i], str(s7d8_q_type[i]))

custom_legend = [
    Line2D([0], [0], color = 'w', markersize = 10, markerfacecolor = 'k', marker = 'X', label = 'Fixed point'),
    Line2D([0], [0], color = 'w', markersize = 10, markerfacecolor = 'b', marker = 'X', label = 'Saddle point (1 unst dim)'),
    Line2D([0], [0], color = 'w', markersize = 10, markerfacecolor = 'g', marker = 'X', label = 'Saddle point (2 unst dims)'),
    Line2D([0], [0], color = 'w', markersize = 10, markerfacecolor = 'y', marker = 'X', label = 'Saddle point (3 unst dims)'),
    Line2D([0], [0], color = 'w', markersize = 10, markerfacecolor = 'r', marker = '^', label = 'Saddle point (>3 unst dims)'),
    Line2D([0], [0], color = 'darkgray', linestyle='--', label = 'Approx. test trajectory')
]
ax.legend(handles = custom_legend, loc=9, ncol=2)
ax.set_xlabel('PC1')
ax.set_ylabel('PC2')
ax.set_zlabel('PC3')
ax.view_init(51,-43)_____no_output_____

---
Slow point analysis of RNN delayed by 8 time-steps. There are now slow points with more than three unstable dimensions. Those are marked by red triangles and the number of unstable dimensions is written above.

---_____no_output_____# Triggered average analysis_____no_output_____0 delay_____no_output_____

time_delay = 0
nb_before = time_delay+10
nb_after = 11
pattern = np.concatenate((np.ones(nb_before)*tension_to_index([-1,-1,-1]),np.ones(nb_after+time_delay)*tension_to_index([-1,-1,1]))).astype('int')
triggered_indexes = matchPattern(s7d0_out_type,pattern)
triggered_indexes = np.array([np.arange(i,i+nb_before+nb_after) for i in triggered_indexes])
s7d0_triggered_states = np.mean(s7d0_states[triggered_indexes],axis=0)_____no_output_____s7d0_triggered_pca = PCA(3)
s7d0_triggered_pca.fit(s7d0_triggered_states)
s7d0_trig_pc1, s7d0_trig_pc2, s7d0_trig_pc3 = s7d0_triggered_pca.transform(s7d0_triggered_states).T_____no_output_____fig = plt.figure(figsize=(8,6))
ax = fig.add_subplot(111, projection='3d')

for edge in s7d0_cube:
    edge_pc1, edge_pc2, edge_pc3 = s7d0_triggered_pca.transform(edge).T
    ax.plot(edge_pc1, edge_pc2, edge_pc3, '--', c='darkgray', lw=1)

for i in range(nb_before+nb_after):
    ax.scatter(s7d0_trig_pc1[i], s7d0_trig_pc2[i], s7d0_trig_pc3[i], c=colors[pattern[i]], alpha=0.3)
ax.plot(s7d0_trig_pc1, s7d0_trig_pc2, s7d0_trig_pc3, '-k', linewidth=0.5, alpha=0.3)

custom_legend = [
    Line2D([0], [0], color = 'w', markersize = 15, markerfacecolor = colors[tension_to_index([-1,-1,-1])], marker = '.', label = 'Out (-1,-1,-1)'),
    Line2D([0], [0], color = 'w', markersize = 15, markerfacecolor = colors[tension_to_index([-1,-1,1])], marker = '.', label = 'Out (-1,-1,1)'),
    Line2D([0], [0], color = 'gray', label = 'States trajectory'),
    Line2D([0], [0], color = 'darkgray', linestyle='--', label = 'Approx. test trajectory')
]
ax.legend(handles = custom_legend, loc=9, ncol=2)
ax.set_xlabel('PC1')
ax.set_ylabel('PC2')
ax.set_zlabel('PC3')
ax.view_init(23,146)_____no_output_____

---
The switch from output (-1,-1,-1) to output (-1,-1,1) for the RNN without delay was studied using a triggered average. Here is the PCA analysis of the transition state. A few points before and after the switch are also plotted to show the relative movement of the transition to that of the two quasi-stable states.

---_____no_output_____4 delay_____no_output_____

time_delay = 4
nb_before = time_delay+10
nb_after = 11
pattern = np.concatenate((np.ones(nb_before)*tension_to_index([-1,-1,-1]),np.ones(nb_after+time_delay)*tension_to_index([-1,-1,1]))).astype('int')
triggered_indexes = matchPattern(s7d4_out_type,pattern)
triggered_indexes = np.array([np.arange(i,i+nb_before+nb_after) for i in triggered_indexes])
s7d4_triggered_states = np.mean(s7d4_states[triggered_indexes],axis=0)_____no_output_____s7d4_triggered_pca = PCA(3)
s7d4_triggered_pca.fit(s7d4_triggered_states)
s7d4_trig_pc1, s7d4_trig_pc2, s7d4_trig_pc3 = s7d4_triggered_pca.transform(s7d4_triggered_states).T_____no_output_____fig = plt.figure(figsize=(8,6))
ax = fig.add_subplot(111, projection='3d')

for edge in s7d4_cube:
    edge_pc1, edge_pc2, edge_pc3 = s7d4_triggered_pca.transform(edge).T
    ax.plot(edge_pc1, edge_pc2, edge_pc3, '--', c='darkgray', lw=1)

for i in range(nb_before+nb_after):
    ax.scatter(s7d4_trig_pc1[i], s7d4_trig_pc2[i], s7d4_trig_pc3[i], c=colors[pattern[i]], alpha=0.3)
ax.plot(s7d4_trig_pc1, s7d4_trig_pc2, s7d4_trig_pc3, '-k', linewidth=0.5, alpha=0.3)

custom_legend = [
    Line2D([0], [0], color = 'w', markersize = 15, markerfacecolor = colors[tension_to_index([-1,-1,-1])], marker = '.', label = 'Out (-1,-1,-1)'),
    Line2D([0], [0], color = 'w', markersize = 15, markerfacecolor = colors[tension_to_index([-1,-1,1])], marker = '.', label = 'Out (-1,-1,1)'),
    Line2D([0], [0], color = 'gray', label = 'States trajectory'),
    Line2D([0], [0], color = 'darkgray', linestyle='--', label = 'Approx. test trajectory')
]
ax.legend(handles = custom_legend, loc=9, ncol=2)
ax.set_xlabel('PC1')
ax.set_ylabel('PC2')
ax.set_zlabel('PC3')
ax.view_init(32,-146)_____no_output_____

---
The switch from output (-1,-1,-1) to output (-1,-1,1) for RNN delayed by 4 time-steps.

---_____no_output_____8 delay_____no_output_____

time_delay = 8
nb_before = time_delay+10
nb_after = 11
pattern = np.concatenate((np.ones(nb_before)*tension_to_index([-1,-1,-1]),np.ones(nb_after+time_delay)*tension_to_index([-1,-1,1]))).astype('int')
triggered_indexes = matchPattern(s7d8_out_type,pattern)
triggered_indexes = np.array([np.arange(i,i+nb_before+nb_after) for i in triggered_indexes])
s7d8_triggered_states = np.mean(s7d8_states[triggered_indexes],axis=0)_____no_output_____s7d8_triggered_pca = PCA(3)
s7d8_triggered_pca.fit(s7d8_triggered_states)
s7d8_trig_pc1, s7d8_trig_pc2, s7d8_trig_pc3 = s7d8_triggered_pca.transform(s7d8_triggered_states).T_____no_output_____fig = plt.figure(figsize=(8,6))
ax = fig.add_subplot(111, projection='3d')

for edge in s7d8_cube:
    edge_pc1, edge_pc2, edge_pc3 = s7d8_triggered_pca.transform(edge).T
    ax.plot(edge_pc1, edge_pc2, edge_pc3, '--', c='darkgray', lw=1)

for i in range(nb_before+nb_after):
    ax.scatter(s7d8_trig_pc1[i], s7d8_trig_pc2[i], s7d8_trig_pc3[i], c=colors[pattern[i]], alpha=0.3)
ax.plot(s7d8_trig_pc1, s7d8_trig_pc2, s7d8_trig_pc3, '-k', linewidth=0.5, alpha=0.3)

custom_legend = [
    Line2D([0], [0], color = 'w', markersize = 15, markerfacecolor = colors[tension_to_index([-1,-1,-1])], marker = '.', label = 'Out (-1,-1,-1)'),
    Line2D([0], [0], color = 'w', markersize = 15, markerfacecolor = colors[tension_to_index([-1,-1,1])], marker = '.', label = 'Out (-1,-1,1)'),
    Line2D([0], [0], color = 'gray', label = 'States trajectory'),
    Line2D([0], [0], color = 'darkgray', linestyle='--', label = 'Approx. test trajectory')
]
ax.legend(handles = custom_legend, loc=9, ncol=2)
ax.set_xlabel('PC1')
ax.set_ylabel('PC2')
ax.set_zlabel('PC3')
ax.view_init(14,-125)_____no_output_____

---
The switch from output (-1,-1,-1) to output (-1,-1,1) for RNN delayed by 8 time-steps.

---_____no_output_____

s7d0_triggered_pca_long = PCA(4)
s7d0_triggered_pca_long.fit(s7d0_triggered_states)
s7d0_output_array = np.tanh(np.matmul(s7d0_triggered_states,s7d0_wout))

fig, ax_array = plt.subplots(6,1,sharex=True,figsize=(8,8))
fig.subplots_adjust(hspace=0)

trig_states = s7d0_triggered_pca_long.transform(s7d0_triggered_states).T
input_array = np.zeros(21)
input_array[10] = -1
output_array = s7d0_output_array
lim = np.max(np.abs(trig_states))

for ax in ax_array:
    ax.grid(True)
    ax.get_xaxis().set_ticks([])
    ax.axvline(0,linestyle='--',color='gray',linewidth=1)

for j in range(4):
    ax_array[j].plot(np.arange(-10,11),trig_states[j],'k-',linewidth=0.5)
    ax_array[j].plot(np.arange(-10,11),trig_states[j],'b.',markersize=5)
    ax_array[j].get_yaxis().set_ticks([-2,0,2])
    ax_array[j].set_ylim(-1.2*lim,1.2*lim)
    ax_array[j].set_ylabel('PC '+str(j),fontsize=10)

ax_array[4].plot(np.arange(-10,11),output_array[:,2],'-k')
ax_array[4].get_yaxis().set_ticks([])
ax_array[4].set_ylabel('Output ch 3',fontsize=10)

ax_array[5].plot(np.arange(-10,11),input_array,'-k')
ax_array[5].get_yaxis().set_ticks([])
ax_array[5].set_ylabel('Input ch 3',fontsize=10)

ax_array[5].set_xlabel('Elapsed time-steps')_____no_output_____

---
The PCA analysis plotted as tension over time for the main primary components.

---_____no_output_____

s7d4_triggered_pca_long = PCA(7)
s7d4_triggered_pca_long.fit(s7d4_triggered_states)
s7d4_output_array = np.tanh(np.matmul(s7d4_triggered_states,s7d4_wout))

fig, ax_array = plt.subplots(9,1,sharex=True,figsize=(8,12))
fig.subplots_adjust(hspace=0)

trig_states = s7d4_triggered_pca_long.transform(s7d4_triggered_states).T
input_array = np.zeros(21+4)
input_array[10] = -1
output_array = s7d4_output_array
lim = np.max(np.abs(trig_states))

for ax in ax_array:
    ax.grid(True)
    ax.get_xaxis().set_ticks([])
    ax.axvline(0,linestyle='--',color='gray',linewidth=1)
    ax.axvline(4,linestyle='--',color='gray',linewidth=1)

for j in range(7):
    ax_array[j].plot(np.arange(-10,11+4),trig_states[j],'k-',linewidth=0.5)
    ax_array[j].plot(np.arange(-10,11+4),trig_states[j],'b.',markersize=5)
    ax_array[j].get_yaxis().set_ticks([-2,0,2])
    ax_array[j].set_ylim(-1.2*lim,1.2*lim)
    ax_array[j].set_ylabel('PC '+str(j),fontsize=10)

ax_array[7].plot(np.arange(-10,11+4),output_array[:,2],'-k')
ax_array[7].get_yaxis().set_ticks([])
ax_array[7].set_ylabel('Output channel 3',fontsize=5)

ax_array[8].plot(np.arange(-10,11+4),input_array,'-k')
ax_array[8].get_yaxis().set_ticks([])
ax_array[8].set_ylabel('Input channel 3',fontsize=5)

ax_array[8].set_xlabel('Elapsed time-steps')_____no_output_____s7d8_triggered_pca_long = PCA(9)
s7d8_triggered_pca_long.fit(s7d8_triggered_states)
s7d8_output_array = np.tanh(np.matmul(s7d8_triggered_states,s7d8_wout))

fig, ax_array = plt.subplots(11,1,sharex=True,figsize=(8,15))
fig.subplots_adjust(hspace=0)

trig_states = s7d8_triggered_pca_long.transform(s7d8_triggered_states).T
input_array = np.zeros(21+8)
input_array[10] = -1
output_array = s7d8_output_array
lim = np.max(np.abs(trig_states))

for ax in ax_array:
    ax.grid(True)
    ax.get_xaxis().set_ticks([])
    ax.axvline(0,linestyle='--',color='gray',linewidth=1)
    ax.axvline(8,linestyle='--',color='gray',linewidth=1)

for j in range(9):
    ax_array[j].plot(np.arange(-10,11+8),trig_states[j],'k-',linewidth=0.5)
    ax_array[j].plot(np.arange(-10,11+8),trig_states[j],'b.',markersize=5)
    ax_array[j].get_yaxis().set_ticks([-2,0,2])
    ax_array[j].set_ylim(-1.2*lim,1.2*lim)
    ax_array[j].set_ylabel('PC '+str(j),fontsize=10)

ax_array[9].plot(np.arange(-10,11+8),output_array[:,2],'-k')
ax_array[9].get_yaxis().set_ticks([])
ax_array[9].set_ylabel('Output channel 3',fontsize=5)

ax_array[10].plot(np.arange(-10,11+8),input_array,'-k')
ax_array[10].get_yaxis().set_ticks([])
ax_array[10].set_ylabel('Input channel 3',fontsize=5)

ax_array[10].set_xlabel('Elapsed time-steps')_____no_output_____

# Eigenvalues analysis_____no_output_____

s7d0_eigvals = np.linalg.eig(s7d0_w)[0]
s7d4_eigvals = np.linalg.eig(s7d4_w)[0]
s7d8_eigvals = np.linalg.eig(s7d8_w)[0]

fig, (ax0,ax1,ax2) = plt.subplots(1, 3, sharey=True, figsize=(8,3))

ax0.axvline(0,linestyle='--',color='gray',linewidth=1)
ax0.axhline(0,linestyle='--',color='gray',linewidth=1)
ax0.plot(s7d0_eigvals.real,s7d0_eigvals.imag,'b+',ms=5)
ax0.set_xticks([-0.5,0,0.5,1])

ax1.axvline(0,linestyle='--',color='gray',linewidth=1)
ax1.axhline(0,linestyle='--',color='gray',linewidth=1)
ax1.plot(s7d4_eigvals.real,s7d4_eigvals.imag,'b+',ms=5)
ax1.set_xticks([-1,-0.5,0,0.5,1,1.5])

ax2.axvline(0,linestyle='--',color='gray',linewidth=1)
ax2.axhline(0,linestyle='--',color='gray',linewidth=1)
ax2.plot(s7d8_eigvals.real,s7d8_eigvals.imag,'b+',ms=5)
ax2.set_xticks([-1,0,1,2])

ax0.set_ylabel('Complex value')
ax1.set_xlabel('Real value')

fig.tight_layout()_____no_output_____

---
Analysis of the eigenvalues for the three studied RNNs. The eigenvalues of the network without delay is diplayed to the left, the ones for 4 time-steps at the center and the ones for 8 time-steps to the right.

---_____no_output_____


# Notebook from NLM-Reproducibility-Project/May_2019_Repo3
Path: Notebooks/Untitled.ipynb

<code>
R.version_____no_output_____install.packages("ggplot2")Updating HTML index of packages in '.Library'
Making 'packages.html' ... done
install.packages("deseq2")Warning message:
“package ‘deseq2’ is not available (for R version 3.5.1)”IRkernel::installspec()_____no_output_____library(ggplot2)

ggplot(mpg, aes(displ, hwy, colour = class)) +
  geom_point()_____no_output_____library(ggplot2)_____no_output_____
</code>


# Notebook from amitmac/Language-Detection
Path: src/n-gram/3-gram-model.ipynb

import numpy as np
import pandas as pd
import os
import json
import pickle
import sys

sys.path.insert(0,'/workspace/lang-detect/')_____no_output_____with open('../data/lang_dict_3_gram.json', 'r') as f:
    lang_dict_n_gram = pickle.load(f)_____no_output_____print(len(lang_dict_n_gram['en'].keys()))
print(lang_dict_n_gram['en']['the'])4100
5480
test_data, test_labels = [], []
with open("../../europarl.test", "r") as f:
    for line in f:
        line = line.split()
        test_data.append(" ".join(line[1:]))
        test_labels.append(line[0])
print(test_data[0], test_labels[0])('"\xd0\x95\xd0\xb2\xd1\x80\xd0\xbe\xd0\xbf\xd0\xb0 2020" \xd0\xbd\xd0\xb5 \xd1\x82\xd1\x80\xd1\x8f\xd0\xb1\xd0\xb2\xd0\xb0 \xd0\xb4\xd0\xb0 \xd1\x81\xd1\x82\xd0\xb0\xd1\x80\xd1\x82\xd0\xb8\xd1\x80\xd0\xb0 \xd0\xbd\xd0\xbe\xd0\xb2 \xd0\xba\xd0\xbe\xd0\xbd\xd0\xba\xd1\x83\xd1\x80\xd0\xb5\xd0\xbd\xd1\x82\xd0\xb5\xd0\xbd \xd0\xbc\xd0\xb0\xd1\x80\xd0\xb0\xd1\x82\xd0\xbe\xd0\xbd \xd0\xb8 \xd0\xb8\xd0\xb7\xd1\x85\xd0\xbe\xd0\xb4 \xd1\x81 \xd0\xbf\xd1\x80\xd0\xb8\xd0\xb2\xd0\xb0\xd1\x82\xd0\xb8\xd0\xb7\xd0\xb0\xd1\x86\xd0\xb8\xd1\x8f.', 'bg')
print(len(test_data))21000
from src import utils

for i in range(len(test_data)):
    test_data[i] = utils.preprocess(test_data[i])

new_data = [0]*len(test_data)

for i in range(len(test_data)):
    new_data[i] = utils.create_n_gram(test_data[i], 3)_____no_output_____new_data[0]_____no_output_____# Based on total occurence
output_labels = []
for i in range(len(new_data)):
    count = {}
    for key in lang_dict_n_gram.keys():
        count[key] = 0
    for token in new_data[i].split():
        for key in lang_dict_n_gram.keys():
            if token in lang_dict_n_gram[key]:
                count[key] += lang_dict_n_gram[key][token]
    best_lang, max_count = "NN", 0
    for key in count.keys():
        if count[key] > max_count:
            best_lang = key
            max_count = count[key]
    output_labels.append(best_lang)_____no_output_____print(len(output_labels), len(test_labels))(21000, 21000)
correct_preds = 0
for i in range(len(output_labels)):
    if output_labels[i] == test_labels[i]:
        correct_preds += 1
print("Accuracy", float(correct_preds)/len(output_labels)*100)('Accuracy', 91.85238095238095)
# Based on 0-1 occurence
output_labels = []
for i in range(len(new_data)):
    count = {}
    for key in lang_dict_n_gram.keys():
        count[key] = 0
    for token in new_data[i].split():
        for key in lang_dict_n_gram.keys():
            if token in lang_dict_n_gram[key]:
                count[key] += 1
    best_lang, max_count = "NN", 0
    for key in count.keys():
        if count[key] > max_count:
            best_lang = key
            max_count = count[key]
    output_labels.append(best_lang)_____no_output_____correct_preds = 0
for i in range(len(output_labels)):
    if output_labels[i] == test_labels[i]:
        correct_preds += 1
print("Accuracy", float(correct_preds)/len(output_labels)*100)('Accuracy', 97.64285714285714)

#### **Accuracy based on 0-1 occurence is 97.6% which is awesome while if we include total frequency of each token, then the accuracy reduces to 91.8% which is not bad either.**_____no_output_____

# Test few cases where this model didn't work
num_case = 0
for i in range(len(output_labels)):
    if output_labels[i] != test_labels[i]:
        print(new_data[i])
        print("Actual Language {0} | Predicted Language {1}".format(test_labels[i], output_labels[i]))
        count = {}
        for key in lang_dict_n_gram.keys():
            count[key] = 0
        for token in new_data[i].split():
            for key in lang_dict_n_gram.keys():
                if token in lang_dict_n_gram[key]:
                    count[key] += 1
        print(count)
        print("--------------------------------")
        num_case += 1
        if num_case == 5:
            breakco jsm sme tí ím teh ehd hdy dal ali naj aje jev evo
Actual Language cs | Predicted Language sk
{'pl': 7, 'el': 4, 'fr': 7, 'bg': 3, 'nl': 7, 'ro': 8, 'pt': 9, 'de': 8, 'sv': 7, 'da': 8, 'lv': 9, 'cs': 10, 'lt': 8, 'en': 7, 'sk': 11, 'sl': 9, 'it': 7, 'et': 9, 'hu': 10, 'fi': 10, 'es': 8}
--------------------------------
jme men eno nov ova val ali se wil ill lli lia iam mey eye yer ber ern rna nar ard sta tar ari rie reg egi gin ina nal ald pik ike tho hom oma mas sha haw jam ame mes mcl cle lei eis ish arc rch chi hib iba bal ald bar arr rro row owm wma man a alb lbe ber ert rob obe ber ert rts a vš ši �ic ich chn hni bud udo dou v sob obo bot otu vyz yzn zna nam ame men en� ná án �ni
Actual Language cs | Predicted Language sk
{'pl': 76, 'el': 44, 'fr': 74, 'bg': 38, 'nl': 74, 'ro': 65, 'pt': 73, 'de': 73, 'sv': 72, 'da': 69, 'lv': 68, 'cs': 78, 'lt': 75, 'en': 74, 'sk': 79, 'sl': 72, 'it': 66, 'et': 75, 'hu': 75, 'fi': 74, 'es': 73}
--------------------------------
nem em� má ám �me ča �as pro roj oje jed edn dn� ná áv �va vat roz ozv zvo voj pro kiv ivu če �el el� lí ím �me zde hum uma man ani nit it� tá ár �rn rn� ní kri riz izi
Actual Language cs | Predicted Language sk
{'pl': 18, 'el': 9, 'fr': 22, 'bg': 6, 'nl': 20, 'ro': 15, 'pt': 25, 'de': 24, 'sv': 23, 'da': 24, 'lv': 26, 'cs': 43, 'lt': 23, 'en': 18, 'sk': 44, 'sl': 27, 'it': 15, 'et': 20, 'hu': 36, 'fi': 22, 'es': 27}
--------------------------------
pro roh ohl hla la� aš šu �uj uji zas ase sed ed� dá án �n� ní evr vro rop ops psk sk� ké éh �ho par arl rla lam ame men ent ntu za uko kon on� nč če �en en� né
Actual Language cs | Predicted Language sk
{'pl': 24, 'el': 10, 'fr': 20, 'bg': 15, 'nl': 20, 'ro': 20, 'pt': 22, 'de': 23, 'sv': 22, 'da': 22, 'lv': 26, 'cs': 41, 'lt': 29, 'en': 18, 'sk': 42, 'sl': 31, 'it': 18, 'et': 22, 'hu': 28, 'fi': 23, 'es': 22}
--------------------------------
rea eal ali lit ita je drs rsn sn� ná nee eex exi xis ist stu tuj uj� jí alt lte ter ern rna nat ati tiv ivn vn� ní fos osi sil iln ln� ní pal ali liv iva kte ter er� rá jso sou lac aci cin in� ná a ú� �č či �in inn nn� ná
Actual Language cs | Predicted Language sk
{'pl': 30, 'el': 15, 'fr': 34, 'bg': 14, 'nl': 32, 'ro': 32, 'pt': 39, 'de': 35, 'sv': 38, 'da': 37, 'lv': 30, 'cs': 54, 'lt': 33, 'en': 33, 'sk': 55, 'sl': 41, 'it': 31, 'et': 35, 'hu': 41, 'fi': 36, 'es': 39}
--------------------------------

#### **If we actually notice the difference between actual language and predicted language, we see very less difference in terms of occurence of words/trigrams.**_____no_output_____

# Lets check which language has most wrong predictions
wrong_pred = {}
for key in lang_dict_n_gram.keys():
    wrong_pred[key] = 0
for i in range(len(output_labels)):
    if output_labels[i] != test_labels[i]:
        wrong_pred[test_labels[i]] += 1
print(wrong_pred){'pl': 0, 'el': 0, 'fr': 12, 'bg': 0, 'nl': 7, 'ro': 9, 'pt': 20, 'de': 21, 'sv': 21, 'da': 32, 'lv': 1, 'cs': 8, 'lt': 4, 'en': 66, 'sk': 62, 'sl': 34, 'it': 92, 'et': 27, 'hu': 1, 'fi': 16, 'es': 62}
%matplotlib inline
import matplotlib.pyplot as plt_____no_output_____plt.bar(wrong_pred.keys(), wrong_pred.values())
plt.show()_____no_output_____

The most number of mismatches happened with italian(it) - 92, english(en) - 66, slovak(sk) - 62 and spanish(es) - 62. While bulgarian(bg), polish(pl) and greek(el) has true positive rate 1. Another thing to notice from the base model is that, for that bulgarian and greek had the least TPR while in this model they have the highest TPR. For italian, english and spanish language, TPR was higher in base model while they have least TPR in the n-gram model._____no_output_____

# Convert the classes to numbers
classes_map = {}
classes = []
for ind, key in enumerate(count.keys()):
    classes_map[key] = ind
    classes.append(key)

test_labels_n, output_labels_n = [], []
for i in range(len(test_labels)):
    test_labels_n.append(classes_map[test_labels[i]])
    output_labels_n.append(classes_map[output_labels[i]])_____no_output_____# Plot confusion matrix
from sklearn.metrics import confusion_matrix
cnf_matrix = confusion_matrix(test_labels_n, output_labels_n)

import seaborn as sns
fig = plt.figure(figsize=(16,12))
ax = fig.add_subplot(111)
sns.heatmap(cnf_matrix, annot=True, fmt='g',ax=ax)
ax.xaxis.set_ticklabels(classes)
ax.yaxis.set_ticklabels(classes)
plt.xlabel('Predicted')
plt.ylabel('True')_____no_output_____

From confusion matrix we can seee that 42 English(en) sentences are classified as Dutch(nl). Lets see what are those sentences._____no_output_____

counter = 0
for i in range(len(output_labels)):
    if (output_labels[i] == 'nl') and (test_labels[i] == 'en'):
        print(test_data[i])
        print(new_data[i])
        print("----------------------------------------------")
        counter += 1
    if counter == 5:
        breakSK dear colleagues it saddens me very much that the situation concerning democracy human rights and the rule of law has not been improving in belarus
SK dea ear col oll lle lea eag agu gue ues it sad add dde den ens me ver ery muc uch tha hat the sit itu tua uat ati tio ion con onc nce cer ern rni nin ing dem emo moc ocr cra rac acy hum uma man rig igh ght hts and the rul ule of law has not bee een imp mpr pro rov ovi vin ing in bel ela lar aru rus
----------------------------------------------
A constructive response to it from the european union is overdue and necessary
A con ons nst str tru ruc uct cti tiv ive res esp spo pon ons nse to it fro rom the eur uro rop ope pea ean uni nio ion is ove ver erd rdu due and nec ece ces ess ssa sar ary
----------------------------------------------
and he is right of course
and he is rig igh ght of cou our urs rse
----------------------------------------------
capital adequacy of financial institutions
cap api pit ita tal ade deq equ qua uac acy of fin ina nan anc nci cia ial ins nst sti tit itu tut uti tio ion ons
----------------------------------------------
clarification is undoubtedly necessary
cla lar ari rif ifi fic ica cat ati tio ion is und ndo dou oub ubt bte ted edl dly nec ece ces ess ssa sar ary
----------------------------------------------

**Could 4-gram model help?**_____no_output_____