# Notebook from wconnell/metrx
Path: .ipynb_checkpoints/2019.12.17_rna-seq_pan_diseased-tissue_metastatic-SKCM-checkpoint.ipynb

import numpy as np
import pandas as pd

# viz
import matplotlib.pyplot as plt

# notebook settings
from IPython.core.interactiveshell import InteractiveShell
InteractiveShell.ast_node_interactivity = "all"

pd.set_option('display.max_columns', 1000)_____no_output_____

## Sample Prep_____no_output_____

samples = pd.read_csv('../data/TCGA/rna-seq_pan/meta/gdc_sample_sheet.2019-12-12.tsv', sep="\t")
# get file type
samples['data'] = [val[1] for i,val in samples['File Name'].str.split(".").items()]
samples['project'] = [val[1] for i,val in samples['Project ID'].str.split("-").items()]
samples['project'].value_counts()_____no_output_____samples['Sample Type'].value_counts()_____no_output_____

## New Model based on Tissues with available metastatic samples_____no_output_____

samples[samples['Sample Type']=='Metastatic']['project'].value_counts()
samples[samples['Sample Type']=='Primary Tumor']['project'].value_counts().head(9)_____no_output_____proj = np.append(samples['project'].value_counts().head(9).index.values, ['SKCM'])

cases = samples[samples['Sample Type']=='Primary Tumor'].sample(frac=1).copy()
cases.shape
cases = cases[cases['project'].isin(proj)]
cases['project'].value_counts()
cases.shape_____no_output_____

## Dataset Prep_____no_output_____

from sklearn.model_selection import train_test_split

target = 'project'
cases[target] = cases[target].astype('category')
train, test = train_test_split(cases)_____no_output_____import torch
import torch.nn as nn
from torch.optim import lr_scheduler
import torch.optim as optim
from torch.autograd import Variable

from trainer import fit
import visualization as vis
import numpy as np

if torch.cuda.is_available():
    cuda = torch.cuda.is_available()
    print("{} GPUs available".format(torch.cuda.device_count()))

classes = train[target].cat.categories.values
8 GPUs available
from tcga_datasets import TCGA, SiameseTCGA
root_dir = "../data/TCGA/rna-seq_pan/"
batch_size = 1

train_dataset = TCGA(root_dir, samples=train, train=True, target=target)
test_dataset = TCGA(root_dir, samples=test, train=False, target=target)
print('Loaded')
kwargs = {'num_workers': 1, 'pin_memory': True} if cuda else {}
train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=False, **kwargs)
test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False, **kwargs)Loaded

## Siamese Network_____no_output_____

# Step 1 set up dataloader
siamese_train_dataset = SiameseTCGA(train_dataset) # Returns pairs of images and target same/different
siamese_test_dataset = SiameseTCGA(test_dataset)
batch_size = 64
kwargs = {'num_workers': 20, 'pin_memory': True} if cuda else {}
siamese_train_loader = torch.utils.data.DataLoader(siamese_train_dataset, batch_size=batch_size, shuffle=True, **kwargs)
siamese_test_loader = torch.utils.data.DataLoader(siamese_test_dataset, batch_size=batch_size, shuffle=False, **kwargs)

# Set up the network and training parameters
from tcga_networks import EmbeddingNet, SiameseNet
from losses import ContrastiveLoss
from metrics import AccumulatedAccuracyMetric

# Step 2
embedding_net = EmbeddingNet()
# Step 3
model = SiameseNet(embedding_net)
if cuda:
    model = nn.DataParallel(model)
    model.cuda()

# Step 4
margin = 1.
loss_fn = ContrastiveLoss(margin)
lr = 1e-3
optimizer = optim.Adam(model.parameters(), lr=lr)
scheduler = lr_scheduler.StepLR(optimizer, 8, gamma=0.1, last_epoch=-1)
n_epochs = 8
# print training metrics every log_interval * batch_size
log_interval = 4_____no_output_____train_loss, val_loss = fit(siamese_train_loader, siamese_test_loader, model, loss_fn, optimizer, scheduler,
    n_epochs, cuda, log_interval)Train: [0/3842 (0%)]	Loss: 0.154764
Train: [256/3842 (7%)]	Loss: 5.473493
Train: [512/3842 (13%)]	Loss: 0.770664
Train: [768/3842 (20%)]	Loss: 3.586725
Train: [1024/3842 (26%)]	Loss: 0.429813
Train: [1280/3842 (33%)]	Loss: 1.186276
Train: [1536/3842 (39%)]	Loss: 1.381489
Train: [1792/3842 (46%)]	Loss: 0.410460
Train: [2048/3842 (52%)]	Loss: 1.311575
Train: [2304/3842 (59%)]	Loss: 0.689415
Train: [2560/3842 (66%)]	Loss: 1.807112
Train: [2816/3842 (72%)]	Loss: 0.843931
Train: [3072/3842 (79%)]	Loss: 0.441966
Train: [3328/3842 (85%)]	Loss: 0.202272
Train: [3584/3842 (92%)]	Loss: 0.217853
Train: [120/3842 (98%)]	Loss: 0.938776
Epoch: 1/8. Train set: Average loss: 1.2938
Epoch: 1/8. Validation set: Average loss: 4.2793
Train: [0/3842 (0%)]	Loss: 11.663042
Train: [256/3842 (7%)]	Loss: 0.767174
Train: [512/3842 (13%)]	Loss: 0.702323
Train: [768/3842 (20%)]	Loss: 13.336773
Train: [1024/3842 (26%)]	Loss: 2.797228
Train: [1280/3842 (33%)]	Loss: 1.121683
Train: [1536/3842 (39%)]	Loss: 0.719549
Train: [1792/3842 (46%)]	Loss: 1.497713
Train: [2048/3842 (52%)]	Loss: 0.432077
Train: [2304/3842 (59%)]	Loss: 1.577474
Train: [2560/3842 (66%)]	Loss: 7.316233
Train: [2816/3842 (72%)]	Loss: 2.262206
Train: [3072/3842 (79%)]	Loss: 3.030925
Train: [3328/3842 (85%)]	Loss: 1.188261
Train: [3584/3842 (92%)]	Loss: 0.781908
Train: [120/3842 (98%)]	Loss: 0.446683
Epoch: 2/8. Train set: Average loss: 2.6816
Epoch: 2/8. Validation set: Average loss: 0.1545
Train: [0/3842 (0%)]	Loss: 0.176105
Train: [256/3842 (7%)]	Loss: 0.178326
Train: [512/3842 (13%)]	Loss: 0.279621
Train: [768/3842 (20%)]	Loss: 0.136791
Train: [1024/3842 (26%)]	Loss: 0.148906
Train: [1280/3842 (33%)]	Loss: 0.146629
Train: [1536/3842 (39%)]	Loss: 0.152748
Train: [1792/3842 (46%)]	Loss: 0.169032
Train: [2048/3842 (52%)]	Loss: 0.121632
Train: [2304/3842 (59%)]	Loss: 0.127311
Train: [2560/3842 (66%)]	Loss: 0.174492
Train: [2816/3842 (72%)]	Loss: 0.110927
Train: [3072/3842 (79%)]	Loss: 0.110610
Train: [3328/3842 (85%)]	Loss: 0.236859
Train: [3584/3842 (92%)]	Loss: 0.173859
Train: [120/3842 (98%)]	Loss: 0.083620
Epoch: 3/8. Train set: Average loss: 0.1571
Epoch: 3/8. Validation set: Average loss: 0.1001
Train: [0/3842 (0%)]	Loss: 0.144654
Train: [256/3842 (7%)]	Loss: 0.069552
Train: [512/3842 (13%)]	Loss: 0.075354
Train: [768/3842 (20%)]	Loss: 0.069079
Train: [1024/3842 (26%)]	Loss: 0.062714
Train: [1280/3842 (33%)]	Loss: 0.085847
Train: [1536/3842 (39%)]	Loss: 0.068812
Train: [1792/3842 (46%)]	Loss: 0.065267
Train: [2048/3842 (52%)]	Loss: 0.068256
Train: [2304/3842 (59%)]	Loss: 0.059904
Train: [2560/3842 (66%)]	Loss: 0.064222
Train: [2816/3842 (72%)]	Loss: 0.056450
Train: [3072/3842 (79%)]	Loss: 0.057656
Train: [3328/3842 (85%)]	Loss: 0.065219
Train: [3584/3842 (92%)]	Loss: 0.047660
Train: [120/3842 (98%)]	Loss: 0.036634
Epoch: 4/8. Train set: Average loss: 0.0648
Epoch: 4/8. Validation set: Average loss: 0.0621
Train: [0/3842 (0%)]	Loss: 0.064000
Train: [256/3842 (7%)]	Loss: 0.055790
Train: [512/3842 (13%)]	Loss: 0.129598
Train: [768/3842 (20%)]	Loss: 0.131458
Train: [1024/3842 (26%)]	Loss: 0.183881
Train: [1280/3842 (33%)]	Loss: 0.072684
Train: [1536/3842 (39%)]	Loss: 0.045016
Train: [1792/3842 (46%)]	Loss: 0.078769
Train: [2048/3842 (52%)]	Loss: 0.071546
Train: [2304/3842 (59%)]	Loss: 0.072028
Train: [2560/3842 (66%)]	Loss: 0.050866
Train: [2816/3842 (72%)]	Loss: 0.058598
Train: [3072/3842 (79%)]	Loss: 0.044501
Train: [3328/3842 (85%)]	Loss: 0.045276
Train: [3584/3842 (92%)]	Loss: 0.045073
Train: [120/3842 (98%)]	Loss: 0.044643
Epoch: 5/8. Train set: Average loss: 0.0751
Epoch: 5/8. Validation set: Average loss: 0.0501
Train: [0/3842 (0%)]	Loss: 0.023416
Train: [256/3842 (7%)]	Loss: 0.043218
Train: [512/3842 (13%)]	Loss: 0.039415
Train: [768/3842 (20%)]	Loss: 0.043040
Train: [1024/3842 (26%)]	Loss: 0.035299
Train: [1280/3842 (33%)]	Loss: 0.041390
Train: [1536/3842 (39%)]	Loss: 0.032511
Train: [1792/3842 (46%)]	Loss: 0.036165
Train: [2048/3842 (52%)]	Loss: 0.036369
Train: [2304/3842 (59%)]	Loss: 0.032227
Train: [2560/3842 (66%)]	Loss: 0.039405
Train: [2816/3842 (72%)]	Loss: 0.031171
Train: [3072/3842 (79%)]	Loss: 0.061018
Train: [3328/3842 (85%)]	Loss: 0.071696
Train: [3584/3842 (92%)]	Loss: 0.103974
Train: [120/3842 (98%)]	Loss: 0.044169
plt.plot(range(0, n_epochs), train_loss, 'rx-')
plt.plot(range(0, n_epochs), val_loss, 'bx-')_____no_output_____classes = train[target].cat.categories.values
train_embeddings_cl, train_labels_cl = vis.extract_embeddings(train_loader, model)
vis.plot_embeddings(train_embeddings_cl, train_labels_cl, classes)_____no_output_____val_embeddings_baseline, val_labels_baseline = vis.extract_embeddings(test_loader, model)
vis.plot_embeddings(val_embeddings_baseline, val_labels_baseline, classes)_____no_output_____

### Project Metastatic SKCM onto learned space_____no_output_____

skcm_cat = np.where(cases['project'].cat.categories.values=='SKCM')[0][0]_____no_output_____ms = samples[(samples['Sample Type']=='Metastatic') & (samples['project']=='SKCM')].sample(frac=1).copy()
ms[target] = [i + '-MET' for i in ms[target]]
ms[target] = ms[target].astype('category')

met_classes = ms[target].cat.categories.values_____no_output_____root_dir = "../data/TCGA/rna-seq_pan/"
batch_size = 1

ms_dataset = TCGA(root_dir, samples=ms, train=False, target=target)
print('Loaded')
kwargs = {'num_workers': 1, 'pin_memory': True} if cuda else {}
ms_loader = torch.utils.data.DataLoader(ms_dataset, batch_size=batch_size, shuffle=False, **kwargs)_____no_output_____
ms_embeddings_baseline, ms_labels_baseline = vis.extract_embeddings(ms_loader, model)_____no_output_____comb_classes = np.append(classes, met_classes)

comb_embeddings = np.append(train_embeddings_cl, ms_embeddings_baseline, axis=0)
comb_embeddings.shape

ms_labels = np.repeat(np.unique(train_labels_cl).max() + 1, len(ms_labels_baseline))
comb_labels = np.append(train_labels_cl, ms_labels, axis=0)
comb_labels.shape_____no_output_____vis.plot_embeddings(comb_embeddings, comb_labels, comb_classes)_____no_output_____


# Notebook from gerhajdu/rrl_binaries_1
Path: 13534_oc.ipynb

# Example usage of the O-C tools

## This example shows how to construct and fit with MCMC the O-C diagram of the RR Lyrae star OGLE-BLG-RRLYR-02950_____no_output_____### We start with importing some libraries_____no_output_____

import numpy as np
import oc_tools as octs_____no_output_____

### We read in the data, set the period used to construct the O-C diagram (and to fold the light curve to construct the template curves, etc.), and the orders of the Fourier series we will fit to the light curve in the first and second iterations in the process_____no_output_____

who    = "13534"
period = 0.463105
order1 = 10
order2 = 15

jd3, mag3 = np.loadtxt('data/{:s}.o3'.format(who), usecols=[0,1], unpack=True)
jd4, mag4 = np.loadtxt('data/{:s}.o4'.format(who), usecols=[0,1], unpack=True)_____no_output_____

### We correct for possible average magnitude and amplitude differences between The OGLE-III and IV photometries by moving the intensity average of the former to the intensity average measured for the latter
### The variables "jd" and "mag" contain the merged timings and magnitudes of the OGLE-III + IV photometry, wich are used from hereon to calculate the O-C values_____no_output_____

mag3_shift=octs.shift_int(jd3, mag3, jd4, mag4, order1, period, plot=True)
jd  = np.hstack((jd3,jd4))
mag = np.hstack((mag3_shift, mag4))_____no_output_____

### Calling the split_lc_seasons() function provides us with an array containing masks splitting the combined light curve into short sections, depending on the number of points

### Optionally, the default splitting can be overriden by using the optional parameters "limits" and "into". For example, calling the function as:

octs.split_lc_seasons(jd, plot=True, mag = mag, limits = np.array((0, 8, np.inf)), into = np.array((0, 2)))
### will always split seasons with at least nine points into two separate segments_____no_output_____

splits = octs.split_lc_seasons(jd, plot=True, mag = mag)#,
                               #limits = np.array((0, 8, 80, 120, 160, 240, np.inf)),
                               #into   = np.array((0, 1, 2,  3,   4,   5)))_____no_output_____

### The function calc_oc_points() fits the light curve of the variable to produce a template, and uses it to determine the O-C points of the individual segments_____no_output_____

oc_jd, oc_oc = octs.calc_oc_points(jd, mag, period, order1, splits, figure=True)_____no_output_____

### We make a guess at the binary parameters _____no_output_____

e      = 0.49
P_orb  = 3230.
T_peri = 7650.
a_sini = 0.0048
omega  = -np.pi
a= 7e-3
b= -3e-06
c= 2.5e-10
params = np.asarray((e, P_orb, T_peri, a_sini, omega, a, b, c))

lower_bounds = np.array((0.,     100., -np.inf, 0.0, -np.inf, -np.inf, -np.inf, -np.inf))
upper_bounds = np.array((0.99,  7000.,  np.inf, 1.0,  np.inf,  np.inf,  np.inf,  np.inf))_____no_output_____

### We use the above guesses as the starting point (dashed grey line on the plot below) to find the O-C LTTE solution of the first iteration of our procedure. The yellow line on the plot shows the fit. The vertical blue bar shows the timing of the periastron passage

### Note that in this function also provides the timings of the individual observations corrected for this initial O-C solution_____no_output_____

params2, jd2 = octs.fit_oc1(oc_jd, oc_oc, jd, params, lower_bounds, upper_bounds)_____no_output_____

### We use the initial solution as the starting point for the MCMC fit, therefore we prepare it first by transforming $e$ and $\omega$ to $\sqrt{e}\sin{\omega}$ and $\sqrt{e}\sin{\omega}$
### For each parameter, we also have a lower and higher limit in its prior, but the values given for $\sqrt{e}\sin{\omega}$ and $\sqrt{e}\sin{\omega}$ are ignored, as these are handled separately within the function checking the priors_____no_output_____

start      = np.zeros_like(params2)
start[0:3] = params2[1:4]
start[3]   = np.sqrt(params2[0]) * np.sin(params2[4])
start[4]   = np.sqrt(params2[0]) * np.cos(params2[4])
start[5:]  = params2[5:]

prior_ranges = np.asanyarray([[start[0]*0.9,         start[0]*1.1],
                              [start[1]-start[0]/4., start[1]+start[0]/4.],
                              [0.,                   0.057754266],
                              [0.,                   0.],
                              [0.,                   0.],
                              [-1.,                  1.],
                              [-1e-4,                1e-4],
                              [-1e-8,                1e-8]])_____no_output_____

### We set a random seed to get reproducible results, then prepare the initial positions of the 200 walkers we are using during the fitting. During this, we check explicitly that these correspond to a position with a finite prior (i.e., they are not outside of the prior ranges defined above)_____no_output_____

np.random.seed(0)
walkers = 200
random_scales = np.array((1e+1, 1e+1, 1e-4, 1e-2, 1e-2, 1e-3, 2e-7, 5e-11))
pos = np.zeros((walkers, start.size))

for i in range(walkers):
    pos[i,:]     = start + random_scales * np.random.normal(size=8)
    while np.isinf(octs.log_prior(pos[i,:], prior_ranges)):
        pos[i,:] = start + random_scales * np.random.normal(size=8)_____no_output_____

### We recalculate the O-C points, but this time we use a higher-order Fourier series to fit the light curve with the modified timings, and we also calculate errors using bootstrapping_____no_output_____

oc_jd, oc_oc, oc_sd = octs.calc_oc_points(jd, mag, period, order2, splits, 500, jd2, figure=True)_____no_output_____

### We fit the O-C points measured above using MCMC by calling the run_mcmc() function
### We plot both the fit, as well as the triangle plot showing the two- (and one-)dimensional posterior distributions (these can be suppressed by setting the optional parameters "plot_oc" and "plot_triangle" to False)_____no_output_____

sampler, fit_mcmc, oc_sigmas, param_means, param_sigmas, fit_at_points, K =\
octs.run_mcmc(oc_jd, oc_oc, oc_sd,
              prior_ranges, pos,
              nsteps = 31000, discard = 1000,
              thin = 300, processes=1)100%|██████████| 31000/31000 [02:32<00:00, 203.36it/s]
100%|███████████████████████████████████| 20000/20000 [00:02<00:00, 7865.74it/s]

## The estimated LTTE parameters are:_____no_output_____

print("Orbital period:             {:d} +- {:d}     [d]".format(int(param_means[0]),
                                                                int(param_sigmas[0])))
print("Projected semi-major axis: {:.3f} +- {:.3f} [AU]".format(param_means[2]*173.144633,
                                                                param_sigmas[2]*173.144633))
print("Eccentricity:              {:.3f} +- {:.3f}".format(param_means[3],
                                                           param_sigmas[3]))
print("Argumen of periastron:      {:+4d} +- {:d} [deg]".format(int(param_means[4]*180/np.pi),
                                                         int(param_sigmas[4]*180/np.pi)))
print("Periastron passage time:    {:d} +- {:d} [HJD-2450000]".format(int(param_means[1]),
                                                                      int(param_sigmas[1])))
print("Period-change rate:       {:+.3f} +- {:.3f} [d/Myr] ".format(param_means[7]*365.2422*2e6*period,
                                                                    param_sigmas[7]*365.2422*2e6*period))
print("RV semi-amplitude:         {:5.2f} +- {:.2f}  [km/s]".format(K[0], K[1]))
print("Mass function:           {:.5f} +- {:.5f} [M_Sun]".format(K[2], K[3]))Orbital period:             3672 +- 12     [d]
Projected semi-major axis: 0.943 +- 0.011 [AU]
Eccentricity:              0.271 +- 0.022
Argumen of periastron:       +22 +- 4 [deg]
Periastron passage time:    7454 +- 41 [HJD-2450000]
Period-change rate:       +0.003 +- 0.004 [d/Myr]
RV semi-amplitude:          2.90 +- 0.04  [km/s]
Mass function:           0.00830 +- 0.00031 [M_Sun]


# Notebook from CatChenal/CatChenal.github.io
Path: _posts/notebooks/Age Pyramid in Altair.ipynb

# Charting an Age Pyramid in [Altair](https://altair-viz.github.io/)

>### Altair is a declarative statistical visualization library for Python, based on Vega and Vega-Lite, and the source is available on [GitHub](https://github.com/altair-viz/altair)._____no_output_____

import altair as alt
from vega_datasets import data
from altair.expr import datum, if______no_output_____

### In the Jupyter notebook only (not in the JupyterLab), uncomment and run this command once per session:_____no_output_____

#alt.renderers.enable('notebook')_____no_output_____

*This is a correction of the Gallery [example](https://altair-viz.github.io/gallery/us_population_pyramid_over_time.html), which renders an **inverted** age pyramid.*_____no_output_____

pop = data.population()

# Get the min and max of the slider tool from the dataset:
slider = alt.binding_range(min=pop.year.min(), max=pop.year.max(), step=10)

# If name is None or not given, the default slider title of "selector<nnn>" will be used;
# Note 1: The <nnn> portion change as per the number of time the chart has been refreshed.
# Note 2: name (or default string) is automatically concatenated with "_" (?) and fields.
select_year = alt.selection_single(name='Select', fields=['year'], bind=slider)

base = ( alt.Chart(pop).add_selection(select_year)
                       .transform_filter(select_year)
                       .transform_calculate(gender=if_(datum.sex == 1, 'Male', 'Female')) )

title = alt.Axis(title='population')
color_scale = alt.Scale(domain=['Male', 'Female'], range=["steelblue", "salmon"])

left = ( base.transform_filter(datum.gender == 'Female')
             .encode(y=alt.X('age:O', axis=None, sort='descending'),
                     x=alt.Y('sum(people):Q',
                     axis=title,
                     sort=alt.SortOrder('descending')),
                     color=alt.Color('gender:N', scale=color_scale, legend=None))
              .mark_bar().properties(title='Female') )

middle = base.encode(y=alt.Y('age:O', axis=None, sort='descending'),
                     text=alt.Text('age:Q')).mark_text().properties(width=20, title='Age')

right = ( base.transform_filter(datum.gender == 'Male')
              .encode(y=alt.Y('age:O', axis=None, sort='descending'),
                      x=alt.X('sum(people):Q', axis=title),
                      color=alt.Color('gender:N', scale=color_scale, legend=None))
               .mark_bar().properties(title='Male') )

# Concatenate the three charts horizontally, same as using alt.hconcat(left, middle, right):
left | middle | right_____no_output_____


# Notebook from SalishSeaCast/analysis-keegan
Path: notebooks/Thalweg_work/Puget_Thalweg_Plotting.ipynb

This script contains thalweg plots for Puget sound for a thalweg drawn through adrimarlty inlet to Main basin. These thalweg plots show how several model variables change along the thalwegs such as salinity and temperature._____no_output_____

import numpy as np
import matplotlib.pyplot as plt
from matplotlib import pylab
from salishsea_tools import viz_tools, places, visualisations
import netCDF4 as nc # unless you prefer xarray
import datetime as dt
import os
import xarray as xr
import glob
import cmocean

%matplotlib inline
#Try changing things, try looking at the model output, and try comparing output from different times.
#In the 2018 files you have the years 2015-2018
#There is this star thing that people use in terminal (*) That is useful for finding files and you should try and find that._____no_output_____

## Thalweg Path_____no_output_____

grid = nc.Dataset('/data/nsoontie/MEOPAR/NEMO-forcing/grid/bathymetry_201702.nc','r')
bathy = grid.variables['Bathymetry'][:,:]
thalweg = np.loadtxt('puget_thalweg.txt', delimiter=' ', dtype=int)_____no_output_____fig,ax = plt.subplots(1,1,figsize=(8,10))
cmin=0; cmax=270;
ps=ax.pcolormesh(bathy,vmin=cmin,vmax=cmax,cmap='viridis') # This will have to be changed in orderto get a clearer view of the Puget thalweg.
fig.colorbar(ps)

ax.plot(thalweg[:,1]+1,thalweg[:,0]+1,marker='o',color='k',MarkerSize=2)
ax.set_xlim(150,350)
ax.set_ylim(-20,330)

for k in range (1,6):
    pylab.plot(thalweg[k*39,1]+1, thalweg[k*39,0]+1,'o')
    print(thalweg[k*39,1] + 1, thalweg[k*39,0] + 1)<ipython-input-12-87d1c3906156>:6: MatplotlibDeprecationWarning: Case-insensitive properties were deprecated in 3.3 and support will be removed two minor releases later
  ax.plot(thalweg[:,1]+1,thalweg[:,0]+1,marker='o',color='k',MarkerSize=2)

## Thalweg plots
  method using contour_thalweg from visualisations.py in tools repo_____no_output_____

#open bathy file and meshmask
f=nc.Dataset('/results/SalishSea/nowcast-green.201812/01apr16/SalishSea_1h_20160401_20160401_ptrc_T.nc')
g=nc.Dataset('/results/SalishSea/nowcast-green.201812/01apr16/SalishSea_1h_20160401_20160401_grid_T.nc')
fbathy=nc.Dataset('/data/eolson/results/MEOPAR/NEMO-forcing-new/grid/bathymetry_201702.nc')
fmesh=nc.Dataset('/data/eolson/results/MEOPAR/NEMO-forcing-new/grid/mesh_mask201702.nc')
il=5_____no_output_____

### Salinity_____no_output_____

fig,ax=plt.subplots(1,1,figsize=(13,4))
modvar='vosaline'
cb=visualisations.contour_thalweg(ax,g.variables[modvar][il,...],fbathy,fmesh,clevels='salinity',cmap=cmocean.cm.amp,thalweg_file='puget_thalweg.txt',
                                  cbar_args={'label':modvar})
_____no_output_____fbathy_____no_output_____fig,ax=plt.subplots(1,1,figsize=(13,4))
modvar='votemper'
cb=visualisations.contour_thalweg(ax,g.variables[modvar][il,...],fbathy,fmesh,clevels='temperature',cmap=cmocean.cm.amp,thalweg_file='puget_thalweg.txt',
                                  cbar_args={'label':modvar})_____no_output_____fig,ax=plt.subplots(1,1,figsize=(13,4))
modvar='microzooplankton'
cb=visualisations.contour_thalweg(ax,f.variables[modvar][il,...],fbathy,fmesh,clevels=50,cmap=cmocean.cm.amp,thalweg_file='puget_thalweg.txt',
                                  cbar_args={'label':modvar})_____no_output_____fig,ax=plt.subplots(1,1,figsize=(13,4))
modvar='mesozooplankton'
cb=visualisations.contour_thalweg(ax,f.variables[modvar][il,...],fbathy,fmesh,clevels=50,cmap=cmocean.cm.amp,thalweg_file='puget_thalweg.txt',
                                  cbar_args={'label':modvar})_____no_output_____fig,ax=plt.subplots(1,1,figsize=(13,4))
modvar='nitrate'
cb=visualisations.contour_thalweg(ax,f.variables[modvar][il,...],fbathy,fmesh,clevels=25,cmap=cmocean.cm.amp,thalweg_file='puget_thalweg.txt',
                                  cbar_args={'label':modvar})_____no_output_____fig,ax=plt.subplots(1,1,figsize=(13,4))
modvar='silicon'
cb=visualisations.contour_thalweg(ax,f.variables[modvar][il,...],fbathy,fmesh,clevels=20,cmap=cmocean.cm.amp,thalweg_file='puget_thalweg.txt',
                                  cbar_args={'label':modvar})_____no_output_____fig,ax=plt.subplots(1,1,figsize=(13,4))
modvar='ammonium'
cb=visualisations.contour_thalweg(ax,f.variables[modvar][il,...],fbathy,fmesh,clevels=50,cmap=cmocean.cm.amp,thalweg_file='puget_thalweg.txt',
                                  cbar_args={'label':modvar})_____no_output_____fbathy.close()
fmesh.close()
f.close()
g.close()_____no_output_____


# Notebook from sivajyothi1/DeepLearning.ai_Assignments
Path: Sequence Models/Week1/Dinosaurus Island- Character level language model final.ipynb

# Character level language model - Dinosaurus land

Welcome to Dinosaurus Island! 65 million years ago, dinosaurs existed, and in this assignment they are back. You are in charge of a special task. Leading biology researchers are creating new breeds of dinosaurs and bringing them to life on earth, and your job is to give names to these dinosaurs. If a dinosaur does not like its name, it might go beserk, so choose wisely!

Luckily you have learned some deep learning and you will use it to save the day. Your assistant has collected a list of all the dinosaur names they could find, and compiled them into this [dataset](dinos.txt). (Feel free to take a look by clicking the previous link.) To create new dinosaur names, you will build a character level language model to generate new names. Your algorithm will learn the different name patterns, and randomly generate new names. Hopefully this algorithm will keep you and your team safe from the dinosaurs' wrath!

By completing this assignment you will learn:

- How to store text data for processing using an RNN
- How to synthesize data, by sampling predictions at each time step and passing it to the next RNN-cell unit
- How to build a character-level text generation recurrent neural network
- Why clipping the gradients is important

We will begin by loading in some functions that we have provided for you in `rnn_utils`. Specifically, you have access to functions such as `rnn_forward` and `rnn_backward` which are equivalent to those you've implemented in the previous assignment. _____no_output_____

import numpy as np
from utils import *
import random_____no_output_____

## 1 - Problem Statement

### 1.1 - Dataset and Preprocessing

Run the following cell to read the dataset of dinosaur names, create a list of unique characters (such as a-z), and compute the dataset and vocabulary size. _____no_output_____

data = open('dinos.txt', 'r').read()
data= data.lower()
chars = list(set(data))
data_size, vocab_size = len(data), len(chars)
print('There are %d total characters and %d unique characters in your data.' % (data_size, vocab_size))There are 19909 total characters and 27 unique characters in your data.

The characters are a-z (26 characters) plus the "\n" (or newline character), which in this assignment plays a role similar to the `<EOS>` (or "End of sentence") token we had discussed in lecture, only here it indicates the end of the dinosaur name rather than the end of a sentence. In the cell below, we create a python dictionary (i.e., a hash table) to map each character to an index from 0-26. We also create a second python dictionary that maps each index back to the corresponding character character. This will help you figure out what index corresponds to what character in the probability distribution output of the softmax layer. Below, `char_to_ix` and `ix_to_char` are the python dictionaries. _____no_output_____

char_to_ix = { ch:i for i,ch in enumerate(sorted(chars)) }
ix_to_char = { i:ch for i,ch in enumerate(sorted(chars)) }
print(ix_to_char){0: '\n', 1: 'a', 2: 'b', 3: 'c', 4: 'd', 5: 'e', 6: 'f', 7: 'g', 8: 'h', 9: 'i', 10: 'j', 11: 'k', 12: 'l', 13: 'm', 14: 'n', 15: 'o', 16: 'p', 17: 'q', 18: 'r', 19: 's', 20: 't', 21: 'u', 22: 'v', 23: 'w', 24: 'x', 25: 'y', 26: 'z'}

### 1.2 - Overview of the model

Your model will have the following structure:

- Initialize parameters
- Run the optimization loop
    - Forward propagation to compute the loss function
    - Backward propagation to compute the gradients with respect to the loss function
    - Clip the gradients to avoid exploding gradients
    - Using the gradients, update your parameter with the gradient descent update rule.
- Return the learned parameters

 **Figure 1**: Recurrent Neural Network, similar to what you had built in the previous notebook "Building a RNN - Step by Step".

At each time-step, the RNN tries to predict what is the next character given the previous characters. The dataset $X = (x^{\langle 1 \rangle}, x^{\langle 2 \rangle}, ..., x^{\langle T_x \rangle})$ is a list of characters in the training set, while $Y = (y^{\langle 1 \rangle}, y^{\langle 2 \rangle}, ..., y^{\langle T_x \rangle})$ is such that at every time-step $t$, we have $y^{\langle t \rangle} = x^{\langle t+1 \rangle}$. _____no_output_____## 2 - Building blocks of the model

In this part, you will build two important blocks of the overall model:
- Gradient clipping: to avoid exploding gradients
- Sampling: a technique used to generate characters

You will then apply these two functions to build the model._____no_output_____### 2.1 - Clipping the gradients in the optimization loop

In this section you will implement the `clip` function that you will call inside of your optimization loop. Recall that your overall loop structure usually consists of a forward pass, a cost computation, a backward pass, and a parameter update. Before updating the parameters, you will perform gradient clipping when needed to make sure that your gradients are not "exploding," meaning taking on overly large values.

In the exercise below, you will implement a function `clip` that takes in a dictionary of gradients and returns a clipped version of gradients if needed. There are different ways to clip gradients; we will use a simple element-wise clipping procedure, in which every element of the gradient vector is clipped to lie between some range [-N, N]. More generally, you will provide a `maxValue` (say 10). In this example, if any component of the gradient vector is greater than 10, it would be set to 10; and if any component of the gradient vector is less than -10, it would be set to -10. If it is between -10 and 10, it is left alone.

 **Figure 2**: Visualization of gradient descent with and without gradient clipping, in a case where the network is running into slight "exploding gradient" problems.

**Exercise**: Implement the function below to return the clipped gradients of your dictionary `gradients`. Your function takes in a maximum threshold and returns the clipped versions of your gradients. You can check out this [hint](https://docs.scipy.org/doc/numpy-1.13.0/reference/generated/numpy.clip.html) for examples of how to clip in numpy. You will need to use the argument `out = ...`._____no_output_____

### GRADED FUNCTION: clip

def clip(gradients, maxValue):
    '''
    Clips the gradients' values between minimum and maximum.

    Arguments:
    gradients -- a dictionary containing the gradients "dWaa", "dWax", "dWya", "db", "dby"
    maxValue -- everything above this number is set to this number, and everything less than -maxValue is set to -maxValue

    Returns:
    gradients -- a dictionary with the clipped gradients.
    '''

    dWaa, dWax, dWya, db, dby = gradients['dWaa'], gradients['dWax'], gradients['dWya'], gradients['db'], gradients['dby']

    ### START CODE HERE ###
    # clip to mitigate exploding gradients, loop over [dWax, dWaa, dWya, db, dby]. (≈2 lines)
    for gradient in [dWax, dWaa, dWya, db, dby]:
        gradient = np.clip(gradient, -maxValue, maxValue, out = gradient)
    ### END CODE HERE ###

    gradients = {"dWaa": dWaa, "dWax": dWax, "dWya": dWya, "db": db, "dby": dby}

    return gradients_____no_output_____np.random.seed(3)
dWax = np.random.randn(5,3)*10
dWaa = np.random.randn(5,5)*10
dWya = np.random.randn(2,5)*10
db = np.random.randn(5,1)*10
dby = np.random.randn(2,1)*10
gradients = {"dWax": dWax, "dWaa": dWaa, "dWya": dWya, "db": db, "dby": dby}
gradients = clip(gradients, 10)
print("gradients[\"dWaa\"][1][2] =", gradients["dWaa"][1][2])
print("gradients[\"dWax\"][3][1] =", gradients["dWax"][3][1])
print("gradients[\"dWya\"][1][2] =", gradients["dWya"][1][2])
print("gradients[\"db\"][4] =", gradients["db"][4])
print("gradients[\"dby\"][1] =", gradients["dby"][1])gradients["dWaa"][1][2] = 10.0
gradients["dWax"][3][1] = -10.0
gradients["dWya"][1][2] = 0.29713815361
gradients["db"][4] = [ 10.]
gradients["dby"][1] = [ 8.45833407]

** Expected output:**

    **gradients["dWaa"][1][2] **

    10.0

    **gradients["dWax"][3][1]**

    -10.0

    **gradients["dWya"][1][2]**

0.29713815361

    **gradients["db"][4]**

[ 10.]

    **gradients["dby"][1]**

[ 8.45833407]

_____no_output_____### 2.2 - Sampling

Now assume that your model is trained. You would like to generate new text (characters). The process of generation is explained in the picture below:

 **Figure 3**: In this picture, we assume the model is already trained. We pass in $x^{\langle 1\rangle} = \vec{0}$ at the first time step, and have the network then sample one character at a time.

**Exercise**: Implement the `sample` function below to sample characters. You need to carry out 4 steps:

- **Step 1**: Pass the network the first "dummy" input $x^{\langle 1 \rangle} = \vec{0}$ (the vector of zeros). This is the default input before we've generated any characters. We also set $a^{\langle 0 \rangle} = \vec{0}$

- **Step 2**: Run one step of forward propagation to get $a^{\langle 1 \rangle}$ and $\hat{y}^{\langle 1 \rangle}$. Here are the equations:

$$ a^{\langle t+1 \rangle} = \tanh(W_{ax}  x^{\langle t \rangle } + W_{aa} a^{\langle t \rangle } + b)\tag{1}$$

$$ z^{\langle t + 1 \rangle } = W_{ya}  a^{\langle t + 1 \rangle } + b_y \tag{2}$$

$$ \hat{y}^{\langle t+1 \rangle } = softmax(z^{\langle t + 1 \rangle })\tag{3}$$

Note that $\hat{y}^{\langle t+1 \rangle }$ is a (softmax) probability vector (its entries are between 0 and 1 and sum to 1). $\hat{y}^{\langle t+1 \rangle}_i$ represents the probability that the character indexed by "i" is the next character.  We have provided a `softmax()` function that you can use.

- **Step 3**: Carry out sampling: Pick the next character's index according to the probability distribution specified by $\hat{y}^{\langle t+1 \rangle }$. This means that if $\hat{y}^{\langle t+1 \rangle }_i = 0.16$, you will pick the index "i" with 16% probability. To implement it, you can use [`np.random.choice`](https://docs.scipy.org/doc/numpy-1.13.0/reference/generated/numpy.random.choice.html).

Here is an example of how to use `np.random.choice()`:
```python
np.random.seed(0)
p = np.array([0.1, 0.0, 0.7, 0.2])
index = np.random.choice([0, 1, 2, 3], p = p.ravel())
```
This means that you will pick the `index` according to the distribution:
$P(index = 0) = 0.1, P(index = 1) = 0.0, P(index = 2) = 0.7, P(index = 3) = 0.2$.

- **Step 4**: The last step to implement in `sample()` is to overwrite the variable `x`, which currently stores $x^{\langle t \rangle }$, with the value of $x^{\langle t + 1 \rangle }$. You will represent $x^{\langle t + 1 \rangle }$ by creating a one-hot vector corresponding to the character you've chosen as your prediction. You will then forward propagate $x^{\langle t + 1 \rangle }$ in Step 1 and keep repeating the process until you get a "\n" character, indicating you've reached the end of the dinosaur name. _____no_output_____

# GRADED FUNCTION: sample

def sample(parameters, char_to_ix, seed):
    """
    Sample a sequence of characters according to a sequence of probability distributions output of the RNN

    Arguments:
    parameters -- python dictionary containing the parameters Waa, Wax, Wya, by, and b.
    char_to_ix -- python dictionary mapping each character to an index.
    seed -- used for grading purposes. Do not worry about it.

    Returns:
    indices -- a list of length n containing the indices of the sampled characters.
    """

    # Retrieve parameters and relevant shapes from "parameters" dictionary
    Waa, Wax, Wya, by, b = parameters['Waa'], parameters['Wax'], parameters['Wya'], parameters['by'], parameters['b']
    vocab_size = by.shape[0]
    n_a = Waa.shape[1]

    ### START CODE HERE ###
    # Step 1: Create the one-hot vector x for the first character (initializing the sequence generation). (≈1 line)
    x = np.zeros((vocab_size,1))
    # Step 1': Initialize a_prev as zeros (≈1 line)
    a_prev = np.zeros((n_a,1))

    # Create an empty list of indices, this is the list which will contain the list of indices of the characters to generate (≈1 line)
    indices = []

    # Idx is a flag to detect a newline character, we initialize it to -1
    idx = -1

    # Loop over time-steps t. At each time-step, sample a character from a probability distribution and append
    # its index to "indices". We'll stop if we reach 50 characters (which should be very unlikely with a well
    # trained model), which helps debugging and prevents entering an infinite loop.
    counter = 0
    newline_character = char_to_ix['\n']

    while (idx != newline_character and counter != 50):

        # Step 2: Forward propagate x using the equations (1), (2) and (3)
        a = np.tanh(np.dot(Wax,x) + np.dot(Waa,a_prev) + b)
        z = np.dot(Wya,a) + by
        y = softmax(z)

        # for grading purposes
        np.random.seed(counter+seed)

        # Step 3: Sample the index of a character within the vocabulary from the probability distribution y
        idx = np.random.choice(list(range(vocab_size)), p = y.ravel())

        # Append the index to "indices"
        indices.append(idx)

        # Step 4: Overwrite the input character as the one corresponding to the sampled index.
        x = np.zeros((vocab_size, 1))
        x[idx] = 1

        # Update "a_prev" to be "a"
        a_prev = a

        # for grading purposes
        seed += 1
        counter +=1

    ### END CODE HERE ###

    if (counter == 50):
        indices.append(char_to_ix['\n'])

    return indices_____no_output_____np.random.seed(2)
_, n_a = 20, 100
Wax, Waa, Wya = np.random.randn(n_a, vocab_size), np.random.randn(n_a, n_a), np.random.randn(vocab_size, n_a)
b, by = np.random.randn(n_a, 1), np.random.randn(vocab_size, 1)
parameters = {"Wax": Wax, "Waa": Waa, "Wya": Wya, "b": b, "by": by}

indices = sample(parameters, char_to_ix, 0)
print("Sampling:")
print("list of sampled indices:", indices)
print("list of sampled characters:", [ix_to_char[i] for i in indices])Sampling:
list of sampled indices: [12, 17, 24, 14, 13, 9, 10, 22, 24, 6, 13, 11, 12, 6, 21, 15, 21, 14, 3, 2, 1, 21, 18, 24, 7, 25, 6, 25, 18, 10, 16, 2, 3, 8, 15, 12, 11, 7, 1, 12, 10, 2, 7, 7, 11, 5, 6, 12, 25, 0, 0]
list of sampled characters: ['l', 'q', 'x', 'n', 'm', 'i', 'j', 'v', 'x', 'f', 'm', 'k', 'l', 'f', 'u', 'o', 'u', 'n', 'c', 'b', 'a', 'u', 'r', 'x', 'g', 'y', 'f', 'y', 'r', 'j', 'p', 'b', 'c', 'h', 'o', 'l', 'k', 'g', 'a', 'l', 'j', 'b', 'g', 'g', 'k', 'e', 'f', 'l', 'y', '\n', '\n']

** Expected output:**

    **list of sampled indices:**

    [12, 17, 24, 14, 13, 9, 10, 22, 24, 6, 13, 11, 12, 6, 21, 15, 21, 14, 3, 2, 1, 21, 18, 24,

    7, 25, 6, 25, 18, 10, 16, 2, 3, 8, 15, 12, 11, 7, 1, 12, 10, 2, 7, 7, 11, 5, 6, 12, 25, 0, 0]

    **list of sampled characters:**

    ['l', 'q', 'x', 'n', 'm', 'i', 'j', 'v', 'x', 'f', 'm', 'k', 'l', 'f', 'u', 'o',

    'u', 'n', 'c', 'b', 'a', 'u', 'r', 'x', 'g', 'y', 'f', 'y', 'r', 'j', 'p', 'b', 'c', 'h', 'o',

    'l', 'k', 'g', 'a', 'l', 'j', 'b', 'g', 'g', 'k', 'e', 'f', 'l', 'y', '\n', '\n']

_____no_output_____## 3 - Building the language model

It is time to build the character-level language model for text generation.

### 3.1 - Gradient descent

In this section you will implement a function performing one step of stochastic gradient descent (with clipped gradients). You will go through the training examples one at a time, so the optimization algorithm will be stochastic gradient descent. As a reminder, here are the steps of a common optimization loop for an RNN:

- Forward propagate through the RNN to compute the loss
- Backward propagate through time to compute the gradients of the loss with respect to the parameters
- Clip the gradients if necessary
- Update your parameters using gradient descent

**Exercise**: Implement this optimization process (one step of stochastic gradient descent).

We provide you with the following functions:

```python
def rnn_forward(X, Y, a_prev, parameters):
    """ Performs the forward propagation through the RNN and computes the cross-entropy loss.
    It returns the loss' value as well as a "cache" storing values to be used in the backpropagation."""
    ....
    return loss, cache

def rnn_backward(X, Y, parameters, cache):
    """ Performs the backward propagation through time to compute the gradients of the loss with respect
    to the parameters. It returns also all the hidden states."""
    ...
    return gradients, a

def update_parameters(parameters, gradients, learning_rate):
    """ Updates parameters using the Gradient Descent Update Rule."""
    ...
    return parameters
```_____no_output_____

# GRADED FUNCTION: optimize

def optimize(X, Y, a_prev, parameters, learning_rate = 0.01):
    """
    Execute one step of the optimization to train the model.

    Arguments:
    X -- list of integers, where each integer is a number that maps to a character in the vocabulary.
    Y -- list of integers, exactly the same as X but shifted one index to the left.
    a_prev -- previous hidden state.
    parameters -- python dictionary containing:
                        Wax -- Weight matrix multiplying the input, numpy array of shape (n_a, n_x)
                        Waa -- Weight matrix multiplying the hidden state, numpy array of shape (n_a, n_a)
                        Wya -- Weight matrix relating the hidden-state to the output, numpy array of shape (n_y, n_a)
                        b --  Bias, numpy array of shape (n_a, 1)
                        by -- Bias relating the hidden-state to the output, numpy array of shape (n_y, 1)
    learning_rate -- learning rate for the model.

    Returns:
    loss -- value of the loss function (cross-entropy)
    gradients -- python dictionary containing:
                        dWax -- Gradients of input-to-hidden weights, of shape (n_a, n_x)
                        dWaa -- Gradients of hidden-to-hidden weights, of shape (n_a, n_a)
                        dWya -- Gradients of hidden-to-output weights, of shape (n_y, n_a)
                        db -- Gradients of bias vector, of shape (n_a, 1)
                        dby -- Gradients of output bias vector, of shape (n_y, 1)
    a[len(X)-1] -- the last hidden state, of shape (n_a, 1)
    """

    ### START CODE HERE ###

    # Forward propagate through time (≈1 line)
    loss, cache = rnn_forward(X,Y,a_prev,parameters)

    # Backpropagate through time (≈1 line)
    gradients, a = rnn_backward(X,Y,parameters, cache)

    # Clip your gradients between -5 (min) and 5 (max) (≈1 line)
    gradients = clip(gradients,5)

    # Update parameters (≈1 line)
    parameters = update_parameters(parameters,gradients,learning_rate)

    ### END CODE HERE ###

    return loss, gradients, a[len(X)-1]_____no_output_____np.random.seed(1)
vocab_size, n_a = 27, 100
a_prev = np.random.randn(n_a, 1)
Wax, Waa, Wya = np.random.randn(n_a, vocab_size), np.random.randn(n_a, n_a), np.random.randn(vocab_size, n_a)
b, by = np.random.randn(n_a, 1), np.random.randn(vocab_size, 1)
parameters = {"Wax": Wax, "Waa": Waa, "Wya": Wya, "b": b, "by": by}
X = [12,3,5,11,22,3]
Y = [4,14,11,22,25, 26]

loss, gradients, a_last = optimize(X, Y, a_prev, parameters, learning_rate = 0.01)
print("Loss =", loss)
print("gradients[\"dWaa\"][1][2] =", gradients["dWaa"][1][2])
print("np.argmax(gradients[\"dWax\"]) =", np.argmax(gradients["dWax"]))
print("gradients[\"dWya\"][1][2] =", gradients["dWya"][1][2])
print("gradients[\"db\"][4] =", gradients["db"][4])
print("gradients[\"dby\"][1] =", gradients["dby"][1])
print("a_last[4] =", a_last[4])Loss = 126.503975722
gradients["dWaa"][1][2] = 0.194709315347
np.argmax(gradients["dWax"]) = 93
gradients["dWya"][1][2] = -0.007773876032
gradients["db"][4] = [-0.06809825]
gradients["dby"][1] = [ 0.01538192]
a_last[4] = [-1.]

** Expected output:**

    **Loss **

    126.503975722

    **gradients["dWaa"][1][2]**

    0.194709315347

    **np.argmax(gradients["dWax"])**

     93

    **gradients["dWya"][1][2]**

     -0.007773876032

    **gradients["db"][4]**

     [-0.06809825]

    **gradients["dby"][1]**

    [ 0.01538192]

    **a_last[4]**

     [-1.]

_____no_output_____### 3.2 - Training the model _____no_output_____Given the dataset of dinosaur names, we use each line of the dataset (one name) as one training example. Every 100 steps of stochastic gradient descent, you will sample 10 randomly chosen names to see how the algorithm is doing. Remember to shuffle the dataset, so that stochastic gradient descent visits the examples in random order.

**Exercise**: Follow the instructions and implement `model()`. When `examples[index]` contains one dinosaur name (string), to create an example (X, Y), you can use this:
```python
        index = j % len(examples)
        X = [None] + [char_to_ix[ch] for ch in examples[index]]
        Y = X[1:] + [char_to_ix["\n"]]
```
Note that we use: `index= j % len(examples)`, where `j = 1....num_iterations`, to make sure that `examples[index]` is always a valid statement (`index` is smaller than `len(examples)`).
The first entry of `X` being `None` will be interpreted by `rnn_forward()` as setting $x^{\langle 0 \rangle} = \vec{0}$. Further, this ensures that `Y` is equal to `X` but shifted one step to the left, and with an additional "\n" appended to signify the end of the dinosaur name. _____no_output_____

# GRADED FUNCTION: model

def model(data, ix_to_char, char_to_ix, num_iterations = 35000, n_a = 50, dino_names = 7, vocab_size = 27):
    """
    Trains the model and generates dinosaur names.

    Arguments:
    data -- text corpus
    ix_to_char -- dictionary that maps the index to a character
    char_to_ix -- dictionary that maps a character to an index
    num_iterations -- number of iterations to train the model for
    n_a -- number of units of the RNN cell
    dino_names -- number of dinosaur names you want to sample at each iteration.
    vocab_size -- number of unique characters found in the text, size of the vocabulary

    Returns:
    parameters -- learned parameters
    """

    # Retrieve n_x and n_y from vocab_size
    n_x, n_y = vocab_size, vocab_size

    # Initialize parameters
    parameters = initialize_parameters(n_a, n_x, n_y)

    # Initialize loss (this is required because we want to smooth our loss, don't worry about it)
    loss = get_initial_loss(vocab_size, dino_names)

    # Build list of all dinosaur names (training examples).
    with open("dinos.txt") as f:
        examples = f.readlines()
    examples = [x.lower().strip() for x in examples]

    # Shuffle list of all dinosaur names
    np.random.seed(0)
    np.random.shuffle(examples)

    # Initialize the hidden state of your LSTM
    a_prev = np.zeros((n_a, 1))

    # Optimization loop
    for j in range(num_iterations):

        ### START CODE HERE ###

        # Use the hint above to define one training example (X,Y) (≈ 2 lines)
        index = j % len(examples)
        X = [None] + [char_to_ix[ch] for ch in examples[index]]
        Y = X[1:] + [char_to_ix['\n']]

        # Perform one optimization step: Forward-prop -> Backward-prop -> Clip -> Update parameters
        # Choose a learning rate of 0.01
        curr_loss, gradients, a_prev = optimize(X,Y,a_prev,parameters)
        ### END CODE HERE ###

        # Use a latency trick to keep the loss smooth. It happens here to accelerate the training.
        loss = smooth(loss, curr_loss)

        # Every 2000 Iteration, generate "n" characters thanks to sample() to check if the model is learning properly
        if j % 2000 == 0:

            print('Iteration: %d, Loss: %f' % (j, loss) + '\n')

            # The number of dinosaur names to print
            seed = 0
            for name in range(dino_names):

                # Sample indices and print them
                sampled_indices = sample(parameters, char_to_ix, seed)
                print_sample(sampled_indices, ix_to_char)

                seed += 1  # To get the same result for grading purposed, increment the seed by one.

            print('\n')

    return parameters_____no_output_____

Run the following cell, you should observe your model outputting random-looking characters at the first iteration. After a few thousand iterations, your model should learn to generate reasonable-looking names. _____no_output_____

parameters = model(data, ix_to_char, char_to_ix)Iteration: 0, Loss: 23.087336

Nkzxwtdmfqoeyhsqwasjkjvu
Kneb
Kzxwtdmfqoeyhsqwasjkjvu
Neb
Zxwtdmfqoeyhsqwasjkjvu
Eb
Xwtdmfqoeyhsqwasjkjvu

Iteration: 2000, Loss: 27.884160

Liusskeomnolxeros
Hmdaairus
Hytroligoraurus
Lecalosapaus
Xusicikoraurus
Abalpsamantisaurus
Tpraneronxeros

Iteration: 4000, Loss: 25.901815

Mivrosaurus
Inee
Ivtroplisaurus
Mbaaisaurus
Wusichisaurus
Cabaselachus
Toraperlethosdarenitochusthiamamumamaon

Iteration: 6000, Loss: 24.608779

Onwusceomosaurus
Lieeaerosaurus
Lxussaurus
Oma
Xusteonosaurus
Eeahosaurus
Toreonosaurus

Iteration: 8000, Loss: 24.070350

Onxusichepriuon
Kilabersaurus
Lutrodon
Omaaerosaurus
Xutrcheps
Edaksoje
Trodiktonus

Iteration: 10000, Loss: 23.844446

Onyusaurus
Klecalosaurus
Lustodon
Ola
Xusodonia
Eeaeosaurus
Troceosaurus

Iteration: 12000, Loss: 23.291971

Onyxosaurus
Kica
Lustrepiosaurus
Olaagrraiansaurus
Yuspangosaurus
Eealosaurus
Trognesaurus

Iteration: 14000, Loss: 23.382339

Meutromodromurus
Inda
Iutroinatorsaurus
Maca
Yusteratoptititan
Ca
Troclosaurus

Iteration: 16000, Loss: 23.288447

Meuspsangosaurus
Ingaa
Iusosaurus
Macalosaurus
Yushanis
Daalosaurus
Trpandon

Iteration: 18000, Loss: 22.823526

Phytrolonhonyg
Mela
Mustrerasaurus
Peg
Ytronorosaurus
Ehalosaurus
Trolomeehus

Iteration: 20000, Loss: 23.041871

Nousmofonosaurus
Loma
Lytrognatiasaurus
Ngaa
Ytroenetiaudostarmilus
Eiafosaurus
Troenchulunosaurus

Iteration: 22000, Loss: 22.728849

Piutyrangosaurus
Midaa
Myroranisaurus
Pedadosaurus
Ytrodon
Eiadosaurus
Trodoniomusitocorces

Iteration: 24000, Loss: 22.683403

Meutromeisaurus
Indeceratlapsaurus
Jurosaurus
Ndaa
Yusicheropterus
Eiaeropectus
Trodonasaurus

Iteration: 26000, Loss: 22.554523

Phyusaurus
Liceceron
Lyusichenodylus
Pegahus
Yustenhtonthosaurus
Elagosaurus
Trodontonsaurus

Iteration: 28000, Loss: 22.484472

Onyutimaerihus
Koia
Lytusaurus
Ola
Ytroheltorus
Eiadosaurus
Trofiashates

Iteration: 30000, Loss: 22.774404

Phytys
Lica
Lysus
Pacalosaurus
Ytrochisaurus
Eiacosaurus
Trochesaurus

Iteration: 32000, Loss: 22.209473

Mawusaurus
Jica
Lustoia
Macaisaurus
Yusolenqtesaurus
Eeaeosaurus
Trnanatrax

Iteration: 34000, Loss: 22.396744

Mavptokekus
Ilabaisaurus
Itosaurus
Macaesaurus
Yrosaurus
Eiaeosaurus
Trodon

## Conclusion

You can see that your algorithm has started to generate plausible dinosaur names towards the end of the training. At first, it was generating random characters, but towards the end you could see dinosaur names with cool endings. Feel free to run the algorithm even longer and play with hyperparameters to see if you can get even better results. Our implemetation generated some really cool names like `maconucon`, `marloralus` and `macingsersaurus`. Your model hopefully also learned that dinosaur names tend to end in `saurus`, `don`, `aura`, `tor`, etc.

If your model generates some non-cool names, don't blame the model entirely--not all actual dinosaur names sound cool. (For example, `dromaeosauroides` is an actual dinosaur name and is in the training set.) But this model should give you a set of candidates from which you can pick the coolest!

This assignment had used a relatively small dataset, so that you could train an RNN quickly on a CPU. Training a model of the english language requires a much bigger dataset, and usually needs much more computation, and could run for many hours on GPUs. We ran our dinosaur name for quite some time, and so far our favoriate name is the great, undefeatable, and fierce: Mangosaurus!

_____no_output_____## 4 - Writing like Shakespeare

The rest of this notebook is optional and is not graded, but we hope you'll do it anyway since it's quite fun and informative.

A similar (but more complicated) task is to generate Shakespeare poems. Instead of learning from a dataset of Dinosaur names you can use a collection of Shakespearian poems. Using LSTM cells, you can learn longer term dependencies that span many characters in the text--e.g., where a character appearing somewhere a sequence can influence what should be a different character much much later in ths sequence. These long term dependencies were less important with dinosaur names, since the names were quite short.

 Let's become poets!

We have implemented a Shakespeare poem generator with Keras. Run the following cell to load the required packages and models. This may take a few minutes. _____no_output_____

from __future__ import print_function
from keras.callbacks import LambdaCallback
from keras.models import Model, load_model, Sequential
from keras.layers import Dense, Activation, Dropout, Input, Masking
from keras.layers import LSTM
from keras.utils.data_utils import get_file
from keras.preprocessing.sequence import pad_sequences
from shakespeare_utils import *
import sys
import ioUsing TensorFlow backend.

To save you some time, we have already trained a model for ~1000 epochs on a collection of Shakespearian poems called [*"The Sonnets"*](shakespeare.txt). _____no_output_____Let's train the model for one more epoch. When it finishes training for an epoch---this will also take a few minutes---you can run `generate_output`, which will prompt asking you for an input (`<`40 characters). The poem will start with your sentence, and our RNN-Shakespeare will complete the rest of the poem for you! For example, try "Forsooth this maketh no sense " (don't enter the quotation marks). Depending on whether you include the space at the end, your results might also differ--try it both ways, and try other inputs as well.
_____no_output_____

print_callback = LambdaCallback(on_epoch_end=on_epoch_end)

model.fit(x, y, batch_size=128, epochs=1, callbacks=[print_callback])Epoch 1/1
  640/31412 [..............................] - ETA: 224s - loss: 2.0881# Run this cell to try with different inputs without having to re-train the model
generate_output()Write the beginning of your poem, the Shakespeare machine will complete it. Your input is: Forsooth this maketh no sense

Here is your poem:

Forsooth this maketh no sense quite,
ow thear serblen comver yout' rithiest dose kind.
il have you my death that i apince is geloll,
and anw, to love with by art is fort'dy nage.
i the prite we artion heapt hith fors medes.
 hos mister befurtaun hing (home my sate,
aw heats be but's and all arfing, to be which, or mine,
the lasth of my meauty nepfech thee to be.

hos swidt with me ac haske fremen the paifty:
but love engus n

The RNN-Shakespeare model is very similar to the one you have built for dinosaur names. The only major differences are:
- LSTMs instead of the basic RNN to capture longer-range dependencies
- The model is a deeper, stacked LSTM model (2 layer)
- Using Keras instead of python to simplify the code

If you want to learn more, you can also check out the Keras Team's text generation implementation on GitHub: https://github.com/keras-team/keras/blob/master/examples/lstm_text_generation.py.

Congratulations on finishing this notebook! _____no_output_____**References**:
- This exercise took inspiration from Andrej Karpathy's implementation: https://gist.github.com/karpathy/d4dee566867f8291f086. To learn more about text generation, also check out Karpathy's [blog post](http://karpathy.github.io/2015/05/21/rnn-effectiveness/).
- For the Shakespearian poem generator, our implementation was based on the implementation of an LSTM text generator by the Keras team: https://github.com/keras-team/keras/blob/master/examples/lstm_text_generation.py _____no_output_____


# Notebook from lucianogsilvestri/sarkas
Path: docs/examples/Ultrafast_EC/Ultrafast_EC_Validation.ipynb

# Ultrafast electron cooling in an expanding ultracold plasma

The YAML input file can be found at [input_file](https://raw.githubusercontent.com/murillo-group/sarkas/master/docs/examples/Ultrafast_EC/input_files/uec.yaml) and this notebook at [notebook](https://raw.githubusercontent.com/murillo-group/sarkas/master/docs/examples/Ultrafast_EC/Ultrafast_EC_Validation.ipynb).

## Ultracold Microplasma

Let us observe the [**Ultrafast electron cooling in an expanding ultracold plasma**](https://doi.org/10.1038/s41467-020-20815-8). Here, a cloud of neutral atoms is trapped and cooled to quantum degeneracy at ~nK temperatures. The Bose-Einstein condensate (BEC) with a peak number density of $n \sim 2 \times 10^{20}\, \textrm{m}^{-3}$ is then rapidly ionized by a femtosecond laser pulse of $511\, \textrm{nm}$ wavelength and $215\, \textrm{fs}$ duration so that $N \sim 4000$ ions and electrons immediately emerge after two-photon ionization. The excess energy of $0.68\, \textrm{eV}$ is distributed between ions and electrons leading to an initial ion temperature of $T_{\rm i} \sim 33\, \textrm{mK}$ and fast escaping electrons of $T_e \sim 5250\, \textrm{K}$.

The ions are initially strongly coupled as the temperture is very small compared to the Coulomb energy of the dense, ionized BEC:

$$ \Gamma_{\rm i} = \frac{(e)^2}{4\pi \varepsilon_0 a_{\rm ws}} \frac{1}{k_B T_{\rm i}} = 4800 $$

where $T_{\rm i}$ is the ion temperature, and $a_{\rm ws} = [3 / (4\pi n)]^{1/3}$ is the Wigner-Seitz radius defined by the number density $n$.

The high density also leads to ultrafast dynamics as the inverse electron plasma frequency is given by:

$$ \nu^{-1}_{\rm p,e} = 2 \pi \sqrt{\frac{m_{\rm e} \varepsilon_0}{n e^2}} \sim 8\, \textrm{ps} $$_____no_output_____## Initialization

We start by importing the usual libraries and creating a path to the input file of our simulation._____no_output_____

%pylab
%matplotlib inline
import os
import pandas as pd

from sarkas.processes import PreProcess, Simulation, PostProcess

plt.style.use('../../../sarkas/mplstyles/MSUstyle.mplstyle')
input_file = os.path.join('input_files', 'uec.yaml')Using matplotlib backend: agg
Populating the interactive namespace from numpy and matplotlib

## Simulation

Let us start the simulation which takes ~30 minutes on 2021's computer._____no_output_____

# Initialize the Simulation class
sim = Simulation(input_file)
# Setup the simulation's parameters
sim.setup(read_yaml=True)
# Run the simulation
sim.run()

     _______.     ___      .______       __  ___      ___           _______.
    /       |    /   \     |   _  \     |  |/  /     /   \         /       |
   |   (----`   /  ^  \    |  |_)  |    |  '  /     /  ^  \       |   (----`
    \   \      /  /_\  \   |      /     |    <     /  /_\  \       \   \
.----)   |    /  _____  \  |  |\  \----.|  .  \   /  _____  \  .----)   |
|_______/    /__/     \__\ | _| `._____||__|\__\ /__/     \__\ |_______/

An open-source pure-python molecular dynamics suite for non-ideal plasmas.

WARNING: Short-range cut-off of 2.0000e-08 enabled. Use this feature with care!

!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! WARNING !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
Use the PP method with care for pure Coulomb interactions.
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
                                   Simulation
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *

Job ID:  UEC_4000
Job directory:  Simulations/UEC_4000

Equilibration dumps directory:
 Simulations/UEC_4000/Simulation/Equilibration/dumps
Production dumps directory:
 Simulations/UEC_4000/Simulation/Production/dumps

Equilibration Thermodynamics file:
 Simulations/UEC_4000/Simulation/Equilibration/EquilibrationEnergy_UEC_4000.csv
Production Thermodynamics file:
 Simulations/UEC_4000/Simulation/Production/ProductionEnergy_UEC_4000.csv

PARTICLES:
Total No. of particles =  8000
No. of species =  2
Species ID: 0
	Name: e
	No. of particles = 4000
	Number density = 2.000000e+20 [N/m^3]
	Atomic weight = 0.0005 [a.u.]
	Mass = 9.109384e-31 [kg]
	Mass density = 1.455190e-05 [kg/m^3]
	Charge number/ionization degree = -1.0000
	Charge = -1.602177e-19 [C]
	Temperature = 5.250000e+03 [K] = 4.524100e-01 [eV]
	Debye Length = 3.535658e-07 [Hz]
	Plasma Frequency = 7.978230e+11 [Hz]
Species ID: 1
	Name: Rb
	No. of particles = 4000
	Number density = 2.000000e+20 [N/m^3]
	Atomic weight = 87.0000 [a.u.]
	Mass = 1.455181e-25 [kg]
	Mass density = 1.455190e-05 [kg/m^3]
	Charge number/ionization degree = 1.0000
	Charge = 1.602177e-19 [C]
	Temperature = 3.300000e-02 [K] = 2.843720e-06 [eV]
	Debye Length = 8.864364e-10 [Hz]
	Plasma Frequency = 1.996147e+09 [Hz]

SIMULATION AND INITIAL PARTICLE BOX:
Units:  mks
Wigner-Seitz radius = 8.419452e-08 [m]
No. of non-zero box dimensions =  3
Box side along x axis = 2.375452e+03 a_ws = 2.000000e-04 [m]
Box side along y axis = 2.375452e+03 a_ws = 2.000000e-04 [m]
Box side along z axis = 2.375452e+03 a_ws = 2.000000e-04 [m]
Box Volume = 8.000000e-12 [m^3]
Initial particle box side along x axis = 2.579187e+01 a_ws = 2.171534e-06 [m]
Initial particle box side along y axis = 2.579187e+01 a_ws = 2.171534e-06 [m]
Initial particle box side along z axis = 5.037475e+01 a_ws = 4.241278e-06 [m]
Initial particle box Volume = 2.000000e-17 [m^3]
Boundary conditions: absorbing

ELECTRON PROPERTIES:
Number density: n_e = 2.000000e+20 [N/m^3]
Wigner-Seitz radius: a_e = 1.060784e-07 [m]
Temperature: T_e = 5.250000e+03 [K] = 4.524100e-01 [eV]
de Broglie wavelength: lambda_deB = 1.028727e-09 [m]
Thomas-Fermi length: lambda_TF = 3.535658e-07 [m]
Fermi wave number: k_F = 1.809188e+07 [1/m]
Fermi Energy: E_F = 1.247068e-05 [eV]
Relativistic parameter: x_F = 6.986347e-06 --> E_F = 1.247066e-05 [eV]
Degeneracy parameter: Theta = 3.627788e+04
Coupling: r_s = 2004.592027,  Gamma_e = 0.030005
Warm Dense Matter Parameter: W = 3.3054e-06
Chemical potential: mu = -1.6033e+01 k_B T_e = -5.8165e+05 E_F

POTENTIAL:  Coulomb
Effective Coupling constant: Gamma_eff = 2386.77
Short-range Cutoff radius: rs = 2.000000e-08 [m]

ALGORITHM:  PP
rcut = 1187.7258 a_ws = 1.000000e-04 [m]
No. of PP cells per dimension =  2,  2,  2
No. of particles in PP loop =  27000
No. of PP neighbors per particle =   4188
Tot Force Error = 0.000000e+00

THERMOSTAT:
Type: Berendsen
First thermostating timestep, i.e. relaxation_timestep = 0
Berendsen parameter tau: 5.000 [timesteps]
Berendsen relaxation rate: 0.200 [1/timesteps]

!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! WARNING !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
Equilibration temperatures not defined. I will use the species's temperatures
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

Thermostating temperatures:
Species ID 0: T_eq = 5.250000e+03 [K] = 4.524100e-01 [eV]
Species ID 1: T_eq = 3.300000e-02 [K] = 2.843720e-06 [eV]

INTEGRATOR:
Type: verlet
Time step = 5.000000e-14 [s]
Total plasma frequency = 7.978255e+11 [Hz]
w_p dt = 0.0399 ~ 1/25

Equilibration:
No. of equilibration steps = 0
Total equilibration time = 0.0000e+00 [s] ~ 0 w_p T_eq
snapshot interval step = 10
snapshot interval time = 5.0000e-13 [s] = 0.3989 w_p T_snap
Total number of snapshots = 0

Production:
No. of production steps = 600
Total production time = 3.0000e-11 [s] ~ 23 w_p T_prod
snapshot interval step = 4
snapshot interval time = 2.0000e-13 [s] = 0.1596 w_p T_snap
Total number of snapshots = 150

------------------------ Initialization Times ------------------------

Potential Initialization Time: 0 sec 193 msec 315 usec 462 nsec

Particles Initialization Time: 0 sec 131 msec 268 usec 760 nsec

Total Simulation Initialization Time: 0 sec 193 msec 315 usec 462 nsec

------------- Equilibration -------------

## PostProcessing

Now let's have a look at the results and run the PostProcessing_____no_output_____

# Initialize the Postprocessing class
postproc = PostProcess(input_file)
# Read the simulation's parameters and assign attributes
postproc.setup(read_yaml=True)
# Calculate observables
postproc.run()

* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
                                 Postprocessing
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *

Job ID:  UEC_4000
Job directory:  Simulations/UEC_4000
PostProcessing directory:
 Simulations/UEC_4000/PostProcessing

Equilibration dumps directory:  Simulations/UEC_4000/Simulation/Equilibration/dumps
Production dumps directory:
 Simulations/UEC_4000/Simulation/Production/dumps

Equilibration Thermodynamics file:
 Simulations/UEC_4000/Simulation/Equilibration/EquilibrationEnergy_UEC_4000.csv
Production Thermodynamics file:
 Simulations/UEC_4000/Simulation/Production/ProductionEnergy_UEC_4000.csv

==================== Radial Distribution Function ====================

Data saved in:
 Simulations/UEC_4000/PostProcessing/RadialDistributionFunction/Production/RadialDistributionFunction_UEC_4000.csv
Data accessible at: self.ra_values, self.dataframe

No. bins = 51
dr = 23.2887 a_ws = 1.9608e-06 [m]
Maximum Distance (i.e. potential.rc)= 1187.7258 a_ws = 1.0000e-04 [m]

Radial Distribution Function Calculation Time: 0 sec 4 msec 89 usec 455 nsec

Let us have a closer look at the energy evolution over time which we can extract from our Sarkas simulation into a [pandas.DataFrame](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.html)._____no_output_____

# Rename the Thermodynamics class for convenience
E = postproc.therm
# Setup its attributes
E.setup(postproc.parameters)
# Grab the data
E.parse()

# Results dataframe
Temp_vel_data = pd.DataFrame()
# Time
Temp_vel_data['Time'] = E.dataframe["Time"] * 1e12 # scale by picosec
# Total energies
eVpp = 1 / (np.abs(E.qe) * postproc.species[0].num) # scale by eV per particle
Temp_vel_data['Total Energy'] = E.dataframe["Total Energy"] * eVpp
Temp_vel_data['Total Kinetic Energy'] = E.dataframe["Total Kinetic Energy"] * eVpp
Temp_vel_data['Potential Energy'] = E.dataframe["Potential Energy"] * eVpp
# Kinetic energies
Temp_vel_data['e Kinetic Energy'] = E.dataframe[' '.join([postproc.species[0].name, 'Kinetic Energy'])] * eVpp
Temp_vel_data['Ion Kinetic Energy'] = E.dataframe[' '.join([postproc.species[1].name, 'Kinetic Energy'])] * eVpp
# Potential energies
Temp_vel_data['ee Potential Energy'] = E.dataframe[' '.join([postproc.species[0].name, 'Potential Energy'])] * eVpp
Temp_vel_data['IonIon Potential Energy'] = E.dataframe[' '.join([postproc.species[1].name, 'Potential Energy'])] * eVpp
Temp_vel_data['eIon Potential Energy'] = Temp_vel_data['Potential Energy'] \
    - Temp_vel_data['ee Potential Energy'] - Temp_vel_data['IonIon Potential Energy']_____no_output_____

It is now time to plot and compare. For this tutorial we grabbed data from Fig. 4f of this [paper](https://doi.org/10.1038/s41467-020-20815-8). The data was obtained using [WebPlotDigitizer](https://automeris.io/WebPlotDigitizer/) and saved into the files ``Ee.csv``, ``Ei.csv`` and ``Ekinp.csv`` in the folder ``uec_data``, reflecting the ultrafast energy evolution for the electrons, ions and plasma electrons, respectively.

We will compare the results with our simulation. The ion energy $E_{\rm i}$ is composed of $E_{\rm i} = E^{\rm kin}_{\rm i} + E^{\rm pot}_{\rm ii} + 0.5 E^{\rm pot}_{\rm ei}$ while the electron energy $E_{\rm e}$ is composed of $E_{\rm e} = E^{\rm kin}_{\rm e} + E^{\rm pot}_{\rm ee} + 0.5 E^{\rm pot}_{\rm ei}$ so that the inter-species potential energy $E^{\rm pot}_{\rm ei}$ is equally shared by the electrons and ions._____no_output_____

# Read paper data
Ee = pd.read_csv(os.path.join('uec_data/', 'Ee.csv'), names=['Time', 'Energy'])
Ei = pd.read_csv(os.path.join('uec_data/', 'Ei.csv'), names=['Time', 'Energy'])
Ekinp = pd.read_csv(os.path.join('uec_data/', 'Ekinp.csv'), names=['Time', 'Energy'])

# Plot
fig_energy, (ax_etot) = plt.subplots(1, 1, figsize=(10, 7))

# Simulation data
plt.plot(Temp_vel_data['Time'], Temp_vel_data['Total Energy'], linestyle='--', color='black', label='Total Energy')
ion_energy = Temp_vel_data['Ion Kinetic Energy'] + Temp_vel_data['IonIon Potential Energy'] \
             + 0.5*Temp_vel_data['eIon Potential Energy']
electron_energy = Temp_vel_data['e Kinetic Energy'] + Temp_vel_data['ee Potential Energy'] \
             + 0.5*Temp_vel_data['eIon Potential Energy']
plt.plot(Temp_vel_data['Time'], electron_energy, linestyle='-', color='skyblue', label='Electron Energy')
plt.plot(Temp_vel_data['Time'], ion_energy, linestyle='-', color='firebrick', label='Ion Energy')

# Paper data
plt.scatter(Ekinp['Time'], Ekinp['Energy'], marker='x', color='cadetblue', alpha=0.5, label=r'$E_{\rm kin,plasma}$')
plt.scatter(Ee['Time'], Ee['Energy'], marker='.', color='skyblue', alpha=0.5, label=r'$E_{\rm e}$')
plt.scatter(Ei['Time'], Ei['Energy'], marker='o', color='firebrick', alpha=0.5, label=r'$E_{\rm i}$')

plt.xlim((0, 30))
plt.ylim((0, 0.8))
ax_etot.legend()
ax_etot.set(xlabel =r'Time (ps)', ylabel = 'Energy (eV)')
fig_energy.tight_layout()
fig_energy.savefig(os.path.join(postproc.io.postprocessing_dir,'Energy_Plot.png'))_____no_output_____

Nice! Sarkas accurately reproduces the ultrafast electron cooling. Minor descrepancies in the amplitude and electron plasma frequency are attributed to the initial particle distribution which is an elongated box in sarkas and a cylinder in the original publication._____no_output_____


# Notebook from bayeslabs/AiGym
Path: Computer Vision/CS131/HW6/hw6.ipynb

# Homework 6
*This notebook includes both coding and written questions. Please hand in this notebook file with all the outputs as a pdf on gradescope for "HW6 pdf". Upload the three files of code (`compression.py`, `k_nearest_neighbor.py` and `features.py`) on gradescope for "HW6 code".*

This assignment covers:
- image compression using SVD
- kNN methods for image recognition.
- PCA and LDA to improve kNN_____no_output_____

# Setup
from time import time
from collections import defaultdict

import numpy as np
import matplotlib.pyplot as plt
from matplotlib import rc
from skimage import io

%matplotlib inline
plt.rcParams['figure.figsize'] = (15.0, 12.0) # set default size of plots
plt.rcParams['image.interpolation'] = 'nearest'
plt.rcParams['image.cmap'] = 'gray'

# for auto-reloading external modules
%load_ext autoreload
%autoreload 2_____no_output_____

## Part 1 - Image Compression (15 points)

Image compression is used to reduce the cost of storage and transmission of images (or videos).
One lossy compression method is to apply Singular Value Decomposition (SVD) to an image, and only keep the top n singular values._____no_output_____

image = io.imread('pitbull.jpg', as_grey=True)
plt.imshow(image)
plt.axis('off')
plt.show()_____no_output_____

Let's implement image compression using SVD.
We first compute the SVD of the image, and as seen in class we keep the `n` largest singular values and singular vectors to reconstruct the image.

Implement function `compress_image` in `compression.py`._____no_output_____

from compression import compress_image

compressed_image, compressed_size = compress_image(image, 100)
compression_ratio = compressed_size / image.size
print('Original image shape:', image.shape)
print('Compressed size: %d' % compressed_size)
print('Compression ratio: %.3f' % compression_ratio)

assert compressed_size == 298500_____no_output_____# Number of singular values to keep
n_values = [10, 50, 100]

for n in n_values:
    # Compress the image using `n` singular values
    compressed_image, compressed_size = compress_image(image, n)

    compression_ratio = compressed_size / image.size

    print("Data size (original): %d" % (image.size))
    print("Data size (compressed): %d" % compressed_size)
    print("Compression ratio: %f" % (compression_ratio))

    plt.imshow(compressed_image, cmap='gray')
    title = "n = %s" % n
    plt.title(title)
    plt.axis('off')
    plt.show()_____no_output_____

## Face Dataset

We will use a dataset of faces of celebrities. Download the dataset using the following command:

    sh get_dataset.sh

The face dataset for CS131 assignment.
The directory containing the dataset has the following structure:

    faces/
        train/
            angelina jolie/
            anne hathaway/
            ...
        test/
            angelina jolie/
            anne hathaway/
            ...

Each class has 50 training images and 10 testing images._____no_output_____

from utils import load_dataset

X_train, y_train, classes_train = load_dataset('faces', train=True, as_grey=True)
X_test, y_test, classes_test = load_dataset('faces', train=False, as_grey=True)

assert classes_train == classes_test
classes = classes_train

print('Class names:', classes)
print('Training data shape:', X_train.shape)
print('Training labels shape: ', y_train.shape)
print('Test data shape:', X_test.shape)
print('Test labels shape: ', y_test.shape)Class names: ['angelina jolie', 'anne hathaway', 'barack obama', 'brad pitt', 'cristiano ronaldo', 'emma watson', 'george clooney', 'hillary clinton', 'jennifer aniston', 'johnny depp', 'justin timberlake', 'leonardo dicaprio', 'natalie portman', 'nicole kidman', 'scarlett johansson', 'tom cruise']
Training data shape: (800, 64, 64)
Training labels shape:  (800,)
Test data shape: (160, 64, 64)
Test labels shape:  (160,)
# Visualize some examples from the dataset.
# We show a few examples of training images from each class.
num_classes = len(classes)
samples_per_class = 10
for y, cls in enumerate(classes):
    idxs = np.flatnonzero(y_train == y)
    idxs = np.random.choice(idxs, samples_per_class, replace=False)
    for i, idx in enumerate(idxs):
        plt_idx = i * num_classes + y + 1
        plt.subplot(samples_per_class, num_classes, plt_idx)
        plt.imshow(X_train[idx])
        plt.axis('off')
        if i == 0:
            plt.title(y)
plt.show()_____no_output_____# Flatten the image data into rows
# we now have one 4096 dimensional featue vector for each example
X_train = np.reshape(X_train, (X_train.shape[0], -1))
X_test = np.reshape(X_test, (X_test.shape[0], -1))
print("Training data shape:", X_train.shape)
print("Test data shape:", X_test.shape)Training data shape: (800, 4096)
Test data shape: (160, 4096)

---
## Part 2 - k-Nearest Neighbor (30 points)

We're now going to try to classify the test images using the k-nearest neighbors algorithm on the **raw features of the images** (i.e. the pixel values themselves). We will see later how we can use kNN on better features.

Here are the steps that we will follow:

1. We compute the L2 distances between every element of X_test and every element of X_train in `compute_distances`.
2. We split the dataset into 5 folds for cross-validation in `split_folds`.
3. For each fold, and for different values of `k`, we predict the labels and measure accuracy.
4. Using the best `k` found through cross-validation, we measure accuracy on the test set._____no_output_____

from k_nearest_neighbor import compute_distances

# Step 1: compute the distances between all features from X_train and from X_test
dists = compute_distances(X_test, X_train)
assert dists.shape == (160, 800)
print("dists shape:", dists.shape)_____no_output_____from k_nearest_neighbor import predict_labels

# We use k = 1 (which corresponds to only taking the nearest neighbor to decide)
y_test_pred = predict_labels(dists, y_train, k=1)

# Compute and print the fraction of correctly predicted examples
num_test = y_test.shape[0]
num_correct = np.sum(y_test_pred == y_test)
accuracy = float(num_correct) / num_test
print('Got %d / %d correct => accuracy: %f' % (num_correct, num_test, accuracy))_____no_output_____

### Cross-Validation

We don't know the best value for our parameter `k`.
There is no theory on how to choose an optimal `k`, and the way to choose it is through cross-validation.

We **cannot** compute any metric on the test set to choose the best `k`, because we want our final test accuracy to reflect a real use case. This real use case would be a setting where we have new examples come and we classify them on the go. There is no way to check the accuracy beforehand on that set of test examples to determine `k`.

Cross-validation will make use split the data into different fold (5 here).
For each fold, if we have a total of 5 folds we will have:
- 80% of the data as training data
- 20% of the data as validation data

We will compute the accuracy on the validation accuracy for each fold, and use the mean of these 5 accuracies to determine the best parameter `k`._____no_output_____

from k_nearest_neighbor import split_folds

# Step 2: split the data into 5 folds to perform cross-validation.
num_folds = 5

X_trains, y_trains, X_vals, y_vals = split_folds(X_train, y_train, num_folds)

assert X_trains.shape == (5, 640, 4096)
assert y_trains.shape == (5, 640)
assert X_vals.shape == (5, 160, 4096)
assert y_vals.shape == (5, 160)_____no_output_____# Step 3: Measure the mean accuracy for each value of `k`

# List of k to choose from
k_choices = list(range(5, 101, 5))

# Dictionnary mapping k values to accuracies
# For each k value, we will have `num_folds` accuracies to compute
# k_to_accuracies[1] will be for instance [0.22, 0.23, 0.19, 0.25, 0.20] for 5 folds
k_to_accuracies = {}

for k in k_choices:
    print("Running for k=%d" % k)
    accuracies = []
    for i in range(num_folds):
        # Make predictions
        fold_dists = compute_distances(X_vals[i], X_trains[i])
        y_pred = predict_labels(fold_dists, y_trains[i], k)

        # Compute and print the fraction of correctly predicted examples
        num_correct = np.sum(y_pred == y_vals[i])
        accuracy = float(num_correct) / len(y_vals[i])
        accuracies.append(accuracy)

    k_to_accuracies[k] = accuracies
_____no_output_____# plot the raw observations
for k in k_choices:
    accuracies = k_to_accuracies[k]
    plt.scatter([k] * len(accuracies), accuracies)

# plot the trend line with error bars that correspond to standard deviation
accuracies_mean = np.array([np.mean(v) for k,v in sorted(k_to_accuracies.items())])
accuracies_std = np.array([np.std(v) for k,v in sorted(k_to_accuracies.items())])
plt.errorbar(k_choices, accuracies_mean, yerr=accuracies_std)
plt.title('Cross-validation on k')
plt.xlabel('k')
plt.ylabel('Cross-validation accuracy')
plt.show()_____no_output_____# Based on the cross-validation results above, choose the best value for k,
# retrain the classifier using all the training data, and test it on the test
# data. You should be able to get above 26% accuracy on the test data.

best_k = None
# YOUR CODE HERE
# Choose the best k based on the cross validation above
pass
# END YOUR CODE

y_test_pred = predict_labels(dists, y_train, k=best_k)

# Compute and display the accuracy
num_correct = np.sum(y_test_pred == y_test)
accuracy = float(num_correct) / num_test
print('For k = %d, got %d / %d correct => accuracy: %f' % (best_k, num_correct, num_test, accuracy))_____no_output_____

---
## Part 3: PCA (30 points)

Principal Component Analysis (PCA) is a simple yet popular and useful linear transformation technique that is used in numerous applications, such as stock market predictions, the analysis of gene expression data, and many more. In this tutorial, we will see that PCA is not just a "black box", and we are going to unravel its internals in 3 basic steps.

### Introduction
The sheer size of data in the modern age is not only a challenge for computer hardware but also a main bottleneck for the performance of many machine learning algorithms. The main goal of a PCA analysis is to identify patterns in data; PCA aims to detect the correlation between variables. If a strong correlation between variables exists, the attempt to reduce the dimensionality only makes sense. In a nutshell, this is what PCA is all about: Finding the directions of maximum variance in high-dimensional data and project it onto a smaller dimensional subspace while retaining most of the information.

### A Summary of the PCA Approach
- Standardize the data.
- Obtain the Eigenvectors and Eigenvalues from the covariance matrix or correlation matrix, or perform Singular Vector Decomposition.
- Sort eigenvalues in descending order and choose the $k$ eigenvectors that correspond to the $k$ largest eigenvalues where $k$ is the number of dimensions of the new feature subspace ($k \leq d$).
- Construct the projection matrix $\mathbf{W}$ from the selected $k$ eigenvectors.
- Transform the original dataset $\mathbf{X}$ via $\mathbf{W}$ to obtain a $k$-dimensional feature subspace Y._____no_output_____

from features import PCA

pca = PCA()_____no_output_____

### 3.1 - Eigendecomposition
The eigenvectors and eigenvalues of a covariance (or correlation) matrix represent the "core" of a PCA: The eigenvectors (principal components) determine the directions of the new feature space, and the eigenvalues determine their magnitude. In other words, the eigenvalues explain the variance of the data along the new feature axes.

Implement **`_eigen_decomp`** in `pca.py`._____no_output_____

# Perform eigenvalue decomposition on the covariance matrix of training data.
e_vecs, e_vals = pca._eigen_decomp(X_train - X_train.mean(axis=0))

print(e_vals.shape)
print(e_vecs.shape)_____no_output_____

### 3.2 - Singular Value Decomposition
Doing an eigendecomposition of the covariance matrix is very expensive, especially when the number of features (`D = 4096` here) gets very high.

To obtain the same eigenvalues and eigenvectors in a more efficient way, we can use Singular Value Decomposition (SVD). If we perform SVD on matrix $X$, we obtain $U$, $S$ and $V$ such that:
$$
X = U S V^T
$$

- the columns of $U$ are the eigenvectors of $X X^T$
- the columns of $V^T$ are the eigenvectors of $X^T X$
- the values of $S$ are the square roots of the eigenvalues of $X^T X$ (or $X X^T$)

Therefore, we can find out the top `k` eigenvectors of the covariance matrix $X^T X$ using SVD.

Implement **`_svd`** in `pca.py`._____no_output_____

# Perform SVD on directly on the training data.
u, s = pca._svd(X_train - X_train.mean(axis=0))

print(s.shape)
print(u.shape)_____no_output_____# Test whether the square of singular values and eigenvalues are the same.
# We also observe that `e_vecs` and `u` are the same (only the sign of each column can differ).
N = X_train.shape[0]
assert np.allclose((s ** 2) / (N - 1), e_vals[:len(s)])

for i in range(len(s) - 1):
    assert np.allclose(e_vecs[:, i], u[:, i]) or np.allclose(e_vecs[:, i], -u[:, i])
    # (the last eigenvector for i = len(s) - 1 is very noisy because the eigenvalue is almost 0,
    #  so imprecisions in the computation build up)_____no_output_____

### 3.3 - Dimensionality Reduction

The top $k$ principal components explain most of the variance of the underlying data.

By projecting our initial data (the images) onto the subspace spanned by the top k principal components,
we can reduce the dimension of our inputs while keeping most of the information.

In the example below, we can see that **using the first two components in PCA is not enough** to allow us to see pattern in the data. All the classes seem placed at random in the 2D plane._____no_output_____

# Dimensionality reduction by projecting the data onto
# lower dimensional subspace spanned by k principal components

# To visualize, we will project in 2 dimensions
n_components = 2
pca.fit(X_train)
X_proj = pca.transform(X_train, n_components)

# Plot the top two principal components
for y in np.unique(y_train):
    plt.scatter(X_proj[y_train==y,0], X_proj[y_train==y,1], label=classes[y])

plt.xlabel('1st component')
plt.ylabel('2nd component')
plt.legend()
plt.show()_____no_output_____

### 3.4 - Visualizing Eigenfaces

The columns of the PCA projection matrix `pca.W_pca` represent the eigenvectors of $X^T X$.

We can visualize the biggest singular values as well as the corresponding vectors to get a sense of what the PCA algorithm is extracting.

If we visualize the top 10 eigenfaces, we can see tht the algorithm focuses on the different shades of the faces. For instance, in face n°2 the light seems to come from the left._____no_output_____

for i in range(10):
    plt.subplot(1, 10, i+1)
    plt.imshow(pca.W_pca[:, i].reshape(64, 64))
    plt.title("%.2f" % s[i])
plt.show()_____no_output_____# Reconstruct data with principal components
n_components = 100  # Experiment with different number of components.
X_proj = pca.transform(X_train, n_components)
X_rec = pca.reconstruct(X_proj)

print(X_rec.shape)
print(classes)

# Visualize reconstructed faces
samples_per_class = 10
for y, cls in enumerate(classes):
    idxs = np.flatnonzero(y_train == y)
    idxs = np.random.choice(idxs, samples_per_class, replace=False)
    for i, idx in enumerate(idxs):
        plt_idx = i * num_classes + y + 1
        plt.subplot(samples_per_class, num_classes, plt_idx)
        plt.imshow((X_rec[idx]).reshape((64, 64)))
        plt.axis('off')
        if i == 0:
            plt.title(y)
plt.show()_____no_output_____

### Written Question 1 (5 points)
*Question*: Consider a dataset of $N$ face images, each with shape $(h, w)$. Then, we need $O(Nhw)$ of memory to store the data. Suppose we perform dimensionality reduction on the dataset with $p$ principal components, and use the components as bases to represent images. Calculate how much memory we need to store the images and the matrix used to get back to the original space.

Said in another way, how much memory does storing the compressed images **and** the uncompresser cost.

*Answer:* TODO_____no_output_____### 3.5 - Reconstruction error and captured variance

We can plot the reconstruction error with respect to the dimension of the projected space.

The reconstruction gets better with more components.

We can see in the plot that the inflexion point is around dimension 200 or 300. This means that using this number of components is a good compromise between good reconstruction and low dimension._____no_output_____

# Plot reconstruction errors for different k
N = X_train.shape[0]
d = X_train.shape[1]

ns = range(1, d, 100)
errors = []

for n in ns:
    X_proj = pca.transform(X_train, n)
    X_rec = pca.reconstruct(X_proj)

    # Compute reconstruction error
    error = np.mean((X_rec - X_train) ** 2)
    errors.append(error)

plt.plot(ns, errors)
plt.xlabel('Number of Components')
plt.ylabel('Reconstruction Error')
plt.show()_____no_output_____

We can do the same process to see how much variance is captured by the projection.

Again, we see that the inflexion point is around 200 or 300 dimensions._____no_output_____

# Plot captured variance
ns = range(1, d, 100)
var_cap = []

for n in ns:
    var_cap.append(np.sum(s[:n] ** 2)/np.sum(s ** 2))

plt.plot(ns, var_cap)
plt.xlabel('Number of Components')
plt.ylabel('Variance Captured')
plt.show()_____no_output_____

### 3.6 - kNN with PCA

Performing kNN on raw features (the pixels of the image) does not yield very good results.
Computing the distance between images in the image space is not a very good metric for actual proximity of images.
For instance, an image of person A with a dark background will be close to an image of B with a dark background, although these people are not the same.

Using a technique like PCA can help discover the real interesting features and perform kNN on them could give better accuracy.

However, we observe here that PCA doesn't really help to disentangle the features and obtain useful distance metrics between the different classes. We basically obtain the same performance as with raw features._____no_output_____

num_test = X_test.shape[0]

# We computed the best k and n for you
best_k = 20
best_n = 500

# PCA
pca = PCA()
pca.fit(X_train)
X_proj = pca.transform(X_train, best_n)
X_test_proj = pca.transform(X_test, best_n)

# kNN
dists = compute_distances(X_test_proj, X_proj)
y_test_pred = predict_labels(dists, y_train, k=best_k)

# Compute and display the accuracy
num_correct = np.sum(y_test_pred == y_test)
accuracy = float(num_correct) / num_test
print('Got %d / %d correct => accuracy: %f' % (num_correct, num_test, accuracy))_____no_output_____

---
## Part 4 - Fisherface: Linear Discriminant Analysis (25 points)

LDA is a linear transformation method like PCA, but with a different goal.
The main difference is that LDA takes information from the labels of the examples to maximize the separation of the different classes in the transformed space.

Therefore, LDA is not totally **unsupervised** since it requires labels. PCA is **fully unsupervised**.

In summary:
- PCA perserves maximum variance in the projected space.
- LDA preserves discrimination between classes in the project space. We want to maximize scatter between classes and minimize scatter intra class._____no_output_____### 4.1 - Dimensionality Reduction via PCA
To apply LDA, we need $D < N$. Since in our dataset, $N = 800$ and $D = 4096$, we first need to reduce the number of dimensions of the images using PCA.
More information at: http://www.scholarpedia.org/article/Fisherfaces_____no_output_____

N = X_train.shape[0]
c = num_classes

pca = PCA()
pca.fit(X_train)
X_train_pca = pca.transform(X_train, N-c)
X_test_pca = pca.transform(X_test, N-c)_____no_output_____

### 4.2 - Scatter matrices

We first need to compute the within-class scatter matrix:
$$
S_W = \sum_{i=1}^c S_i
$$
where $S_i = \sum_{x_k \in Y_i} (x_k - \mu_i)(x_k - \mu_i)^T$ is the scatter of class $i$.

We then need to compute the between-class scatter matrix:
$$
S_B = \sum_{i=1}^c N_i (\mu_i - \mu)(\mu_i - \mu)^T
$$
where $N_i$ is the number of examples in class $i$._____no_output_____

# Compute within-class scatter matrix
S_W = lda._within_class_scatter(X_train_pca, y_train)
print(S_W.shape)_____no_output_____# Compute between-class scatter matrix
S_B = lda._between_class_scatter(X_train_pca, y_train)
print(S_B.shape)_____no_output_____

### 4.3 - Solving generalized Eigenvalue problem

Implement methods `fit` and `transform` of the `LDA` class._____no_output_____

lda.fit(X_train_pca, y_train)_____no_output_____# Dimensionality reduction by projecting the data onto
# lower dimensional subspace spanned by k principal components
n_components = 2
X_proj = lda.transform(X_train_pca, n_components)
X_test_proj = lda.transform(X_test_pca, n_components)

# Plot the top two principal components on the training set
for y in np.unique(y_train):
    plt.scatter(X_proj[y_train==y, 0], X_proj[y_train==y, 1], label=classes[y])

plt.xlabel('1st component')
plt.ylabel('2nd component')
plt.legend()
plt.title("Training set")
plt.show()

# Plot the top two principal components on the test set
for y in np.unique(y_test):
    plt.scatter(X_test_proj[y_test==y, 0], X_test_proj[y_test==y,1], label=classes[y])

plt.xlabel('1st component')
plt.ylabel('2nd component')
plt.legend()
plt.title("Test set")
plt.show()_____no_output_____

### 4.4 - kNN with LDA

Thanks to having the information from the labels, LDA gives a discriminant space where the classes are far apart from each other.
This should help kNN a lot, as the job should just be to find the obvious 10 clusters.

However, as we've seen in the previous plot (section 4.3), the training data gets clustered pretty well, but the test data isn't as nicely clustered as the training data (overfitting?).

Perform cross validation following the code below (you can change the values of `k_choices` and `n_choices` to search). Using the best result from cross validation, obtain the test accuracy._____no_output_____

num_folds = 5

X_trains, y_trains, X_vals, y_vals = split_folds(X_train, y_train, num_folds)

k_choices = [1, 5, 10, 20]
n_choices = [5, 10, 20, 50, 100, 200, 500]
pass

# n_k_to_accuracies[(n, k)] should be a list of length num_folds giving the different
# accuracy values that we found when using that value of n and k.
n_k_to_accuracies = defaultdict(list)

for i in range(num_folds):
    # Fit PCA
    pca = PCA()
    pca.fit(X_trains[i])

    N = len(X_trains[i])
    X_train_pca = pca.transform(X_trains[i], N-c)
    X_val_pca = pca.transform(X_vals[i], N-c)

    # Fit LDA
    lda = LDA()
    lda.fit(X_train_pca, y_trains[i])

    for n in n_choices:
        X_train_proj = lda.transform(X_train_pca, n)
        X_val_proj = lda.transform(X_val_pca, n)

        dists = compute_distances(X_val_proj, X_train_proj)

        for k in k_choices:
            y_pred = predict_labels(dists, y_trains[i], k=k)

            # Compute and print the fraction of correctly predicted examples
            num_correct = np.sum(y_pred == y_vals[i])
            accuracy = float(num_correct) / len(y_vals[i])
            n_k_to_accuracies[(n, k)].append(accuracy)

for n in n_choices:
    print()
    for k in k_choices:
        accuracies = n_k_to_accuracies[(n, k)]
        print("For n=%d, k=%d: average accuracy is %f" % (n, k, np.mean(accuracies)))_____no_output_____# Based on the cross-validation results above, choose the best value for k,
# retrain the classifier using all the training data, and test it on the test
# data. You should be able to get above 40% accuracy on the test data.

best_k = None
best_n = None
# YOUR CODE HERE
# Choose the best k based on the cross validation above
pass
# END YOUR CODE

N = len(X_train)

# Fit PCA
pca = PCA()
pca.fit(X_train)
X_train_pca = pca.transform(X_train, N-c)
X_test_pca = pca.transform(X_test, N-c)

# Fit LDA
lda = LDA()
lda.fit(X_train_pca, y_train)

# Project using LDA
X_train_proj = lda.transform(X_train_pca, best_n)
X_test_proj = lda.transform(X_test_pca, best_n)

dists = compute_distances(X_test_proj, X_train_proj)
y_test_pred = predict_labels(dists, y_train, k=best_k)

# Compute and display the accuracy
num_correct = np.sum(y_test_pred == y_test)
accuracy = float(num_correct) / num_test
print("For k=%d and n=%d" % (best_k, best_n))
print('Got %d / %d correct => accuracy: %f' % (num_correct, num_test, accuracy))_____no_output_____