# Notebook from mariomorvan/deepARTransit
Path: notebooks/post_processing.ipynb

## Post-Processing Notebook

Providing a use-case example for analysing and fitting the predictions of a previously ran TLCD-LSTM model._____no_output_____

# General imports and widgets
import os
import sys
import numpy as np

import warnings
warnings.filterwarnings('ignore')

import matplotlib.pyplot as plt
%matplotlib inline
%matplotlib inline
plt.rcParams['figure.figsize'] = (12, 6)
plt.rcParams['font.size'] = 15
warnings.simplefilter("ignore")

import ipywidgets as widgets_____no_output_____project_dir = os.path.join('/','Users','mario','git-repos','deepARTransit')  # change to yours_____no_output_____# Radial aperture to help excluding the main star and compute background flux
def radial_aperture(array, cent, radius=3):
    aperture = np.empty_like(array, dtype=bool)
    if len(array.shape) == 2:
        aperture[:,:] = False
        for i in np.arange(array.shape[-2]):
            for j in np.arange(array.shape[-1]) :
                if (i + 0.5 - cent[0]) ** 2 + (j + 0.5  -cent[1]) ** 2 <= radius ** 2:
                    aperture[i,j] = True
    elif len(array.shape) == 3:
        aperture[:,:,:] = False
        for t in range(array.shape[0]):
            for i in np.arange(array.shape[-2]):
                for j in np.arange(array.shape[-1]) :
                    if (i + 0.5 - cent[0]) ** 2 + (j + 0.5  -cent[1]) ** 2 <= radius ** 2:
                        aperture[t, i,j] = True
    return aperture _____no_output_____mask = ~radial_aperture(np.ones(shape=(32,32)), (16,16), 16)_____no_output_____experiment_widget = widgets.Dropdown(
    options=sorted(list({r.split('experiments')[1][1:] for r,d,f in
                         os.walk('./experiments') if 'output' in d and 'summary' in d})),
    description='\t\t',
    disabled=False
)_____no_output_____

### Selecting Experiment Folder_____no_output_____

experiment_widget_____no_output_____from deepartransit.utils.config import get_config_file, process_config
from deepartransit.utils.data_handling import data_generator
from deepartransit.utils.transit import get_transit_model

experiment_folder = experiment_widget.value
experiment_dir = os.path.join('experiments', experiment_folder)
config_file = get_config_file(experiment_dir)

config = process_config(config_file)
data = data_generator.DataGenerator(config)
config = data.update_config()

t1, t2, t3 = config['pretrans_length'], config['trans_length'], config['postrans_length']
n_obs = data.Z.shape[0]_____no_output_____time_arrays = np.arange(len(data.Z))
bts = np.zeros_like(data.Z)_____no_output_____# Here I import time array corresponding to my data., ignore this cell or replace with your imports
from spitzerlc.data_handling import combine_fits_files
# For HD189733B
channel = 4
aorkey_list = ['22807296', '22807552', '22807808', '24537856', '27603712', '27773440']
parent_dir = '~/git-repos/spitzerLC/spitzerlc/data/agol_hd189733b/'

#WASP 121
channel = 2
aorkey_list = ['62160640']
parent_dir = '/Users/mario/data/IRAC/wasp121/'

from spitzerlc.observation import Observation
from spitzerlc.data_handling import load_data

time_arrays = np.zeros(data.Z.shape[:2])

background_mean = []
bts = []
for i,aorkey in enumerate(aorkey_list):
    time_array, flux, header = combine_fits_files(aorkey, channel, parent_dir = parent_dir)
    obs = Observation(aorkey, channel, header, time_array, flux )
    time_arrays[i] = obs.time_array

    #background_mean.append(np.nanmedian(obs.flux[200:500,mask], axis=1).mean()*25)
    bts.append(np.nanmedian(obs.flux[:,mask], axis=1) * 25)

    #print('background estimate', background_mean[-1])
    plt.plot(bts[-1])
    plt.show()

    obs.select_subregion(radius=2)
    plt.plot(obs.raw_light_curve().flux)
    plt.show()
    plt.plot(obs.raw_light_curve().flux - bts[-1])
    plt.show()
time_arrays = time_arrays - 2454000_____no_output_____

### Plotting input data_____no_output_____

%matplotlib inline
plt.rcParams['font.size'] = 20

fig, ax = plt.subplots(3, max(n_obs,2), figsize=(25,12), sharey='row', sharex='col')
Z_orig = data.scaler_Z.inverse_transform(data.Z)
for obs in range(n_obs):
    ax[0, obs].scatter(time_arrays[obs], data.Z[obs,:,0],
                       s=3, label='raw light curve')
    ax[1, obs].plot(time_arrays[obs], data.X[obs,:,0], label='centroid X position', color='darkblue', alpha=0.5)
    ax[1, obs].plot(time_arrays[obs], data.X[obs,:,1], label='centroid Y position', color='darkgreen', alpha=0.5)
    ax[2, obs].plot(time_arrays[obs], data.X[obs,:,2], label='centroid X position', color='darkblue', alpha=0.5)
    ax[2, obs].plot(time_arrays[obs], data.X[obs,:,3], label='centroid Y position', color='darkgreen', alpha=0.5)

plt.tight_layout()
plt.legend()

fig.add_subplot(111, frameon=False)
# hide tick and tick label of the big axis
plt.tick_params(labelcolor='none', top=False, bottom=False, left=False, right=False)
plt.margins(x = 1)
plt.xlabel("Time [BMJD - 2454000 days]", fontsize=25, labelpad=30)
plt.subplots_adjust(bottom=0.8)

fig.add_subplot(111, frameon=False)
# hide tick and tick label of the big axis
plt.tick_params(labelcolor='none', top=False, bottom=False, left=False, right=False)
plt.margins(x = 1)
plt.ylabel("Normalized time-series", fontsize=25, labelpad=10)
#plt.subplots_adjust(bottom=0.8)

if 'plots' not in os.listdir(experiment_dir):
    os.mkdir(os.path.join(experiment_dir, 'plots'))
fig.savefig(os.path.join(experiment_dir, 'plots','data.png'))
_____no_output_____

# Loading results_____no_output_____

# retrieve evaluation steps
loc_dict = {}
pars_dict = {}
scale_dict = {}
for fn in [s for s in os.listdir(os.path.join(project_dir, config.output_dir)) if (s[:3] == 'loc')]:
    step = fn.split('_')[-1].split('.')[0]
    try:
        loc_dict[int(step)] = np.load(os.path.join(project_dir, config.output_dir, fn)).swapaxes(0,1)#.mean(axis=0)
        scale_dict[int(step)] = np.load(os.path.join(project_dir, config.output_dir, 'scales_array_{}.npy'.format(step))).swapaxes(0,1)#.mean(axis=0)
    except:
        print(fn)
        break
step_list = sorted([int(k) for k in loc_dict if k!='array'])
print(step_list)_____no_output_____# Selection of optimal step
step = step_list[-1] # Last
step = 150_____no_output_____loc_array = loc_dict[step]_____no_output_____def to_real_time(t, true_ta, wrong_ta):
    return true_ta[0] + t * (true_ta[-1] - true_ta[0]) / (wrong_ta[-1] - wrong_ta[0])

time_arrays[0].shape, data.time_array[obs].shape_____no_output_____fig, ax = plt.subplots(1, max(n_obs,2), figsize=(25,5), sharey='row')
for obs in range(n_obs):
    ax[obs].scatter(time_arrays[obs], data.Z[obs,:,0], s=3)
    ax[obs].plot(time_arrays[obs], loc_array[obs,:,0], label='prediction', color='red')
    ax[obs].ticklabel_format(axis='x',style='plain', useOffset=False)

    m, M = data.Z[obs,:,0].min(), data.Z[obs,:,0].max()
    ax[obs].vlines(to_real_time(t1-1, time_arrays[obs], data.time_array),
                   m, M, 'black', linewidth=3, linestyles='dashed')
    ax[obs].vlines(to_real_time(t1+t2 + 1, time_arrays[obs], data.time_array),
                   m, M, 'black', linewidth=3, linestyles='dashed')

#plt.tight_layout()
ax[-1].legend()
ax[0].set_ylabel('Normalized Flux', fontsize=25)

ax1 = fig.add_subplot(111, frameon=False)

plt.tick_params(labelcolor='none', top=False, bottom=False, left=False, right=False)
plt.margins(x = 1)
plt.xlabel("Time [BMJD - 2454000 days]", fontsize=25, labelpad=-30)
#ax2 = plt.gca()
ax1.xaxis.set_label_coords(0.5, -0.13)    _____no_output_____

# Fitting
_____no_output_____### Linear Transit_____no_output_____

from deepartransit.utils.transit import LinearTransit
fig, ax = plt.subplots(2, max(n_obs,2), figsize=(25,9), sharey='row', sharex='col')

transit_component = np.zeros(shape=data.Z.shape)

ltransits = []
for obs in range(n_obs):

    transit_component[obs] = ((data.scaler_Z.inverse_transform(data.Z)[obs,:,0]-bts[obs])
                         / (data.scaler_Z.inverse_transform(loc_array)[obs,:,0]-bts[obs]))[:,np.newaxis]

    ax[0, obs].scatter(time_arrays[obs], data.Z[obs,:,0], s=3)
    ax[0, obs].plot(time_arrays[obs], loc_array[obs,:,0], label='prediction', color='red')
    ax[1, obs].scatter(time_arrays[obs], transit_component[obs,:,0], s=5)

    ltransit = LinearTransit(data.time_array)
    ltransit.fit(transit_component[obs,:,0], time_axis=0)
    ax[1, obs].plot(time_arrays[obs], ltransit.flux, label='prediction', color='red')
    print('{:.2f}'.format(ltransit.delta*100))
    mid_time = time_arrays[obs, 0] + ltransit.t_c * (time_arrays[obs, -1] - time_arrays[obs, 0]) / (data.time_array[-1] - data.time_array[0])
    ax[1, obs].vlines(mid_time, 0.995*(min(transit_component[obs])), 1, color='grey')

ax[0, obs].set_ylabel('Normalized Flux', fontsize=25)
ax[1, obs].set_ylabel('Star-Normalized Flux', fontsize=25)

ax1 = fig.add_subplot(111, frameon=False)
plt.tick_params(labelcolor='none', top=False, bottom=False, left=False, right=False)
plt.margins(x = 1)
plt.xlabel("Time [BMJD - 2454000 days]", fontsize=25, labelpad=-30)
ax1.xaxis.set_label_coords(0.5, -0.13)

if 'plots' not in os.listdir(experiment_dir):
    os.mkdir(os.path.join(experiment_dir, 'plots'))
plt.savefig(os.path.join(experiment_dir, 'plots','linear_fit.png'))_____no_output_____import pandas as pd

y = transit_component[0,1:,0]
y_series = pd.Series(y)
mstd_width = 5
err = np.append(y_series[:5].std(), y_series.rolling(mstd_width,1).std().values[1:])_____no_output_____err.shape_____no_output_____plt.figure(figsize=(15,5))
plt.errorbar(time_arrays[obs,1:], y, err)_____no_output_____np.save('wasp121_detrended_lc.npy', np.stack([time_arrays[obs,1:], y, err]).T)_____no_output_____os.listdir('.')_____no_output_____from pylightcurve import find_oec_parameters
(planet, logg, effective_temperature, metallicity, rp_over_rs, fp_over_fs,
 period, sma_over_rs, eccentricity, inclination, periastron, mid_time) = find_oec_parameters('hd189733b')
_____no_output_____(planet, logg, effective_temperature, metallicity, rp_over_rs, fp_over_fs,
 period, sma_over_rs, eccentricity, inclination, periastron, mid_time)_____no_output_____inclination, sma_over_rs, period_____no_output_____import time

from pylightcurve import TransitAndPolyFitting
import pandas as pd

fitting_list = []
ma_width = 50
mstd_width = 4

i = 0
fitting_folder = 'results_fitting'
if os.path.exists(os.path.join(experiment_dir, fitting_folder)):
    gmtime = time.gmtime(os.path.getmtime(os.path.join(experiment_dir, fitting_folder)))
    t = time.strftime('%y-%m-%d_%H-%M-%S', gmtime)
    os.rename(os.path.join(experiment_dir, fitting_folder),
              os.path.join(experiment_dir, f'{fitting_folder}_{t}'))
os.mkdir(os.path.join(experiment_dir, fitting_folder))

for obs in range(n_obs):
    # estimate mid_time from linear fit
    mid_time = time_arrays[obs, 0] + ltransit.t_c * (time_arrays[obs, -1] - time_arrays[obs, 0]) / (data.time_array[-1] - data.time_array[0])
    print(mid_time)
    #x = (data.scaler_Z.inverse_transform(data.Z)[obs,:,0])
    #y = (x / pd.Series(x).rolling(ma_width,1).mean())
    y = pd.Series(transit_component[obs,:,0])
    err = np.append(y[:5].std(), y.rolling(mstd_width,1).std().values[1:])
    print(err.mean())
    transitFittingObject = TransitAndPolyFitting(data=[[time_arrays[obs],
                                                        transit_component[obs,:,0],
                                                        err
                                                ]],
                                                method='linear',
                                                limb_darkening_coefficients=[0.1],
                                                rp_over_rs=rp_over_rs,
                                                period=period,
                                                sma_over_rs=sma_over_rs,
                                                eccentricity=eccentricity,
                                                inclination=inclination,
                                                periastron=periastron,
                                                mid_time = mid_time,
                                                iterations= 200_000,
                                                walkers= 100,
                                                burn= 100_000,
                                                precision=3,
                                                exp_time=0.,
                                                time_factor=1,
                                                fit_first_order=False,
                                                fit_second_order=False,
                                                fit_rp_over_rs=[0.01, 0.2],
                                                fit_period=False,
                                                fit_sma_over_rs = [sma_over_rs*0.5, sma_over_rs*2],
                                                fit_eccentricity = False,
                                                fit_inclination = [inclination*0.8, 90],
                                                fit_periastron = False,
                                                fit_mid_time = [mid_time - period /50, mid_time + period/50],
                                                fit_ld=[[0.0,0.5]],
                                                counter=True,
                                                counter_window=False)
    transitFittingObject.fit_ld = True
    transitFittingObject.run_mcmc()
    fitting_list.append(transitFittingObject.results)

    # Just printing
    rp_over_rs_new = transitFittingObject.results['parameters']['rp']
    print('Rp/Rs = {:.5f} -{:.6f}/+{:.6f}'.format(rp_over_rs_new['value'],
                                                  rp_over_rs_new['m_error'],
                                                  rp_over_rs_new['p_error']))
    print('(Rp/Rs)**2 = {:.5f} -{:.6f}/+{:.6f}'.format(rp_over_rs_new['value']**2,
                                                       np.abs(2 * rp_over_rs_new['m_error'] * rp_over_rs_new['value']),
                                                       np.abs(2 * rp_over_rs_new['p_error'] * rp_over_rs_new['value'])))

    # Just saving

    transitFittingObject.save_all(os.path.join(experiment_dir, fitting_folder, f'simulation_data_base_{obs}.pickle'))
    transitFittingObject.save_results(os.path.join(experiment_dir,fitting_folder, f'simulation_resultsobs_{obs}.txt'))
    transitFittingObject.plot_corner(os.path.join(experiment_dir,fitting_folder, f'simulation_correlations_{obs}.pdf'))
    transitFittingObject.plot_traces(os.path.join(experiment_dir,fitting_folder, f'simulation_traces_{obs}.pdf'))_____no_output_____# Reload saved results
import pickle
fitting_folder = 'results_fitting'
with open(os.path.join(experiment_dir, fitting_folder, f'simulation_data_base_{obs}.pickle'), 'rb') as pick_file:
    transitFittingObject = pickle.load(pick_file)_____no_output_____fig, ax = plt.subplots(3, max(n_obs,2), figsize=(25,14), sharey='row') #, sharex='col')
rol_win = 50
for obs in range(n_obs):
    ax[0,obs].scatter(time_arrays[obs], transit_component[obs,:,0], s=3)
    ax[0,obs].plot(time_arrays[obs], fitting_list[obs]['output_series']['model'], color='red', label='mcmc fit')

    res = fitting_list[obs]['output_series']['residuals']
    if obs==1:
        res = res.clip(-0.005, 0.005)
    ax[1,obs].scatter(time_arrays[obs], res, s=3)

    ma_res = pd.Series(res).rolling(rol_win, min_periods=1).mean().values
    mstd_res = pd.Series(res).rolling(rol_win, min_periods=1).std().values
    ax[1,obs].plot(time_arrays[obs], ma_res, color='red')
    ax[1,obs].fill_between(time_arrays[obs], ma_res - mstd_res, ma_res+mstd_res, alpha=0.5)

    ax[0,obs].ticklabel_format(axis='x',style='plain', useOffset=False)
    ax[1,obs].ticklabel_format(axis='x',style='plain', useOffset=False)
    ax[0,obs].tick_params(labeltop=True, labelbottom=False, top=True, bottom=False)
    ax[1,obs].tick_params(labeltop=False, labelbottom=False, top=False, bottom=False)
    ax[2, obs].bar(range(data.Z.shape[1]),fitting_list[obs]['detrended_statistics']['res_autocorr'] )

ax[0, 0].set_ylabel('Normalized Flux', fontsize=25)
ax[1, 0].set_ylabel('Residuals', fontsize=25)
ax[2, 0].set_ylabel('ACF', fontsize=25)
ax[2, 0].set_ylim(-0.25,0.25)

ax1 = fig.add_subplot(111, frameon=False)
#ax2 = ax1.twinx()
# hide tick and tick label of the big axis
plt.tick_params(labelcolor='none', top=False, bottom=False, left=False, right=False)
plt.margins(x = 1)
plt.xlabel("Timesteps", fontsize=25, labelpad=-30)
#ax2 = plt.gca()
ax1.xaxis.set_label_coords(0.5, -0.05)

#ax2 = fig.add_subplot(111, frameon=False)
ax2 = ax1.twiny()
# hide tick and tick label of the big axis
plt.tick_params(labelcolor='none', top=False, bottom=False, left=False, right=False)
plt.margins(x = 1)
plt.xlabel("Time [BMJD - 2454000 days]", fontsize=25, labelpad=-30)
#ax2 = plt.gca()
ax2.xaxis.set_label_coords(0.5, 1.07)
#ax.set_xlabel('xlabel', ha='left', va = 'top', )
#plt.subplots_adjust(bottom=0.8)

if 'plots' not in os.listdir(experiment_dir):
    os.mkdir(os.path.join(experiment_dir, 'plots'))
if 'mcmc_fit.png' in os.listdir(os.path.join(experiment_dir, 'plots')):
    ct = time.strftime('%y-%m-%d_%H-%M-%S',
                       time.gmtime(os.path.getmtime(os.path.join(experiment_dir, 'plots','mcmc_fit.png'))))
    os.rename(os.path.join(experiment_dir, 'plots', 'mcmc_fit.png'),
              os.path.join(experiment_dir, 'plots', 'mcmc_fit_{}.png'.format(ct))
             )
plt.savefig(os.path.join(experiment_dir, 'plots','mcmc_fit.png'))_____no_output_____

## Multi-observations: Plotting and printing_____no_output_____

# Defining weighted avg function, which computes both weighted average and std of this weighted average
def weighted_avg(x, sigma):
    x = np.array(x)
    sigma = np.array(sigma)
    try:
        if sigma.shape[1] == 2:
            sigma = sigma.mean(1)

    except IndexError:
        pass
    print(x.shape,sigma.shape)
    assert x.shape == sigma.shape

    weights = 1 / (np.array(sigma)**2)
    weighted_mean_delta = np.sum((x * weights )) / weights.sum()
    std_weighted_mean_delta = np.sqrt(1 / weights.sum())

    return weighted_mean_delta, std_weighted_mean_delta_____no_output_____plt.figure(figsize=(15,5))

deltas = np.array([fitting_list[obs]['parameters']['rp']['value']**2 for obs in range(n_obs)])
deltas_errors = np.array([[fitting_list[obs]['parameters']['rp']['m_error']* rp_over_rs_new['value'] * 2,
        fitting_list[obs]['parameters']['rp']['p_error']* rp_over_rs_new['value'] * 2]
       for obs in range(n_obs)])

mu, sigmu = weighted_avg(deltas, deltas_errors)
plt.errorbar(range(1, n_obs+1),
             deltas,
             yerr=deltas_errors.T,
             markersize=5, fmt='o',)
plt.hlines(mu, 1,6 , color= 'black')
plt.hlines(mu-sigmu, 1,6, linestyle='dashed', color= 'black')
plt.hlines(mu+sigmu, 1,6, linestyle='dashed', color= 'black')

_____no_output_____par_names_pylc = ['rp', 'ldc1', 'a','i']
par_names = ['$R_p/R_s$', '$u$', '$a/R_s$','$i$']
weighted_mu_dict = {}
f,ax = plt.subplots(len(par_names)//2,len(par_names)//2, figsize=(15, 8), sharex='col')
for i, parameter in enumerate(par_names_pylc):
    print(parameter)
    #param = fitting_list[obs]['parameters'][]

    par_values = [fitting_list[obs]['parameters'][parameter]['value'] for obs in range(n_obs)]
    par_errors = np.array([[fitting_list[obs]['parameters'][parameter]['m_error'],
                            fitting_list[obs]['parameters'][parameter]['p_error']] for obs in range(n_obs)])
    ix = i//2
    iy = (i % 2)
    print(ix,iy)
    ax[ix,iy].errorbar(range(1, n_obs+1), par_values, yerr=par_errors.T, markersize=5, fmt='o')

    mu, sigmu = weighted_avg(par_values, par_errors)
    ax[ix,iy].hlines(mu, 1,6 , color= 'black',
                     label="$\overline{{{}}}={:.4g}$".format(par_names[i][1:-1], mu))
    ax[ix,iy].hlines(mu-sigmu, 1,6, linestyle='dashed', color= 'black')
    ax[ix,iy].hlines(mu+sigmu, 1,6, linestyle='dashed', color= 'black')
    ax[ix,iy].legend()
    ax[ix,iy].set_ylabel(par_names[i], fontsize=20)

    weighted_mu_dict[parameter] = mu

ax[-1, 0].set_xlabel('Observation number', fontsize=20)
ax[-1, 1].set_xlabel('Observation number', fontsize=20)

plt.show()_____no_output_____

# Latex Print_____no_output_____

latex_string = ""
prec_val = [6,5,4,3,3]
for obs in range(6):
    latex_val_list = []
    latex_err_list = []
    for parameter in ['mt', 'rp', 'i', 'a', 'ldc1']:
        latex_val_list.append(fitting_list[obs]['parameters'][parameter]['value'])
        latex_err_list.append(max(fitting_list[obs]['parameters'][parameter]['m_error'], fitting_list[obs]['parameters'][parameter]['p_error']))
        if parameter == "mt":
            #latex_val_list[-1] -= 2454000
            pass
    latex_string += '\t'*2
    latex_string += " & ".join(["${0:.{2}f}\pm{{{1:.{3}f}}}$".format(latex_val_list[i],
                                                                     latex_err_list[i],
                                                                     prec_val[i],
                                                                     prec_val[i]
                                                                    ) for i in range(len(latex_val_list))])
    latex_string += ' \\\\\n'
print(latex_string)_____no_output_____# Just testing Latex rendering
from IPython.display import Latex
Latex(latex_string)_____no_output_____

## Plot with Agol_____no_output_____

agol_deltas = np.array([2.4022, 2.4253, 2.4333, 2.4224, 2.3984, 2.3965])
agol_deltas_unc = np.array([47,63,51,49,62,74])/10_000
agol_deltas.shape, agol_deltas_unc.shape, deltas.shape, deltas_errors.shape_____no_output_____# Scatter
agol_deltas.std(), np.array(deltas).std()_____no_output_____deltas, agol_deltas_____no_output_____wm, ws = weighted_avg(deltas*100, deltas_errors*100)
wma,wsa = weighted_avg(agol_deltas, agol_deltas_unc)

wm, wma, (wma - wm), (wma-wm)/ws_____no_output_____plt.figure(figsize=(15,5))
p = plt.errorbar(np.arange(6)+0.02, 100*np.array(deltas), yerr=100*deltas_errors.T,
             fmt='o', label='This paper', markersize = 8)
p2 = plt.errorbar(np.arange(6)-0.02, agol_deltas, yerr=agol_deltas_unc,  markersize = 8,
             fmt='o', label='Agol et al.')

plt.hlines(weighted_avg(100*deltas, 100*deltas_errors)[0], 0,5 , color= p[0].get_color())
plt.hlines(weighted_avg(100*deltas, 100*deltas_errors)[0] + weighted_avg(100*deltas, 100*deltas_errors)[1],
           0,5, linestyle='dashed', color= p[0].get_color())
plt.hlines(weighted_avg(100*deltas, 100*deltas_errors)[0] - weighted_avg(100*deltas, 100*deltas_errors)[1],
           0,5, linestyle='dashed', color= p[0].get_color())

plt.hlines(weighted_avg(agol_deltas, agol_deltas_unc)[0], 0,5 , color= p2[0].get_color())
plt.hlines(weighted_avg(agol_deltas, agol_deltas_unc)[0] + weighted_avg(deltas, agol_deltas_unc)[1],
           0,5, linestyle='dashed', color= p2[0].get_color())
plt.hlines(weighted_avg(agol_deltas, agol_deltas_unc)[0] - weighted_avg(deltas, agol_deltas_unc)[1],
           0,5, linestyle='dashed', color= p2[0].get_color())

plt.legend()
plt.xlabel('Observation Number', fontsize=25)

plt.ylabel('$(R_p/R_*)^2$  $[\%]$', fontsize=25)
pass_____no_output_____

## Second run with $i$, $u$ and $a/R_S$ fixed
_____no_output_____

fitting_list_2 = []
ma_width = 50
mstd_width = 5
for obs in range(n_obs):
    # estimate mid_time from linear fit
    mid_time = time_arrays[obs, 0] + ltransit.t_c * (time_arrays[obs, -1] - time_arrays[obs, 0]) / (data.time_array[-1] - data.time_array[0])

    y = pd.Series(transit_component[obs,:,0])
    err = np.append(y[:5].std(), y.rolling(mstd_width,1).std().values[1:])
    transitFittingObject = TransitAndPolyFitting(data=[[time_arrays[obs],
                                                        transit_component[obs,:,0],
                                                        err
                                                ]],
                                                method='linear',
                                                limb_darkening_coefficients=[weighted_mu_dict['ldc1']], ##########
                                                rp_over_rs=weighted_mu_dict['rp'],
                                                period=period,
                                                sma_over_rs=weighted_mu_dict['a'], #########
                                                eccentricity=eccentricity,
                                                inclination=weighted_mu_dict['i'], ########
                                                periastron=periastron,
                                                mid_time = mid_time,
                                                iterations= 50_000,
                                                walkers= 100,
                                                burn= 25_000,
                                                precision=3,
                                                exp_time=0.,
                                                time_factor=1,
                                                fit_first_order=False,
                                                fit_second_order=False,
                                                fit_rp_over_rs=[0.14, 0.17],
                                                fit_period=False,
                                                fit_sma_over_rs = False,
                                                fit_eccentricity = False,
                                                fit_inclination = False,
                                                fit_periastron = False,
                                                fit_mid_time = [mid_time - period /50, mid_time + period/50],
                                                fit_ld=False,
                                                counter=True,
                                                counter_window=False)
    transitFittingObject.fit_ld = True
    transitFittingObject.run_mcmc()
    fitting_list_2.append(transitFittingObject.results)

    # Just printing
    rp_over_rs_new = transitFittingObject.results['parameters']['rp']
    print('Rp/Rs = {:.5f} -{:.6f}/+{:.6f}'.format(rp_over_rs_new['value'],
                                                  rp_over_rs_new['m_error'],
                                                  rp_over_rs_new['p_error']))
    print('(Rp/Rs)**2 = {:.5f} -{:.6f}/+{:.6f}'.format(rp_over_rs_new['value']**2,
                                                       np.abs(2 * rp_over_rs_new['m_error'] * rp_over_rs_new['value']),
                                                       np.abs(2 * rp_over_rs_new['p_error'] * rp_over_rs_new['value'])))

    # Just saving
    transitFittingObject.save_all(f'fit_results/simulation_data_base_{obs}_2.pickle')
    transitFittingObject.save_results(f'fit_results/simulation_resultsobs_{obs}_2.txt')
    transitFittingObject.plot_corner(f'fit_results/simulation_correlations_{obs}_2.pdf')
    transitFittingObject.plot_traces(f'fit_results/simulation_traces_{obs}_2.pdf')_____no_output_____plt.figure(figsize=(15,5))

deltas = np.array([fitting_list_2[obs]['parameters']['rp']['value']**2 for obs in range(n_obs)])
deltas_errors = np.array([[fitting_list_2[obs]['parameters']['rp']['m_error']* rp_over_rs_new['value'] * 2,
        fitting_list[obs]['parameters']['rp']['p_error']* rp_over_rs_new['value'] * 2]
       for obs in range(n_obs)])

mu, sigmu = weighted_avg(deltas, deltas_errors)
plt.errorbar(range(1, n_obs+1),
             deltas,
             yerr=deltas_errors.T,
             markersize=5, fmt='-o',)
plt.hlines(mu, 1,6 , color= p[0].get_color())
plt.hlines(mu-sigmu, 1,6, linestyle='dashed', color= 'black')
plt.hlines(mu+sigmu, 1,6, linestyle='dashed', color= 'black')
_____no_output_____


# Notebook from prisae/aoc20
Path: 2021/Day-08.ipynb

# Day 08_____no_output_____

import numpy as np
from aocd.models import Puzzle_____no_output_____

## Data_____no_output_____

puzzle = Puzzle(year=2021, day=8)
data = puzzle.input_data.split('\n')_____no_output_____

## Part One_____no_output_____

output = []
for d in data:
    output.extend([len(o) for o in d.split('|')[1].split()])_____no_output_____answer_a = output.count(2) + output.count(3) + output.count(4) + output.count(7)
answer_a_____no_output_____puzzle.answer_a = answer_aThat's the right answer!  You are one gold star closer to finding the sleigh keys. [Continue to Part Two]

## Part Two_____no_output_____

def contains(key, name):
    """Returns true if all characters of <key> are in <name>."""
    return all([x in name for x in [c for c in key]])

def get_out_number(number):

    # Get input to deciffer
    inp, out = number.split('|')

    key = {}

    umap = {2: 1, 3: 7, 4: 4, 7: 8}
    for r in inp.split():
        if len(r) in [2, 3, 4, 7]:
            key[umap[len(r)]] = r

    for r in inp.split():
        if len(r) == 6:
            if contains(key[4], r):
                key[9] = r
            elif contains(key[1], r):
                key[0] = r
            else:
                key[6] = r

    for r in inp.split():
        if len(r) == 5:
            if contains(key[1], r):
                key[3] = r
            elif contains(r, key[6]):
                key[5] = r
            else:
                key[2] = r

    # Invert key-dict, sort characters
    k = {''.join(sorted(v)): k for k, v in key.items()}

    # Get and sum numbers
    sol = ''
    for ir in out.split():
        sol += str(k[''.join(sorted(ir))])

    return int(sol)

count = 0
for number in data:
    count += get_out_number(number)_____no_output_____answer_b = count
answer_b_____no_output_____puzzle.answer_b = answer_bThat's the right answer!  You are one gold star closer to finding the sleigh keys.You have completed Day 8! You can [Shareon
  Twitter
Mastodon] this victory or [Return to Your Advent Calendar].
import scooby
scooby.Report('aocd')_____no_output_____


# Notebook from jilljenn/lectures-labs
Path: labs/06_deep_nlp/Transformers_Joint_Intent_Classification_Slot_Filling_rendered.ipynb

# Joint Intent Classification and Slot Filling with Transformers

The goal of this notebook is to fine-tune a pretrained transformer-based neural network model to convert a user query expressed in English into
a representation that is structured enough to be processed by an automated service.

Here is an example of interpretation computed by such a Natural Language Understanding system:

```python
>>> nlu("Book a table for two at Le Ritz for Friday night",
        tokenizer, joint_model, intent_names, slot_names)
```
```
{
    'intent': 'BookRestaurant',
    'slots': {
        'party_size_number': 'two',
        'restaurant_name': 'Le Ritz',
        'timeRange': 'Friday night'
    }
}
```

Intent classification is a simple sequence classification problem. The trick is to treat the structured knowledge extraction part ("Slot Filling") as token-level classification problem using BIO-annotations:

```python
>>> show_predictions("Book a table for two at Le Ritz for Friday night!",
...                  tokenizer, joint_model, intent_names, slot_names)
```
```
## Intent: BookRestaurant
## Slots:
      Book : O
         a : O
     table : O
       for : O
       two : B-party_size_number
        at : O
        Le : B-restaurant_name
         R : I-restaurant_name
     ##itz : I-restaurant_name
       for : O
    Friday : B-timeRange
     night : I-timeRange
         ! : O
```

We will show how to train a such as join "sequence classification" and "token classification" joint model on a [voice command dataset]() published by snips.ai.

This notebook is a partial reproduction of some of the results presented in this paper:

BERT for Joint Intent Classification and Slot Filling
Qian Chen, Zhu Zhuo, Wen Wang

https://arxiv.org/abs/1902.10909_____no_output_____

%tensorflow_version 2.xTensorFlow 2.x selected.
!nvidia-smiWed Jan  8 22:19:37 2020
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 440.44       Driver Version: 418.67       CUDA Version: 10.1     |
|-------------------------------+----------------------+----------------------+
| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
|===============================+======================+======================|
|   0  Tesla P100-PCIE...  Off  | 00000000:00:04.0 Off |                    0 |
| N/A   33C    P0    25W / 250W |      0MiB / 16280MiB |      0%      Default |
+-------------------------------+----------------------+----------------------+

+-----------------------------------------------------------------------------+
| Processes:                                                       GPU Memory |
|  GPU       PID   Type   Process name                             Usage      |
|=============================================================================|
|  No running processes found                                                 |
+-----------------------------------------------------------------------------+
Wed Jan  8 22:19:46 2020
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 440.44       Driver Version: 418.67       CUDA Version: 10.1     |
|-------------------------------+----------------------+----------------------+
| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
|===============================+======================+======================|
|   0  Tesla P100-PCIE...  Off  | 00000000:00:04.0 Off |                    0 |
| N/A   33C    P0    25W / 250W |      0MiB / 16280MiB |      0%      Default |
+-------------------------------+----------------------+----------------------+

+-----------------------------------------------------------------------------+
| Processes:                                                       GPU Memory |
|  GPU       PID   Type   Process name                             Usage      |
|=============================================================================|
|  No running processes found                                                 |
+-----------------------------------------------------------------------------+
%pip install -q transformers     |████████████████████████████████| 450kB 9.6MB/s eta 0:00:01
     |████████████████████████████████| 870kB 32.2MB/s
     |████████████████████████████████| 1.0MB 60.3MB/s
  Building wheel for sacremoses (setup.py) ... done

## The Data

We will use a speech command dataset collected, annotated and published by French startup SNIPS.ai (bought in 2019 by Audio device manufacturer Sonos).

The original dataset comes in [YAML format with inline markdown annotations](https://snips-nlu.readthedocs.io/en/latest/dataset.html).

Instead we will use a preprocessed variant with token level B-I-O annotations closer the representation our model will predict. This variant of the SNIPS
dataset was prepared by [Su Zhu](https://github.com/sz128).

_____no_output_____

from urllib.request import urlretrieve
from pathlib import Path

SNIPS_DATA_BASE_URL = (
    "https://github.com/ogrisel/slot_filling_and_intent_detection_of_SLU/blob/"
    "master/data/snips/"
)
for filename in ["train", "valid", "test", "vocab.intent", "vocab.slot"]:
    path = Path(filename)
    if not path.exists():
        print(f"Downloading {filename}...")
        urlretrieve(SNIPS_DATA_BASE_URL + filename + "?raw=true", path)Downloading train...
Downloading valid...
Downloading test...
Downloading vocab.intent...
Downloading vocab.slot...

Let's have a look at the first lines from the training set:_____no_output_____

lines_train = Path("train").read_text().strip().splitlines()
lines_train[:5]_____no_output_____

Some remarks:

- The class label for the voice command appears at the end of each line (after the "<=>" marker).
- Each word-level token is annotated with B-I-O labels using the ":" separator.
- B/I/O stand for "Beginning" / "Inside" / "Outside"
- "Add:O" means that the token "Add" is "Outside" of any annotation span
- "Don:B-entity_name" means that "Don" is the "Beginning" of an annotation of type "entity-name".
- "and:I-entity_name" means that "and" is "Inside" the previously started annotation of type "entity-name".

Let's write a parsing function and test it on the first line:_____no_output_____

def parse_line(line):
    utterance_data, intent_label = line.split(" <=> ")
    items = utterance_data.split()
    words = [item.rsplit(":", 1)[0]for item in items]
    word_labels = [item.rsplit(":", 1)[1]for item in items]
    return {
        "intent_label": intent_label,
        "words": " ".join(words),
        "word_labels": " ".join(word_labels),
        "length": len(words),
    }_____no_output_____parse_line(lines_train[0])_____no_output_____

This utterance is a voice command of type "AddToPlaylist" with to annotations:

- an entity-name: "Don and Sherri",
- a playlist: "Medidate to Sounds of Nature".

The goal of this project is to build a baseline Natural Understanding model to analyse such voice commands and predict:

- the intent of the speaker: the sentence level class label ("AddToPlaylist");
- extract the interesting "slots" (typed named entities) from the sentence by performing word level classification using the B-I-O tags as target classes. This second task is often referred to as "NER" (Named Entity Recognition) in the Natural Language Processing literature. Alternatively this is also known as "slot filling" when we expect a fixed set of named entity per sentence of a given class.

The list of possible classes for the sentence level and the word level classification problems are given as:

_____no_output_____

print(Path("vocab.intent").read_text())AddToPlaylist
BookRestaurant
GetWeather
PlayMusic
RateBook
SearchCreativeWork
SearchScreeningEvent

print(Path("vocab.slot").read_text())B-album
B-artist
B-best_rating
B-city
B-condition_description
B-condition_temperature
B-country
B-cuisine
B-current_location
B-entity_name
B-facility
B-genre
B-geographic_poi
B-location_name
B-movie_name
B-movie_type
B-music_item
B-object_location_type
B-object_name
B-object_part_of_series_type
B-object_select
B-object_type
B-party_size_description
B-party_size_number
B-playlist
B-playlist_owner
B-poi
B-rating_unit
B-rating_value
B-restaurant_name
B-restaurant_type
B-served_dish
B-service
B-sort
B-spatial_relation
B-state
B-timeRange
B-track
B-year
I-album
I-artist
I-city
I-country
I-cuisine
I-current_location
I-entity_name
I-facility
I-genre
I-geographic_poi
I-location_name
I-movie_name
I-movie_type
I-music_item
I-object_location_type
I-object_name
I-object_part_of_series_type
I-object_select
I-object_type
I-party_size_description
I-playlist
I-playlist_owner
I-poi
I-restaurant_name
I-restaurant_type
I-served_dish
I-service
I-sort
I-spatial_relation
I-state
I-timeRange
I-track
O

"POI" stands for "Point of Interest".

Let's parse all the lines and store the results in pandas DataFrames:_____no_output_____

import pandas as pd

parsed = [parse_line(line) for line in lines_train]

df_train = pd.DataFrame([p for p in parsed if p is not None])
df_train_____no_output_____df_train_____no_output_____df_train.groupby("intent_label").count()_____no_output_____df_train.hist("length", bins=30);_____no_output_____lines_valid = Path("valid").read_text().strip().splitlines()
lines_test = Path("test").read_text().strip().splitlines()

df_valid = pd.DataFrame([parse_line(line) for line in lines_valid])
df_test = pd.DataFrame([parse_line(line) for line in lines_test])_____no_output_____

## A First Model: Intent Classification (Sentence Level)

Let's ignore the slot filling task for now and let's try to build a sentence level classifier by fine-tuning a pre-trained Transformer-based model using the `huggingface/transformers` package that provides both TF2/Keras and Pytorch APIs.

### The BERT Tokenizer

First let's load a pre-trained tokenizer and test it on a test sentence from the training set:_____no_output_____

from transformers import BertTokenizer

model_name = "bert-base-cased"
tokenizer = BertTokenizer.from_pretrained(model_name)_____no_output_____first_sentence = df_train.iloc[0]["words"]
first_sentence_____no_output_____tokenizer.tokenize(first_sentence)_____no_output_____

Notice that BERT uses subword tokens so the length of the tokenized sentence is likely to be larger than the number of words in the sentence.

**Question**:

- why is it particulary interesting to use subword tokenization for general purpose language models such as BERT?

Each token string is mapped to a unique integer id that makes it fast to lookup the right column in the input layer token embedding:_____no_output_____

tokenizer.encode(first_sentence)_____no_output_____tokenizer.decode(tokenizer.encode(first_sentence))_____no_output_____

Remarks:

- The first token `[CLS]` is used by the pre-training task for sequence classification.
- The last token `[SEP]` is a separator for the pre-training task that classifiies if a pair of sentences are consecutive in a corpus or not (next sentence prediction).
- Here we want to use BERT to compute a representation of a single voice command at a time
- We could reuse the representation of the `[CLS]` token for sequence classification.
- Alternatively we can pool the representations of all the tokens of the voice command (e.g. global average) and use that as the input of the final sequence classification layer._____no_output_____

import matplotlib.pyplot as plt

train_sequence_lengths = [len(tokenizer.encode(text))
                          for text in df_train["words"]]
plt.hist(train_sequence_lengths, bins=30)
plt.title(f"max sequence length: {max(train_sequence_lengths)}");_____no_output_____

To perform transfer learning, we will need to work with padded sequences so they all have the same sizes. The above histograms, shows that after tokenization, 43 tokens are enough to represent all the voice commands in the training set._____no_output_____The mapping can be introspected in the `tokenizer.vocab` attribute:_____no_output_____

tokenizer.vocab_size_____no_output_____bert_vocab_items = list(tokenizer.vocab.items())
bert_vocab_items[:10]_____no_output_____bert_vocab_items[100:110]_____no_output_____bert_vocab_items[900:910]_____no_output_____bert_vocab_items[1100:1110]_____no_output_____bert_vocab_items[20000:20010]_____no_output_____bert_vocab_items[-10:]_____no_output_____

Couple of remarks:

- 30K is a reasonable vocabulary size and is small enough to be used in a softmax output layer;
- it can represent multi-lingual sentences, including non-Western alphabets;
- subword tokenization makes it possible to deal with typos and morphological variations with a small vocabulary size and without any language-specific preprocessing;
- subword tokenization makes it unlikely to use the `[UNK]` special token as rare words can often be represented as a sequence of frequent enough short subwords in a meaningful way._____no_output_____### Encoding the Dataset with the Tokenizer

Let's now encode the full train / valid and test sets with our tokenizer to get a padded integer numpy arrays:_____no_output_____

import numpy as np

def encode_dataset(tokenizer, text_sequences, max_length):
    token_ids = np.zeros(shape=(len(text_sequences), max_length),
                         dtype=np.int32)
    for i, text_sequence in enumerate(text_sequences):
        encoded = tokenizer.encode(text_sequence)
        token_ids[i, 0:len(encoded)] = encoded
    attention_masks = (token_ids != 0).astype(np.int32)
    return {"input_ids": token_ids, "attention_masks": attention_masks}

encoded_train = encode_dataset(tokenizer, df_train["words"], 45)
encoded_train["input_ids"]_____no_output_____encoded_train["attention_masks"]_____no_output_____encoded_valid = encode_dataset(tokenizer, df_valid["words"], 45)
encoded_test = encode_dataset(tokenizer, df_test["words"], 45)_____no_output_____

### Encoding the Sequence Classification Targets

To do so we build a simple mapping from the auxiliary files:_____no_output_____

intent_names = Path("vocab.intent").read_text().split()
intent_map = dict((label, idx) for idx, label in enumerate(intent_names))
intent_map_____no_output_____intent_train = df_train["intent_label"].map(intent_map).values
intent_train_____no_output_____intent_valid = df_valid["intent_label"].map(intent_map).values
intent_test = df_test["intent_label"].map(intent_map).values_____no_output_____

### Loading and Feeding a Pretrained BERT model

Let's load a pretrained BERT model using the [huggingface transformers](https://github.com/huggingface/transformers) package:_____no_output_____

from transformers import TFBertModel

base_bert_model = TFBertModel.from_pretrained("bert-base-cased")
base_bert_model.summary()Model: "tf_bert_model"
_________________________________________________________________
Layer (type)                 Output Shape              Param #
=================================================================
bert (TFBertMainLayer)       multiple                  108310272
=================================================================
Total params: 108,310,272
Trainable params: 108,310,272
Non-trainable params: 0
_________________________________________________________________
encoded_valid_____no_output_____outputs = base_bert_model(encoded_valid)
len(outputs)_____no_output_____

The **first ouput** of the BERT model is a tensor with shape: `(batch_size, seq_len, output_dim)` which computes **features for each token in the input sequence**:_____no_output_____

outputs[0].shape_____no_output_____

The **second output** of the BERT model is a tensor with shape `(batch_size, output_dim)` which is the vector representation of the special token `[CLS]`. This vector is typically used as a **pooled representation for the sequence as a whole**. This is will be used as the features of our Intent classifier:_____no_output_____

outputs[1].shape_____no_output_____

### Exercise

Use the following code template to build and train a **sequence classification model** using to predict the **intent class**.

Use the `self.bert` pre-trained model in the `call` method and only consider the pooled features (ignore the token-wise features for now)._____no_output_____

import tensorflow as tf
from transformers import TFBertModel
from tensorflow.keras.layers import Dropout, Dense
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.losses import SparseCategoricalCrossentropy
from tensorflow.keras.metrics import SparseCategoricalAccuracy

class IntentClassificationModel(tf.keras.Model):

    def __init__(self, intent_num_labels=None, model_name="bert-base-cased",
                 dropout_prob=0.1):
        super().__init__(name="joint_intent_slot")
        # Let's preload the pretrained model BERT in the constructor of our
        # classifier model
        self.bert = TFBertModel.from_pretrained(model_name)

        # TODO: define a (Dense) classification layer to compute the
        # for each sequence in a batch the batch of samples. The number of
        # output classes is given by the intent_num_labels parameter.

        # Use the default linear activation (no softmax) to compute logits.
        # The softmax normalization will be computed in the loss function
        # instead of the model itself.

    def call(self, inputs, **kwargs):
        # Use the pretrained model to extract features from our encoded inputs:
        sequence_output, pooled_output = self.bert(inputs, **kwargs)

        # The second output of the main BERT layer has shape:
        # (batch_size, output_dim)
        # and gives a "pooled" representation for the full sequence from the
        # hidden state that corresponds to the "[CLS]" token.

        # TODO: use the classifier layer to compute the logits from the pooled
        # features.
        intent_logits = None
        return intent_logits

intent_model = IntentClassificationModel(intent_num_labels=len(intent_map))

intent_model.compile(optimizer=Adam(learning_rate=3e-5, epsilon=1e-08),
                     loss=SparseCategoricalCrossentropy(from_logits=True),
                     metrics=[SparseCategoricalAccuracy('accuracy')])

# TODO: uncomment to train the model:

# history = intent_model.fit(encoded_train, intent_train, epochs=2, batch_size=32,
#                            validation_data=(encoded_valid, intent_valid))_____no_output_____

_____no_output_____

### Solution_____no_output_____

import tensorflow as tf
from transformers import TFBertModel
from tensorflow.keras.layers import Dropout, Dense

class IntentClassificationModel(tf.keras.Model):

    def __init__(self, intent_num_labels=None, model_name="bert-base-cased",
                 dropout_prob=0.1):
        super().__init__(name="joint_intent_slot")
        self.bert = TFBertModel.from_pretrained(model_name)
        self.dropout = Dropout(dropout_prob)

        # Use the default linear activation (no softmax) to compute logits.
        # The softmax normalization will be computed in the loss function
        # instead of the model itself.
        self.intent_classifier = Dense(intent_num_labels)

    def call(self, inputs, **kwargs):
        sequence_output, pooled_output = self.bert(inputs, **kwargs)

        pooled_output = self.dropout(pooled_output,
                                     training=kwargs.get("training", False))

        intent_logits = self.intent_classifier(pooled_output)
        return intent_logits

intent_model = IntentClassificationModel(intent_num_labels=len(intent_map))_____no_output_____

Our classification model outputs logits instead of probabilities. The final softmax normalization layer is implicit, that is, included in the loss function instead of the model directly.

We need to configure the loss function `SparseCategoricalCrossentropy(from_logits=True)` accordingly:_____no_output_____

intent_model.compile(optimizer=Adam(learning_rate=3e-5, epsilon=1e-08),
                     loss=SparseCategoricalCrossentropy(from_logits=True),
                     metrics=[SparseCategoricalAccuracy('accuracy')])_____no_output_____history = intent_model.fit(encoded_train, intent_train, epochs=2, batch_size=32,
                           validation_data=(encoded_valid, intent_valid))Train on 13084 samples, validate on 700 samples
Epoch 1/2
13084/13084 [==============================] - 95s 7ms/sample - loss: 0.3259 - accuracy: 0.8827 - val_loss: 0.0647 - val_accuracy: 0.9829
Epoch 2/2
13084/13084 [==============================] - 80s 6ms/sample - loss: 0.0366 - accuracy: 0.9883 - val_loss: 0.0724 - val_accuracy: 0.9829
def classify(text, tokenizer, model, intent_names):
    inputs = tf.constant(tokenizer.encode(text))[None, :]  # batch_size = 1
    class_id = model(inputs).numpy().argmax(axis=1)[0]
    return intent_names[class_id]

classify("Book a table for two at La Tour d'Argent for Friday night.",
         tokenizer, intent_model, intent_names)_____no_output_____classify("I would like to listen to Anima by Thom Yorke.",
         tokenizer, intent_model, intent_names)_____no_output_____classify("Will it snow tomorrow in Saclay?",
         tokenizer, intent_model, intent_names)_____no_output_____classify("Where can I see to the last Star Wars near Odéon tonight?",
         tokenizer, intent_model, intent_names)_____no_output_____

## Join Intent Classification and Slot Filling

Let's now refine our Natural Language Understanding system by trying the retrieve the important structured elements of each voice command.

To do so we will perform word level (or token level) classification of the BIO labels.

Since we have word level tags but BERT uses a wordpiece tokenizer, we need to align the BIO labels with the BERT tokens.

Let's load the list of possible word token labels and augment it with an additional padding label to be able to ignore special tokens:_____no_output_____

slot_names = ["[PAD]"]
slot_names += Path("vocab.slot").read_text().strip().splitlines()
slot_map = {}
for label in slot_names:
    slot_map[label] = len(slot_map)
slot_map_____no_output_____

The following function generates token-aligned integer labels from the BIO word-level annotations. In particular, if a specific word is too long to be represented as a single token, we expand its label for all the tokens of that word while taking care of using "B-" labels only for the first token and then use "I-" for the matching slot type for subsequent tokens of the same word:_____no_output_____

def encode_token_labels(text_sequences, slot_names, tokenizer, slot_map,
                        max_length):
    encoded = np.zeros(shape=(len(text_sequences), max_length), dtype=np.int32)
    for i, (text_sequence, word_labels) in enumerate(
            zip(text_sequences, slot_names)):
        encoded_labels = []
        for word, word_label in zip(text_sequence.split(), word_labels.split()):
            tokens = tokenizer.tokenize(word)
            encoded_labels.append(slot_map[word_label])
            expand_label = word_label.replace("B-", "I-")
            if not expand_label in slot_map:
                expand_label = word_label
            encoded_labels.extend([slot_map[expand_label]] * (len(tokens) - 1))
        encoded[i, 1:len(encoded_labels) + 1] = encoded_labels
    return encoded

slot_train = encode_token_labels(
    df_train["words"], df_train["word_labels"], tokenizer, slot_map, 45)
slot_valid = encode_token_labels(
    df_valid["words"], df_valid["word_labels"], tokenizer, slot_map, 45)
slot_test = encode_token_labels(
    df_test["words"], df_test["word_labels"], tokenizer, slot_map, 45)_____no_output_____slot_train[0]_____no_output_____slot_valid[0]_____no_output_____

Note that the special tokens such as "[PAD]" and "[SEP]" and all padded positions recieve a 0 label._____no_output_____### Exercise

Use the following code template to build a joint sequence and token classification model suitable for training on our encoded dataset with slot labels:

_____no_output_____

from transformers import TFBertModel
from tensorflow.keras.layers import Dropout, Dense

class JointIntentAndSlotFillingModel(tf.keras.Model):

    def __init__(self, intent_num_labels=None, slot_num_labels=None,
                 model_name="bert-base-cased", dropout_prob=0.1):
        super().__init__(name="joint_intent_slot")
        self.bert = TFBertModel.from_pretrained(model_name)
        # TODO: define all the needed layers here.

    def call(self, inputs, **kwargs):
        # TODO: extrac the features from the inputs using the pre-trained
        # BERT model here.

        # TODO: use the new layers to predict slot class (logits) for each
        # token position in the input sequence:
        slot_logits = None  # (batch_size, seq_len, slot_num_labels)

        # TODO: define a second classification head for the sequence-wise
        # predictions:
        intent_logits = None  # (batch_size, intent_num_labels)

        return slot_logits, intent_logits

joint_model = JointIntentAndSlotFillingModel(
    intent_num_labels=len(intent_map), slot_num_labels=len(slot_map))

# Define one classification loss for each output:
losses = [SparseCategoricalCrossentropy(from_logits=True),
          SparseCategoricalCrossentropy(from_logits=True)]
joint_model.compile(optimizer=Adam(learning_rate=3e-5, epsilon=1e-08),
                    loss=losses)

# TODO: uncomment to train the model:
# history = joint_model.fit(
#     encoded_train, (slot_train, intent_train),
#     validation_data=(encoded_valid, (slot_valid, intent_valid)),
#     epochs=2, batch_size=32)_____no_output_____

_____no_output_____

### Solution:
_____no_output_____

from transformers import TFBertModel
from tensorflow.keras.layers import Dropout, Dense

class JointIntentAndSlotFillingModel(tf.keras.Model):

    def __init__(self, intent_num_labels=None, slot_num_labels=None,
                 model_name="bert-base-cased", dropout_prob=0.1):
        super().__init__(name="joint_intent_slot")
        self.bert = TFBertModel.from_pretrained(model_name)
        self.dropout = Dropout(dropout_prob)
        self.intent_classifier = Dense(intent_num_labels,
                                       name="intent_classifier")
        self.slot_classifier = Dense(slot_num_labels,
                                     name="slot_classifier")

    def call(self, inputs, **kwargs):
        sequence_output, pooled_output = self.bert(inputs, **kwargs)

        # The first output of the main BERT layer has shape:
        # (batch_size, max_length, output_dim)
        sequence_output = self.dropout(sequence_output,
                                       training=kwargs.get("training", False))
        slot_logits = self.slot_classifier(sequence_output)

        # The second output of the main BERT layer has shape:
        # (batch_size, output_dim)
        # and gives a "pooled" representation for the full sequence from the
        # hidden state that corresponds to the "[CLS]" token.
        pooled_output = self.dropout(pooled_output,
                                     training=kwargs.get("training", False))
        intent_logits = self.intent_classifier(pooled_output)

        return slot_logits, intent_logits

joint_model = JointIntentAndSlotFillingModel(
    intent_num_labels=len(intent_map), slot_num_labels=len(slot_map))_____no_output_____opt = Adam(learning_rate=3e-5, epsilon=1e-08)
losses = [SparseCategoricalCrossentropy(from_logits=True),
          SparseCategoricalCrossentropy(from_logits=True)]
metrics = [SparseCategoricalAccuracy('accuracy')]
joint_model.compile(optimizer=opt, loss=losses, metrics=metrics)_____no_output_____history = joint_model.fit(
    encoded_train, (slot_train, intent_train),
    validation_data=(encoded_valid, (slot_valid, intent_valid)),
    epochs=2, batch_size=32)Train on 13084 samples, validate on 700 samples
Epoch 1/2
13084/13084 [==============================] - 95s 7ms/sample - loss: 0.5970 - output_1_loss: 0.3107 - output_2_loss: 0.2862 - output_1_accuracy: 0.9315 - output_2_accuracy: 0.8997 - val_loss: 0.1009 - val_output_1_loss: 0.0518 - val_output_2_loss: 0.0489 - val_output_1_accuracy: 0.9862 - val_output_2_accuracy: 0.9886
Epoch 2/2
13084/13084 [==============================] - 80s 6ms/sample - loss: 0.0768 - output_1_loss: 0.0467 - output_2_loss: 0.0300 - output_1_accuracy: 0.9878 - output_2_accuracy: 0.9908 - val_loss: 0.0698 - val_output_1_loss: 0.0266 - val_output_2_loss: 0.0428 - val_output_1_accuracy: 0.9922 - val_output_2_accuracy: 0.9886

The following function uses our trained model to make a prediction on a single text sequence and display both the sequence-wise and the token-wise class labels:_____no_output_____

def show_predictions(text, tokenizer, model, intent_names, slot_names):
    inputs = tf.constant(tokenizer.encode(text))[None, :]  # batch_size = 1
    outputs = model(inputs)
    slot_logits, intent_logits = outputs
    slot_ids = slot_logits.numpy().argmax(axis=-1)[0, 1:-1]
    intent_id = intent_logits.numpy().argmax(axis=-1)[0]
    print("## Intent:", intent_names[intent_id])
    print("## Slots:")
    for token, slot_id in zip(tokenizer.tokenize(text), slot_ids):
        print(f"{token:>10} : {slot_names[slot_id]}")_____no_output_____show_predictions("Book a table for two at Le Ritz for Friday night!",
                 tokenizer, joint_model, intent_names, slot_names)## Intent: BookRestaurant
## Slots:
      Book : O
         a : O
     table : O
       for : O
       two : B-party_size_number
        at : O
        Le : B-restaurant_name
         R : I-restaurant_name
     ##itz : I-restaurant_name
       for : O
    Friday : B-timeRange
     night : O
         ! : O
show_predictions("Will it snow tomorrow in Saclay?",
                 tokenizer, joint_model, intent_names, slot_names)## Intent: GetWeather
## Slots:
      Will : O
        it : O
      snow : B-condition_description
  tomorrow : B-timeRange
        in : O
        Sa : B-city
       ##c : I-city
     ##lay : I-city
         ? : O
show_predictions("I would like to listen to Anima by Thom Yorke.",
                 tokenizer, joint_model, intent_names, slot_names)## Intent: PlayMusic
## Slots:
         I : O
     would : O
      like : O
        to : O
    listen : O
        to : O
        An : B-track
     ##ima : I-track
        by : O
      Thom : B-artist
      York : I-artist
       ##e : I-artist
         . : O

### Decoding Predictions into Structured Knowledge

For completeness, here a minimal function to naively decode the predicted BIO slot ids and convert it into a structured representation for the detected slots as a Python dictionaries: _____no_output_____

def decode_predictions(text, tokenizer, intent_names, slot_names,
                       intent_id, slot_ids):
    info = {"intent": intent_names[intent_id]}
    collected_slots = {}
    active_slot_words = []
    active_slot_name = None
    for word in text.split():
        tokens = tokenizer.tokenize(word)
        current_word_slot_ids = slot_ids[:len(tokens)]
        slot_ids = slot_ids[len(tokens):]
        current_word_slot_name = slot_names[current_word_slot_ids[0]]
        if current_word_slot_name == "O":
            if active_slot_name:
                collected_slots[active_slot_name] = " ".join(active_slot_words)
                active_slot_words = []
                active_slot_name = None
        else:
            # Naive BIO: handling: treat B- and I- the same...
            new_slot_name = current_word_slot_name[2:]
            if active_slot_name is None:
                active_slot_words.append(word)
                active_slot_name = new_slot_name
            elif new_slot_name == active_slot_name:
                active_slot_words.append(word)
            else:
                collected_slots[active_slot_name] = " ".join(active_slot_words)
                active_slot_words = [word]
                active_slot_name = new_slot_name
    if active_slot_name:
        collected_slots[active_slot_name] = " ".join(active_slot_words)
    info["slots"] = collected_slots
    return info_____no_output_____def nlu(text, tokenizer, model, intent_names, slot_names):
    inputs = tf.constant(tokenizer.encode(text))[None, :]  # batch_size = 1
    outputs = model(inputs)
    slot_logits, intent_logits = outputs
    slot_ids = slot_logits.numpy().argmax(axis=-1)[0, 1:-1]
    intent_id = intent_logits.numpy().argmax(axis=-1)[0]

    return decode_predictions(text, tokenizer, intent_names, slot_names,
                              intent_id, slot_ids)

nlu("Book a table for two at Le Ritz for Friday night",
    tokenizer, joint_model, intent_names, slot_names)_____no_output_____nlu("Will it snow tomorrow in Saclay",
    tokenizer, joint_model, intent_names, slot_names)_____no_output_____nlu("I would like to listen to Anima by Thom Yorke",
    tokenizer, joint_model, intent_names, slot_names)_____no_output_____

## Limitations

### Language

BERT is pretrained primarily on English content. It can therefore only extract meaningful features on text written in English.

Note that there exists alternative pretrained model that use a mix of different languages (e.g. [XLM](https://github.com/facebookresearch/XLM/)) and others that have been trained on other languages. For instance [CamemBERT](https://camembert-model.fr/) is pretrained on French text. Both kinds of models are available in the transformers package:

https://github.com/huggingface/transformers#model-architectures

The public snips.ai dataset used for fine-tuning is English only. To build a model for another language we would need to collect and annotate a similar corpus with tens of thousands of diverse, representative samples.

### Biases Embedded in the Pre-Trained Model

The original data used to pre-trained BERT was collected from the Internet and contains all kinds of data, including offensive and hateful speech.

While using BERT for or voice command understanding system is quite unlikely to be significantly impacted by those biases, it could be a serious problem for other kinds of applications such as Machine Translation for instance.

It is therefore strongly recommended to spend time auditing the biases that are embedded in such pre-trained models before deciding to deploy system that derive from them.

### Computational Resources

The original BERT model has many parameters which uses a lot of memory and can be prohibitive to deploy on small devices such as mobile phones. It is also very computationally intensive and typically requires powerful GPUs or TPUs to process text data at a reasonable speed (both for training and at inference time).

Designing alternative architectures with fewer parameters or more efficient training and inference procedures is still a very active area of research.

Depending of on the problems, it might be the case that simpler architectures based on convolutional neural networks and LSTMs might offer a better speed / accuracy trade-off.
_____no_output_____


# Notebook from boyali/pytorch-mixture_of_density_networks
Path: pytorch_notebooks-master/mnist_es/pytorch_mnist_mini_es_rank_oes.ipynb

<code>
import numpy as np
import math
import multiprocessing as mp

import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torchvision import datasets, transforms
from torch.autograd import Variable
from collections import namedtuple

from PIL import Image
import os
import os.path
import errno
import codecs
import copy_____no_output_____torch.manual_seed(0)
np.random.seed(0)
print("torch.cuda.device_count()", torch.cuda.device_count())
print("torch.cuda.current_device()", torch.cuda.current_device())
torch.cuda.set_device(1)
print("torch.cuda.current_device()", torch.cuda.current_device())torch.cuda.device_count() 4
torch.cuda.current_device() 0
torch.cuda.current_device() 1
def compute_ranks(x):
  """
  Returns ranks in [0, len(x))
  Note: This is different from scipy.stats.rankdata, which returns ranks in [1, len(x)].
  (https://github.com/openai/evolution-strategies-starter/blob/master/es_distributed/es.py)
  """
  assert x.ndim == 1
  ranks = np.empty(len(x), dtype=int)
  ranks[x.argsort()] = np.arange(len(x))
  return ranks

def compute_centered_ranks(x):
  """
  https://github.com/openai/evolution-strategies-starter/blob/master/es_distributed/es.py
  """
  y = compute_ranks(x.ravel()).reshape(x.shape).astype(np.float32)
  y /= (x.size - 1)
  y -= .5
  return y

def compute_weight_decay(weight_decay, model_param_list):
  model_param_grid = np.array(model_param_list)
  return - weight_decay * np.mean(model_param_grid * model_param_grid, axis=1)

class CMAES:
  '''CMA-ES wrapper.'''
  def __init__(self, num_params,      # number of model parameters
               sigma_init=0.10,       # initial standard deviation
               popsize=255):          # population size

    self.num_params = num_params
    self.sigma_init = sigma_init
    self.popsize = popsize

    self.solutions = None

    import cma
    self.es = cma.CMAEvolutionStrategy( self.num_params * [0],
                                        self.sigma_init,
                                        {'popsize': self.popsize})

  def rms_stdev(self):
    sigma = self.es.result()[6]
    return np.mean(np.sqrt(sigma*sigma))

  def ask(self):
    '''returns a list of parameters'''
    self.solutions = np.array(self.es.ask())
    return self.solutions

  def tell(self, reward_table_result):
    reward_table = reward_table_result
    self.es.tell(self.solutions, (-reward_table).tolist()) # convert minimizer to maximizer.

  def done(self):
    return self.es.stop()

  def current_param(self):
    return self.es.result()[5] # mean solution, presumably better with noise

  def best_param(self):
    return self.es.result()[0] # best evaluated solution

  def result(self): # return best params so far, along with historically best reward, curr reward, sigma
    r = self.es.result()
    return (r[0], -r[1], -r[1], r[6])

class SimpleES:
  '''Simple Evolution Strategies.'''
  def __init__(self, num_params,      # number of model parameters
               sigma_init=0.10,       # initial standard deviation
               sigma_alpha=0.20,      # learning rate for standard deviation
               sigma_decay=0.999,     # anneal standard deviation
               sigma_limit=0.01,      # stop annealing if less than this
               popsize=255,           # population size
               elite_ratio=0.1,       # percentage of the elites
               done_threshold=1e-6,   # threshold when we say we are done
               average_baseline=True, # set baseline to average of batch
               forget_best=True):     # only use the best from latest generation

    self.num_params = num_params
    self.sigma_init = sigma_init
    self.sigma_alpha = sigma_alpha
    self.sigma_decay = sigma_decay
    self.sigma_limit = sigma_limit
    self.popsize = popsize
    self.average_baseline = average_baseline
    if self.average_baseline:
      assert (self.popsize & 2), "Population size must be even"
      self.batch_size = int(self.popsize / 2)
    else:
      assert (self.popsize & 1), "Population size must be odd"
      self.batch_size = int((self.popsize - 1) / 2)
    self.elite_ratio = elite_ratio
    self.elite_popsize = int(self.popsize * self.elite_ratio)
    self.forget_best = forget_best
    self.batch_reward = np.zeros(self.batch_size * 2)
    self.mu = np.zeros(self.num_params)
    self.sigma = np.ones(self.num_params) * self.sigma_init
    self.curr_best_mu = np.zeros(self.num_params)
    self.best_mu = np.zeros(self.num_params)
    self.best_reward = 0
    self.first_interation = True
    self.done_threshold = done_threshold

  def rms_stdev(self):
    sigma = self.sigma
    return np.mean(np.sqrt(sigma*sigma))

  def ask(self):
    '''returns a list of parameters'''
    # antithetic sampling
    self.epsilon = np.random.randn(self.batch_size, self.num_params) * self.sigma.reshape(1, self.num_params)
    self.epsilon_full = np.concatenate([self.epsilon, - self.epsilon])
    if self.average_baseline:
      epsilon = self.epsilon_full
    else:
      # first population is mu, then positive epsilon, then negative epsilon
      epsilon = np.concatenate([np.zeros((1, self.num_params)), self.epsilon_full])
    solutions = self.mu.reshape(1, self.num_params) + epsilon
    return solutions

  def tell(self, reward_table_result):
    # input must be a numpy float array
    assert(len(reward_table_result) == self.popsize), "Inconsistent reward_table size reported."

    reward_table = reward_table_result

    reward_offset = 1
    if self.average_baseline:
      b = np.mean(reward_table)
      reward_offset = 0
    else:
      b = reward_table[0] # baseline

    reward = reward_table[reward_offset:]
    idx = np.argsort(reward)[::-1][0:self.elite_popsize]

    best_reward = reward[idx[0]]
    if (best_reward > b or self.average_baseline):
      best_mu = self.mu + self.epsilon_full[idx[0]]
      best_reward = reward[idx[0]]
    else:
      best_mu = self.mu
      best_reward = b

    self.curr_best_reward = best_reward
    self.curr_best_mu = best_mu

    if self.first_interation:
      self.first_interation = False
      self.best_reward = self.curr_best_reward
      self.best_mu = best_mu
    else:
      if self.forget_best or (self.curr_best_reward > self.best_reward):
        self.best_mu = best_mu
        self.best_reward = self.curr_best_reward

    # adaptive sigma
    # normalization
    stdev_reward = reward.std()
    epsilon = self.epsilon
    sigma = self.sigma
    S = ((epsilon * epsilon - (sigma * sigma).reshape(1, self.num_params)) / sigma.reshape(1, self.num_params))
    reward_avg = (reward[:self.batch_size] + reward[self.batch_size:]) / 2.0
    rS = reward_avg - b
    delta_sigma = (np.dot(rS, S)) / (2 * self.batch_size * stdev_reward)

    # move mean to the average of the best idx means
    self.mu += self.epsilon_full[idx].mean(axis=0)

    # adjust sigma according to the adaptive sigma calculation
    change_sigma = self.sigma_alpha * delta_sigma
    change_sigma = np.minimum(change_sigma, self.sigma)
    change_sigma = np.maximum(change_sigma, - 0.5 * self.sigma)
    self.sigma += change_sigma
    self.sigma[self.sigma > self.sigma_limit] *= self.sigma_decay

  def done(self):
    return (self.rms_stdev() < self.done_threshold)

  def current_param(self):
    return self.curr_best_mu

  def best_param(self):
    return self.best_mu

  def result(self): # return best params so far, along with historically best reward, curr reward, sigma
    return (self.best_mu, self.best_reward, self.curr_best_reward, self.sigma)

class SimpleGA:
  '''Simple Genetic Algorithm.'''
  def __init__(self, num_params,      # number of model parameters
               sigma_init=0.1,        # initial standard deviation
               sigma_decay=0.999,     # anneal standard deviation
               sigma_limit=0.01,      # stop annealing if less than this
               popsize=255,           # population size
               elite_ratio=0.1,       # percentage of the elites
               forget_best=True,      # forget the historical best elites
               done_threshold=1e-6):  # threshold when we say we are done

    self.num_params = num_params
    self.sigma_init = sigma_init
    self.sigma_decay = sigma_decay
    self.sigma_limit = sigma_limit
    self.popsize = popsize

    self.elite_ratio = elite_ratio
    self.elite_popsize = int(self.popsize * self.elite_ratio)

    self.sigma = self.sigma_init
    self.elite_params = np.zeros((self.elite_popsize, self.num_params))
    self.elite_rewards = np.zeros(self.elite_popsize)
    self.best_param = np.zeros(self.num_params)
    self.best_reward = 0
    self.first_iteration = True
    self.forget_best = forget_best
    self.done_threshold = done_threshold

  def rms_stdev(self):
    return self.sigma # same sigma for all parameters.

  def ask(self):
    '''returns a list of parameters'''
    # antithetic sampling
    self.epsilon = np.random.randn(self.popsize, self.num_params) * self.sigma
    solutions = []

    def mate(a, b):
      c = np.copy(a)
      idx = np.where(np.random.rand((c.size)) > 0.5)
      c[idx] = b[idx]
      return c

    elite_range = range(self.elite_popsize)
    for i in range(self.popsize):
      idx_a = np.random.choice(elite_range)
      idx_b = np.random.choice(elite_range)
      child_params = mate(self.elite_params[idx_a], self.elite_params[idx_b])
      solutions.append(child_params + self.epsilon[i])

    solutions = np.array(solutions)
    self.solutions = solutions

    return solutions

  def tell(self, reward_table_result):
    # input must be a numpy float array
    assert(len(reward_table_result) == self.popsize), "Inconsistent reward_table size reported."

    if (not forget_best or self.first_iteration):
      reward = reward_table_result
      solution = self.solutions
    else:
      reward = np.concatenate([reward_table_result, self.elite_rewards])
      solution = np.concatenate([self.solutions, self.elite_params])

    idx = np.argsort(reward)[::-1][0:self.elite_popsize]

    self.elite_rewards = reward[idx]
    self.elite_params = solution[idx]

    self.curr_best_reward = self.elite_rewards[0]

    if self.first_iteration or (self.curr_best_reward > self.best_reward):
      self.first_iteration = False
      self.best_reward = self.elite_rewards[0]
      self.best_param = np.copy(self.elite_params[0])

    if (self.sigma > self.sigma_limit):
      self.sigma *= self.sigma_decay

  def done(self):
    return (self.rms_stdev() < self.done_threshold)

  def current_param(self):
    return self.elite_params[0]

  def best_param(self):
    return self.best_param

  def result(self): # return best params so far, along with historically best reward, curr reward, sigma
    return (self.best_param, self.best_reward, self.curr_best_reward, self.sigma)

class OpenES:
  ''' Basic Version of OpenAI Evolution Strategies.'''
  def __init__(self, num_params,             # number of model parameters
               sigma_init=0.1,               # initial standard deviation
               sigma_decay=0.999,            # anneal standard deviation
               sigma_limit=0.01,             # stop annealing if less than this
               learning_rate=0.001,          # learning rate for standard deviation
               learning_rate_decay = 0.9999, # annealing the learning rate
               learning_rate_limit = 0.001,  # stop annealing learning rate
               popsize=255,                  # population size
               antithetic=False,             # whether to use antithetic sampling
               forget_best=True):           # forget historical best

    self.num_params = num_params
    self.sigma_decay = sigma_decay
    self.sigma = sigma_init
    self.sigma_limit = sigma_limit
    self.learning_rate = learning_rate
    self.learning_rate_decay = learning_rate_decay
    self.learning_rate_limit = learning_rate_limit
    self.popsize = popsize
    self.antithetic = antithetic
    if self.antithetic:
      assert (self.popsize & 2), "Population size must be even"
      self.half_popsize = int(self.popsize / 2)

    self.reward = np.zeros(self.popsize)
    self.mu = np.zeros(self.num_params)
    self.best_mu = np.zeros(self.num_params)
    self.best_reward = 0
    self.first_interation = True
    self.forget_best = forget_best

  def rms_stdev(self):
    sigma = self.sigma
    return np.mean(np.sqrt(sigma*sigma))

  def ask(self):
    '''returns a list of parameters'''
    # antithetic sampling
    if self.antithetic:
      self.epsilon_half = np.random.randn(self.half_popsize, self.num_params)
      self.epsilon = np.concatenate([self.epsilon_half, - self.epsilon_half])
    else:
      self.epsilon = np.random.randn(self.popsize, self.num_params)

    self.solutions = self.mu.reshape(1, self.num_params) + self.epsilon * self.sigma

    return self.solutions

  def tell(self, reward):
    # input must be a numpy float array
    assert(len(reward) == self.popsize), "Inconsistent reward_table size reported."

    idx = np.argsort(reward)[::-1]

    best_reward = reward[idx[0]]
    best_mu = self.solutions[idx[0]]

    self.curr_best_reward = best_reward
    self.curr_best_mu = best_mu

    if self.first_interation:
      self.first_interation = False
      self.best_reward = self.curr_best_reward
      self.best_mu = best_mu
    else:
      if self.forget_best or (self.curr_best_reward > self.best_reward):
        self.best_mu = best_mu
        self.best_reward = self.curr_best_reward

    # main bit:
    # standardize the rewards to have a gaussian distribution
    normalized_reward = (reward - np.mean(reward)) / np.std(reward)
    self.mu += self.learning_rate/(self.popsize*self.sigma)*np.dot(self.epsilon.T, normalized_reward)

    # adjust sigma according to the adaptive sigma calculation
    if (self.sigma > self.sigma_limit):
      self.sigma *= self.sigma_decay

    if (self.learning_rate > self.learning_rate_limit):
      self.learning_rate *= self.learning_rate_decay

  def done(self):
    return False

  def current_param(self):
    return self.curr_best_mu

  def best_param(self):
    return self.best_mu

  def result(self): # return best params so far, along with historically best reward, curr reward, sigma
    return (self.best_mu, self.best_reward, self.curr_best_reward, self.sigma)_____no_output_____Args = namedtuple('Args', ['batch_size', 'test_batch_size', 'epochs', 'lr', 'cuda', 'seed', 'log_interval'])_____no_output_____args = Args(batch_size=1000, test_batch_size=1000, epochs=30, lr=0.001, cuda=True, seed=0, log_interval=10)_____no_output_____torch.manual_seed(args.seed)
if args.cuda:
  torch.cuda.manual_seed(args.seed)_____no_output_____kwargs = {'num_workers': 1, 'pin_memory': True} if args.cuda else {}

train_loader = torch.utils.data.DataLoader(
  datasets.MNIST('MNIST_data', train=True, download=True, transform=transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,))])),
  batch_size=args.batch_size, shuffle=True, **kwargs)

valid_loader = train_loader

test_loader = torch.utils.data.DataLoader(
  datasets.MNIST('MNIST_data', train=False, transform=transforms.Compose([transforms.ToTensor(),transforms.Normalize((0.1307,), (0.3081,))])),
  batch_size=args.batch_size, shuffle=True, **kwargs)_____no_output_____class Net(nn.Module):
  def __init__(self):
    super(Net, self).__init__()
    self.num_filter1 = 8
    self.num_filter2 = 16
    self.num_padding = 2
    # input is 28x28
    # padding=2 for same padding
    self.conv1 = nn.Conv2d(1, self.num_filter1, 5, padding=self.num_padding)
    # feature map size is 14*14 by pooling
    # padding=2 for same padding
    self.conv2 = nn.Conv2d(self.num_filter1, self.num_filter2, 5, padding=self.num_padding)
    # feature map size is 7*7 by pooling
    self.fc = nn.Linear(self.num_filter2*7*7, 10)

  def forward(self, x):
    x = F.max_pool2d(F.relu(self.conv1(x)), 2)
    x = F.max_pool2d(F.relu(self.conv2(x)), 2)
    x = x.view(-1, self.num_filter2*7*7)   # reshape Variable
    x = self.fc(x)
    return F.log_softmax(x)_____no_output_____NPOPULATION = 101
weight_decay_coef = 0.1_____no_output_____'''
models = []
for i in range(NPOPULATION):
  model = Net()
  if args.cuda:
    model.cuda()
  model.eval()
  models.append(model)
'''

model = Net()
if args.cuda:
  model.cuda()

orig_model = copy.deepcopy(model)_____no_output_____# get init params
orig_params = []
model_shapes = []
for param in orig_model.parameters():
  p = param.data.cpu().numpy()
  model_shapes.append(p.shape)
  orig_params.append(p.flatten())
orig_params_flat = np.concatenate(orig_params)
NPARAMS = len(orig_params_flat)
print(NPARAMS)11274
def update_model(flat_param, model, model_shapes):
  idx = 0
  i = 0
  for param in model.parameters():
    delta = np.product(model_shapes[i])
    block = flat_param[idx:idx+delta]
    block = np.reshape(block, model_shapes[i])
    i += 1
    idx += delta
    block_data = torch.from_numpy(block).float()
    if args.cuda:
      block_data = block_data.cuda()
    param.data = block_data_____no_output_____def evaluate(model, test_loader, print_mode=True, return_loss=False):
  model.eval()
  test_loss = 0
  correct = 0
  for data, target in test_loader:
    if args.cuda:
      data, target = data.cuda(), target.cuda()
    data, target = Variable(data, volatile=True), Variable(target)
    output = model(data)
    test_loss += F.nll_loss(output, target, size_average=False).data[0] # sum up batch loss
    pred = output.data.max(1, keepdim=True)[1] # get the index of the max log-probability
    correct += pred.eq(target.data.view_as(pred)).cpu().sum()

  test_loss /= len(test_loader.dataset)
  acc = correct / len(test_loader.dataset)

  if print_mode:
    print('\nAverage loss: {:.4f}, Accuracy: {}/{} ({:.4f}%)\n'.format(
      test_loss, correct, len(test_loader.dataset),
      100. * acc))

  if return_loss:
    return test_loss
  return acc_____no_output_____"""
es = SimpleES(NPARAMS,
              popsize=NPOPULATION,
              sigma_init=0.01,
              sigma_decay=0.999,
              sigma_alpha=0.2,
              sigma_limit=0.001,
              elite_ratio=0.1,
              average_baseline=False,
              forget_best=True
             )
"""
es = OpenES(NPARAMS,
              popsize=NPOPULATION,
              sigma_init=0.01,
              sigma_decay=0.999,
              sigma_limit=0.01,
              forget_best=False,
              learning_rate=0.001,
              learning_rate_decay = 0.9999,
              learning_rate_limit = 0.0001,
             )_____no_output_____def worker(procnum, model, solution, data, target, send_end):
  update_model(solution, model, model_shapes)
  output = model(data)
  loss = F.nll_loss(output, target)
  reward = - loss.data[0]
  send_end.send(reward)

def batch_simulation(model_list, solutions, data, target, process_count):
  jobs = []
  pipe_list = []

  for i in range(process_count):
    recv_end, send_end = mp.Pipe(False)
    p = mp.Process(target=worker, args=(i, model_list[i], solutions[i], data, target, send_end))
    jobs.append(p)
    pipe_list.append(recv_end)

  for p in jobs:
    p.start()

  for p in jobs:
    p.join()

  result_list = [x.recv() for x in pipe_list]
  return np.array(result_list)

def batch_simulation_sequential(model_list, solutions, data, target, process_count):
  result_list = []
  for i in range(process_count):
    update_model(solutions[i], model_list[i], model_shapes)
    output = model_list[i](data)
    loss = F.nll_loss(output, target)
    reward = - loss.data[0]
    result_list.append(reward)
  return np.array(result_list)
_____no_output_____#'''
best_valid_acc = 0
training_log = []
for epoch in range(1, 10*args.epochs + 1):

  # train loop
  model.eval()
  for batch_idx, (data, target) in enumerate(train_loader):
    if args.cuda:
      data, target = data.cuda(), target.cuda()
    data, target = Variable(data), Variable(target)

    solutions = es.ask()
    reward = np.zeros(es.popsize)

    for i in range(es.popsize):
      update_model(solutions[i], model, model_shapes)
      output = model(data)
      loss = F.nll_loss(output, target)
      reward[i] = - loss.data[0]

    best_raw_reward = reward.max()
    #reward = compute_centered_ranks(reward)
    l2_decay = compute_weight_decay(weight_decay_coef, solutions)
    reward += l2_decay

    es.tell(reward)

    result = es.result()

    if (batch_idx % 5 == 0):
      print(epoch, batch_idx, best_raw_reward, result[0].mean(), result[3])

  curr_solution = es.current_param()
  update_model(curr_solution, model, model_shapes)

  valid_acc = evaluate(model, valid_loader, print_mode=False)
  training_log.append([epoch, valid_acc])
  print('valid_acc', valid_acc * 100.)
  if valid_acc >= best_valid_acc:
    best_valid_acc = valid_acc
    best_model = copy.deepcopy(model)
    print('best valid_acc', best_valid_acc * 100.)
#'''1 0 -2.30193543434 -7.93079584148e-05 0.01
1 5 -2.25925588608 0.000434028402925 0.01
1 10 -1.88105452061 0.000863433082393 0.01
1 15 -1.42825007439 0.00091040908483 0.01
1 20 -1.19715201855 0.000916270582721 0.01
1 25 -1.08275806904 0.000974892798043 0.01
1 30 -0.983183026314 0.00104386101784 0.01
1 35 -0.919698655605 0.000678370184286 0.01
1 40 -0.844817101955 0.000926555426733 0.01
1 45 -0.817394971848 0.000922090877786 0.01
1 50 -0.814984381199 0.00116070012036 0.01
1 55 -0.805679738522 0.00116070012036 0.01
valid_acc 74.995
best valid_acc 74.995
2 0 -0.84048551321 0.00116070012036 0.01
2 5 -0.790781915188 0.00128451260662 0.01
2 10 -0.77977091074 0.00128451260662 0.01
2 15 -0.741036474705 0.00128451260662 0.01
2 20 -0.7792776227 0.00123299907838 0.01
2 25 -0.75918161869 0.00123299907838 0.01
2 30 -0.747699975967 0.00123299907838 0.01
2 35 -0.720850110054 0.00123299907838 0.01
2 40 -0.724429726601 0.00123299907838 0.01
2 45 -0.742388427258 0.00123299907838 0.01
2 50 -0.724133014679 7.19565741462e-05 0.01
2 55 -0.652908325195 3.64853758096e-05 0.01
valid_acc 77.13166666666666
best valid_acc 77.13166666666666
3 0 -0.645577549934 1.33878905309e-05 0.01
3 5 -0.682914495468 1.33878905309e-05 0.01
3 10 -0.648366212845 1.33878905309e-05 0.01
3 15 -0.638106167316 0.000125426673433 0.01
3 20 -0.540650069714 -9.21696463994e-05 0.01
3 25 -0.556898653507 -9.21696463994e-05 0.01
3 30 -0.638110458851 -9.21696463994e-05 0.01
3 35 -0.713058412075 -9.21696463994e-05 0.01
3 40 -0.622235655785 -9.21696463994e-05 0.01
3 45 -0.675054490566 -9.21696463994e-05 0.01
3 50 -0.584510922432 -9.21696463994e-05 0.01
3 55 -0.576616287231 -0.000112087453943 0.01
valid_acc 82.30499999999999
best valid_acc 82.30499999999999
4 0 -0.600017428398 -0.000112087453943 0.01
4 5 -0.619903504848 -0.000112087453943 0.01
4 10 -0.605637669563 -0.000112087453943 0.01
4 15 -0.557456731796 -0.000112087453943 0.01
4 20 -0.603248476982 -0.000112087453943 0.01
4 25 -0.516546428204 -0.000130976206558 0.01
4 30 -0.662497699261 -0.000419869767099 0.01
4 35 -0.497017562389 -0.000537615356387 0.01
4 40 -0.48434561491 -0.000537615356387 0.01
4 45 -0.572362482548 -0.000537615356387 0.01
4 50 -0.484995305538 -0.000537615356387 0.01
4 55 -0.402304798365 2.71777749911e-05 0.01
valid_acc 85.425
best valid_acc 85.425
5 0 -0.512217462063 2.71777749911e-05 0.01
5 5 -0.47760027647 0.000290281907917 0.01
5 10 -0.469509631395 0.000290281907917 0.01
5 15 -0.430136412382 0.000290281907917 0.01
5 20 -0.506188929081 -0.000288672359689 0.01
5 25 -0.466771543026 -0.000288672359689 0.01
5 30 -0.468196004629 -0.000288672359689 0.01
5 35 -0.437986165285 -0.000288672359689 0.01
5 40 -0.438537508249 -0.000288672359689 0.01
5 45 -0.387450665236 -0.000288672359689 0.01
5 50 -0.397666871548 -0.000288672359689 0.01
5 55 -0.439944893122 -0.000288672359689 0.01
valid_acc 86.67833333333334
best valid_acc 86.67833333333334
6 0 -0.347905814648 -0.00149817072964 0.01
6 5 -0.447554171085 -0.00149817072964 0.01
6 10 -0.402372598648 -0.00149817072964 0.01
6 15 -0.409267932177 -0.00149817072964 0.01
6 20 -0.4146451056 -0.00149817072964 0.01
6 25 -0.4839220047 -0.00149817072964 0.01
6 30 -0.454222351313 -0.000433241448375 0.01
6 35 -0.387279421091 -0.000433241448375 0.01
6 40 -0.334012746811 -0.000433241448375 0.01
6 45 -0.365834981203 -0.000433241448375 0.01
6 50 -0.404047071934 -0.000433241448375 0.01
6 55 -0.400403231382 -0.000433241448375 0.01
valid_acc 86.65166666666667
7 0 -0.363019198179 -0.000433241448375 0.01
7 5 -0.414384096861 -0.000433241448375 0.01
7 10 -0.379683673382 -0.000433241448375 0.01
7 15 -0.393472790718 -0.000433241448375 0.01
7 20 -0.4494561553 -0.000433241448375 0.01
7 25 -0.392005860806 -0.000433241448375 0.01
7 30 -0.335558623075 -0.000433241448375 0.01
7 35 -0.371694803238 -0.000433241448375 0.01
7 40 -0.416715025902 -0.000433241448375 0.01
7 45 -0.314037531614 -0.00198473241621 0.01
7 50 -0.318436920643 -0.00198473241621 0.01
7 55 -0.394181460142 -0.00198473241621 0.01
valid_acc 89.13333333333333
best valid_acc 89.13333333333333
8 0 -0.331167161465 -0.00198473241621 0.01
8 5 -0.384443312883 -0.00198473241621 0.01
8 10 -0.372689723969 -0.00259837116625 0.01
8 15 -0.373687326908 -0.00259837116625 0.01
8 20 -0.281910330057 -0.00291375822061 0.01
8 25 -0.323569297791 -0.00291375822061 0.01
8 30 -0.368499070406 -0.00291375822061 0.01
8 35 -0.337300509214 -0.00291375822061 0.01
8 40 -0.350602418184 -0.00291375822061 0.01
8 45 -0.389239996672 -0.00297230081081 0.01
8 50 -0.374930620193 -0.00297230081081 0.01
8 55 -0.296674877405 -0.00297230081081 0.01
valid_acc 89.74166666666666
best valid_acc 89.74166666666666
9 0 -0.320743650198 -0.00297230081081 0.01
9 5 -0.323158413172 -0.00297230081081 0.01
9 10 -0.336762040854 -0.00297230081081 0.01
9 15 -0.301319152117 -0.00297230081081 0.01
9 20 -0.282345205545 -0.00297230081081 0.01
9 25 -0.355873435736 -0.00297230081081 0.01
9 30 -0.357389628887 -0.00297230081081 0.01
9 35 -0.339281797409 -0.00297230081081 0.01
9 40 -0.308460146189 -0.00297230081081 0.01
9 45 -0.327920734882 -0.00355290293571 0.01
9 50 -0.308028161526 -0.00359936536822 0.01
9 55 -0.325912952423 -0.00359936536822 0.01
valid_acc 91.23166666666667
best valid_acc 91.23166666666667
10 0 -0.301850050688 -0.00359936536822 0.01
10 5 -0.319415986538 -0.00359936536822 0.01
10 10 -0.268859952688 -0.00359936536822 0.01
10 15 -0.26133787632 -0.00359936536822 0.01
10 20 -0.297950655222 -0.00359936536822 0.01
10 25 -0.307303011417 -0.00359936536822 0.01
10 30 -0.270721167326 -0.00266184739491 0.01
10 35 -0.265403568745 -0.00266184739491 0.01
10 40 -0.234723746777 -0.00266184739491 0.01
10 45 -0.238956734538 -0.00266184739491 0.01
10 50 -0.226362392306 -0.00285833689813 0.01
10 55 -0.216390743852 -0.00282641573113 0.01
valid_acc 92.28333333333333
best valid_acc 92.28333333333333
11 0 -0.313869148493 -0.00282641573113 0.01
11 5 -0.298807680607 -0.0022941715027 0.01
11 10 -0.261306345463 -0.0022941715027 0.01
11 15 -0.266971409321 -0.0022941715027 0.01
11 20 -0.281724721193 -0.0022941715027 0.01
11 25 -0.265448004007 -0.0022941715027 0.01
11 30 -0.235448226333 -0.0022941715027 0.01
11 35 -0.286105573177 -0.0022941715027 0.01
11 40 -0.341850072145 -0.0022941715027 0.01
11 45 -0.290374726057 -0.0022941715027 0.01
11 50 -0.237797632813 -0.0022941715027 0.01
11 55 -0.284512996674 -0.0022941715027 0.01
valid_acc 92.49333333333334
best valid_acc 92.49333333333334
12 0 -0.254266709089 -0.0022941715027 0.01
12 5 -0.284009665251 -0.0022941715027 0.01
12 10 -0.264256238937 -0.0022941715027 0.01
12 15 -0.244095250964 -0.0022941715027 0.01
12 20 -0.241942957044 -0.0022941715027 0.01
12 25 -0.259648025036 -0.0022941715027 0.01
12 30 -0.268098890781 -0.0022941715027 0.01
12 35 -0.271785199642 -0.0022941715027 0.01
12 40 -0.226829901338 -0.0022941715027 0.01
12 45 -0.269428253174 -0.0022941715027 0.01
12 50 -0.283169299364 -0.0022941715027 0.01
12 55 -0.263792365789 -0.0022941715027 0.01
valid_acc 92.30166666666668
13 0 -0.260377705097 -0.0022941715027 0.01
13 5 -0.230912268162 -0.0022941715027 0.01
13 10 -0.277780115604 -0.0022941715027 0.01
13 15 -0.252786666155 -0.0022941715027 0.01
13 20 -0.321997731924 -0.0022941715027 0.01
13 25 -0.224732428789 -0.0022941715027 0.01
13 30 -0.245754793286 -0.0022941715027 0.01
13 35 -0.249502882361 -0.0022941715027 0.01
13 40 -0.287863254547 -0.0022941715027 0.01
13 45 -0.256801664829 -0.0022941715027 0.01
13 50 -0.231633469462 -0.0022941715027 0.01
13 55 -0.281965464354 -0.0022941715027 0.01
valid_acc 92.78166666666667
best valid_acc 92.78166666666667
14 0 -0.28423923254 -0.0022941715027 0.01
14 5 -0.29552334547 -0.0022941715027 0.01
14 10 -0.215178668499 -0.0022941715027 0.01
14 15 -0.22211818397 -0.0022941715027 0.01
14 20 -0.27687934041 -0.0022941715027 0.01
14 25 -0.289502352476 -0.0022941715027 0.01
14 30 -0.224296152592 -0.0022941715027 0.01
14 35 -0.284197866917 -0.0022941715027 0.01
14 40 -0.218760222197 -0.0022941715027 0.01
14 45 -0.23111987114 -0.0022941715027 0.01
14 50 -0.245130345225 -0.0022941715027 0.01
14 55 -0.274078726768 -0.0022941715027 0.01
valid_acc 92.27166666666666
15 0 -0.23385155201 -0.0022941715027 0.01
15 5 -0.311232149601 -0.0022941715027 0.01
15 10 -0.371193230152 -0.0022941715027 0.01
evaluate(best_model, valid_loader, print_mode=True)
Average loss: 0.1493, Accuracy: 57571/60000 (95.9517%)

evaluate(best_model, test_loader, print_mode=True)
Average loss: 0.1398, Accuracy: 9623/10000 (96.2300%)

evaluate(best_model, train_loader, print_mode=True)
Average loss: 0.1493, Accuracy: 57571/60000 (95.9517%)

update_model(es.best_param(), model, model_shapes)_____no_output_____evaluate(model, valid_loader, print_mode=True)
Average loss: 0.1689, Accuracy: 57364/60000 (95.6067%)

evaluate(model, test_loader, print_mode=True)
Average loss: 0.1591, Accuracy: 9558/10000 (95.5800%)

evaluate(model, train_loader, print_mode=True)
Average loss: 0.1689, Accuracy: 57364/60000 (95.6067%)

update_model(es.current_param(), model, model_shapes)_____no_output_____evaluate(model, valid_loader, print_mode=True)
Average loss: 0.1554, Accuracy: 57321/60000 (95.5350%)

evaluate(model, test_loader, print_mode=True)
Average loss: 0.1479, Accuracy: 9569/10000 (95.6900%)

evaluate(model, train_loader, print_mode=True)
Average loss: 0.1554, Accuracy: 57321/60000 (95.5350%)

eval_acc = evaluate(best_model, test_loader)
print('final test acc', eval_acc * 100.)
Average loss: 0.1398, Accuracy: 9623/10000 (96.2300%)

final test acc 96.23
param_count = 0
for param in model.parameters():
  print(param.data.shape)
  param_count += np.product(param.data.shape)
print(param_count)torch.Size([8, 1, 5, 5])
torch.Size([8])
torch.Size([16, 8, 5, 5])
torch.Size([16])
torch.Size([10, 784])
torch.Size([10])
11274
orig_params = []
for param in orig_model.parameters():
  orig_params.append(param.data.cpu().numpy().flatten())_____no_output_____orig_params_flat = np.concatenate(orig_params)_____no_output_____import matplotlib.pyplot as plt_____no_output______ = plt.hist(orig_params_flat, bins=200)
plt.show()_____no_output_____final_params = []
for param in best_model.parameters():
  final_params.append(param.data.cpu().numpy().flatten())
final_params_flat = np.concatenate(final_params)_____no_output______ = plt.hist(final_params_flat, bins=200)
plt.show()_____no_output_____
</code>