# Notebook from nesl/tinyodom
Path: RoNIN/TinyOdom_RoNIN.ipynb

import os
import warnings
warnings.filterwarnings('ignore')
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
os.environ["CUDA_VISIBLE_DEVICES"]="1"
import pandas as pd
import numpy as np
from gtda.time_series import SlidingWindow
import matplotlib.pyplot as plt
from math import atan2, pi, sqrt, cos, sin, floor
from data_utils import *
import tensorflow as tf
from tensorflow.python.keras.backend import set_session
config = tf.compat.v1.ConfigProto()
config.gpu_options.allow_growth = True
config.log_device_placement = True
sess2 = tf.compat.v1.Session(config=config)
set_session(sess2)
from tensorflow.keras.layers import Dense, MaxPooling1D, Flatten
from tensorflow.keras import Input, Model
from tensorflow.keras.callbacks import ModelCheckpoint
import tensorflow.compat.v1.keras.backend as K
from tensorflow.keras.models import load_model
from tcn import TCN, tcn_full_summary
from sklearn.metrics import mean_squared_error
from mango.tuner import Tuner
from scipy.stats import uniform
from keras_flops import get_flops
import pickle
import csv
import random
import itertools
import quaternion
import math
from hardware_utils import *
import time_____no_output_____

## Import Training, Validation and Test Set_____no_output_____

sampling_rate = 200
window_size = 400
stride = 20
f = '/home/nesl/swapnil/Aquamote_Baselines/Human/datasets/ronin/' #dataset directory_____no_output_____#Training Set
X, Y_disp, Y_head, Y_pos, x0_list, y0_list, size_of_each, x_vel, y_vel, head_s, head_c, X_orig =import_ronin_dataset(type_flag = 2,
                         useMagnetometer = True, useStepCounter = True, AugmentationCopies = 0,
                         dataset_folder = f, sampling_rate = sampling_rate, window_size = window_size, stride = stride,verbose=False)_____no_output_____#Validation Set
X_val, Y_disp_val, Y_head_val, Y_pos_val, x0_list_val, y0_list_val, size_of_each_val, x_vel_val, y_vel_val, head_s_val, head_c_val, X_orig_val = X, Y_disp, Y_head, Y_pos, x0_list, y0_list, size_of_each, x_vel, y_vel, head_s, head_c, X_orig =import_ronin_dataset(type_flag = 3,
                         useMagnetometer = True, useStepCounter = True, AugmentationCopies = 0,
                         dataset_folder = f, sampling_rate = sampling_rate, window_size = window_size, stride = stride,verbose=False)_____no_output_____#Seen Subject Test Set
X_test, Y_disp_test, Y_head_test, Y_pos_test, x0_list_test, y0_list_test, size_of_each_test, x_vel_test, y_vel_test, head_s_test, head_c_test, X_orig_test = X, Y_disp, Y_head, Y_pos, x0_list, y0_list, size_of_each, x_vel, y_vel, head_s, head_c, X_orig =import_ronin_dataset(type_flag = 4,
                         useMagnetometer = True, useStepCounter = True, AugmentationCopies = 0,
                         dataset_folder = f, sampling_rate = sampling_rate, window_size = window_size, stride = stride,verbose=False)_____no_output_____

## Training and NAS_____no_output_____

device = "NUCLEO_F746ZG" #hardware name
model_name = 'TD_RoNIN_'+device+'.hdf5'
dirpath="/home/nesl/Mbed Programs/tinyodom_tcn/" #hardware program directory
HIL = True #use real hardware or proxy?
quantization = False #use quantization or not?
model_epochs = 900 #epochs to train each model for
NAS_epochs = 50 #epochs for hyperparameter tuning
output_name = 'g_model.tflite'
log_file_name = 'log_NAS_RoNiN_'+device+'.csv'
if os.path.exists(log_file_name):
    os.remove(log_file_name)
row_write = ['score', 'rmse_vel_x','rmse_vel_y','RAM','Flash','Flops','Latency',
                 'nb_filters','kernel_size','dilations','dropout_rate','use_skip_connections','norm_flag']
with open(log_file_name, 'a', newline='') as csvfile:
    csvwriter = csv.writer(csvfile)
    csvwriter.writerow(row_write)
if os.path.exists(log_file_name[0:-4]+'.p'):
    os.remove(log_file_name[0:-4]+'.p')_____no_output_____def objective_NN(epochs=500,nb_filters=32,kernel_size=7,dilations=[1, 2, 4, 8, 16, 32, 64, 128],dropout_rate=0,
                 use_skip_connections=False,norm_flag=0):

    inval = 0
    rmse_vel_x = 'inf'
    rmse_vel_y = 'inf'
    batch_size, timesteps, input_dim = 256, window_size, X.shape[2]
    i = Input(shape=(timesteps, input_dim))

    if(norm_flag==1):
        m = TCN(nb_filters=nb_filters,kernel_size=kernel_size,dilations=dilations,dropout_rate=dropout_rate,
                use_skip_connections=use_skip_connections,use_batch_norm=True)(i)
    else:
        m = TCN(nb_filters=nb_filters,kernel_size=kernel_size,dilations=dilations,dropout_rate=dropout_rate,
                use_skip_connections=use_skip_connections)(i)

    m = tf.reshape(m, [-1, nb_filters, 1])

    m = MaxPooling1D(pool_size=(2))(m)
    m = Flatten()(m)
    m = Dense(32, activation='linear', name='pre')(m)
    output1 = Dense(1, activation='linear', name='velx')(m)
    output2 = Dense(1, activation='linear', name='vely')(m)
    model = Model(inputs=[i], outputs=[output1, output2])
    opt = tf.keras.optimizers.Adam()
    model.compile(loss={'velx': 'mse','vely':'mse'},optimizer=opt)
    Flops = get_flops(model, batch_size=1)
    convert_to_tflite_model(model=model,training_data=X,quantization=quantization,output_name=output_name)
    maxRAM, maxFlash = return_hardware_specs(device)

    if(HIL==True):
        convert_to_cpp_model(dirpath)
        RAM, Flash, Latency, idealArenaSize, errorCode = HIL_controller(dirpath=dirpath,
                                                                       chosen_device=device,
                                                                       window_size=window_size,
                                                                    number_of_channels = input_dim,
                                                                   quantization=quantization)
        score = -5.0
        if(Flash==-1):
            row_write = [score, rmse_vel_x,rmse_vel_y,RAM,Flash,Flops,Latency,
                 nb_filters,kernel_size,dilations,dropout_rate,use_skip_connections,norm_flag]
            print('Design choice:',row_write)
            with open(log_file_name, 'a', newline='') as csvfile:
                csvwriter = csv.writer(csvfile)
                csvwriter.writerow(row_write)
            return score

        elif(Flash!=-1):
            checkpoint = ModelCheckpoint(model_name, monitor='loss', verbose=1, save_best_only=True)
            model.fit(x=X, y=[x_vel, y_vel],epochs=epochs, shuffle=True,callbacks=[checkpoint],batch_size=batch_size)
            model = load_model(model_name,custom_objects={'TCN': TCN})
            y_pred = model.predict(X_val)
            rmse_vel_x = mean_squared_error(x_vel_val, y_pred[0], squared=False)
            rmse_vel_y = mean_squared_error(y_vel_val, y_pred[1], squared=False)
            model_acc = -(rmse_vel_x+rmse_vel_y)
            resource_usage = (RAM/maxRAM) + (Flash/maxFlash)
            score = model_acc + 0.01*resource_usage - 0.05*Latency  #weigh each component as you like

            row_write = [score, rmse_vel_x,rmse_vel_y,RAM,Flash,Flops,Latency,
                 nb_filters,kernel_size,dilations,dropout_rate,use_skip_connections,norm_flag]
            print('Design choice:',row_write)
            with open(log_file_name, 'a', newline='') as csvfile:
                csvwriter = csv.writer(csvfile)
                csvwriter.writerow(row_write)

    else:
        score = -5.0
        Flash = os.path.getsize(output_name)
        RAM = get_model_memory_usage(batch_size=1,model=model)
        Latency=-1
        max_flops = (30e6)

        if(RAM < maxRAM and Flash<maxFlash):
            checkpoint = ModelCheckpoint(model_name, monitor='loss', verbose=1, save_best_only=True)
            model.fit(x=X, y=[x_vel, y_vel],epochs=epochs, shuffle=True,callbacks=[checkpoint],batch_size=batch_size)
            model = load_model(model_name,custom_objects={'TCN': TCN})
            y_pred = model.predict(X_val)
            rmse_vel_x = mean_squared_error(x_vel_val, y_pred[0], squared=False)
            rmse_vel_y = mean_squared_error(y_vel_val, y_pred[1], squared=False)
            model_acc = -(rmse_vel_x+rmse_vel_y)
            resource_usage = (RAM/maxRAM) + (Flash/maxFlash)
            score = model_acc + 0.01*resource_usage - 0.05*(Flops/max_flops)  #weigh each component as you like

        row_write = [score, rmse_vel_x,rmse_vel_y,RAM,Flash,Flops,Latency,
                 nb_filters,kernel_size,dilations,dropout_rate,use_skip_connections,norm_flag]
        print('Design choice:',row_write)
        with open(log_file_name, 'a', newline='') as csvfile:
            csvwriter = csv.writer(csvfile)
            csvwriter.writerow(row_write)

    return score_____no_output_____import pickle

def save_res(data, file_name):
    pickle.dump( data, open( file_name, "wb" ) )

min_layer = 3
max_layer = 8
a_list = [1,2,4,8,16,32,64,128,256]
all_combinations = []
dil_list = []
for r in range(len(a_list) + 1):
    combinations_object = itertools.combinations(a_list, r)
    combinations_list = list(combinations_object)
    all_combinations += combinations_list
all_combinations = all_combinations[1:]
for item in all_combinations:
    if(len(item) >= min_layer and len(item) <= max_layer):
        dil_list.append(list(item))

param_dict = {
    'nb_filters': range(2,64),
    'kernel_size': range(2,16),
    'dropout_rate': np.arange(0.0,0.5,0.1),
    'use_skip_connections': [True, False],
    'norm_flag': np.arange(0,1),
    'dil_list': dil_list
}

def objfunc(args_list):

    objective_evaluated = []

    start_time = time.time()

    for hyper_par in args_list:
        nb_filters = hyper_par['nb_filters']
        kernel_size = hyper_par['kernel_size']
        dropout_rate = hyper_par['dropout_rate']
        use_skip_connections = hyper_par['use_skip_connections']
        norm_flag=hyper_par['norm_flag']
        dil_list = hyper_par['dil_list']

        objective = objective_NN(epochs=model_epochs,nb_filters=nb_filters,kernel_size=kernel_size,
                                 dilations=dil_list,
                                 dropout_rate=dropout_rate,use_skip_connections=use_skip_connections,
                                 norm_flag=norm_flag)
        objective_evaluated.append(objective)

        end_time = time.time()
        print('objective:', objective, ' time:',end_time-start_time)

    return objective_evaluated

conf_Dict = dict()
conf_Dict['batch_size'] = 1
conf_Dict['num_iteration'] = NAS_epochs
conf_Dict['initial_random']= 5
tuner = Tuner(param_dict, objfunc,conf_Dict)
all_runs = []
results = tuner.maximize()
all_runs.append(results)
save_res(all_runs,log_file_name[0:-4]+'.p')_____no_output_____

## Train the Best Model_____no_output_____

nb_filters = results['best_params']['nb_filters']
kernel_size = results['best_params']['kernel_size']
dilations = results['best_params']['dilations']
dropout_rate = results['best_params']['dropout_rate']
use_skip_connections = results['best_params']['use_skip_connections']
norm_flag = results['best_params']['norm_flag']

batch_size, timesteps, input_dim = 256, window_size, X.shape[2]
i = Input(shape=(timesteps, input_dim))
if(norm_flag==1):
    m = TCN(nb_filters=nb_filters,kernel_size=kernel_size,dilations=dilations,dropout_rate=dropout_rate,
            use_skip_connections=use_skip_connections,use_batch_norm=True)(i)
else:
    m = TCN(nb_filters=nb_filters,kernel_size=kernel_size,dilations=dilations,dropout_rate=dropout_rate,
            use_skip_connections=use_skip_connections)(i)

m = tf.reshape(m, [-1, nb_filters, 1])
m = MaxPooling1D(pool_size=(2))(m)
m = Flatten()(m)
m = Dense(32, activation='linear', name='pre')(m)
output1 = Dense(1, activation='linear', name='velx')(m)
output2 = Dense(1, activation='linear', name='vely')(m)
model = Model(inputs=[i], outputs=[output1, output2])
opt = tf.keras.optimizers.Adam()
model.compile(loss={'velx': 'mse','vely':'mse'},optimizer=opt)
checkpoint = ModelCheckpoint(model_name, monitor='loss', verbose=1, save_best_only=True)
model.fit(x=X, y=[x_vel, y_vel],epochs=model_epochs, shuffle=True,callbacks=[checkpoint],batch_size=batch_size)     _____no_output_____

## Evaluate the Best Model_____no_output_____#### Velocity Prediction RMSE_____no_output_____

model = load_model(model_name,custom_objects={'TCN': TCN})
y_pred = model.predict(X_test)
rmse_vel_x = mean_squared_error(x_vel_test, y_pred[0], squared=False)
rmse_vel_y = mean_squared_error(y_vel_test, y_pred[1], squared=False)
print('Vel_X RMSE, Vel_Y RMSE:',rmse_vel_x,rmse_vel_y)_____no_output_____

#### ATE and RTE Metrics_____no_output_____

a = 0
b = size_of_each_test[0]
ATE = []
RTE = []
ATE_dist = []
RTE_dist = []
for i in range(len(size_of_each_test)):
    X_test_sel = X_test[a:b,:,:]
    x_vel_test_sel = x_vel_test[a:b]
    y_vel_test_sel = y_vel_test[a:b]
    Y_head_test_sel = Y_head_test[a:b]
    Y_disp_test_sel = Y_disp_test[a:b]
    if(i!=len(size_of_each_test)-1):
        a += size_of_each_test[i]
        b += size_of_each_test[i]

    y_pred = model.predict(X_test_sel)

    pointx = []
    pointy = []
    Lx =  x0_list_test[i]
    Ly = y0_list_test[i]
    for j in range(len(x_vel_test_sel)):
        Lx = Lx + (x_vel_test_sel[j]/(((window_size-stride)/stride)))
        Ly = Ly + (y_vel_test_sel[j]/(((window_size-stride)/stride)))
        pointx.append(Lx)
        pointy.append(Ly)
    Gvx = pointx
    Gvy = pointy

    pointx = []
    pointy = []
    Lx =  x0_list_test[i]
    Ly = y0_list_test[i]
    for j in range(len(x_vel_test_sel)):
        Lx = Lx + (y_pred[0][j]/(((window_size-stride)/stride)))
        Ly = Ly + (y_pred[1][j]/(((window_size-stride)/stride)))
        pointx.append(Lx)
        pointy.append(Ly)
    Pvx = pointx
    Pvy = pointy

    at, rt, at_all, rt_all = Cal_TE(Gvx, Gvy, Pvx, Pvy,
                                    sampling_rate=sampling_rate,window_size=window_size,stride=stride)
    ATE.append(at)
    RTE.append(rt)
    ATE_dist.append(Cal_len_meters(Gvx, Gvy))
    RTE_dist.append(Cal_len_meters(Gvx, Gvy, 600))
    print('ATE, RTE, Trajectory Length, Trajectory Length (60 seconds)',ATE[i],RTE[i],ATE_dist[i],RTE_dist[i])

print('Median ATE and RTE', np.median(ATE),np.median(RTE))_____no_output_____

#### Sample Trajectory Plotting_____no_output_____

#you can use the size_of_each_test variable to control the region to plot. We plot for the last trajectory
a = sum(size_of_each_test[0:5])
b = sum(size_of_each_test[0:5])+900

X_test_sel = X_test[a:b,:,:]
x_vel_test_sel = x_vel_test[a:b]
y_vel_test_sel = y_vel_test[a:b]
Y_head_test_sel = Y_head_test[a:b]
Y_disp_test_sel = Y_disp_test[a:b]

y_pred = model.predict(X_test_sel)

pointx = []
pointy = []
Lx =  x0_list_test[i]
Ly = y0_list_test[i]
for j in range(len(x_vel_test_sel)):
    Lx = Lx + (x_vel_test_sel[j]/(((window_size-stride)/stride)))
    Ly = Ly + (y_vel_test_sel[j]/(((window_size-stride)/stride)))
    pointx.append(Lx)
    pointy.append(Ly)
Gvx = pointx
Gvy = pointy

pointx = []
pointy = []
Lx =  x0_list_test[i]
Ly = y0_list_test[i]
for j in range(len(x_vel_test_sel)):
    Lx = Lx + (y_pred[0][j]/(((window_size-stride)/stride)))
    Ly = Ly + (y_pred[1][j]/(((window_size-stride)/stride)))
    pointx.append(Lx)
    pointy.append(Ly)
Pvx = pointx
Pvy = pointy

print('Plotting Trajectory of length (meters): ',Cal_len_meters(Gvx, Gvy))

ptox = Pvx
ptoy = Pvy

plt.plot(Gvx,Gvy,label='Ground Truth',color='salmon')
plt.plot(ptox,ptoy,label='TinyOdom',color='green',linestyle='-')
plt.grid()
plt.legend(loc='best')
plt.title('PDR - RoNIN Dataset')
plt.xlabel('East (m)')
plt.ylabel('North (m)')
plt.show()_____no_output_____

#### Error Evolution_____no_output_____

#For the last trajectory

a = sum(size_of_each_test[0:5])
b = sum(size_of_each_test[0:5])+size_of_each_test[5]

X_test_sel = X_test[a:b,:,:]
x_vel_test_sel = x_vel_test[a:b]
y_vel_test_sel = y_vel_test[a:b]
Y_head_test_sel = Y_head_test[a:b]
Y_disp_test_sel = Y_disp_test[a:b]

y_pred = model.predict(X_test_sel)

pointx = []
pointy = []
Lx =  x0_list_test[i]
Ly = y0_list_test[i]
for j in range(len(x_vel_test_sel)):
    Lx = Lx + (x_vel_test_sel[j]/(((window_size-stride)/stride)))
    Ly = Ly + (y_vel_test_sel[j]/(((window_size-stride)/stride)))
    pointx.append(Lx)
    pointy.append(Ly)
Gvx = pointx
Gvy = pointy

pointx = []
pointy = []
Lx =  x0_list_test[i]
Ly = y0_list_test[i]
for j in range(len(x_vel_test_sel)):
    Lx = Lx + (y_pred[0][j]/(((window_size-stride)/stride)))
    Ly = Ly + (y_pred[1][j]/(((window_size-stride)/stride)))
    pointx.append(Lx)
    pointy.append(Ly)
Pvx = pointx
Pvy = pointy

at, rt, at_all, rt_all = Cal_TE(Gvx, Gvy, Pvx, Pvy,
                                    sampling_rate=sampling_rate,window_size=window_size,stride=stride)

x_ax = np.linspace(0,60,len(rt_all))
print('Plotting for trajectory of length (meters): ',Cal_len_meters(Gvx, Gvy))

plt.plot(x_ax,rt_all,label='TinyOdom',color='green',linestyle='-')
plt.legend()
plt.xlabel('Time (seconds)')
plt.ylabel('Position Error (m)')
plt.title('PDR - RoNIN Dataset')
plt.grid()
plt.show()_____no_output_____

## Deployment_____no_output_____#### Conversion to TFLite_____no_output_____

convert_to_tflite_model(model=model,training_data=X_tr,quantization=quantization,output_name='g_model.tflite') _____no_output_____

#### Conversion to C++_____no_output_____

convert_to_cpp_model(dirpath)_____no_output_____


# Notebook from DaneseAnna/scATAC-benchmarking
Path: Extra/10x_PBMC_5k/test_firstPC/run_clustering_10xpbmc5k.ipynb

<code>
import pandas as pd
import numpy as np
import scanpy as sc
from scipy import stats
import os
from sklearn.cluster import KMeans
from sklearn.cluster import AgglomerativeClustering
from sklearn.metrics.cluster import adjusted_rand_score
from sklearn.metrics.cluster import adjusted_mutual_info_score
from sklearn.metrics.cluster import homogeneity_score
import rpy2.robjects as robjects
from rpy2.robjects import pandas2ri_____no_output_____# df_metrics = pd.DataFrame(columns=['ARI_Louvain','ARI_kmeans','ARI_HC',
#                                    'AMI_Louvain','AMI_kmeans','AMI_HC',
#                                    'Homogeneity_Louvain','Homogeneity_kmeans','Homogeneity_HC'])_____no_output_____workdir = './output_new/'
path_fm = os.path.join(workdir,'feature_matrices/')
path_clusters = os.path.join(workdir,'clusters/')
path_metrics = os.path.join(workdir,'metrics/')
os.system('mkdir -p '+path_clusters)
os.system('mkdir -p '+path_metrics)_____no_output_____metadata = pd.read_csv('../input/metadata.tsv',sep='\t',index_col=0)
num_clusters = len(np.unique(metadata['label']))
print(num_clusters)8
files = [x for x in os.listdir(path_fm) if x.startswith('fm')]
len(files)_____no_output_____files_____no_output_____def getNClusters(adata,n_cluster,range_min=0,range_max=3,max_steps=20):
    this_step = 0
    this_min = float(range_min)
    this_max = float(range_max)
    while this_step < max_steps:
        print('step ' + str(this_step))
        this_resolution = this_min + ((this_max-this_min)/2)
        sc.tl.louvain(adata,resolution=this_resolution)
        this_clusters = adata.obs['louvain'].nunique()

        print('got ' + str(this_clusters) + ' at resolution ' + str(this_resolution))

        if this_clusters > n_cluster:
            this_max = this_resolution
        elif this_clusters < n_cluster:
            this_min = this_resolution
        else:
            return(this_resolution, adata)
        this_step += 1

    print('Cannot find the number of clusters')
    print('Clustering solution from last iteration is used:' + str(this_clusters) + ' at resolution ' + str(this_resolution))_____no_output_____for file in files:
    file_split = file[:-4].split('_')
    method = file_split[1]
    if(len(file_split)>2):
        method = method + '_'+''.join(file_split[2:]).replace('2','_pca')
    print(method)

    pandas2ri.activate()
    readRDS = robjects.r['readRDS']
    df_rds = readRDS(os.path.join(path_fm,file))
    fm_mat = pandas2ri.ri2py(robjects.r['data.frame'](robjects.r['as.matrix'](df_rds)))
    fm_mat.fillna(0,inplace=True)
    fm_mat.columns = metadata.index

    adata = sc.AnnData(fm_mat.T)
    adata.var_names_make_unique()
    adata.obs = metadata.loc[adata.obs.index,]
#     df_metrics.loc[method,] = ""
    #Louvain
    sc.pp.neighbors(adata, n_neighbors=15,use_rep='X')
#     sc.tl.louvain(adata)
    getNClusters(adata,n_cluster=num_clusters)
    #kmeans
    kmeans = KMeans(n_clusters=num_clusters, random_state=2019).fit(adata.X)
    adata.obs['kmeans'] = pd.Series(kmeans.labels_,index=adata.obs.index).astype('category')
    #hierachical clustering
    hc = AgglomerativeClustering(n_clusters=num_clusters).fit(adata.X)
    adata.obs['hc'] = pd.Series(hc.labels_,index=adata.obs.index).astype('category')
#     #clustering metrics

#     #adjusted rank index
#     ari_louvain = adjusted_rand_score(adata.obs['label'], adata.obs['louvain'])
#     ari_kmeans = adjusted_rand_score(adata.obs['label'], adata.obs['kmeans'])
#     ari_hc = adjusted_rand_score(adata.obs['label'], adata.obs['hc'])
#     #adjusted mutual information
#     ami_louvain = adjusted_mutual_info_score(adata.obs['label'], adata.obs['louvain'],average_method='arithmetic')
#     ami_kmeans = adjusted_mutual_info_score(adata.obs['label'], adata.obs['kmeans'],average_method='arithmetic')
#     ami_hc = adjusted_mutual_info_score(adata.obs['label'], adata.obs['hc'],average_method='arithmetic')
#     #homogeneity
#     homo_louvain = homogeneity_score(adata.obs['label'], adata.obs['louvain'])
#     homo_kmeans = homogeneity_score(adata.obs['label'], adata.obs['kmeans'])
#     homo_hc = homogeneity_score(adata.obs['label'], adata.obs['hc'])

#     df_metrics.loc[method,['ARI_Louvain','ARI_kmeans','ARI_HC']] = [ari_louvain,ari_kmeans,ari_hc]
#     df_metrics.loc[method,['AMI_Louvain','AMI_kmeans','AMI_HC']] = [ami_louvain,ami_kmeans,ami_hc]
#     df_metrics.loc[method,['Homogeneity_Louvain','Homogeneity_kmeans','Homogeneity_HC']] = [homo_louvain,homo_kmeans,homo_hc]
    adata.obs[['louvain','kmeans','hc']].to_csv(os.path.join(path_clusters ,method + '_clusters.tsv'),sep='\t')Control
def residual_average_gini_index(gene_scores,folder_clusters,
                                housekeeping_genes,marker_genes,
                                min_cells_per_cluster=10):

    #Subset from the main matrix the housekeeping genes and marker genes
    df_matrix_housekeeping=gene_scores.loc[gene_scores.index.intersection(housekeeping_genes),]
    df_matrix_marker=gene_scores.loc[gene_scores.index.intersection(marker_genes),]

    #Define a function to compute the Gini score
    def gini(list_of_values):
        sorted_list = sorted(list_of_values)
        height, area = 0, 0
        for value in sorted_list:
            height += value
            area += height - value / 2.
            fair_area = height * len(list_of_values) / 2.
        return (fair_area - area) / fair_area

    #Function to calculate Gini value for all the genes
    def calculate_gini(df_matrix, gene_name,clustering_info):
        return gini(get_avg_per_cluster(df_matrix,gene_name,clustering_info,use_log2=False))

    #Function to calculate Gini value for all the genes
    def calculate_gini_values(df_matrix,clustering_info):
        gini_values=[]
        for gene_name in df_matrix.index:
            gini_values.append(calculate_gini(df_matrix, gene_name,clustering_info))
        return gini_values

    #Write a function to compute delta difference of the average accessibility in Marker vs Housekeeping and Kolmogorov Smirnov test
    def score_clustering_solution(df_matrix_marker,df_matrix_housekeeping,clustering_info):
        gini_values_housekeeping=calculate_gini_values(df_matrix_housekeeping,clustering_info)
        gini_values_marker=calculate_gini_values(df_matrix_marker,clustering_info)
        statistic,p_value=stats.ks_2samp(gini_values_marker,gini_values_housekeeping)

        return  np.mean(gini_values_marker), np.mean(gini_values_housekeeping),np.mean(gini_values_marker)-np.mean(gini_values_housekeeping), statistic,p_value

    #Function to compute the average accessibility value per cluster
    def get_avg_per_cluster(df_matrix, gene_name, clustering_info,use_log2=False):
        N_clusters=len(clustering_info.index.unique())
        avg_per_cluster=np.zeros(N_clusters)
        for idx,idx_cluster in enumerate(sorted(np.unique(clustering_info.index.unique()))):
            if use_log2:
                values_cluster=df_matrix.loc[gene_name,clustering_info.loc[idx_cluster,:].values.flatten()].apply(lambda x:np.log2(x+1))
            else:
                values_cluster=df_matrix.loc[gene_name,clustering_info.loc[idx_cluster,:].values.flatten()]

            avg_per_cluster[idx]=values_cluster.mean()
            if avg_per_cluster[idx]>0:
                  avg_per_cluster[idx]=avg_per_cluster[idx]#/values_cluster.std()

        return avg_per_cluster

    #Run the method for all the clustering solutions

    df_metrics = pd.DataFrame(columns=['Method','Clustering','Gini_Marker_Genes','Gini_Housekeeping_Genes','Difference','KS_statistics','p-value'])

    for clusters_filename in os.listdir(folder_clusters):
        method = '_'.join(clusters_filename.split('_')[:-1])
        print(method)
        df_clusters = pd.read_csv(os.path.join(folder_clusters,clusters_filename),sep='\t',index_col=0)
        for clustering_method in df_clusters.columns:
            clustering_info = pd.DataFrame(df_clusters[clustering_method])
            clustering_info['Barcode'] = clustering_info.index
            clustering_info=clustering_info.set_index(clustering_method)

            #REMOVE CLUSTERS WITH FEW CELLS
            cluster_sizes=pd.value_counts(clustering_info.index)
            clustering_info=clustering_info.loc[cluster_sizes[cluster_sizes>min_cells_per_cluster].index.values,:]

            mean_gini_marker,mean_gini_housekeeping,mean_gini_difference,statistics,p_value=score_clustering_solution(df_matrix_marker,df_matrix_housekeeping,clustering_info)

            df_metrics = df_metrics.append({'Method': method,'Clustering':clustering_method,
                               'Gini_Marker_Genes':mean_gini_marker,'Gini_Housekeeping_Genes':mean_gini_housekeeping,
                               'Difference':mean_gini_difference,'KS_statistics':statistics,'p-value':p_value},
                              ignore_index=True)

    return df_metrics_____no_output_____gene_scores = pd.read_csv('./FM_GeneScoring_10xpbmc5k.tsv',
                         sep = '\t',index_col=0)_____no_output_____gene_scores.head()_____no_output_____#https://www.tau.ac.il/~elieis/Housekeeping_genes.html List of Housekeeping genes
housekeeping_genes=['ACTB','ALDOA','GAPDH','PGK1','LDHA','RPS27A','RPL19','RPL11','NONO','ARHGDIA','RPL32','RPS18','HSPCB',
    'C1orf43','CHMP2A','EMC7','GPI','PSMB2,''PSMB4','RAB7A','REEP5','SNRPD3','VCP','VPS29']

#List of Marker Genes
marker_genes=['CD209', 'ENG', 'FOXP3', 'CD34', 'BATF3', 'S100A12', 'THBD','CD3D', 'THY1', 'CD8A', 'CD8B', 'CD14', 'PROM1', 'IL2RA', 'FCGR3A',
'IL3RA', 'FCGR1A', 'CD19', 'IL7R', 'CD79A', 'MS4A1', 'NCAM1','CD3E', 'CD3G', 'KIT', 'CD1C', 'CD68', 'CD4']_____no_output_____folder_clusters = './output_new/clusters/'_____no_output_____df_metrics = residual_average_gini_index(gene_scores,folder_clusters,
                                housekeeping_genes,marker_genes,
                                min_cells_per_cluster=10)Control
Control_new
BROCKMAN
BROCKMAN_new
Cusanovich2018
Cusanovich2018_new
chromVAR_kmers_pca
chromVAR_kmers_pcanew
chromVAR_motifs_pca
chromVAR_motifs_pcanew
GeneScoring2
GeneScoring2_new
Cicero2
Cicero2_new
SnapATAC
SnapATAC_new
SCRAT2
SCRAT2_new
df_metrics.to_csv(path_metrics+'clustering_RAGI_scores.csv')_____no_output_____df_metrics_____no_output_____
</code>


# Notebook from nicovandenhooff/wids-datathon-2022
Path: wids-datathon-2022.ipynb

_____no_output_____# 2022 Women in Data Science ("WiDS") Kaggle Competition

- **Purpose**: This notebook contains my final solution for the 2022 WiDS Kaggle Competition.
- **Problem Scope**: Using machine learning to predict building energy usage (regression).
- **Final leaderboard rank**: 82 / 829 (Top 10%) after removing deleted accounts from leaderboard
    - Apparently the deleted accounts were deleted for cheating based on the discussions thread.
- **Related GitHub repository**: https://github.com/nicovandenhooff/wids-datathon-2022
- **Author**: [Nico Van den Hooff, CPA](https://www.linkedin.com/in/nicovandenhooff/)
- **Occupaction**: Graduate Student in the Master of Data Science program at the University of British Columbia._____no_output_____# Abstract

> *This Kaggle notebook contains my final solution to the 2022 WiDS competition.  The competition involved predicting a buildings Site Energy Usage Intensity metric, which was the `site_eui` feature in the dataset (regression problem).  To solve this problem, I first separated the dataset into twelve individual datasets based on buildings with similar `site_eui` usage patterns and other characteristics.  I then engineer features, perform leave one group out cross validation, and finally train an ensemble model (XGBoost, LightGBM, and CatBoost regressors) for each dataset on the most powerful features.  My final solution ended up in the top 10% of the final leaderboard.*_____no_output_____# Table of Contents

* [Section 1: Notebook Set Up](#Section-1:-Notebook-Set-Up)
* [Section 2: Helper Functions](#Section-2:-Helper-Functions)
* [Section 3: Data Preprocessing](#Section-3:-Data-Preprocessing)
* [Section 4: Feature Engineering](#Section-4:-Feature-Engineering)
* [Section 5: Modelling](#Section-5:-Modelling)
* [Section 6: Final models and predictions](#Section-6:-Final-models-and-predictions)_____no_output_____# Section 1: Notebook Set Up

In this section I import the packages that are used throughout my solution and I perform general notebook set up._____no_output_____## 1.1 Imports_____no_output_____

import numpy as np
import pandas as pd
import xgboost as xgb
import catboost as cb
import lightgbm as lgb
import matplotlib.pyplot as plt

from copy import deepcopy
from collections import defaultdict

from sklearn.linear_model import Ridge
from sklearn.pipeline import make_pipeline
from sklearn.ensemble import VotingRegressor
from sklearn.metrics import mean_squared_error
from sklearn.compose import make_column_transformer
from sklearn.preprocessing import OneHotEncoder, StandardScaler
from sklearn.model_selection import RandomizedSearchCV, train_test_split, LeaveOneGroupOut

from sklearn.experimental import enable_iterative_imputer
from sklearn.impute import IterativeImputer_____no_output_____

## 1.2 Global notebook variables_____no_output_____

TRAIN_PATH = "../input/widsdatathon2022/train.csv"
TEST_PATH = "../input/widsdatathon2022/test.csv"
SUBMISSION_PATH = "../output/"
TARGET = "site_eui"
SCORE = "neg_root_mean_squared_error"
SEED = 777_____no_output_____

# Section 2: Helper Functions

This section contains several helper functions that are used throughout my notebook.  I have hidden the cell on the Kaggle viewer, as there are several hundreds of lines of code here.  Please feel free to expand it and read the code.  You can also find all of the function organized into scripts on my [GitHub repository](https://github.com/nicovandenhooff/wids-datathon-2022)._____no_output_____

def read_data(train_path, test_path):
    """Read in train and test data for a kaggle competition.

    Parameters
    ----------
    train_path : str
        The path to the training data.
    test_path : str
        The path to the test data

    Returns
    -------
    train_df, test_df : pandas DataFrames
        The train and test datasets.
    """
    train_df = pd.read_csv(train_path)
    test_df = pd.read_csv(test_path)
    return train_df, test_df

def get_duplicates(df, drop_cols=None):
    """Determine and return the duplicated values in a dataframe.

    Parameters
    ----------
    df : pandas DataFrame
        The dataframe to check
    drop_cols : str or list of str, optional
        The columns to drop before returning duplicates.

    Returns
    -------
    pandas DataFrame
        A dataframe containing the rows with duplicated values.
    """
    if drop_cols is not None:
        return df[df.drop(columns=drop_cols).duplicated()]
    else:
        return df[df.duplicated()]

def remove_duplicates(df, drop_cols=None):
    """Removes the duplicated values in a dataframe.

    Parameters
    ----------
    df : pandas DataFrame
        The dataframe to check
    drop_cols : str or list of str, optional
        The columns to drop before removing duplicates.
    Returns
    -------
    pandas DataFrame
        A dataframe without duplicated.
    """
    df_clean = df.copy()

    if drop_cols is not None:
        df_clean = df_clean[~df_clean.drop(columns=drop_cols).duplicated()]
    else:
        df_clean = df_clean[~df_clean.duplicated()]

    return df_clean.reset_index(drop=True)

def split_building_data(df, groups):
    """Splits the WiDS 2022 dataset based on groups of facility types.

    Parameters
    ----------
    df : pandas DataFrame
        The WiDS train or test dataset.
    groups : dict (key: str, value: set)
        The dictionary of facility types.  The key should be the desired group name.
        The value should be a set containing the facility types in a given group.

    Returns
    -------
    dfs : dict
        A dictionary of pandas DataFrames, one for each group.
    """
    dfs = {}

    for name, group in groups.items():
        group_df = df.query("facility_type in @group")
        dfs[name] = group_df.reset_index(drop=True)

    return dfs

def get_manual_facility_groups():
    """Returns the manual facility groups used in my final WiDS solution.

    Returns
    -------
    The dictionary of facility types.
        The key is group name (first word in facility type).
        The value is a set containing the facility types in a given group.
    """

    facility_groups = {
        "2to4_5plus_Mixed": {
            "2to4_Unit_Building",
            "5plus_Unit_Building",
            "Mixed_Use_Predominantly_Residential",
        },
        "Commercial_Education_Mixed_Industrial_Parking": {
            "Education_College_or_university",
            "Education_Other_classroom",
            "Education_Preschool_or_daycare",
            "Education_Uncategorized",
            "Commercial_Other",
            "Commercial_Unknown",
            "Mixed_Use_Commercial_and_Residential",
            "Mixed_Use_Predominantly_Commercial",
            "Industrial",
            "Parking_Garage",
        },
        "Food_Grocery": {
            "Food_Sales",
            "Food_Service_Other",
            "Food_Service_Restaurant_or_cafeteria",
            "Food_Service_Uncategorized",
            "Grocery_store_or_food_market",
        },
        "Health": {
            "Health_Care_Inpatient",
            "Health_Care_Outpatient_Clinic",
            "Health_Care_Outpatient_Uncategorized",
            "Health_Care_Uncategorized",
        },
        "Laboratory_Data": {"Laboratory", "Data_Center"},
        "Lodging": {
            "Lodging_Dormitory_or_fraternity_sorority",
            "Lodging_Hotel",
            "Lodging_Other",
            "Lodging_Uncategorized",
        },
        "Multifamily": {"Multifamily_Uncategorized"},
        "Office_Nursing": {
            "Office_Bank_or_other_financial",
            "Office_Medical_non_diagnostic",
            "Office_Mixed_use",
            "Office_Uncategorized",
            "Nursing_Home",
        },
        "Public": {
            "Public_Assembly_Drama_theater",
            "Public_Assembly_Entertainment_culture",
            "Public_Assembly_Library",
            "Public_Assembly_Movie_Theater",
            "Public_Assembly_Other",
            "Public_Assembly_Recreation",
            "Public_Assembly_Social_meeting",
            "Public_Assembly_Stadium",
            "Public_Assembly_Uncategorized",
            "Public_Safety_Courthouse",
            "Public_Safety_Fire_or_police_station",
            "Public_Safety_Penitentiary",
            "Public_Safety_Uncategorized",
        },
        "Religious": {"Religious_worship"},
        "Retail": {
            "Retail_Enclosed_mall",
            "Retail_Strip_shopping_mall",
            "Retail_Uncategorized",
            "Retail_Vehicle_dealership_showroom",
        },
        "Warehouse_Service": {
            "Warehouse_Distribution_or_Shipping_center",
            "Warehouse_Nonrefrigerated",
            "Warehouse_Refrigerated",
            "Warehouse_Selfstorage",
            "Warehouse_Uncategorized",
            "Service_Drycleaning_or_Laundry",
            "Service_Uncategorized",
            "Service_Vehicle_service_repair_shop",
        },
    }

    return facility_groups

def count_missing(df):
    """Counts the missing data in a dataframe.

    Parameters
    ----------
    df : pandas DataFrame
        The dataframe to count the missing data in.

    Returns
    -------
    pandas DataFrame
        A summary of missing data (counts and %)
    """
    missing_df = pd.DataFrame(
        df.isna().sum().sort_values(ascending=False), columns=["count"]
    )
    missing_df["percent"] = missing_df["count"] / df.shape[0]
    return missing_df.query("count != 0")

def iterative_impute(train_df, test_df, model, ct, target, feat_names, seed):
    """Imputes missing data into train and test datasets with a ML model of choice.

    Parameters
    ----------
    train_df : pandas DataFrame
        The training dataset
    test_df : pandas DataFrame
        The test dataset
    model : sklearn estimator
        The machine learning model to use for imputation
    ct : sklearn ColumnTransformer
        The column transformer to perform on the dataset
    target : str
        The target variable (removed before imputation)
    feat_names : list
        Names of features to append to OHE features from column transformer.
    seed : int
        The random seed for imputation.

    Returns
    -------
    train_imp, test_imp
        Train and test datasets with imputed values.
    """
    train_imp = ct.fit_transform(train_df.drop(columns=[target]))
    test_imp = ct.transform(test_df)

    imputer = IterativeImputer(estimator=model, random_state=seed)

    cols = (
        ct.named_transformers_["onehotencoder"].get_feature_names().tolist()
        + feat_names
    )

    train_imp = pd.DataFrame(imputer.fit_transform(train_imp), columns=cols)
    test_imp = pd.DataFrame(imputer.transform(test_imp), columns=cols)

    return train_imp, test_imp

def replace_columns(df, df_imp, columns):
    """Replace columns in a dataframe with columns from another.

    Note: Meant for use with imputed datasets for WiDS 2022.

    Parameters
    ----------
    df : pandas DataFrame
        The original dataframe.
    df_imp : pandas DataFrame
        The imputed dataframe.
    columns : str or list of str
        The columns to replace between dataframes.

    Returns
    -------
    df_replaced : pandas DataFrame
        Dataframe with replaced columns.

    """
    df_replaced = df.copy()

    for col in columns:
        df_replaced[col] = df_imp[col]

    return df_replaced

def impute_and_replace(
    train_dfs, test_dfs, model, ct, target, feat_names, replace, seed
):
    """Iteratively impute multiple dataframes.

    Note: Meant for use solely with WiDS 2022 data.  This simple calls
    the `iterative_impute` and `replace_column` functions for each
    individual dataset in my final WiDS solution.
    """
    train_dfs_imp = {}
    test_dfs_imp = {}

    # iterative imputation
    for (name1, train_df), (name2, test_df) in zip(train_dfs.items(), test_dfs.items()):
        train_imp, test_imp = iterative_impute(
            train_df, test_df, model, ct, target, feat_names, seed
        )
        train_dfs_imp[name1] = train_imp
        test_dfs_imp[name1] = test_imp

    # replace train columns with missing values
    for (name1, df), (name2, imp_df) in zip(train_dfs.items(), train_dfs_imp.items()):
        train_dfs[name1] = replace_columns(df, imp_df, replace)

    # replace test columns with missing values
    for (name1, df), (name2, imp_df) in zip(test_dfs.items(), test_dfs_imp.items()):
        test_dfs[name1] = replace_columns(df, imp_df, replace)

    return train_dfs, test_dfs

def has_feature(df, feat):
    """Adds a boolean feature if a feature exists or now"""
    return df[feat].notna().astype(int)

def group_by_feature(train_df, test_df, group, agg_feat, transform, name):
    """Adds a new aggregated feature based on a categorical variable"""
    train_df_new = train_df.copy()
    test_df_new = test_df.copy()

    fill_values = train_df_new.groupby(group).aggregate(transform)[agg_feat].to_dict()

    train_df_new[name] = train_df_new[group].map(fill_values)
    test_df_new[name] = test_df_new[group].map(fill_values)

    return train_df_new, test_df_new

def bin_days_below_above(df):
    """Bins the days below and above features into 4 bins"""
    df_eng = df.copy()

    df_eng["freezing_days"] = df_eng['days_below_0F'] + df_eng['days_below_10F']
    df_eng["cold_days"] = df_eng['days_below_20F'] + df_eng['days_below_30F']
    df_eng["warm_days"] = df_eng['days_above_80F'] + df_eng['days_above_90F']
    df_eng["hot_days"] = df_eng['days_above_100F'] + df_eng['days_above_110F']

    return df_eng

def seasonal_temps(df, stat):
    """Add seasonal temps for a given stat (e.g. "avg_temp")"""
    df_eng = df.copy()

    df_eng[f"winter_{stat}"] = (
        df_eng[f"december_{stat}"]
        + df_eng[f"january_{stat}"]
        + df_eng[f"february_{stat}"]
    ) / 3

    df_eng[f"spring_{stat}"] = (
        df_eng[f"march_{stat}"] + df_eng[f"april_{stat}"] + df_eng[f"june_{stat}"]
    ) / 3

    df_eng[f"summer_{stat}"] = (
        df_eng[f"june_{stat}"] + df_eng[f"july_{stat}"] + df_eng[f"august_{stat}"]
    ) / 3

    df_eng[f"autumn_{stat}"] = (
        df_eng[f"september_{stat}"]
        + df_eng[f"october_{stat}"]
        + df_eng[f"november_{stat}"]
    ) / 3

    return df_eng

def feature_engineer(train_df, test_df):
    """Feature engineering function for the WiDS 2022 kaggle competition.

    Note: The feature engineering here was done in a "quick and dirty" method
    in pandas, as speed is key in Kaggle competitions.  In real life ML, I would
    not do it this way, but rather, with custom sklearn Transformers, which is
    much cleaner.
    """
    train_df_eng = train_df.copy()
    test_df_eng = test_df.copy()

    # whether or not a building has a fog detector
    train_df_eng["has_fog_detector"] = has_feature(train_df_eng, "days_with_fog")
    test_df_eng["has_fog_detector"] = has_feature(test_df_eng, "days_with_fog")

    # whether or not a building has a wind detector
    train_df_eng["has_wind_detector"] = has_feature(train_df_eng, "max_wind_speed")
    test_df_eng["has_wind_detector"] = has_feature(test_df_eng, "max_wind_speed")

    # bin days above/below temperature
    train_df_eng = bin_days_below_above(train_df_eng)
    test_df_eng = bin_days_below_above(test_df_eng)

    # seasonal avg temps
    train_df_eng = seasonal_temps(train_df_eng, "avg_temp")
    test_df_eng = seasonal_temps(test_df_eng, "avg_temp")

    # aggregate features
    agg_feats = ["energy_star_rating", "floor_area", "ELEVATION"]

    for agg_feat in agg_feats:
        name = "mean_" + agg_feat
        train_df_eng, test_df_eng = group_by_feature(
            train_df_eng, test_df_eng, "facility_type", agg_feat, "mean", name
        )

    # whether or not energy star is better than mean for facility
    train_df_eng["e_star_better_than_mean"] = (
        train_df_eng["energy_star_rating"] > train_df_eng["mean_energy_star_rating"]
    ).astype(int)
    test_df_eng["e_star_better_than_mean"] = (
        test_df_eng["energy_star_rating"] > test_df_eng["mean_energy_star_rating"]
    ).astype(int)

    # total snow and rain
    train_df_eng["snow_rain_inches"] = (
        train_df_eng["snowfall_inches"] + train_df_eng["precipitation_inches"]
    )
    test_df_eng["snow_rain_inches"] = (
        test_df_eng["snowfall_inches"] + test_df_eng["precipitation_inches"]
    )

    # total degree days
    train_df_eng["degree_days"] = (
        train_df_eng["cooling_degree_days"] + train_df_eng["heating_degree_days"]
    )
    test_df_eng["degree_days"] = (
        test_df_eng["cooling_degree_days"] + test_df_eng["heating_degree_days"]
    )

    # floor area interaction with e-star
    train_df_eng["e_star_floor_area"] = (
        train_df_eng["floor_area"] * train_df_eng["energy_star_rating"] + 1
    )
    test_df_eng["e_star_floor_area"] = (
        test_df_eng["floor_area"] * test_df_eng["energy_star_rating"] + 1
    )

    # elevation interaction with e-star
    train_df_eng["e_star_elevation"] = (
        train_df_eng["ELEVATION"] * train_df_eng["energy_star_rating"] + 1
    )
    test_df_eng["e_star_elevation"] = (
        test_df_eng["ELEVATION"] * test_df_eng["energy_star_rating"] + 1
    )

    # year_built interaction with e-star
    train_df_eng["e_star_year_built"] = (
        train_df_eng["year_built"] * train_df_eng["energy_star_rating"] + 1
    )
    test_df_eng["e_star_year_built"] = (
        test_df_eng["year_built"] * test_df_eng["energy_star_rating"] + 1
    )

    # cooling degree days interaction with energy star
    train_df_eng["cooling_e_star"] = (
        train_df_eng["cooling_degree_days"] * train_df_eng["energy_star_rating"] + 1
    )
    test_df_eng["cooling_e_star"] = (
        test_df_eng["cooling_degree_days"] * test_df_eng["energy_star_rating"] + 1
    )

    # heating degree days interaction with energy star
    train_df_eng["heating_e_star"] = (
        train_df_eng["heating_degree_days"] * train_df_eng["energy_star_rating"] + 1
    )
    test_df_eng["heating_e_star"] = (
        test_df_eng["heating_degree_days"] * test_df_eng["energy_star_rating"] + 1
    )

    # floor area interaction with year built
    train_df_eng["floor_area_year_built"] = (
        train_df_eng["floor_area"] * train_df_eng["year_built"] + 1
    )
    test_df_eng["floor_area_year_built"] = (
        test_df_eng["floor_area"] * test_df_eng["year_built"] + 1
    )

    # concatenated state and facility type and floor area
    train_df_eng["facility_floor"] = (
        train_df_eng["facility_type"] + "_" + train_df_eng["floor_area"].astype(str)
    )
    test_df_eng["facility_floor"] = (
        test_df_eng["facility_type"] + "_" + test_df_eng["floor_area"].astype(str)
    )

    # concatenated state, facility, year built
    train_df_eng["facility_year"] = train_df_eng["facility_type"] + train_df_eng[
        "year_built"
    ].astype(str)
    test_df_eng["facility_year"] = test_df_eng["facility_type"] + test_df_eng[
        "year_built"
    ].astype(str)

    # concatenated state and facility type and floor area and year
    train_df_eng["facility_floor_year"] = (
        train_df_eng["facility_floor"] + "_" + train_df_eng["year_built"].astype(str)
    )
    test_df_eng["facility_floor_year"] = (
        test_df_eng["facility_floor"] + "_" + test_df_eng["year_built"].astype(str)
    )

    # grouped mean site_eui
    groups = ["facility_floor", "facility_year", "facility_floor_year"]

    for group in groups:
        name = "median_" + group + "_site_eui"
        train_df_eng, test_df_eng = group_by_feature(
            train_df_eng, test_df_eng, group, "site_eui", "median", name
        )

    return train_df_eng, test_df_eng

def feature_engineer_multiple(train_dfs, test_dfs):
    """Feature engineer multiple dataframes at once"""
    train_dfs_eng = {}
    test_dfs_eng = {}

    for (name1, train_df), (name2, test_df) in zip(train_dfs.items(), test_dfs.items()):
        assert name1 == name2
        train_dfs_eng[name1], test_dfs_eng[name2] = feature_engineer(train_df, test_df)

    return train_dfs_eng, test_dfs_eng

def create_X_y(dfs, target, group_col=None):
    """Separates each WiDS 2022 dataframe into X and y train sets.

    Parameters
    ----------
    dfs : dict
        A dictionary of pandas DataFrames, one for each group.
    target : str
        The target variable.
    group_col : str, optional
        The columns to use for CV groups, by default None

    Returns
    -------
    X_dfs, y_dfs, groups
        Dictionaries of X and y train sets, and groups.
    """

    X_dfs = {}
    y_dfs = {}
    groups = {}

    for name, df in dfs.items():
        X_dfs[name] = df.drop(columns=target)
        y_dfs[name] = df[target]

        if group_col is not None:
            groups[name] = df[group_col]

    if group_col is not None:
        return X_dfs, y_dfs, groups
    else:
        return X_dfs, y_dfs

def get_ct_feat_names(ct, other_names):
    """Get the feature names from a ColumnTrasformer and combine
    with other feature names.

    Note: In recent versions of sklearn there is support for this
    without having to create a function.  However, Kaggle was running
    an older version of sklearn in their kernels during this competition,
    hence creation of this function.

    Parameters
    ----------
    ct : sklearn ColumnTransformer
        A fitted sklearn ColumnTransformer.
    other_names : list of str
        The other feature names to append

    Returns
    -------
    names : list of str
        The list of all feature names after a ColumnTransformer
        transforms a dataset.
    """
    names = []
    names += other_names
    return names

def logo_cv_lgb(
    model, X, y, groups, ct, early_stopping_rounds, return_models=True, other_names=None
):
    """Custom cross validation function for the WiDS 2022 Kaggle Competition.

    Parameters
    ----------
    model : lightGBM regressor
        The lgbm regressor model to perform cv with.
    X : pandas DataFrame
        X train set.
    y : pandas DataFrame
        y train set.
    groups : pandas DataFrame
        The leave one group out cv groups.
    ct : sklearn ColumnTransformer
        The column transformer for the dataset.
    early_stopping_rounds : int
        The number of early stopping rounds for lightGBM
    return_models : bool, optional
        Whether or not to return the indivdual lightGBM models, by default True
    other_names : list of str, optional
        Other feature names to append to CT names, by default None

    Returns
    -------
    results, models : pandas DataFrame, dict
        Results dataframe and dictionary of lgbm models.
    """

    models = []
    results = []
    train_scores = []
    val_scores = []
    fold = 0
    logo = LeaveOneGroupOut()

    for t, v in logo.split(X, groups=groups):
        fold += 1

        # to avoid lightgbm not saving new fit model (weird??? might be bug in lgb...)
        current_model = deepcopy(model)

        # train and val sets for current fold
        X_train, y_train = X.iloc[t], y.iloc[t]
        X_val, y_val = X.iloc[v], y.iloc[v]

        # year factors for current folds
        X_train_yf = np.sort(X_train["Year_Factor"].unique())
        X_val_yf = X_val["Year_Factor"].unique()

        # transform train and val sets
        X_train_tsf = ct.fit_transform(X_train)
        X_val_tsf = ct.transform(X_val)

        if other_names is None:
            feature_names = ct.get_feature_names()
        else:
            feature_names = get_ct_feat_names(ct, other_names)

        # fit and score model w/ early stopping
        current_model.fit(
            X_train_tsf,
            y_train,
            eval_set=[(X_val_tsf, y_val)],
            eval_metric=["rmse"],
            feature_name=feature_names,
            callbacks=[lgb.early_stopping(early_stopping_rounds, verbose=False)],
        )

        # train/val rmse scores
        train_rmse = mean_squared_error(
            y_true=y_train, y_pred=current_model.predict(X_train_tsf), squared=False
        )

        val_rmse = current_model.best_score_["valid_0"]["rmse"]

        fold_results = {
            "Fold": fold,
            "Train_shape": X_train_tsf.shape,
            "Val_shape": X_val_tsf.shape,
            "Train_years": X_train_yf,
            "Val_years": X_val_yf,
            "Train_RMSE": train_rmse,
            "Val_RMSE": val_rmse,
            "RMSE_spread": train_rmse - val_rmse,
        }

        results.append(fold_results)
        train_scores.append(train_rmse)
        val_scores.append(val_rmse)
        models.append(current_model)

    # mean validation score
    results.append(
        {
            "Fold": "MEAN",
            "Train_shape": "N/A",
            "Val_shape": "N/A",
            "Train_years": "N/A",
            "Val_years": "N/A",
            "Train_RMSE": np.mean(train_scores),
            "Val_RMSE": np.mean(val_scores),
            "RMSE_spread": np.mean(train_scores) - np.mean(val_scores),
        }
    )

    results = pd.DataFrame(results)

    if return_models:
        return results, models
    else:
        return results

def logo_cv_lgb_all(
    model,
    X_trains,
    y_trains,
    groups,
    ct,
    early_stopping_rounds,
    return_models=True,
    other_names=None,
):
    """Performs CV for each dataset in the WiDS kaggle competition."""
    results = {}
    models = {}

    for (n1, X), (n2, y), (n3, group) in zip(
        X_trains.items(), y_trains.items(), groups.items()
    ):
        assert n1 == n2 == n3
        results[n1], models[n1] = logo_cv_lgb(
            model, X, y, group, ct, early_stopping_rounds, return_models, other_names
        )

    return results, models

def process_cv_results(results):
    """Process cv results for multiple datasets into one dataframe"""
    results_df = pd.DataFrame()

    for name, result in results.items():
        result.insert(0, "facility_group", name)
        results_df = pd.concat([results_df, result])

    return results_df

def train_and_predict(model, X_trains, y_trains, X_tests, ct, target):
    """Train final WiDS models and get predictions.

    Parameters
    ----------
    model : sklearn estimator
        The model to train and predict.
    X_trains : dict
        Dictionary of X train sets.
    y_trains : dict
        Dictionary of y train sets.
    X_tests : dict
        Dictionary of X test sets.
    ct : sklearn ColumnTransformer
        Column transformer with tranformations to perform on data.
    target : str
        The target variable.

    Returns
    -------
    predictions : pandas DataFrame
        Dataframe containing the final predictions.
    """
    predictions = pd.DataFrame()

    for (n1, X_train), (n2, y_train), (n3, X_test) in zip(
        X_trains.items(), y_trains.items(), X_tests.items()
    ):
        assert n1 == n2 == n3

        current_model = deepcopy(model)
        pipe = make_pipeline(ct, current_model)

        pipe.fit(X_train, y_train)

        pred = {"id": X_test["id"], target: pipe.predict(X_test)}

        predictions = pd.concat([predictions, pd.DataFrame(pred)])

    predictions = predictions.sort_values("id")

    return predictions_____no_output_____

# Section 3: Data Preprocessing

In this section I preprocess the dataset used in the competition, which includes:
- Reading in the data.
- Dealing with duplicated data.
- Creating separate datasets based on building facility type.
- Dealing with missing data._____no_output_____## 3.1 Reading in Datasets_____no_output_____

train, test = read_data(TRAIN_PATH, TEST_PATH)_____no_output_____train.shape_____no_output_____test.shape_____no_output_____

## 3.2 Duplicated Data

In this section I deal with the duplicated data in the dataset:

- We will see that there are 39 duplicated buildings in the train set, and 5 duplicated buildings in the test set
- As the train set has 75k+ rows, I removed the duplicated buildings in the train set
- I just left the duplicated buildings for the test set as is (removing them would break the final predictions for submission)_____no_output_____

train_duplicates = get_duplicates(train, ["id"])
test_duplicates = get_duplicates(test, ["id"])_____no_output_____print(f"There are {train_duplicates.shape[0]} duplicates in the train set")
print(f"There are {test_duplicates.shape[0]} duplicates in the train set")There are 39 duplicates in the train set
There are 5 duplicates in the train set
train = remove_duplicates(train, ["id"])_____no_output_____

## 3.3 Create Individual Datasets Based on Facility Types

In this section I split the datasets into 12 individual ones based on facility types with similar `site_eui` characteristics.

**Why I did this:**
- In this competition, facility type was a feature that described what kind of facility a building was, and there were 60 total types.
- When I was doing exploratory data analysis, I noticed that there were an uneven number of facilities of different types.
- For example, there were 40k Multifamily buildings in the dataset, while other facility types such as Industrial only had a few hundred (or even less).
- Further, the distribution of site energy usage for different facility types was drastically different than others, and certain feature distributions were different as well.
- Therefore, it intuitively did not make sense to me to just train one machine learning model, since for example, how could a model trained with a significant portion of the data being related to Multifamily homes, make accurate predictions on other types where there was only 100 examples?
- So my idea behind this was to create smaller datasets of similar buildings that had similar energy usage patterns and feature distributions, and then to use these to train individual machine learning models on each dataset, with the ultimate goal of getting more accurate predictions overall in the end.

**How I did it:**
- In developing the individual datasets I tried a bunch of things, including:
    1. Naively grouping buildings based on the first word in their facility type, for example the types `Food_Sales` and `Food_Sales_Other` were grouped together in a dataset called `Food`.
    2. Using KMeans clustering to cluster similar buildings together, and then group the buildings based on cluster labels.
    3. Manually grouping buildings based on exploratory data analysis and iterating to optmize the Kaggle score

**What worked best:**
- After reading a few papers, my intuition was that the clustering approach would be the best, however, it turned out that the manual grouping approach resulted in the lowest scoring notebook, and that the clustering approach resulted in poor scores.
- In my final solution, I separate the train and test datasets into 12 individual data sets (each).
- You can see the exact groups of facility types that I used by reading the `get_manual_facility_groups` function in [Section 2](#Section-2:-Helper-Functions) of this notebook._____no_output_____

facility_groups = get_manual_facility_groups()_____no_output_____train_dfs = split_building_data(train, facility_groups)
test_dfs = split_building_data(test, facility_groups)_____no_output_____

## 3. 4 Missing Data

In this section I simply identify what data is missing in the dataset:

- Since this is a Kaggle competition, I look at both the train and test set here.
- We'll see that features related to fog or wind, have over 50% of the data missing in both the train and test set, and due to this, I don't use these features in my final solution.
- We'll also see that there is missing data in energy star rating and year built, which I deal with in the next section of this notebook._____no_output_____

count_missing(train)_____no_output_____count_missing(test)_____no_output_____

## 3.5 Data Imputation

In this section, I impute the missing data in the `energy_star_rating` and `year_built` features:
- I use Ridge regression to impute the missing values, as I found this gave me the best results in terms of optimizing the final score of my solution.
- I also tried using XGBoost and LightGBM models to impute, but these did not do as well.
- First, I create a sklearn `ColumnTransformer` which one hot encodes the categorical features I used for imputation, as well as removes features that I do not use for imputation.
- Then, I use sklearn's `IterativeImputer` to impute the missing data.
- Note that I have abstracted the actual code away into functions, which are included in [Section 2](#Section-2:-Helper-Functions) of this notebook._____no_output_____

cat_imp = [
    'Year_Factor', 'State_Factor', 'building_class', 'facility_type'
]

num_imp = [
    'floor_area', 'year_built', 'energy_star_rating', 'ELEVATION',
    'cooling_degree_days', 'heating_degree_days', 'precipitation_inches',
    'snowfall_inches', 'snowdepth_inches', 'avg_temp'
]

drop_imp = [
    'january_min_temp', 'january_avg_temp', 'january_max_temp',
    'february_min_temp', 'february_avg_temp', 'february_max_temp',
    'march_min_temp', 'march_avg_temp', 'march_max_temp', 'april_min_temp',
    'april_avg_temp', 'april_max_temp', 'may_min_temp', 'may_avg_temp',
    'may_max_temp', 'june_min_temp', 'june_avg_temp', 'june_max_temp',
    'july_min_temp', 'july_avg_temp', 'july_max_temp', 'august_min_temp',
    'august_avg_temp', 'august_max_temp', 'september_min_temp',
    'september_avg_temp', 'september_max_temp', 'october_min_temp',
    'october_avg_temp', 'october_max_temp', 'november_min_temp',
    'november_avg_temp', 'november_max_temp', 'december_min_temp',
    'december_avg_temp', 'december_max_temp', 'days_below_30F', 'days_below_20F',
    'days_below_10F', 'days_below_0F', 'days_above_80F', 'days_above_90F',
    'days_above_100F', 'days_above_110F', 'direction_max_wind_speed',
    'direction_peak_wind_speed', 'max_wind_speed', 'days_with_fog',
    'id'
]_____no_output_____model = Ridge()
replace = ["energy_star_rating", "year_built"]

ct_imp = make_column_transformer(
    (OneHotEncoder(handle_unknown="ignore", sparse=False), cat_imp),
    ("passthrough", num_imp),
    ("drop", drop_imp)
)

train_dfs, test_dfs = impute_and_replace(
    train_dfs, test_dfs, model, ct_imp, TARGET, num_imp, replace, SEED
)_____no_output_____

# Section 4: Feature Engineering

In this section I perform feature engineering and add many features to the data:
- I've abstracted all the code away into functions, please see [Section 2](#Section-2:-Helper-Functions) of this notebook for the actual code.
- In total, I engineer 28 new features, for which the names of each are included below.
- `median_facility_floor_site_eui`, `median_facility_year_site_eui`, and `median_facility_floor_year_site_eui` were the most powerful features for prediction.
- These features worked by concatenating a combination of the features `facility_type`, `floor_area`, and/or `year_built`, and then encoding them with a grouped median value of the target `site_eui`
- I thought that this would cause data leakage, since we are encoding the target variable as a feature (and in a sense, just identifying exact buildings in the dataset and using past `site_eui` to predict future `site_eui`, but in the end these three features were the most powerful in my entire notebook.
- This would suggest that a buildings prior site energy usage reading can be used as a key predictor in predicting it's future energy usage (similar to pure time series data)._____no_output_____

train_dfs_eng, test_dfs_eng = feature_engineer_multiple(train_dfs, test_dfs)_____no_output_____print("New Features:")
print("-----------------------------------------------------------")
for i, j in enumerate(train_dfs_eng['2to4_5plus_Mixed'].columns[64:]):
    print(f"New Feature {i+1}: {j}")New Features:
-----------------------------------------------------------
New Feature 1: has_fog_detector
New Feature 2: has_wind_detector
New Feature 3: freezing_days
New Feature 4: cold_days
New Feature 5: warm_days
New Feature 6: hot_days
New Feature 7: winter_avg_temp
New Feature 8: spring_avg_temp
New Feature 9: summer_avg_temp
New Feature 10: autumn_avg_temp
New Feature 11: mean_energy_star_rating
New Feature 12: mean_floor_area
New Feature 13: mean_ELEVATION
New Feature 14: e_star_better_than_mean
New Feature 15: snow_rain_inches
New Feature 16: degree_days
New Feature 17: e_star_floor_area
New Feature 18: e_star_elevation
New Feature 19: e_star_year_built
New Feature 20: cooling_e_star
New Feature 21: heating_e_star
New Feature 22: floor_area_year_built
New Feature 23: facility_floor
New Feature 24: facility_year
New Feature 25: facility_floor_year
New Feature 26: median_facility_floor_site_eui
New Feature 27: median_facility_year_site_eui
New Feature 28: median_facility_floor_year_site_eui

# Section 5: Modelling

This section is where I trained my initial machine learning models._____no_output_____## 5.1 Create X and y Datasets

Here I create the `X_train`, `y_train`, and `X_test` for each of the twelve datasets.  I also create the groups needed for leave one group one cross validation._____no_output_____

X_trains, y_trains, groups = create_X_y(train_dfs_eng, TARGET, "Year_Factor")
X_tests = test_dfs_eng.copy()_____no_output_____

## 5.2 Column Transformer

A few things to note on the final column transformer:
- You'll notice only a subset of features are used in the final model, and the majority of features are actually dropped.
- There were no categorical features used in my final solution.
- All features were numeric, and the only further preprocessing was to standardize them (not required for tree based learning, but can slightly improve performance)._____no_output_____

num = [
    'year_built', 'energy_star_rating', 'ELEVATION', 'floor_area',
    'cooling_degree_days', 'heating_degree_days', 'precipitation_inches',
    'snowdepth_inches', 'e_star_floor_area', 'e_star_elevation',
    'e_star_year_built', 'cooling_e_star', 'heating_e_star', 'floor_area_year_built',
    'median_facility_floor_site_eui', 'median_facility_floor_year_site_eui',
    'median_facility_year_site_eui'
]

passthrough = [
    'has_fog_detector', 'has_wind_detector',
]

drop = [
    'Year_Factor', 'State_Factor', 'january_min_temp', 'january_avg_temp', 'january_max_temp',
    'february_min_temp', 'february_avg_temp', 'february_max_temp',
    'march_min_temp', 'march_avg_temp', 'march_max_temp', 'april_min_temp',
    'april_avg_temp', 'april_max_temp', 'may_min_temp', 'may_avg_temp',
    'may_max_temp', 'june_min_temp', 'june_avg_temp', 'june_max_temp',
    'july_min_temp', 'july_avg_temp', 'july_max_temp', 'august_min_temp',
    'august_avg_temp', 'august_max_temp', 'september_min_temp',
    'september_avg_temp', 'september_max_temp', 'october_min_temp',
    'october_avg_temp', 'october_max_temp', 'november_min_temp',
    'november_avg_temp', 'november_max_temp', 'december_min_temp',
    'december_avg_temp', 'december_max_temp',  'direction_max_wind_speed',
    'direction_peak_wind_speed', 'max_wind_speed', 'days_with_fog',
    'id', 'days_below_30F', 'days_below_20F', 'days_below_10F', 'days_below_0F',
    'days_above_80F', 'days_above_90F', 'days_above_100F', 'days_above_110F', 'State_Factor', 'avg_temp',
    'facility_floor', 'facility_floor_year', 'facility_year','snowfall_inches',
    'freezing_days', 'cold_days', 'warm_days', 'hot_days', 'winter_avg_temp',
    'spring_avg_temp', 'summer_avg_temp', 'autumn_avg_temp',
    'mean_energy_star_rating', 'mean_floor_area', 'mean_ELEVATION', 'e_star_better_than_mean',
    'snow_rain_inches', 'degree_days', 'building_class', 'facility_type'
]

assert set(num + passthrough + drop) == set(X_trains[list(X_trains.keys())[0]].columns)_____no_output_____ct = make_column_transformer(
    (StandardScaler(), num),
    ("passthrough", passthrough),
    ("drop", drop),
)_____no_output_____

## 5.3 Cross Validation

In this section, I perform cross validation.  A few notes:
- In the interest of speed, I used an out of the box light GBM regressor to perform five fold leave one group out cross validation ("LOGO cv").
- I performed LOGO cv based on the `year_factor` column in the dataset - in other words I trained and predicted on separate year groups, for example one fold would train on years 1 to 5, then predict on 6.
- I did this since the test dataset only included buildings from year 7, so I was trying to emulate this in cross validation by leaving one year out.  In a perfect world with more time, I would have also tried a fancier time series cross validation method.
- The mean cross validation scores for each grouped dataset are included below, we can see that for some groups, the train and validation scores are quite good, except for a few, namely `Food_Grocery`, `Health`, and `Laboratory_Data`._____no_output_____

model = lgb.LGBMRegressor(force_row_wise=True, random_state=SEED)
results, models = logo_cv_lgb_all(model, X_trains, y_trains, groups, ct, 5, other_names=num+passthrough)
results_df = process_cv_results(results)_____no_output_____print("Mean Cross Validation Scores:")
pd.concat(
    [
        results_df[results_df["Fold"] == "MEAN"]["facility_group"],
        results_df[results_df["Fold"] == "MEAN"].loc[:, "Train_RMSE": ]
    ],
    axis=1
)Mean Cross Validation Scores:

## 5.4 Feature Importance

In this section:
- I plot the feature importance graphs from the lightGBM regressor models.
- I only include an example for the four `2to4_5plus_Mixed` models that resulted from the cross validation process, otherwise there would be too many plots to look at in this notebook.
- We'll see that in each fold the grouped median features and interaction features with `energy_star_rating` are the most important_____no_output_____

fig, axes = plt.subplots(2, 2, figsize=(15, 10))

for n, (i, ax) in enumerate(zip(range(len(models["2to4_5plus_Mixed"])), axes.ravel())):
    lgb.plot_importance(models["2to4_5plus_Mixed"][i], figsize=(5, 5), height=0.5, max_num_features=20, ax=ax);
    ax.set_title(f"Model {n+1}")

fig.suptitle("2to4_5plus_Mixed CV LightGBM Model Feature Importances", fontsize=20)
plt.tight_layout()_____no_output_____

# Section 6: Final models and predictions

This section contains my final ML models and their resulting predictions:

- The final model I used was an ensemble of XGB, lightGBM, and CatBoost regressors.
- You will need to use a GPU to run this section of the notebook, or, comment out the GPU code lines (although this will take a long time if you only use a CPU).
- If you re-run this notebook multiple times, the predictions are a bit unstable after the 4th decimal point, I'm not 100% sure why this is since the random seeds are set (if anyone does know, please feel free to comment and let me know why)._____no_output_____

regressor = VotingRegressor([
    (
        "xgb",
        xgb.XGBRegressor(
            n_estimators=1000,
            max_depth=8,
            learning_rate=0.15,
            verbosity=0,
            tree_method="gpu_hist",
            random_state=SEED,
        )
    ),
    (
        "lgb",
        lgb.LGBMRegressor(
            force_row_wise=True,
            max_depth=8,
            learning_rate=0.15,
            n_estimators=1000,
            num_leaves=256,
            device="gpu",
            random_state=SEED
        )
    ),
    (
        "cb",
        cb.CatBoostRegressor(
            n_estimators=1000,
            learning_rate=0.15,
            max_depth=8,
            verbose=False,
            random_seed=SEED,
            task_type="GPU"
        )
    )
])_____no_output_____predictions = train_and_predict(regressor, X_trains, y_trains, X_tests, ct, TARGET)
assert sum(predictions.isna().sum()) == 0_____no_output_____predictions_____no_output_____predictions.to_csv("submission.csv", index=False)_____no_output_____

# Thank You!

Thank you for taking the time to review my 2022 WiDS Kaggle Notebook.  As always, any feedback or comments are welcomed.  Also, please feel free to reach out to me to just chat about data science or machine learning in general!_____no_output_____


# Notebook from sanchobarriga/course-content
Path: tutorials/W1D1_ModelTypes/student/W1D1_Tutorial1.ipynb

_____no_output_____
#  NMA Model Types Tutorial 1: "What" models

Welcome to Neuromatch Academy!

Here are your objectives for this tutorial:

- Load a dataset with spiking activity from hundreds of neurons and understand how it is organized
- Make plots to visualize characteristics of the spiking activity across the population
- Compute the distribution of "inter-spike intervals" (ISIs) for a single neuron
- Consider several formal models of this distribution's shape and fit them to the data "by hand"_____no_output_____

#@title Video: Intro
from IPython.display import YouTubeVideo
video = YouTubeVideo(id='6ft5nyDZIh0', width=854, height=480, fs=1)
print("Video available at https://youtube.com/watch?v=" + video.id)
videoVideo available at https://youtube.com/watch?v=6ft5nyDZIh0

## Setup: Imports, helper functions, and data retreival

Tutorial notebooks typically begin with several set-up steps that are hidden from view by default.

**Important:** Even though the code is hidden, you still need to run it so that the rest of the notebook can work properly. Step through each cell, either by pressing the play button in the upper-left-hand corner or with a keyboard shortcut (`Cmd-Return` on a Mac, `Ctrl-Enter` otherwise). A number will appear inside the brackets (e.g. `[3]`) to tell you that the cell was executed and what order that happened in.

If you are curious to see what is going on inside each cell, you can double click to expand. Once expanded, double-click the white space to the right of the editor to collapse again._____no_output_____

#@title Imports
#@markdown Python requires you to explictly "import" libraries
#@markdown before their functions are available to use.

#@markdown We always do that at the beginning of each notebook or script.
import functools
import io
import requests

import ipywidgets as widgets
import matplotlib.pyplot as plt
from matplotlib.ticker import MaxNLocator
import numpy as np
import scipy as sp
from scipy.io import loadmat
from scipy.optimize import curve_fit
import scipy.stats as stats

from IPython.display import display, Markdown

# numpy print formatting
np.set_printoptions(formatter={'float': '{:7.3f}'.format})_____no_output_____#@title Helper functions

#@markdown Most of the tutorials make use of helper functions
#@markdown to simplify the code that you need to write. They are defined here.

# Please don't edit these, or worry about understanding them now!

def restrict_spike_times(spike_times, interval):
  """Given a spike_time dataset, restrict to spikes within given interval.

  Args:
    spike_times (sequence of np.ndarray): List or array of arrays,
      each inner array has spike times for a single neuron.
    interval (tuple): Min, max time values; keep min <= t < max.

  Returns:
    np.ndarray: like `spike_times`, but only within `interval`
  """
  t_interval = (5, 15)  # units are seconds after start of recording
  interval_spike_times = []
  for spikes in spike_times:
    interval_mask = (spikes >= t_interval[0]) & (spikes < t_interval[1])
    interval_spike_times.append(spikes[interval_mask])
  return np.array(interval_spike_times, object)_____no_output_____#@title Figure parameters
#@markdown This cell defines the way that figures will appear by default.

%matplotlib inline
%config InlineBackend.figure_format='retina'

fig_w, fig_h = (6, 4)
plt.rcParams.update({'figure.figsize': (fig_w, fig_h)})_____no_output_____#@title Data retrieval
#@markdown This cell downloads the example dataset that we will use in this tutorial.
r = requests.get('https://osf.io/sy5xt/download')
if r.status_code != 200:
    print('oops')
spike_times = np.load(io.BytesIO(r.content), allow_pickle=True)['spike_times']_____no_output_____

---

## Exploring the Steinmetz dataset

In this tutorial we will explore the structure of a neuroscience dataset.

We consider a subset of data from a study of [Steinmetz _et al._ (2019)](https://www.nature.com/articles/s41586-019-1787-x). In this study, Neuropixels probes were implanted in the brains of mice. Electrical potentials were measured by hundreds of electrodes along the length of each probe. Each electrode's measurements captured local variations in the electric field due to nearby spiking neurons. A spike sorting algorithm was used to infer spike times and cluster spikes according to common origin: a single cluster of sorted spikes is causally attributed to a single neuron.

In particular, a single recording session of spike times and neuron assignments was loaded and assigned to `spike_times` in the preceding setup.

Typically a dataset comes with some information about its structure. However, this information may be incomplete. You might also apply some transformations or "pre-processing" to create a working representation of the data of interest, which might go partly undocumented depending on the circumstances. In any case it is important to be able to use the available tools to investigate unfamiliar aspects of a data structure.

Let's see what our data looks like..._____no_output_____### Warming up with `spike_times`_____no_output_____What is the Python type of our variable?_____no_output_____

type(spike_times)_____no_output_____

You should see `numpy.ndarray`, which means that it's a normal NumPy array.

If you see an error message, it probably means that you did not execute the set-up cells at the top of the notebook. So go ahead and make sure to do that.

Once everything is running properly, we can ask the next question about the dataset: what's its shape?_____no_output_____

spike_times.shape_____no_output_____

There are 734 entries in one dimension, and no other dimensions. What is the Python type of the first entry, and what is *its* shape?_____no_output_____

idx = 0
print(
  type(spike_times[idx]),
  spike_times[idx].shape,
  sep="\n",
)<class 'numpy.ndarray'>
(826,)

It's also a NumPy array with a 1D shape! Why didn't this show up as a second dimension in the shape of `spike_times`? That is, why not `spike_times.shape == (734, 826)`?

To investigate, let's check another entry._____no_output_____

idx = 321
print(
  type(spike_times[idx]),
  spike_times[idx].shape,
  sep="\n",
)<class 'numpy.ndarray'>
(9723,)

It's also a 1D NumPy array, but it has a different shape. Checking the NumPy types of the values in these arrays, and their first few elements, we see they are composed of floating point numbers (not another level of `np.ndarray`):_____no_output_____

i_neurons = [0, 321]
i_print = slice(0, 5)

for i in i_neurons:
  print(
    "Neuron {}:".format(i),
    spike_times[i].dtype,
    spike_times[i][i_print],
    "\n",
    sep="\n"
  )
Neuron 0:
float32
[  0.815  14.822  24.965  25.144  38.871]

Neuron 321:
float32
[  1.070   1.154   1.240   1.707   1.799]

Note that this time we've checked the NumPy `dtype` rather than the Python variable type. These two arrays contain floating point numbers ("floats") with 32 bits of precision.

The basic picture is coming together:
- `spike_times` is 1D, its entries are NumPy arrays, and its length is the number of neurons (734): by indexing it, we select a subset of neurons.
- An array in `spike_times` is also 1D and corresponds to a single neuron; its entries are floating point numbers, and its length is the number of spikes attributed to that neuron. By indexing it, we select a subset of spike times for that neuron.

Visually, you can think of the data structure as looking something like this:

```
| . . . . . |
| . . . . . . . . |
| . . . |
| . . . . . . . |
```

Before moving on, we'll calculate and store the number of neurons in the dataset and the number of spikes per neuron:_____no_output_____

n_neurons = len(spike_times)
total_spikes_per_neuron = [len(spike_times_i) for spike_times_i in spike_times]

print(f"Number of neurons: {n_neurons}")
print(f"Number of spikes for first five neurons: {total_spikes_per_neuron[:5]}")Number of neurons: 734
Number of spikes for first five neurons: [826, 2818, 3953, 646, 1115]

If the second line in that cell confused you, it's called a "list comprehension", which is a shorthand way to write

```python
total_spikes_per_neuron = []
for spike_times_i in spike_times:
  total_spikes_per_neuron.append(len(spike_times_i)
```_____no_output_____

#@title Video: Exploring the dataset
from IPython.display import YouTubeVideo
video = YouTubeVideo(id='sHp98o22GHM', width=854, height=480, fs=1)
print("Video available at https://youtube.com/watch?v=" + video.id)
videoVideo available at https://youtube.com/watch?v=sHp98o22GHM

### Getting warmer: counting and plotting total spike counts

As we've seen, the number of spikes over the entire recording is variable between neurons. More generally, some neurons tend to spike more than others in a given period. Lets explore what the distribution of spiking looks like across all the neurons in the dataset._____no_output_____Are most neurons "loud" or "quiet", compared to the average? To see, we'll define bins of constant width in terms of total spikes and count the neurons that fall in each bin. This is known as a "histogram".

You can plot a histogram with the matplotlib function `plt.hist`. If you just need to compute it, you can use the numpy function `np.histogram` instead._____no_output_____

n_bins = 50
plt.hist(total_spikes_per_neuron, n_bins)
plt.xlabel("Total spikes per neuron")
plt.ylabel("Number of neurons")_____no_output_____

Let's see what percentage of neurons have a below-average spike count:_____no_output_____

mean_spike_count = np.mean(total_spikes_per_neuron)
frac_below_mean = (total_spikes_per_neuron < mean_spike_count).mean()
print(f"{frac_below_mean:2.1%} of neurons are below the mean")68.0% of neurons are below the mean

We can also see this by adding the average spike count to the histogram plot:_____no_output_____

plt.hist(total_spikes_per_neuron, n_bins)
plt.xlabel("Total spikes per neuron")
plt.ylabel("Number of neurons")
plt.axvline(mean_spike_count, color="orange", label="Mean neuron")
plt.legend()_____no_output_____

This shows that the majority of neurons are relatively "quiet" compared to the mean, while a small number of neurons are exceptionally "loud": they must have spiked more often to reach a large count.

### Exercise: Comparing mean and median neurons

If the mean neuron is more active than 68% of the population, what does that imply about the relationship between the mean neuron and the median neuron?

*Exercise objective:* Reproduce the plot above, but add the median neuron.
_____no_output_____

# To complete the exercise, uncomment the code and fill in the missing parts (...)

# median_spike_count = ...  # Hint: Try the function np.median

# plt.hist(...)
# plt.axvline(..., color="orange", label="Mean neuron")
# plt.axvline(..., color="limegreen", label="Median neuron")
# plt.xlabel("Total spikes per neuron")
# plt.ylabel("Number of neurons")
# plt.legend()_____no_output_____

[*Click for solution*](https://github.com/NeuromatchAcademy/course-content/tree/master//tutorials/W1D1_ModelTypes/solutions/W1D1_Tutorial1_Solution_7183720f.py)

*Example output:*

_____no_output_____
*Bonus:* The median is the 50th percentile. What about other percentiles? Can you show the interquartile range on the histogram?_____no_output_____---

## Visualizing neuronal spiking activity_____no_output_____#### Getting a subset of the data

Now we'll visualize trains of spikes. Because the recordings are long, we will first define a short time interval and restrict the visualization to only the spikes in this interval. We defined a utility function, `restrict_spike_times`, to do this for you. If you call `help()` on the function, it will tell you a little bit about itself:_____no_output_____

help(restrict_spike_times)Help on function restrict_spike_times in module __main__:

restrict_spike_times(spike_times, interval)
    Given a spike_time dataset, restrict to spikes within given interval.

    Args:
      spike_times (sequence of np.ndarray): List or array of arrays,
        each inner array has spike times for a single neuron.
      interval (tuple): Min, max time values; keep min <= t < max.

    Returns:
      np.ndarray: like `spike_times`, but only within `interval`

t_interval = 5, 15  # units are seconds after start of recording
interval_spike_times = restrict_spike_times(spike_times, t_interval)_____no_output_____

Is this a representative interval? What fraction of the total spikes fall in this interval?_____no_output_____

original_counts = sum([len(spikes) for spikes in spike_times])
interval_counts = sum([len(spikes) for spikes in interval_spike_times])
frac_interval_spikes = interval_counts / original_counts
print(f"{frac_interval_spikes:.2%} of the total spikes are in the interval")0.33% of the total spikes are in the interval

How does this compare to the ratio between the interval duration and the experiment duration?

We can approximate the experiment duration by taking the minimum and maximum spike time in the whole dataset. To do that, we "concatenate" all of the neurons into one array and then use `np.ptp` ("peak-to-peak") to get the difference between the maximum and minimum value:_____no_output_____

spike_times_flat = np.concatenate(spike_times)
experiment_duration = np.ptp(spike_times_flat)
interval_duration = t_interval[1] - t_interval[0]

frac_interval_time = interval_duration / experiment_duration
print(f"{frac_interval_time:.2%} of the total time is in the interval")0.37% of the total time is in the interval

These two values are similar. This suggests the average spike rate of the neuronal population is not very different in this interval compared to the entire recording.

### Plotting spike trains and rasters

Now that we have a representative subset, we're ready to plot the spikes, using the matplotlib `plt.eventplot` function. Let's look at a single neuron first:_____no_output_____

neuron_idx = 1
plt.eventplot(interval_spike_times[neuron_idx], color=".2")
plt.xlabel("Time (s)")
plt.yticks([])_____no_output_____

We can also plot multiple neurons. Here are three:_____no_output_____

neuron_idx = [1, 11, 51]
plt.eventplot(interval_spike_times[neuron_idx], color=".2")
plt.xlabel("Time (s)")
plt.yticks([])_____no_output_____

This makes a "raster" plot, where the spikes from each neuron appear in a different row.

Plotting a large number of neurons can give you a sense for the characteristics in the population. Let's show every 5th neuron that was recorded:_____no_output_____

neuron_idx = np.arange(0, len(spike_times), 5)
plt.eventplot(interval_spike_times[neuron_idx], color=".2")
plt.xlabel("Time (s)")
plt.yticks([])_____no_output_____

*Question*: How does the information in this plot relate to the histogram of total spike counts that you saw above?_____no_output_____

#@title Video: Visualizing spiking activity
from IPython.display import YouTubeVideo
video = YouTubeVideo(id='4jNqXqr79o0', width=854, height=480, fs=1)
print("Video available at https://youtube.com/watch?v=" + video.id)
videoVideo available at https://youtube.com/watch?v=4jNqXqr79o0

---

## Inter-spike intervals and their distributions_____no_output_____Given the ordered arrays of spike times for each neuron in `spike_times`, which we've just visualized, what can we ask next?

Scientific questions are informed by existing models. So, what knowledge do we already have that can inform questions about this data?

We know that there are physical constraints on neuron spiking. It is thermodynamically inevitable that a neuron must spend energy to drive the cellular machinery that produces a spike. A neuron's reserves of energy are not bottomless, but must be actively replenished by metabolism. Therefore neurons should have a refractory period: they can only fire as quickly as these processes can support, and there is a minimum delay between consecutive spikes of the same neuron.

More generally, we can ask "how long does a neuron wait to spike again?" or "what is the longest a neuron will wait?" Can we transform spike times into something else, to address questions like these more directly?

We can consider the inter-spike times (or interspike intervals: ISIs). These are simply the time differences between consecutive spikes of the same neuron.

### Exercise: Plot the distribution of ISIs for a single neuron

*Exercise objective:* make a histogram, like we did for spike counts, to show the distribution of ISIs for one of the neurons in the dataset.

Do this in three steps:

1. Extract the spike times for one of the neurons
2. Compute the ISIs (the amount of time between spikes, or equivalently, the difference between adjacent spike times)
3. Plot a histogram with the array of individual ISIs_____no_output_____

# To complete the exercise, uncomment the code and fill missing parts (...)

# 1. Extract spike times for one neuron
single_neuron_idx = 283
# single_neuron_spikes = ...

# 2. Compute the ISIs
# single_neuron_isis = ...  # Hint: try the function np.diff

# 3. Plot the histogram with the array
n_bins = 50
# plt.hist(...)
# plt.xlabel(...)
# plt.ylabel(...)
# plt.axvline(..., color="orange", label="Mean ISI")
# plt.legend()_____no_output_____

[*Click for solution*](https://github.com/NeuromatchAcademy/course-content/tree/master//tutorials/W1D1_ModelTypes/solutions/W1D1_Tutorial1_Solution_4a35d3da.py)

*Example output:*

_____no_output_____---

In general, the shorter ISIs are predominant, with counts decreasing rapidly (and smoothly, more or less) with increasing ISI. However, counts also rapidly decrease to zero with _decreasing_ ISI, below the maximum of the distribution (8-11 ms). The absence of these very low ISIs agrees with the refractory period hypothesis: the neuron cannot fire quickly enough to populate this region of the ISI distribution.

Check the distributions of some other neurons. To resolve various features of the distributions, you might need to play with the value of `n_bins`. Using too few bins might smooth over interesting details, but if you use too many bins, the random variability will start to dominate.

You might also want to restrict the range to see the shape of the distribution when focusing on relatively short or long ISIs. *Hint:* the third argument to `plt.hist` sets the interval to define bins over._____no_output_____---

## What is the functional form of an ISI distribution?_____no_output_____

#@title Video: What models
from IPython.display import YouTubeVideo
video = YouTubeVideo(id='4NoqWMWC1ZY', width=854, height=480, fs=1)
print("Video available at https://youtube.com/watch?v=" + video.id)
videoVideo available at https://youtube.com/watch?v=4NoqWMWC1ZY

The ISI histograms seem to follow continuous, monotonically decreasing functions above their maxima. The function is clearly non-linear. Could it belong to a single family of functions?

To motivate the idea of using a mathematical function to explain physiological phenomena, let's define a few different function forms that we might expect the relationship to follow: exponential, inverse, and linear._____no_output_____

def exponential(xs, scale, rate, x0):
  """A simple parametrized exponential function, applied element-wise.

  Args:
    xs (np.ndarray or float): Input(s) to the function.
    scale (float): Linear scaling factor.
    rate (float): Exponential growth (positive) or decay (negative) rate.
    x0 (float): Horizontal offset.

  """
  ys = scale * np.exp(rate * (xs - x0))
  return ys

def inverse(xs, scale, x0):
  """A simple parametrized inverse function (`1/x`), applied element-wise.

  Args:
    xs (np.ndarray or float): Input(s) to the function.
    scale (float): Linear scaling factor.
    x0 (float): Horizontal offset.

  """
  ys = scale / (xs - x0)
  return ys

def linear(xs, slope, y0):
  """A simple linear function, applied element-wise.

  Args:
    xs (np.ndarray or float): Input(s) to the function.
    slope (float): Slope of the line.
    y0 (float): y-intercept of the line.

  """
  ys = slope * xs + y0
  return ys_____no_output_____

Here is an interactive demo where you can vary the parameters of these functions and see how well the resulting outputs correspond to the data. Adjust the parameters by moving the sliders and see how close you can get the lines to follow the falling curve of the histogram. This will give you a taste of what you're trying to do when you *fit a model* to data._____no_output_____

#@title ISI functions explorer
#@markdown When cells have sliders, there's hidden code that implements an interactive demo.

#@markdown You don't need to worry about how the code works – but you do need to **run the cell** to enable the sliders.
# Don't worry about understanding this code! It's to setup an interactive plot.

single_neuron_idx = 283
single_neuron_spikes = spike_times[single_neuron_idx]
single_neuron_isis = np.diff(single_neuron_spikes)

counts, edges = np.histogram(
  single_neuron_isis,
  n_bins,
  (0, single_neuron_isis.max())
)

functions = dict(
  exponential=exponential,
  inverse=inverse,
  linear=linear,
)

colors = dict(
  exponential="C1",
  inverse="C2",
  linear="C4",
)

@widgets.interact(
  exp_scale=widgets.FloatSlider(1000, min=0, max=20000, step=1000),
  exp_rate=widgets.FloatSlider(-10, min=-200, max=50, step=10),
  exp_x0=widgets.FloatSlider(0.1, min=-0.5, max=0.5, step=0.05),
  inv_scale=widgets.FloatSlider(1000, min=0, max=2e2, step=100),
  inv_x0=widgets.FloatSlider(0, min=-1, max=1, step=0.1),
  lin_slope=widgets.FloatSlider(-1e5, min=-5e5, max=1e5, step=100),
  lin_y0=widgets.FloatSlider(10000, min=0, max=20000, step=1000),
)
def fit_plot(
  exp_scale=1000, exp_rate=-10, exp_x0=0.1,
  inv_scale=1000, inv_x0=0,
  lin_slope=-1e5, lin_y0=2000,
):
  """Helper function for plotting function fits with interactive sliders."""
  func_params = dict(
    exponential=(exp_scale, exp_rate, exp_x0),
    inverse=(inv_scale, inv_x0),
    linear=(lin_slope, lin_y0),
  )
  f, ax = plt.subplots()
  ax.fill_between(edges[:-1], counts, step="post", alpha=.5)
  xs = np.linspace(1e-10, edges.max())
  for name, function in functions.items():
    ys = function(xs, *func_params[name])
    ax.plot(xs, ys, lw=3, color=colors[name], label=name);
  ax.set(
      xlim=(edges.min(), edges.max()),
      ylim=(0, counts.max() * 1.1),
      xlabel="ISI (s)",
      ylabel="Number of spikes",
  )
  ax.legend()_____no_output_____#@title Video: Fitting models by hand
from IPython.display import YouTubeVideo
video = YouTubeVideo(id='gZ3YcChh-CY', width=854, height=480, fs=1)
print("Video available at https://youtube.com/watch?v=" + video.id)
videoVideo available at https://youtube.com/watch?v=gZ3YcChh-CY

## Summary

In this tutorial, we loaded some neural data and poked at it to understand how the dataset is organized. Then we made some basic plots to visualize (1) the average level of activity across the population and (2) the distribution of ISIs for an individual neuron. In the very last bit, we started to think about using mathematical formalisms to understand or explain some physiological phenomenon.

This is the first step towards developing models that can tell us something about the brain. That's what we'll focus on in the next two tutorials._____no_output_____