# Notebook from muharif/ArifEtAll_2020_MultiTissueMI
Path: Codes/Codes.ipynb

## Important!

To be able to follow the script, please download the data from the GSEXXXXX. Please make sure that the name and filepath in the code is correct as per your downloaded files and file locations. If you encounter any problem, please feel free to contact muhammad.arif@scilifelab.se

The codes include all of the analysis steps for differential expression and functional analysis and also network generation and analysis. Codes and data for multi-tissue modelling will be placed under "WBM" folder

This code was re-tested on Python 3.7 with all of the required modules installed via conda._____no_output_____## Differential Expression and Functional Analysis_____no_output_____

<code>
import pandas as pd
import rpy2,os,re
import matplotlib.pyplot as plt
import matplotlib.patches as mpatches
from matplotlib.backends.backend_pdf import PdfPages
import matplotlib_venn as venn
import numpy as np
import seaborn as sns
import matplotlib
matplotlib.rcParams['pdf.fonttype'] = 42
matplotlib.rcParams['ps.fonttype'] = 42
from sklearn.decomposition import PCA as sklearnPCA
import seaborn as sns
os.environ['KMP_DUPLICATE_LIB_OK']='True'
import pickle
#rpy2
import rpy2.robjects as robjects
import rpy2.robjects.numpy2ri
rpy2.robjects.numpy2ri.activate()
from rpy2.robjects import pandas2ri
pandas2ri.activate()
import warnings
from rpy2.rinterface import RRuntimeWarning
warnings.filterwarnings("ignore", category=RRuntimeWarning)_____no_output_____class DESeq_Piano:
    def __init__(self,respath='.'):
        if respath=='.':
            print('Result Path is Not Defined, any result will be written in this directory')
            self.respath=respath
        elif os.path.isdir(respath):
            print('Result Path Exists, please be careful of overwriting')
            self.respath=respath
        else:
            print('Result Path Does Not Exists, creating new directory')
            os.mkdir(respath)
            self.respath=respath
        self.__Init()

    def __Init(self):

        robjects.r('''
            library('DESeq2')
            library('piano')
            library('Biobase')
            library('snow')
            library('RColorBrewer')
            library('gplots')
            library('snowfall')
        ''')
        self.GSC_KEGG=None
        self.GSC_TF=None
        self.GSC_RM=None
        self.GSC_GO=None
        self.GSC_extraGSC=None
        self.KEGG=None
        self.TF=None
        self.RM=None
        self.GO=None
        self.extraGSC=None
        self.dds=None
        self.deseq_res=None
        self.conds=None
        self.count=None
        self.tpm=None

    def __check_dir(self,extrapath):
        if os.path.isdir('%s/%s/' % (self.respath,extrapath)):
            print('Path %s/%s/ exist, writing in the folder' % (self.respath,extrapath))
        else:
            print('Path %s/%s/ does not exist, making the folder to write the results' % (self.respath,extrapath))
            os.mkdir('%s/%s/' % (self.respath,extrapath))

    def DEseq(self,count_df=None,conds_series=None):
        if count_df is not None:
            self.count=count_df
        if conds_series is not None:
            self.conds=conds_series
        self.gene_names=count_df.index
        robjects.globalenv['conds']=np.array(self.conds)
        robjects.globalenv['subject']=np.array(self.conds.index)
        robjects.globalenv['count']=self.count.astype(int).values
        robjects.r('''
            library('DESeq2')
            conds=as.factor(conds)
            coldata <- data.frame(row.names=subject,conds)
            dds <- DESeqDataSetFromMatrix(countData=as.matrix(count),colData=coldata,design=~conds)
            dds <- DESeq(dds)
            print(resultsNames(dds))
            #baseMeanPerLvl <- sapply( levels(conds), function(lvl) rowMeans( counts(dds,normalized=TRUE)[,conds == lvl] ) )
            #conds2=colnames(baseMeanPerLvl)
            #gene_names=rownames(baseMeanPerLvl)

        ''')
        self.dds=robjects.globalenv['dds']

    def DEseq_Compare(self, cond1, cond2,save=True):
        if self.dds is None:
            raise ValueError('No DESeq data found, run the DEseq function')
        else:
            print('Comparing %s and %s' % (cond1,cond2))
            robjects.globalenv['dds']=self.dds
            robjects.globalenv['cond1']=cond1
            robjects.globalenv['cond2']=cond2
            robjects.globalenv['gene_names']=np.array(self.gene_names)
            robjects.r('''
                res=results(dds,contrast=c('conds',cond1,cond2))
                res=data.frame(res)
                res$rownames=gene_names
            ''')
            res=pandas2ri.rpy2py_dataframe(robjects.globalenv['res']).set_index('rownames')
            self.deseq_df=res
            if save:
                self.__check_dir('deseq')
                res.to_csv('%s/deseq/deseq_%s_%s.txt' % (self.respath,cond1,cond2),sep='\t')

    def loadGSC(self,KEGG='',TF='',RM='',GO='',extraGSC=''):
        if KEGG != '':
            name='KEGG'
            self.__check_dir(name)
            self.GSC_KEGG=self.__loadGSC_execute(KEGG)
        if TF != '':
            name='TF'
            self.__check_dir(name)
            self.GSC_TF=self.__loadGSC_execute(TF)
        if RM != '':
            name='RM'
            self.__check_dir(name)
            self.GSC_RM=self.__loadGSC_execute(RM)
        if GO != '':
            name='GO'
            self.__check_dir(name)
            self.GSC_GO=self.__loadGSC_execute(GO)
        if extraGSC != '':
            name='extraGSC'
            self.__check_dir(name)
            self.GSC_extraGSC=self.__loadGSC_execute(extraGSC)

    def __loadGSC_execute(self,GSC):
        robjects.globalenv['GSC']=GSC
        robjects.r('''
            y=loadGSC(GSC)
        ''')
        return robjects.globalenv['y']

    def __PIANO_execute(self,cond1,cond2,GSCtype='',save=True):
        robjects.globalenv['deseq_file']=pandas2ri.py2rpy_pandasdataframe(self.deseq_df)
        if GSCtype == 'KEGG':
            robjects.globalenv['y']=self.GSC_KEGG
            deseq_df=self.deseq_df
            deseq_df.index=[i.upper() for i in deseq_df.index]
            robjects.globalenv['deseq_file']=pandas2ri.py2rpy_pandasdataframe(deseq_df)
        elif GSCtype == 'TF':
            robjects.globalenv['y']=self.GSC_TF
        elif GSCtype == 'RM':
            robjects.globalenv['y']=self.GSC_RM
        elif GSCtype == 'GO':
            robjects.globalenv['y']=self.GSC_GO
            deseq_df=self.deseq_df
            deseq_df.index=[i.upper() for i in deseq_df.index]
            robjects.globalenv['deseq_file']=pandas2ri.py2rpy_pandasdataframe(deseq_df)
        else:
            robjects.globalenv['y']=self.GSC_extraGSC
        robjects.r('''
            DESeq_file=deseq_file
            DESeq_file=DESeq_file[ ,c('log2FoldChange','pvalue')]
            logFC=as.matrix(DESeq_file[,1])
            pval=as.matrix(DESeq_file[,2])
            rownames(logFC)=(rownames(DESeq_file))
            rownames(pval)=(rownames(DESeq_file))
            logFC[is.na(logFC)] <- 0
            pval[is.na(pval)] <- 1
            gsaRes <- runGSA(pval,logFC,gsc=y, geneSetStat="reporter", signifMethod="nullDist", nPerm=1000,ncpus=8)
            res_piano=GSAsummaryTable(gsaRes)
            res_piano$rownames=rownames(res_piano)
        ''')
        res=pandas2ri.rpy2py_dataframe(robjects.globalenv['res_piano']).set_index('rownames').set_index('Name').iloc[0:,0:]#.groupby('Name').sum()
        if save:
            res.to_csv('%s/%s/piano_%s_%s.txt' % (self.respath,GSCtype,cond1,cond2),sep='\t')
        return res

    def PIANO_heatmap(self,cond1,cond2,GSCtype,save=True):
        if GSCtype == 'KEGG':
            deseq=self.KEGG
        elif GSCtype == 'TF':
            deseq=self.TF
        elif GSCtype == 'RM':
            deseq=self.RM
        elif GSCtype == 'GO':
            deseq=self.GO
        else:
            deseq=self.extraGSC
        deseq=deseq[['p adj (dist.dir.up)','p adj (dist.dir.dn)']]
        thr=0.05
        dn=deseq[(deseq.loc[:,deseq.columns.str.contains('dn')]<thr)].drop_duplicates()
        dn=dn.loc[:,dn.columns.str.contains('dn')]
        up=deseq[(deseq.loc[:,deseq.columns.str.contains('up')]<thr)].drop_duplicates()
        up=up.loc[:,up.columns.str.contains('up')]
        temp_dn=np.log10(dn.loc[:,dn.columns.str.contains('adj')][(dn.loc[:,dn.columns.str.contains('adj')]<0.05).sum(1)>0])
        temp_dn.index=[i.split('&')[0] if len(i)>100 else i for i in temp_dn.index]
        temp_up=-np.log10(up.loc[:,up.columns.str.contains('adj')][(up.loc[:,up.columns.str.contains('adj')]<0.05).sum(1)>0])
        temp_up.index=[i.split('&')[0] if len(i)>100 else i for i in temp_up.index]
        name=list(set(temp_up.index).intersection(set(temp_dn.index)))
        temp_df=pd.concat([temp_up.loc[name],temp_dn.loc[name]])
        temp_up=temp_up.loc[list(set(temp_up.index).difference(set(name)))]
        temp_dn=(temp_dn.loc[list(set(temp_dn.index).difference(set(name)))])
        temp_df=temp_df.groupby(temp_df.index).max()
        temp=pd.concat([temp_dn,temp_up,temp_df]).T
        temp=pd.DataFrame(temp.sum(),columns=['%s vs %s' % (cond1,cond2)])
        temp=temp.replace(np.inf,temp.replace([np.inf, -np.inf],np.nan).max().max()*1.1)
        temp=temp.replace(-np.inf,temp.replace([np.inf, -np.inf],np.nan).min().min()*1.1)
        temp=temp.replace(-0.0,0)
        temp=temp.loc[:,sorted(temp.columns)]
        min1=temp.min().min()
        max1=temp.max().max()
        cmap=matplotlib.colors.LinearSegmentedColormap.from_list("", ["#0000a5",'#0000d8',"#FFFAF0",'#d80000',"#a50000"])
        temp=temp.replace(np.inf,temp.replace([np.inf, -np.inf],np.nan).max().max()*1.1)
        temp=temp.replace(-np.inf,temp.replace([np.inf, -np.inf],np.nan).min().min()*1.1)
        temp=temp[(temp.abs()>-np.log10(thr)).sum(1)>0].fillna(0)
        temp2=temp.reindex(temp[temp.columns[0]].abs().sort_values(ascending=False).index).copy()
        if temp.shape[0] > 100:
            temp=temp.reindex(temp[temp.columns[0]].abs().sort_values(ascending=False).index)
            temp_d=temp[temp[temp.columns[0]] < 0]
            temp_u=temp[temp[temp.columns[0]] > 0]
            if temp_d.shape[0]<=0:
                temp_u=temp_u.iloc[0:(100-temp_d.shape[0])]
            elif temp_u.shape[0]<=0:
                temp_d=temp_d.iloc[0:(100-temp_u.shape[0])]
            else:
                temp_d=temp_d.iloc[0:50]
                temp_u=temp_u.iloc[0:50]
            temp=pd.concat([temp_d,temp_u])
            g=sns.clustermap(temp,figsize=(2.5, 10),col_cluster=False,row_cluster=True,linewidths=0.1,cmap=cmap,center=0.0,cbar_kws={'label': '-Log10(P-Adjusted)'})
            g.ax_heatmap.set(yticks=[i-0.5 for i in range(1,temp.shape[0]+1)])
            g.ax_heatmap.set_yticklabels(labels=temp.index[list(map(int,g.dendrogram_row.calculate_dendrogram()['ivl']))],size=6)
        else:
            g=sns.clustermap(temp,figsize=(2.5, 10),col_cluster=False,row_cluster=True,linewidths=0.1,cmap=cmap,center=0.0,cbar_kws={'label': '-Log10(P-Adjusted)'})
            g.ax_heatmap.set(yticks=[i-0.5 for i in range(1,temp.shape[0]+1)])
            g.ax_heatmap.set_yticklabels(labels=temp.index[list(map(int,g.dendrogram_row.calculate_dendrogram()['ivl']))],size=8)
        g.ax_heatmap.xaxis.tick_top()
        path=self.respath+'/Figures/'+GSCtype
        if save:
            g.savefig('%s/Heatmap_%s_%s.pdf' % (path,cond1,cond2))

    def PIANO(self,cond1,cond2,heatmap=True,save=True):
        if save:
            self.__check_dir('Figures')
        if self.deseq_df is None:
            raise ValueError('No DEseq comparison found, run DEseq_Compare first or assign deseq result dataframe to deseq_res')
        else:
            if self.GSC_KEGG is not None:
                name='KEGG'
                self.KEGG = self.__PIANO_execute(cond1,cond2,GSCtype=name,save=save)
                if heatmap == True:
                    self.__check_dir('/Figures/%s' % name)
                    try:
                        self.PIANO_heatmap(cond1,cond2,GSCtype=name,save=save)
                    except ValueError:
                        pass
            if self.GSC_GO is not None:
                name='GO'
                self.GO = self.__PIANO_execute(cond1,cond2,GSCtype=name,save=save)
                if heatmap == True:
                    self.__check_dir('/Figures/%s' % name)
                    try:
                        self.PIANO_heatmap(cond1,cond2,GSCtype=name,save=save)
                    except ValueError:
                        pass
            if self.GSC_TF is not None:
                name='TF'
                self.TF = self.__PIANO_execute(cond1,cond2,GSCtype=name,save=save)
                if heatmap == True:
                    self.__check_dir('/Figures/%s' % name)
                    try:
                        self.PIANO_heatmap(cond1,cond2,GSCtype=name,save=save)
                    except ValueError:
                        pass
            if self.GSC_RM is not None:
                name='RM'
                self.RM = self.__PIANO_execute(cond1,cond2,GSCtype=name,save=save)
                if heatmap == True:
                    self.__check_dir('/Figures/%s' % name)
                    try:
                        self.PIANO_heatmap(cond1,cond2,GSCtype=name,save=save)
                    except ValueError:
                        pass
            if self.GSC_extraGSC is not None:
                name='extraGSC'
                self.extraGSC = self.__PIANO_execute(cond1,cond2,GSCtype=name,save=save)
                if heatmap == True:
                    self.__check_dir('/Figures/%s' % name)
                    try:
                        self.PIANO_heatmap(cond1,cond2,GSCtype=name,save=save)
                    except ValueError:
                        pass

    def __piano2xlsx(self,name,part='ALL'):
        if part != 'ALL':
            files=[i for i in sorted(os.listdir('%s/%s' % (self.respath,name))) if part in i]
        else:
            files=sorted(os.listdir('%s/%s' % (self.respath,name)))
        writer = pd.ExcelWriter('%s/%s_%s.xlsx' % (self.respath,name,part), engine='xlsxwriter')
        for i in files:
            if i.startswith('.'):
                continue
            deseq1=pd.read_csv('%s/%s/%s' % (self.respath,name,i),index_col='Name',sep='\t')[['Genes (tot)','p adj (dist.dir.up)']]
            deseq1.columns=['# of Genes','P-Adj']
            deseq1=deseq1[deseq1['P-Adj']<0.05]
            deseq1['Direction']='UP'
            deseq2=pd.read_csv('%s/%s/%s' % (self.respath,name,i),index_col='Name',sep='\t')[['Genes (tot)','p adj (dist.dir.dn)']]
            deseq2.columns=['# of Genes','P-Adj']
            deseq2=deseq2[deseq2['P-Adj']<0.05]
            deseq2['Direction']='DOWN'
            deseq1=pd.concat([deseq1,deseq2])
            if len(i.replace('piano_','').replace('.txt',''))>31:
                print('sheet name too big, trimming to 30 character')
                n_temp=i.replace('piano_','').replace('.txt','')[0:31]
            else:
                n_temp=i.replace('piano_','').replace('.txt','')
            deseq1.sort_values('P-Adj').to_excel(writer, sheet_name=n_temp)
        writer.save()

    def summarizePiano(self,names=['KEGG','GO','TF','RM','extraGSC'],part='ALL'):
        for name in names:
            if os.path.isdir(self.respath+name):
                self.__piano2xlsx(name,part=part)
            else:
                continue

    def summarizeDEseq(self,deseq_result='deseq',part='ALL'):
        if os.path.isdir(self.respath+deseq_result):
            if part != 'ALL':
                files=[i for i in sorted(os.listdir(self.respath+'/deseq/')) if part in i]
            else:
                files=sorted(os.listdir(self.respath+'/deseq/'))
            writer = pd.ExcelWriter(self.respath+'/DifferentialExpression_%s.xlsx' % part, engine='xlsxwriter')
            fin=0
            for i in files:
                if i.startswith('.'):
                    continue
                deseq=pd.read_csv(self.respath+'/deseq/%s' % i,index_col=0,sep='\t')
                deseq['abs']=deseq['log2FoldChange'].abs()
                deseq=deseq.sort_values('abs',ascending=False)[['log2FoldChange','pvalue','padj']]

                deseq['Direction']=['UP' if i > 0 else 'DOWN' for i in deseq['log2FoldChange']]
                deseq.columns=['L2FC','P-VALUE','P-ADJ','DIRECTION']
                deseq=deseq.sort_values('P-VALUE')
                if len(i.replace('deseq_','').replace('.txt',''))>31:
                    print('sheet name too big, trimming to 30 character')
                    n_temp=i.replace('deseq_','').replace('.txt','')[0:31]
                else:
                    n_temp=i.replace('deseq_','').replace('.txt','')
                deseq.to_excel(writer, sheet_name=n_temp)
            writer.save()
        else:
            raise ValueError('No DESEq result found')

    def save_object(self,filename='DEseq_PIANO.pkl'):
        with open('%s/%s' % (self.respath,filename), 'wb') as file:
            pickle.dump(self, file)

    def load_object(self,filename='DEseq_PIANO.pkl'):
        with open(filename, 'rb') as file:
            res=pickle.load(file)
        return res

    def PCA(self,tpm='',conds='',annot='',respath='',save=False,part='',not_part=None):
        if type(conds) != pd.core.series.Series:
            conds = self.conds
        if type(tpm) != pd.core.frame.DataFrame:
            tpm = self.tpm
        filt=sum(list(map(lambda x: (tpm[conds[conds==x].index].mean(1)>1),set(conds))))
        tpm=tpm[(filt>0)]
        if ((part == '') & (not_part == None)):
            part='ALL'
        elif part != '':
            conds=conds[conds.str.contains(part)]
            tpm=tpm[conds.index]
        elif not_part != None:
            part='NOT %s' % not_part
            conds=conds[~conds.str.contains(part)]
            tpm=tpm[conds.index]
        sklearn_pca = sklearnPCA(n_components=2)
        Y_sklearn = sklearn_pca.fit_transform(np.log10(tpm.T+1))
        with plt.style.context('seaborn-white'):
            fig = plt.figure(figsize=(15,8))
            ax = fig.add_subplot(111, aspect='auto')
            ax.grid(False)
            for lab,m,col in zip(sorted(list(set(conds))),['o','*','v','^']*4,(['k']*4)+(['lightblue']*4)+(['yellow']*4)+(['green']*4)):
                x=Y_sklearn[conds==lab, 0]
                y=Y_sklearn[conds==lab, 1]
                plt.scatter(x,y,c=col,label=lab,marker=m,s=200)
        if annot != '':
            for i, txt in enumerate(annot):
                ax.annotate(txt, (Y_sklearn[i, 0], Y_sklearn[i, 1]))
        plt.xlabel('PC1 (%.2f)' % (sklearn_pca.explained_variance_ratio_[0]*100))
        plt.ylabel('PC2 (%.2f)' % (sklearn_pca.explained_variance_ratio_[1]*100))
        plt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)
        plt.tight_layout()
        if save:
            self.__check_dir('Figures')
            plt.savefig('%s/Figures/PCA_%s.pdf' % (self.respath,part))
        loadings=sklearn_pca.components_.T * np.sqrt(sklearn_pca.explained_variance_)
        loadings=pd.DataFrame(loadings,columns=['x','y'],index=tpm.index)
        return loadings_____no_output_____count=pd.read_csv('../Data/count_geneName.txt',sep='\t',index_col=0)
tpm=pd.read_csv('../Data/tpm_geneName.txt',sep='\t',index_col=0)
conds=pd.read_csv('../Data/metadata.txt',sep='\t',index_col=0)
conds['Conditions']=conds['Tissue']+'_'+conds['Treatment']+'_'+conds['Time']
conds=conds['Conditions']_____no_output_____conds_____no_output_____#Initialize the class
k=DESeq_Piano('../Results/')

#Generate PCA
loadings=k.PCA(tpm,conds,save=True)

#Execute Differential Expression Analysis
k.DEseq(count,conds)

#Loading gene set collection for functional analysis
k.loadGSC(KEGG='../Library/KEGG.gmt' ,GO='../../MainLibrary/Current/GO_BP.gmt')_____no_output_____#Define conditions to compare, example in heart 24h, SHAM vs MI
cond1='heart_MI_24h'
cond2='heart_SHAM_24h'

#Retrieve DEGs specific for the defined comparion
k.DEseq_Compare(cond1,cond2,save=True)

#Retrieve Functional analysis results specific for the defined comparion
k.PIANO(cond1,cond2,heatmap=True,save=True)_____no_output_____#Combine all results to a single excel file for each type of analysis
k.summarizeDEseq()
k.summarizePiano()_____no_output_____
</code>

## Generation of Tissue-specific CN_____no_output_____

<code>
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib, os, shutil
matplotlib.rcParams['pdf.fonttype'] = 42
matplotlib.rcParams['ps.fonttype'] = 42
from scipy.stats import spearmanr
from statsmodels.sandbox.stats.multicomp import multipletests
from tqdm import tqdm_notebook
import leidenalg
import igraph as ig
import gseapy as gp
import pickle
from matplotlib import pyplot as plt_____no_output_____def calc(tpm,pval_thr=0.05,pos_only=False):
    print('Calculating Correlation..')
    temp=spearmanr(tpm.T)
    corr=pd.DataFrame(temp[0],columns=list(tpm.index),index=list(tpm.index))
    pval=pd.DataFrame(temp[1],columns=list(tpm.index),index=list(tpm.index))
    print('Filtering the matrix Correlation..')
    corr=corr.where(np.triu(np.ones(corr.shape)).astype(np.bool))
    pval=pval.where(np.triu(np.ones(pval.shape)).astype(np.bool))
    print('Making long table of Correlation..')
    corr2=corr.unstack().reset_index(name='weight')
    pval2=pval.unstack().reset_index(name='pval')
    res=corr2.merge(pval2,on=['level_0','level_1'])
    res=res[res['level_0'] != res['level_1']]
    res=res.dropna()
    print('Adjusting P-val')
    res['padj']=multipletests(res['pval'],method='fdr_bh')[1]
    res=res[res.padj < pval_thr].reset_index(drop=True)
    if pos_only:
        res=res[res['weight']>0]
    res=res[['level_0','level_1','weight']]
    print('Done!!')
    return res
def filter_tpm(tpm,conds,thr=1):
    #making sure that in at least 1 condition we have TPM > threshold (default 1)
    return tpm[(pd.concat([tpm.T,conds],1).groupby(conds.name).mean().T>thr).sum(1)>0]_____no_output_____tpm=pd.read_csv('../Data/tpm_geneName.txt',sep='\t',index_col=0)
conds=pd.read_csv('../Data/metadata.txt',sep='\t',index_col=0)
conds=conds['Tissue']
tpm=tpm[conds.index]_____no_output_____# Filter the specific tissue, example heart
tissue='heart'
conds2=conds[conds == tissue]
tpm2=tpm[conds2.index]
_____no_output_____#Calculate co-expression network
corr=calc(filter_tpm(tpm2,conds2))
corr.to_csv('../Results/coexp_%s.txt' % tissue,sep='\t',index=False)_____no_output_____
</code>

## CN Analysis_____no_output_____

<code>
class Network_Analysis:
    def __init__(self,filename,respath,KEGG='',GO='',topx=0.05,cluster_size=30):
        self.network_ori=pd.read_csv(filename,sep='\t')
        self.res_path=respath
        if os.path.isdir(self.res_path):
            pass
        else:
            os.mkdir(self.res_path)
#         self.min_weight=min_weight
        self.cluster_size=cluster_size
        self.topx=topx
        self.GSC_KEGG=KEGG
        self.GSC_GO=GO
        self.__net_analysis()
        self.__net_analysis_neg()
        print('Enriching Clusters...')
        self.__cluster_enrichr()
        self.__cluster_enrichr_neg()

    def __net_analysis(self):
        print('Loading The Network...')
#         temp=self.network_ori[self.network_ori['weight']>self.min_weight]
        temp=self.network_ori[self.network_ori['weight']>0]
        temp=temp[temp['weight']>temp['weight'].quantile(1-self.topx)]
        g= ig.Graph.TupleList(zip(temp['level_0'],temp['level_1']))
        print('Cluster Analysis...')
        optimiser = leidenalg.Optimiser()
        clust_calc = leidenalg.ModularityVertexPartition(g)
        diff =  optimiser.optimise_partition(clust_calc, n_iterations=-1)
        clust=pd.Series(clust_calc.membership,index=g.vs['name'])
        thr=self.cluster_size
        temp_c=clust.value_counts()[clust.value_counts()>thr].index.tolist()
        self.clustering=clust[clust.isin(temp_c)]
        self.modularity=clust_calc.modularity
        self.network=g

    def __net_analysis_neg(self):
        print('Loading The Network...')
#         temp=self.network_ori[self.network_ori['weight']<(-1*self.min_weight)]
        temp=self.network_ori[self.network_ori['weight']<0]
        temp=temp[temp['weight']<temp['weight'].quantile(self.topx)]
        g= ig.Graph.TupleList(zip(temp['level_0'],temp['level_1']))
        print('Cluster Analysis...')
        optimiser = leidenalg.Optimiser()
        clust_calc = leidenalg.ModularityVertexPartition(g)
        diff =  optimiser.optimise_partition(clust_calc, n_iterations=-1)
        clust=pd.Series(clust_calc.membership,index=g.vs['name'])
        thr=self.cluster_size
        temp_c=clust.value_counts()[clust.value_counts()>thr].index.tolist()
        self.clustering_neg=clust[clust.isin(temp_c)]
        self.modularity_neg=clust_calc.modularity
        self.network_neg=g

    def __enrichr(self,GSC,db_name):
        # Offline GSEA with enrichr
        final={}
        for i in range(self.clustering.value_counts().shape[0]):
            if db_name == 'GO':
                gene_list=list([i.upper() for i in self.clustering[self.clustering.isin([i])].index])
            else:
                gene_list=list([i.upper() for i in self.clustering[self.clustering.isin([i])].index])
            enr = gp.enrichr(gene_list=gene_list,
                             description='test_name',
                             gene_sets=GSC,
                             background=20000,
                             outdir='test/enrichr_kegg2',
                             cutoff=0.5,
                             verbose=False)

            temp=enr.res2d[enr.res2d['Adjusted P-value']<0.05].sort_values('Adjusted P-value')[['Term','Adjusted P-value']]
            temp.columns=['%s' % db_name,'P-Adj (%s)' % db_name]
            if db_name == 'GO':
                temp['GOID']=[s[s.find("(")+1:s.find(")")] for s in temp['GO']]
                temp=temp[['GO','GOID','P-Adj (GO)']]
            final[i]=temp.reset_index(drop=True)
        return final

    def __cluster_enrichr(self):
        if (self.GSC_KEGG != '') | (self.GSC_GO != ''):
            writer = pd.ExcelWriter('%s/Coexp_Cluste_Enrichr.xlsx' % self.res_path, engine='xlsxwriter')
            if self.GSC_KEGG != '':
                self.KEGG=self.__enrichr(self.GSC_KEGG,'KEGG')
                pd.concat(self.KEGG.values(), axis=1, keys=self.KEGG.keys()).to_excel(writer, sheet_name='KEGG')

            if self.GSC_GO != '':
                self.GO=self.__enrichr(self.GSC_GO,'GO')
                pd.concat(self.GO.values(), axis=1, keys=self.GO.keys()).to_excel(writer, sheet_name='GO')

            if (self.GSC_KEGG != '') & (self.GSC_GO != ''):
                self.Both={i:pd.concat([self.GO[i],self.KEGG[i]],1) for i in range(self.clustering.value_counts().shape[0])}
                pd.concat(self.Both.values(), axis=1, keys=self.Both.keys()).to_excel(writer, sheet_name='Both')
            writer.save()
            shutil.rmtree('test')

    def __enrichr_neg(self,GSC,db_name):
        # Offline GSEA with enrichr
        final={}
        for i in range(self.clustering_neg.value_counts().shape[0]):
            if db_name == 'GO':
                gene_list=list([i.upper() for i in self.clustering_neg[self.clustering_neg.isin([i])].index])
            else:
                gene_list=list([i.upper() for i in self.clustering_neg[self.clustering_neg.isin([i])].index])
            enr = gp.enrichr(gene_list=gene_list,
                             description='test_name',
                             gene_sets=GSC,
                             background=20000,
                             outdir='test/enrichr_kegg2',
                             cutoff=0.5,
                             verbose=False)
            temp=enr.res2d[enr.res2d['Adjusted P-value']<0.05].sort_values('Adjusted P-value')[['Term','Adjusted P-value']]
            temp.columns=['%s' % db_name,'P-Adj (%s)' % db_name]
            if db_name == 'GO':
                temp['GOID']=[s[s.find("(")+1:s.find(")")] for s in temp['GO']]
                temp=temp[['GO','GOID','P-Adj (GO)']]
            final[i]=temp.reset_index(drop=True)
        return final

    def __cluster_enrichr_neg(self):
        if (self.GSC_KEGG != '') | (self.GSC_GO != ''):
            writer = pd.ExcelWriter('%s/Coexp_Cluste_Enrichr_neg.xlsx' % self.res_path, engine='xlsxwriter')
            if self.GSC_KEGG != '':
                self.KEGG_neg=self.__enrichr_neg(self.GSC_KEGG,'KEGG')
                pd.concat(self.KEGG_neg.values(), axis=1, keys=self.KEGG_neg.keys()).to_excel(writer, sheet_name='KEGG')

            if self.GSC_GO != '':
                self.GO_neg=self.__enrichr_neg(self.GSC_GO,'GO')
                pd.concat(self.GO_neg.values(), axis=1, keys=self.GO_neg.keys()).to_excel(writer, sheet_name='GO')

            if (self.GSC_KEGG != '') & (self.GSC_GO != ''):
                self.Both_neg={i:pd.concat([self.GO_neg[i],self.KEGG_neg[i]],1) for i in range(self.clustering_neg.value_counts().shape[0])}
                pd.concat(self.Both_neg.values(), axis=1, keys=self.Both_neg.keys()).to_excel(writer, sheet_name='Both')
            writer.save()
            shutil.rmtree('test')

    def save_network(self):
        pickle_out = open('%s/network_object.pkl' % self.res_path,"wb")
        pickle.dump(self, pickle_out)
        pickle_out.close()

    def export_cluster_cytoscape(self):
        temp=self.network_ori.merge(pd.DataFrame(self.clustering).reset_index(),how='left',left_on='level_0',right_on='index').merge(pd.DataFrame(self.clustering).reset_index(),how='left',left_on='level_1',right_on='index').dropna()
        temp2=temp[['0_x','0_y']]
        temp2['count']=1
        temp2=temp2.groupby(['0_x','0_y']).count().reset_index().pivot_table(index='0_x',columns='0_y',values='count').fillna(0)
        temp2=pd.DataFrame([(i,j,(temp2.loc[i,j]+temp2.loc[j,i]),(temp2.loc[i,j]+temp2.loc[j,i])/(self.clustering.value_counts()[i]+self.clustering.value_counts()[j])) for i in temp2.index for j in temp2.columns if (i < j)],columns=['Source','Target','Weight1','Weight2'])
        temp2.astype(int).to_csv('%s/edge_pos_cluster.txt' % self.res_path,sep='\t')
        pd.DataFrame(self.clustering.value_counts(),columns=['Size']).to_csv('%s/node_pos_cluster.txt' % self.res_path,sep='\t')
        self.clustering.to_csv('%s/clust_pos_cluster.txt' % self.res_path,sep='\t')
        temp=self.network_ori.merge(pd.DataFrame(self.clustering_neg).reset_index(),how='left',left_on='level_0',right_on='index').merge(pd.DataFrame(self.clustering_neg).reset_index(),how='left',left_on='level_1',right_on='index').dropna()
        temp2=temp[['0_x','0_y']]
        temp2['count']=1
        temp2=temp2.groupby(['0_x','0_y']).count().reset_index().pivot_table(index='0_x',columns='0_y',values='count').fillna(0)
        temp2=pd.DataFrame([(i,j,(temp2.loc[i,j]+temp2.loc[j,i]),(temp2.loc[i,j]+temp2.loc[j,i])/(self.clustering_neg.value_counts()[i]+self.clustering_neg.value_counts()[j])) for i in temp2.index for j in temp2.columns if (i < j)],columns=['Source','Target','Weight1','Weight2'])
        temp2.astype(int).to_csv('%s/edge_neg_cluster.txt' % self.res_path,sep='\t')
        pd.DataFrame(self.clustering_neg.value_counts(),columns=['Size']).to_csv('%s/node_neg_cluster.txt' % self.res_path,sep='\t')
        self.clustering_neg.to_csv('%s/clust_neg_cluster.txt' % self.res_path,sep='\t')
    _____no_output_____tissue='heart'

#Self-Explanatory
k=Network_Analysis(filename='../Results/coexp_%s.txt' % tissue,respath='../Results/Coexp_%s/' % tissue,KEGG='../Library/KEGG.gmt',GO='../Library/GO_BP.gmt',topx=topx)
k.export_cluster_cytoscape()
k.save_network()_____no_output_____
</code>


# Notebook from Romit-Maulik/Tutorials-Demos-Practice
Path: ROM_Demos/CAE_LSTM/Single_Parameter/Burgers/Separate/.ipynb_checkpoints/AE_LSTM_1000-checkpoint.ipynb

import numpy as np
import tensorflow as tf

# Set seeds
np.random.seed(10)
tf.random.set_seed(10)

from tensorflow.keras.layers import Input, Dense, LSTM, Lambda, Dropout
from tensorflow.keras.layers import Conv1D, TimeDistributed, UpSampling1D, MaxPooling1D

from tensorflow.keras import optimizers, models, regularizers
from tensorflow.keras import backend as K
from tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping
from tensorflow.keras.models import load_model, Sequential, Model
from tensorflow.keras.regularizers import l1
from tensorflow.keras.utils import plot_model

from scipy.signal import savgol_filter
from sklearn.preprocessing import MinMaxScaler_____no_output_____%matplotlib inline
import seaborn as sns
import re
import matplotlib as mpl
import matplotlib.font_manager as fm
from matplotlib import pyplot as plt
from mpl_toolkits.axes_grid.inset_locator import (inset_axes, InsetPosition,
                                                  mark_inset)
import matplotlib.ticker as mtick
mpl.rcParams['ps.useafm'] = True
#mpl.rcParams['pdf.use14corefonts'] = True
mpl.rcParams['text.usetex'] = True
mpl.rc('text', usetex=True)
mpl.rcParams['text.latex.unicode']=True
print(mpl.pyplot.get_backend())

mpl.rcParams['text.latex.preamble'] = [
       r'\usepackage{siunitx}',   # i need upright \micro symbols, but you need...
       r'\sisetup{detect-all}',   # ...this to force siunitx to actually use your fonts
       r'\usepackage[T1]{fontenc}',
       r'\usepackage[default]{gillius}',    # set the normal font here
       r'\usepackage{sansmath}',  # load up the sansmath so that math -> helvet
       r'\sansmath'               # <- tricky! -- gotta actually tell tex to use!
]

def init_plotting():
    plt.rcParams['figure.figsize'] = (8, 5)
    plt.rcParams['font.size'] = 20
    print(plt.rcParams['font.family'] )
    plt.rcParams['axes.labelsize'] = plt.rcParams['font.size']
    plt.rcParams['axes.titlesize'] = plt.rcParams['font.size']
    plt.rcParams['legend.fontsize'] = 0.9*plt.rcParams['font.size']
    plt.rcParams['xtick.labelsize'] = plt.rcParams['font.size']
    plt.rcParams['ytick.labelsize'] = plt.rcParams['font.size']
#    print(plt.rcParams['savefig.dpi'])
#    plt.rcParams['savefig.dpi'] = 2*plt.rcParams['savefig.dpi']
    plt.rcParams['xtick.minor.visible']=False
    plt.rcParams['ytick.minor.visible']=False
    plt.rcParams['xtick.major.size'] = 6
    plt.rcParams['xtick.minor.size'] = 3
    plt.rcParams['xtick.major.width'] = 2
    plt.rcParams['xtick.minor.width'] = 2
    plt.rcParams['ytick.major.size'] = 6
    plt.rcParams['ytick.minor.size'] = 3
    plt.rcParams['ytick.major.width'] = 2
    plt.rcParams['ytick.minor.width'] = 2
    plt.rcParams['ytick.color'] = "black"#"#808080"
    plt.rcParams['xtick.color'] = "black"#"#808080"
    plt.rcParams['xtick.direction'] = 'in'
    plt.rcParams['ytick.direction'] = 'in'
    #ax.tick_params(direction='out', length=6, width=2, colors='r',
    #           grid_color='r', grid_alpha=0.5)
    plt.rcParams['legend.frameon'] = True
    plt.rcParams['legend.loc'] = 'best'
    plt.rcParams['axes.linewidth'] = 2

init_plotting()

def hide_spines():
    """Hides the top and rightmost axis spines from view for all active
    figures and their respective axes."""
    # Retrieve a list of all current figures.
    figures = [x for x in mpl._pylab_helpers.Gcf.get_all_fig_managers()]
    for figure in figures:
        # Get all Axis instances related to the figure.
        for ax in figure.canvas.figure.get_axes():

            ax.spines['left'].set_color("#808080")
            ax.spines['bottom'].set_color("#808080")
            ax.spines['right'].set_color("#808080")
            ax.spines['top'].set_color("#808080")
            ax.xaxis.set_ticks_position('bottom')
            ax.yaxis.set_ticks_position('left')

            ax.spines['top'].set_linewidth(0.1)
            ax.spines['top'].set_linestyle(':')
            ax.spines['right'].set_linewidth(0.1)
            ax.spines['right'].set_linestyle(':')

hide_spines()module://ipykernel.pylab.backend_inline
['sans-serif']

### Some global variables_____no_output_____

Rnum = 1000
x = np.linspace(0.0,1.0,num=128)
dx = 1.0/np.shape(x)[0]
tsteps = np.linspace(0.0,2.0,num=400)
dt = 2.0/np.shape(tsteps)[0]
time_window = 15 # The window size of the LSTM
mode = 'train'_____no_output_____

### Functions for full-order snapshots of Burgers Equation_____no_output_____

def exact_solution(Rnum,t):
    t0 = np.exp(Rnum/8.0)

    return (x/(t+1))/(1.0+np.sqrt((t+1)/t0)*np.exp(Rnum*(x*x)/(4.0*t+4)))

def collect_snapshots(Rnum):
    snapshot_matrix = np.zeros(shape=(np.shape(x)[0],np.shape(tsteps)[0]))

    trange = np.arange(np.shape(tsteps)[0])
    for t in trange:
        snapshot_matrix[:,t] = exact_solution(Rnum,tsteps[t])[:]

    return np.transpose(snapshot_matrix)_____no_output_____

### Collect some snapshot data for CNN-LSTM embedding_____no_output_____

snapshots_train = collect_snapshots(Rnum)
snapshots_train = np.expand_dims(snapshots_train,-1)_____no_output_____print(np.shape(snapshots_train))

plt.figure()
plt.plot(snapshots_train[-1,:,0],linewidth=3)
plt.ylim((0,0.5))
plt.show()(400, 128, 1)

### ML related presets_____no_output_____

weights_filepath = 'best_weights_ae.h5'
lrate = 0.001

def coeff_determination(y_pred, y_true): #Order of function inputs is important here
    SS_res =  K.sum(K.square( y_true-y_pred ))
    SS_tot = K.sum(K.square( y_true - K.mean(y_true) ) )
    return ( 1 - SS_res/(SS_tot + K.epsilon()) )_____no_output_____

### Autoencoder_____no_output_____

# Define recursive model architecture

## Encoder
encoder_inputs = Input(shape=(128,1),name='Field')
# Encode
hh = Conv1D(25,kernel_size=3,activation='swish',padding='same')(encoder_inputs)
enc_l2 = MaxPooling1D(2,padding='same')(hh)

hh = Conv1D(15,kernel_size=3,activation='swish',padding='same')(enc_l2)
enc_l3 = MaxPooling1D(2,padding='same')(hh)

hh = Conv1D(10,kernel_size=3,activation='swish',padding='same')(enc_l3)
enc_l4 = MaxPooling1D(2,padding='same')(hh)

hh = Conv1D(5,kernel_size=3,activation='swish',padding='same')(enc_l4)
enc_l5 = MaxPooling1D(2,padding='same')(hh)

hh = Conv1D(1,kernel_size=3,activation='swish',padding='same')(enc_l5)
enc_l6 = MaxPooling1D(2,padding='same')(hh)

hh = Conv1D(1,kernel_size=3,activation=None,padding='same')(enc_l6)
encoded = MaxPooling1D(2,padding='same')(hh)

encoder = Model(inputs=encoder_inputs,outputs=encoded)

## Decoder
decoder_inputs = Input(shape=(2,1),name='decoded')

hh = Conv1D(1,kernel_size=3,activation=None,padding='same')(decoder_inputs)
dec_l1 = UpSampling1D(2)(hh)

hh = Conv1D(5,kernel_size=3,activation='relu',padding='same')(dec_l1)
dec_l2 = UpSampling1D(2)(hh)

hh = Conv1D(10,kernel_size=3,activation='relu',padding='same')(dec_l2)
dec_l3 = UpSampling1D(2)(hh)

hh = Conv1D(15,kernel_size=3,activation='relu',padding='same')(dec_l3)
dec_l4 = UpSampling1D(2)(hh)

hh = Conv1D(20,kernel_size=3,activation='relu',padding='same')(dec_l4)
dec_l5 = UpSampling1D(2)(hh)

hh = Conv1D(25,kernel_size=3,activation='relu',padding='same')(dec_l5)
dec_l6 = UpSampling1D(2)(hh)

decoded = Conv1D(1,1,activation=None,padding='same')(dec_l6)

decoder = Model(inputs=decoder_inputs,outputs=decoded)

## Autoencoder
ae_outputs = decoder(encoder(encoder_inputs))

model = Model(inputs=encoder_inputs,outputs=ae_outputs,name='CAE')

# design network
my_adam = optimizers.Adam(lr=lrate, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False)

checkpoint = ModelCheckpoint(weights_filepath, monitor='loss', verbose=1, save_best_only=True, mode='min',save_weights_only=True)
earlystopping = EarlyStopping(monitor='loss', min_delta=0, patience=10, verbose=0, mode='auto', baseline=None, restore_best_weights=False)
callbacks_list = [checkpoint]

# fit network
model.compile(optimizer=my_adam,loss='mean_squared_error',metrics=[coeff_determination])
model.summary()

plot_model(decoder,to_file='Decoder.png')
plot_model(encoder,to_file='Encoder.png')

num_epochs = 500
batch_size = 10

if mode == 'train':
    train_history = model.fit(x=snapshots_train, y=snapshots_train, epochs=num_epochs, batch_size=batch_size, callbacks=callbacks_list)_________________________________________________________________
Layer (type)                 Output Shape              Param #
=================================================================
Field (InputLayer)           (None, 128, 1)            0
_________________________________________________________________
model_2 (Model)              (None, 2, 1)              1875
_________________________________________________________________
model_3 (Model)              (None, 128, 1)            3120
=================================================================
Total params: 4,995
Trainable params: 4,995
Non-trainable params: 0
_________________________________________________________________

### Check accuracy_____no_output_____

model.load_weights(weights_filepath)
recoded_1 = model.predict(snapshots_train[0:,:,0:])

for i in range(0,400,100):
    plt.figure(figsize=(7,6))
    plt.plot(x[:],snapshots_train[i,:,0:],label='Truth',linewidth=3)
    plt.plot(x[:],recoded_1[i,:,0:],'ro',label='Reconstructed')
    plt.xlabel('x')
    plt.ylabel('u')
    plt.ylim((0,0.5))
    plt.tight_layout()

    if i == 0:
        plt.legend()

    plt.show()_____no_output_____

### Generate encoded data for LSTM learning_____no_output_____

encoded = K.eval(encoder(snapshots_train[:,:,0:].astype('float32')))
# Reshape
encoded = encoded.reshape(1,400,2)

from scipy.ndimage.filters import gaussian_filter1d
encoded_f = np.copy(encoded)
encoded_f[0,:,0] = gaussian_filter1d(encoded[0,:,0],sigma=3)
encoded_f[0,:,1] = gaussian_filter1d(encoded[0,:,1],sigma=3)

plt.figure(figsize=(7,6))
plt.plot(encoded_f[0,:,0],label='Latent dimension 1')
plt.plot(encoded_f[0,:,1],label='Latent dimension 2')
plt.legend()
plt.xlabel('Temporal snapshot')
plt.ylabel('Magnitude')
plt.show()
# plt.figure(figsize=(7,6))
# plt.plot(encoded[0,:,0],label='Latent dimension 1')
# plt.plot(encoded[0,:,1],label='Latent dimension 2')
# plt.legend()
# plt.xlabel('Temporal snapshot')
# plt.ylabel('Magnitude')
# plt.show()_____no_output_____

### Train LSTM with parameter information_____no_output_____

lstm_training_data = encoded_f
num_train_snapshots = 1

# Rescale
from sklearn.preprocessing import MinMaxScaler, StandardScaler
scale_lstm = False
if scale_lstm:
    scaler = StandardScaler()
    lstm_training_data_scaled = scaler.fit_transform(lstm_training_data[0,:,:])
else:
    lstm_training_data_scaled = lstm_training_data[0,:,:]

total_size = np.shape(lstm_training_data)[0]*np.shape(lstm_training_data)[1]
total_size_sim = np.shape(lstm_training_data)[1]

# Shape the inputs and outputs
input_seq = np.zeros(shape=(total_size-num_train_snapshots*time_window,time_window,2))
output_seq = np.zeros(shape=(total_size-num_train_snapshots*time_window,2))

# Setting up inputs
sample = 0
for snum in range(num_train_snapshots):
    for t in range(time_window,total_size_sim):
        input_seq[sample,:,:] = lstm_training_data_scaled[t-time_window:t,:]
        output_seq[sample,:] = lstm_training_data_scaled[t,:]
        sample = sample + 1_____no_output_____# Model architecture
lstm_model = models.Sequential()
lstm_model.add(LSTM(20,input_shape=(time_window, 2),return_sequences=True))  #
lstm_model.add(LSTM(20,input_shape=(time_window, 2),return_sequences=False))  #
lstm_model.add(Dense(2, activation=None))

# training parameters
num_epochs = 3000
batch_size = 64

# design network
lstm_filepath = 'lstm_weights.h5'
lstm_adam = optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False)
checkpoint = ModelCheckpoint(lstm_filepath, monitor='loss', verbose=0, save_best_only=True, mode='min',save_weights_only=True)
earlystopping = EarlyStopping(monitor='loss', min_delta=0, patience=5, verbose=0, mode='auto', baseline=None, restore_best_weights=False)
lstm_callbacks_list = [checkpoint]

# fit network
lstm_model.compile(optimizer=lstm_adam,loss='mean_squared_error',metrics=[coeff_determination])

lstm_model.summary()

# if mode == 'train':
lstm_train_history = lstm_model.fit(input_seq, output_seq, epochs=num_epochs, batch_size=batch_size, callbacks=lstm_callbacks_list)_________________________________________________________________
Layer (type)                 Output Shape              Param #
=================================================================
lstm (LSTM)                  (None, 15, 20)            1840
_________________________________________________________________
lstm_1 (LSTM)                (None, 20)                3280
_________________________________________________________________
dense (Dense)                (None, 2)                 42
=================================================================
Total params: 5,162
Trainable params: 5,162
Non-trainable params: 0
_________________________________________________________________
WARNING:tensorflow:From /home/rmlans/anaconda3/envs/deephyper_env/lib/python3.6/site-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Use tf.cast instead.
Epoch 1/3000
385/385 [==============================] - 2s 5ms/sample - loss: 0.0948 - coeff_determination: -213.7870
Epoch 2/3000
385/385 [==============================] - 0s 647us/sample - loss: 0.0461 - coeff_determination: -46.2863
Epoch 3/3000
385/385 [==============================] - 0s 1ms/sample - loss: 0.0137 - coeff_determination: -6.9466
Epoch 4/3000
385/385 [==============================] - 0s 1ms/sample - loss: 0.0102 - coeff_determination: -5.3813
Epoch 5/3000
385/385 [==============================] - 0s 1ms/sample - loss: 0.0058 - coeff_determination: 0.0416
Epoch 6/3000
385/385 [==============================] - 0s 1ms/sample - loss: 0.0020 - coeff_determination: 0.7476
Epoch 7/3000
385/385 [==============================] - 1s 2ms/sample - loss: 0.0010 - coeff_determination: 0.9249
Epoch 8/3000
385/385 [==============================] - 1s 2ms/sample - loss: 2.9911e-04 - coeff_determination: 0.9855
Epoch 9/3000
385/385 [==============================] - 0s 1ms/sample - loss: 5.0327e-04 - coeff_determination: 0.9776
Epoch 10/3000
385/385 [==============================] - 1s 1ms/sample - loss: 1.7925e-04 - coeff_determination: 0.9915
Epoch 11/3000
385/385 [==============================] - 0s 1ms/sample - loss: 4.4374e-05 - coeff_determination: 0.9975
Epoch 12/3000
385/385 [==============================] - 0s 1ms/sample - loss: 6.9572e-05 - coeff_determination: 0.9954
Epoch 13/3000
385/385 [==============================] - 1s 2ms/sample - loss: 4.2698e-05 - coeff_determination: 0.9947
Epoch 14/3000
385/385 [==============================] - 0s 1ms/sample - loss: 3.5260e-05 - coeff_determination: 0.9981
Epoch 15/3000
385/385 [==============================] - 1s 1ms/sample - loss: 3.2552e-05 - coeff_determination: 0.9982
Epoch 16/3000
385/385 [==============================] - 0s 1ms/sample - loss: 3.2232e-05 - coeff_determination: 0.9972
Epoch 17/3000
385/385 [==============================] - 0s 1ms/sample - loss: 3.1423e-05 - coeff_determination: 0.9974
Epoch 18/3000
385/385 [==============================] - 1s 1ms/sample - loss: 3.1006e-05 - coeff_determination: 0.9982
Epoch 19/3000
385/385 [==============================] - 1s 1ms/sample - loss: 3.1107e-05 - coeff_determination: 0.9980
Epoch 20/3000
385/385 [==============================] - 0s 1ms/sample - loss: 3.4915e-05 - coeff_determination: 0.9977
Epoch 21/3000
385/385 [==============================] - 1s 1ms/sample - loss: 3.2252e-05 - coeff_determination: 0.9975
Epoch 22/3000
385/385 [==============================] - 0s 1ms/sample - loss: 3.2864e-05 - coeff_determination: 0.9980
Epoch 23/3000
385/385 [==============================] - 1s 1ms/sample - loss: 3.2294e-05 - coeff_determination: 0.9978
Epoch 24/3000
385/385 [==============================] - 1s 1ms/sample - loss: 2.9931e-05 - coeff_determination: 0.9964
Epoch 25/3000
385/385 [==============================] - 1s 1ms/sample - loss: 2.9945e-05 - coeff_determination: 0.9979
Epoch 26/3000
385/385 [==============================] - 1s 1ms/sample - loss: 3.0217e-05 - coeff_determination: 0.9982
Epoch 27/3000
385/385 [==============================] - 1s 2ms/sample - loss: 2.9541e-05 - coeff_determination: 0.9978
Epoch 28/3000
385/385 [==============================] - 1s 2ms/sample - loss: 2.9442e-05 - coeff_determination: 0.9979
Epoch 29/3000
385/385 [==============================] - 1s 1ms/sample - loss: 2.9325e-05 - coeff_determination: 0.9984
Epoch 30/3000
385/385 [==============================] - 1s 1ms/sample - loss: 2.8939e-05 - coeff_determination: 0.9984
Epoch 31/3000
385/385 [==============================] - 0s 1ms/sample - loss: 2.9160e-05 - coeff_determination: 0.9976
Epoch 32/3000
385/385 [==============================] - 1s 1ms/sample - loss: 2.8777e-05 - coeff_determination: 0.9971
Epoch 33/3000
385/385 [==============================] - 1s 1ms/sample - loss: 2.9043e-05 - coeff_determination: 0.9975
Epoch 34/3000
385/385 [==============================] - 1s 1ms/sample - loss: 2.9342e-05 - coeff_determination: 0.9984
Epoch 35/3000
385/385 [==============================] - 1s 1ms/sample - loss: 3.0315e-05 - coeff_determination: 0.9983
Epoch 36/3000
385/385 [==============================] - 1s 2ms/sample - loss: 2.8585e-05 - coeff_determination: 0.9975
Epoch 37/3000
385/385 [==============================] - 0s 1ms/sample - loss: 2.8975e-05 - coeff_determination: 0.9984
Epoch 38/3000
385/385 [==============================] - 0s 1ms/sample - loss: 2.9443e-05 - coeff_determination: 0.9982
Epoch 39/3000
385/385 [==============================] - 1s 2ms/sample - loss: 2.8544e-05 - coeff_determination: 0.9974
Epoch 40/3000
385/385 [==============================] - 1s 2ms/sample - loss: 2.8042e-05 - coeff_determination: 0.9984
Epoch 41/3000
385/385 [==============================] - 1s 1ms/sample - loss: 2.8157e-05 - coeff_determination: 0.9983
Epoch 42/3000
385/385 [==============================] - 1s 1ms/sample - loss: 2.7678e-05 - coeff_determination: 0.9969
Epoch 43/3000
385/385 [==============================] - 1s 1ms/sample - loss: 3.0147e-05 - coeff_determination: 0.9983
Epoch 44/3000
385/385 [==============================] - 1s 1ms/sample - loss: 2.8773e-05 - coeff_determination: 0.9983
Epoch 45/3000
385/385 [==============================] - 0s 1ms/sample - loss: 2.8445e-05 - coeff_determination: 0.9983
Epoch 46/3000
 64/385 [===>..........................] - ETA: 0s - loss: 2.2868e-05 - coeff_determination: 0.9984# encoded_test = K.eval(encoder(snapshots_train[:,:,0:].astype('float32')))
# Reshape
# encoded_test = encoded_test.reshape(1,400,2)
encoded_test = np.copy(encoded_f)

plt.figure()
plt.plot(encoded_f[0,:,0],label='Mode 1')
plt.plot(encoded_f[0,:,1],label='Mode 2')
plt.legend()
plt.show()_____no_output_____

### Test LSTM with parameter information_____no_output_____

num_train_snapshots = 1
lstm_model.load_weights(lstm_filepath)
lstm_training_data_scaled = np.reshape(lstm_training_data_scaled,newshape=(1,400,2))

total_size = np.shape(lstm_training_data_scaled)[1]
# Shape the inputs and outputs
input_seq = np.zeros(shape=(1,time_window,2))
output_seq_pred = np.zeros(shape=(total_size,2))

input_seq[0,:time_window,:] = lstm_training_data_scaled[0,0:time_window,:]
for t in range(time_window,total_size):
    output_seq_pred[t,:] = lstm_model.predict(input_seq[0:1,0:time_window,:])[0,:]
    input_seq[0,0:time_window-1,:] = input_seq[0,1:,:]
    input_seq[0,time_window-1,:] = output_seq_pred[t,:] _____no_output_____

### Check quality in latent space for testing data_____no_output_____

fig, ax = plt.subplots(nrows=1,ncols=2,figsize=(14,6))
ax[0].plot(lstm_training_data_scaled[0,time_window:,0],'b',label='True',linewidth=3)
ax[0].plot(output_seq_pred[time_window:,0],'r',label='Predicted',linewidth=3)
ax[0].set_xlabel('Temporal snapshot')
ax[0].set_ylabel('Magnitude')

ax[1].plot(lstm_training_data_scaled[0,time_window:,1],'b',label='True',linewidth=3)
ax[1].plot(output_seq_pred[time_window:,1],'r',label='Predicted',linewidth=3)
ax[1].set_xlabel('Temporal snapshot')
ax[1].set_ylabel('Magnitude')

plt.tight_layout()
# ax[0].legend()
ax[1].legend()
plt.show()_____no_output_____# Rescale
if scale_lstm:
    output_seq_pred = scaler.inverse_transform(output_seq_pred)

output_seq_pred = np.expand_dims(output_seq_pred,-1)
print(np.shape(output_seq_pred))(400, 2, 1)

### Evolution in physical space_____no_output_____

decoded = K.eval(decoder(output_seq_pred.astype('float32')))
print(np.shape(decoded))
# Reshape
decoded = decoded.reshape(1,400,128)
snapshot_train_true = snapshots_train[:,:,0:].reshape(1,400,128)

print(np.shape(snapshot_train_true))
print(np.shape(decoded))(400, 128, 1)
(1, 400, 128)
(1, 400, 128)

### Save the data for posterity_____no_output_____

np.save('Training_data_true.npy',snapshot_train_true)
np.save('Training_data_prediction.npy',decoded)_____no_output_____

### A posteriori analysis_____no_output_____

snapshot_test_true = np.load('Training_data_true.npy')
snapshot_test_pred = np.load('Training_data_prediction.npy')
num_sims = np.shape(snapshot_test_true)[0]

pic_num = 0
for i in range(time_window,400,10):
    plt.figure()
    plt.plot(x[:],snapshot_test_pred[0,i,:],'ro',label='Predicted',markersize=4)
    plt.plot(x[:],snapshot_test_true[0,i,:],label='Truth',linewidth=3)
    plt.ylim((0.0,0.5))
    plt.xlabel('x')
    plt.ylabel('u')
    plt.tight_layout()

    if i == 0:
        plt.legend(loc='upper right')

    plt.savefig('pic_'+str(pic_num)+'.png')
    pic_num = pic_num + 1
    plt.show()_____no_output_____

### L2 error_____no_output_____

print(np.shape(snapshot_test_true))
print(np.shape(snapshot_test_pred))

print(np.sum((snapshot_test_true[0,-1,:]-snapshot_test_pred[0,-1,:])**2)/128)(1, 400, 128)
(1, 400, 128)
0.0004181005074002814


# Notebook from Joseantonio-96/BigData_Uni
Path: Deep_Learning/Dense_Neural_Networks/JijonVorbeck_Lab2DNN/JijonVorbeck_Lab_2_Part_1.ipynb

# Lab 2 (Part 1): Image classification with MLPs

**Jose Antonio Jijon Vorbeck - 100438045**

------------------------------------------------------
*Deep Learning. Master in Big Data Analytics*

*Pablo M. Olmos pamartin@ing.uc3m.es*

------------------------------------------------------

In this second lab of the course, you will implement an image classifier using MLPs. We will use the MNIST dataset, which consists of greyscale handwritten digits. Each image is 28x28 pixels, you can see a sample below_____no_output_____

from IPython.display import Image
from IPython.core.display import HTML

Image(url= "https://upload.wikimedia.org/wikipedia/commons/2/27/MnistExamples.png", width=400, height=200)_____no_output_____

Our goal is to build a neural network that can take one of these images and predict the digit in the image.

Note: a big part of the following material is a personal wrap-up of [Facebook's Deep Learning Course in Udacity](https://www.udacity.com/course/deep-learning-pytorch--ud188). So all credit goes for them!!_____no_output_____

%matplotlib inline
%config InlineBackend.figure_format = 'retina'  #To get figures with high quality!

import numpy as np
# pytorch package
import torch
# neural networks
from torch import nn
# optimizer
from torch import optim
import matplotlib.pyplot as plt
import math_____no_output_____

## Part I. Download MNIST with `torchvision`

First up, we need to get our dataset. This is provided through the `torchvision` package. The [torchvision package](https://pytorch.org/docs/stable/torchvision/index.html) consists of popular datasets, model architectures, and common image transformations for computer vision.

The code below will download the MNIST dataset, then create training and test datasets for us. Don't worry too much about the details here, you'll learn more about this later._____no_output_____

### Run this cell

from torchvision import datasets, transforms

# Define a transform to normalize the data
transform = transforms.Compose([transforms.ToTensor(),
                              transforms.Normalize((0.5,), (0.5,)),
                              ])

# Download and load the training  data
# getting all the training data in a variable
trainset = datasets.MNIST('~/.pytorch/MNIST_data/', download=True, train=True, transform=transform)
# need to store this in a trainloder, that contains a list of  xtensors and y tensors
# batch size dictates how many items pass at each time, like training in small pieces
trainloader = torch.utils.data.DataLoader(trainset, batch_size=64, shuffle=True)

# Download and load the test data
testset = datasets.MNIST('~/.pytorch/MNIST_data/', download=True, train=False, transform=transform)
# we do the same for the testing data
testloader = torch.utils.data.DataLoader(testset, batch_size=64, shuffle=True)Downloading http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz to /root/.pytorch/MNIST_data/MNIST/raw/train-images-idx3-ubyte.gz

We have the training data loaded into `trainloader` and we make that an iterator with `iter(trainloader)`. Later, we'll use this to loop through the dataset for training, like

```python
for image, label in trainloader:
    ## do things with images and labels
```

You'll notice I created the `trainloader` with a batch size of 64, and `shuffle=True`. The batch size is the number of images we get in one iteration from the data loader and pass through our network, often called a *batch*. And `shuffle=True` tells it to **shuffle the dataset every time we start going through the data loader again**. But here I'm just grabbing the first batch so we can check out the data. We can see below that `images` is just a tensor with size `(64, 1, 28, 28)`. So, 64 images per batch, **1 color channel**, and 28x28 images._____no_output_____

# iter function creates an iterator
# every element of this iterator contains 64 images of 1 color and 28by28 and label
dataiter = iter(trainloader)   #To iterate through the dataset

images, labels = dataiter.next()
print(type(images))
print(images.shape)
print(labels.shape)
<class 'torch.Tensor'>
torch.Size([64, 1, 28, 28])
torch.Size([64])

This is what one of the images looks like. _____no_output_____

plt.imshow(images[0].view([28,28]), cmap='Greys_r')
print(labels[0])tensor(1)

## Part II. Train a multi-class Logistic Regressor

Our first goal is to train a multi-class logistic regressor to evaluate how good it can do in both the training and the test sets.

The following code is **almost identical** to the one you used for Lab 1 except for two small details:

- We are training a LR classifier with 10 different outputs that implements a **softmax** non-linear function (instead of a binary LR with a sigmoid).

- We are using the MNIST database loaded above.

We first define the Multi-class Logistic Regressor class

> **Exercise**: Complete the following code_____no_output_____

class Multi_LR(nn.Module):
    def __init__(self,dimx,nlabels): #Nlabels will be 10 in our case
        super().__init__()

        # arguments passed for the first layer are: input and output 'neurons'
        # dimx : dimension of x (28*28)
        # nlabels : number of digits we want to predict, so it must be 10
        self.output = nn.Linear(dimx,nlabels)

        self.logsoftmax = nn.LogSoftmax(dim=1)   # NEW w.r.t Lab 1. dim is the dimension along which
                                                 #Softmax will be computed (so every slice along dim will sum to 1)

    def forward(self, x):
        # Pass the input tensor through each of our operations
        x = self.output(x)
        x = self.logsoftmax(x)

        return x_____no_output_____

Note that we use `nn.LogSoftmax` instead of `nn.Softmax()`. In many cases, softmax gives you probabilities which will often be very close to zero or one but floating-point numbers can't accurately represent values near zero or one ([read more here](https://docs.python.org/3/tutorial/floatingpoint.html)). It's usually best to avoid doing calculations with probabilities, typically we use log-probabilities.  The cross entropy loss is obtained by combining `nn.LogSoftmax` with the negative loss likelihood loss `nn.NLLLoss()`.

Alternatively, we can use [`nn.CrossEntropyLoss`](https://pytorch.org/docs/stable/nn.html#torch.nn.CrossEntropyLoss). **This criterion combines `nn.LogSoftmax()` and `nn.NLLLoss()` in one single class.**

This means we need to pass in the raw output of our network into the loss, not the output of the softmax function. This raw output is usually called the *logits* or *scores*.
_____no_output_____Now we implement an extension to the class above (which inheritates from `Multi_LR`) that includes a training method.  One thing, note that MNIST images are of dimension $28\times28=784$. To feed this image as the input to a `nn.Linear` layer, it has to be converted to a $784\times 1$ input tensor.

> **Exercise**: Complete the following code_____no_output_____

''' This class inherits from the `Multi_LR` class. So it has the same atributes
and methods, and some others that we will add.
'''
class Multi_LR_extended(Multi_LR):

    def __init__(self,dimx,nlabels,epochs=100,lr=0.001):

        # pass same parameters than parent class
        super().__init__(dimx,nlabels)  #To initialize `Multi_LR`!,

        self.lr = lr #Learning Rate

        self.optim = optim.Adam(self.parameters(), self.lr)

        self.epochs = epochs

        self.criterion = nn.NLLLoss()               # NEW w.r.t Lab 1

        # A list to store the loss evolution along training

        self.loss_during_training = []

    def train(self,trainloader):

        # Optimization Loop

        for e in range(int(self.epochs)):

            # Random data permutation at each epoch

            running_loss = 0.

            for images, labels in trainloader:              # NEW w.r.t Lab 1

                self.optim.zero_grad()  #TO RESET GRADIENTS!

                # get output of last computed value
                out = self.forward(images.view(images.shape[0], -1))

                #Your code here (multiple lines)

                # Compute the loss with these values:
                loss = self.criterion(out, labels)

                running_loss += loss.item()

                # now we compute the backward error of this:
                loss.backward()

                # finally we run the optimizer:
                self.optim.step()

            self.loss_during_training.append(running_loss/len(trainloader))

            if(e % 1 == 0): # Every 10 epochs

                print("Training loss after %d epochs: %f"
                      %(e,self.loss_during_training[-1]))
_____no_output_____

Ok that was easy, wasn't it? Lets now train the multi-class LR and evaluate the performance in both the training and the test sets.
_____no_output_____

my_LR = Multi_LR_extended(dimx=784,nlabels=10,epochs=10,lr=1e-3)

my_LR.train(trainloader)
Training loss after 0 epochs: 0.465543
Training loss after 1 epochs: 0.326697
Training loss after 2 epochs: 0.307561
Training loss after 3 epochs: 0.300417
Training loss after 4 epochs: 0.295096
Training loss after 5 epochs: 0.290175
Training loss after 6 epochs: 0.286855
Training loss after 7 epochs: 0.284655
Training loss after 8 epochs: 0.282790
Training loss after 9 epochs: 0.279769
plt.plot(my_LR.loss_during_training,'-b',label='Cross Entropy Loss')
plt.xlabel('Iterations')
plt.ylabel('Loss')
plt.grid()_____no_output_____

To evaluate the performance across the entire test dataset, we will implement a for loop using `testloader` and compute errors per mini-batch. The following code will do the work:_____no_output_____

loss = 0
accuracy = 0

# Turn off gradients for validation, saves memory and computations
with torch.no_grad():

    for images,labels in testloader:

        # We use a log-softmax, so what we get are log-probabilities
        logprobs = my_LR.forward(images.view(images.shape[0], -1))

        # get the top probability and class per image
        top_p, top_class = logprobs.topk(1, dim=1)
        # see if these values are the same (match with the groundtruth)
        equals = (top_class == labels.view(images.shape[0], 1))
        # if it does, then is 1, if not it is 0
        accuracy += torch.mean(equals.type(torch.FloatTensor))

print("Test Accuracy %f" %(accuracy/len(testloader)))Test Accuracy 0.920084

With the probabilities, we can get the most likely class using the `probs.topk` method. This returns the $k$ highest values. Since we just want the most likely class, we can use `probs.topk(1)`. This returns a tuple of the top-$k$ values and the top-$k$ indices. If the highest value is the fifth element, we'll get back 4 as the index.

The line
```python
(top_class == labels.view(images.shape[0], 1))
```
returns a boolean vector of `True/False` values, indicanting werther `top_class` is equeal to `labels` at every position. Finally, with the line

```python
equals.type(torch.FloatTensor)
```
we transform it to real a vector in which `True --> 1.0` and `False --> 0.0`, where we can compute the mean using `torch.mean()`._____no_output_____> **Excercise** Modify the code of the `Multi_LR_extended` class so it incorporates a method to evaluate the performance in either the train set or the test set (Use a single method with the proper inputs!). Compute the train/test accuracy using such a method._____no_output_____

#YOUR CODE HERE
# modify the class Multi_LR_extended to add a new method 'performance'
# it has to work in either the test or train sets
# copying the same class from before and modifying it:

class Multi_LR_extended(Multi_LR):

    def __init__(self,dimx,nlabels,epochs=100,lr=0.001):

        # pass same parameters than parent class
        super().__init__(dimx,nlabels)  #To initialize `Multi_LR`!,

        self.lr = lr #Learning Rate

        self.optim = optim.Adam(self.parameters(), self.lr)

        self.epochs = epochs

        self.criterion = nn.NLLLoss()               # NEW w.r.t Lab 1

        # A list to store the loss evolution along training

        self.loss_during_training = []

    def accuracy(self, loader):

        accuracy = 0

        # Turn off gradients for validation, saves memory and computations
        with torch.no_grad():

          for images,labels in loader:

              # We use a log-softmax, so what we get are log-probabilities
              logprobs = self.forward(images.view(images.shape[0], -1))

              # get the top probability and class per image
              top_p, top_class = logprobs.topk(1, dim=1)
              # see if these values are the same (match with the groundtruth)
              equals = (top_class == labels.view(images.shape[0], 1))
              # if it does, then is 1, if not it is 0
              accuracy += torch.mean(equals.type(torch.FloatTensor))

        print("Accuracy %f" %(accuracy/len(loader)))

    def train(self,trainloader):

        # Optimization Loop

        for e in range(int(self.epochs)):

            # Random data permutation at each epoch

            running_loss = 0.

            for images, labels in trainloader:              # NEW w.r.t Lab 1

                self.optim.zero_grad()  #TO RESET GRADIENTS!

                # get output of last computed value
                out = self.forward(images.view(images.shape[0], -1))

                #Your code here (multiple lines)

                # Compute the loss with these values:
                loss = self.criterion(out, labels)

                running_loss += loss.item()

                # now we compute the backward error of this:
                loss.backward()

                # finally we run the optimizer:
                self.optim.step()

            self.loss_during_training.append(running_loss/len(trainloader))

            if(e % 1 == 0): # Every 10 epochs

                print("Training loss after %d epochs: %f"
                      %(e,self.loss_during_training[-1]))
_____no_output_____#YOUR CODE HERE
# try out the method in this part
# performance of train
# performance of test
# dimension must be flat dimension 28*28
dimx = images[0].shape[1]*images[0].shape[1]
# instantiate the model
my_LR = Multi_LR_extended(dimx, 10, epochs=10)
# train it with the trainloader
my_LR.train(trainloader)Training loss after 0 epochs: 0.465284
Training loss after 1 epochs: 0.325387
Training loss after 2 epochs: 0.310647
Training loss after 3 epochs: 0.300118
Training loss after 4 epochs: 0.296137
Training loss after 5 epochs: 0.292431
Training loss after 6 epochs: 0.286664
Training loss after 7 epochs: 0.285157
Training loss after 8 epochs: 0.282413
Training loss after 9 epochs: 0.280112
# now we test the accuracy with testing and training data:
# training data:
my_LR.accuracy(trainloader)
# testing data:
my_LR.accuracy(testloader)Accuracy 0.916445
Accuracy 0.914908

Observe that both values are indeed similar, indicating that the model is not overfitting.

Let's check the values for the weight matrix. For a simpler visualization, we will plot the histogram of all the values in the weight matrix._____no_output_____

plt.hist(my_LR.output.weight.detach().numpy().reshape([-1,]),50) # Modify my_LR by the name of the object you defined above
plt.grid()_____no_output_____# shape of the weights tensor:
my_LR.output.weight.shape
# we see that we have 10 rows (categories), and 784 columns, same as the
# number of pixels we have per image ! 28x28_____no_output_____

> **Exercise**: Plot the histogram of the gradients of the loss function w.r.t. every parameter in the model:_____no_output_____

#YOUR CODE HERE
plt.hist(my_LR.output.weight.grad.view([-1,]),50)
plt.grid()
# we see that all gradients are between [-0.04,0.06], thus they are very small
# and close to zero actually_____no_output_____

As we can see most of the gradients are almost zero_____no_output_____## Part III. Train a MLP to do the same job

Modify the code you have just implemented for a MLP is straightforward. Assume we want to train a MLP with three layers, all using rectified linear units (RELU)s as non-linear activations (except the last layer, that uses a Softmax). The first layer has 128 hidden units and the second 64 of them._____no_output_____

Image(url= "https://pytorch.org/docs/stable/_images/ReLU.png", width=500, height=300)_____no_output_____

As with the LR, we create a small class defining the model and then a larger class than inherites from it to incorporate methods to perform both training and model evaluation.

> **Exercise**: Complete the code for the following class_____no_output_____

class MLP(nn.Module):
    def __init__(self,dimx,hidden1,hidden2,nlabels): #Nlabels will be 10 in our case
        super().__init__()
        # first layer
        self.output1 = nn.Linear(dimx,hidden1)

        # second layer (first hidden)
        self.output2 = nn.Linear(hidden1,hidden2)

        # third layer (second hidden)
        self.output3 = nn.Linear(hidden2,nlabels)

        # Rectified linear units as activation function
        self.relu = nn.ReLU()

        # log_softmax function as activation function for last layer
        self.logsoftmax = nn.LogSoftmax(dim=1)

    def forward(self, x):
        # Pass the input tensor through each of our operations
        # layer 1
        x = self.output1(x)
        x = self.relu(x)
        # layer 2
        x = self.output2(x)
        x = self.relu(x)
        # layer 3
        x = self.output3(x)
        x = self.logsoftmax(x)
        return x_____no_output_____

> **Excercise** Create a class `MLP_extended` that incorporates two methods to the former class. One to perform training and one to perform model evaluation. It is just **one line of code** diferent from the code you have done above for the multi-class LR. This is why I like class and structure my code this way!!_____no_output_____

# now we create a child class that inherits from MLP
class MLP_extended(MLP):

    def __init__(self, dimx, hidden1, hidden2, nlabels, epochs=100,lr=0.001):

        # pass same parameters than parent class
        super().__init__(dimx, hidden1, hidden2, nlabels)

        self.lr = lr #Learning Rate

        self.optim = optim.Adam(self.parameters(), self.lr)

        self.epochs = epochs

        self.criterion = nn.NLLLoss()

        # A list to store the loss evolution along training
        self.loss_during_training = []

    def accuracy(self, loader):

        accuracy = 0

        # Turn off gradients for validation, saves memory and computations
        with torch.no_grad():

          for images,labels in loader:

              # We use a log-softmax, so what we get are log-probabilities
              logprobs = self.forward(images.view(images.shape[0], -1))

              # get the top probability and class per image
              top_p, top_class = logprobs.topk(1, dim=1)
              # see if these values are the same (match with the groundtruth)
              equals = (top_class == labels.view(images.shape[0], 1))
              # if it does, then is 1, if not it is 0
              accuracy += torch.mean(equals.type(torch.FloatTensor))

        print("Accuracy %f" %(accuracy/len(loader)))

    def train(self,trainloader):

        # Optimization Loop

        for e in range(int(self.epochs)):

            # Random data permutation at each epoch

            running_loss = 0.

            for images, labels in trainloader:

                self.optim.zero_grad()

                # get output of last computed value
                out = self.forward(images.view(images.shape[0], -1))

                #Your code here (multiple lines)

                # Compute the loss with these values:
                loss = self.criterion(out, labels)

                running_loss += loss.item()

                # now we compute the backward error of this:
                loss.backward()

                # finally we run the optimizer:
                self.optim.step()

            self.loss_during_training.append(running_loss/len(trainloader))

            if(e % 1 == 0): # Every 10 epochs

                print("Training loss after %d epochs: %f"
                      %(e,self.loss_during_training[-1]))
_____no_output_____

Train the model for 10 epochs and compute the train/test performance. How does it compare with the Logistic Regressor?_____no_output_____

# We initiate and train the model the same way as in the above parts:
# first we instantiate it:
# dimx = 28x28
dimx = images[0].shape[1]*images[0].shape[1]
my_MLP = MLP_extended(dimx, 128, 64, 10, epochs=10, lr= 1e-3)
# now we train it with the trainloader:
my_MLP.train(trainloader)Training loss after 0 epochs: 0.389964
Training loss after 1 epochs: 0.185214
Training loss after 2 epochs: 0.132447
Training loss after 3 epochs: 0.108782
Training loss after 4 epochs: 0.090064
Training loss after 5 epochs: 0.078450
Training loss after 6 epochs: 0.071141
Training loss after 7 epochs: 0.065522
Training loss after 8 epochs: 0.060020
Training loss after 9 epochs: 0.055861
images[0].shape[1]*images[0].shape[1]_____no_output_____# now we get the performances of the models
# training set
my_MLP.accuracy(trainloader)
# testing set
my_MLP.accuracy(testloader)Accuracy 0.989089
Accuracy 0.975617

Wow! Performace is almost perfect with a naive Neural Network!!_____no_output_____> **Exercise:** Lets visualize the activations at the ouput of the first layer for a minibatch test images. This will help to identify possible unused hidden units (allways activated/deactivated) and correlated hidden units, e.g. redundant units. Complete the following code._____no_output_____

# First, we load a test minibatch

x_test,y_test = next(iter(testloader))

# Then, we evaluate the output of the first layer of the network for that mini-batch
# we must also apply the activation function and then detach it and convert it to numpy array
activations = my_MLP.relu(my_MLP.output1(x_test.view(x_test.shape[0],-1))).detach().numpy()

# We plot the matrix using matplotlib

plt.matshow(activations)

plt.colorbar()
# shape of the activations:
activations.shape_____no_output_____

Based on the plot, do you think there are unsued hidden units in the hidden layer? They are characterize by units that are always active (very high values) or unactive (almost zero values). Plot the variance of the hidden units across the test mini-batch to better visualize these unactive hidden units. _____no_output_____

plt.plot(np.var(activations,0))
plt.grid()
plt.xlabel('Hidden Unit')
plt.ylabel('Activation Variance')

print("There are {0:d} hidden units that are unactive".format(np.sum(np.var(activations,0)<=0.1)))
unactive = np.sum(np.var(activations,0)<=0.1)There are 52 hidden units that are unactive

> **Exercise**: Retrain the model reducing accordingly the dimension of the first hidden layer. For that model, repeat the analysis to the activations of both the first and the second layer.
>
>You will notice that in general, unsued activations are prominent in the first layer compared to the second one. >This is in general the case for any NN, as the **loss function is more sensitive to parameter variations in the last >layers**, and hence gradients are higher in magnitude. On the contrary, the **loss function is less senstive to >parameter variations in the first layers** and hence only very relevant parameters are trained (they influence more >in the loss function), while many others vary very little w.r.t. initialization._____no_output_____

my_MLP_lessparams = MLP_extended(dimx, 128-unactive, 64, 10, 10, 1e-3)
my_MLP_lessparams.train(trainloader)_____no_output_____# we compute the activations of the first layer here:
activations_1 = my_MLP_lessparams.relu(my_MLP_lessparams.output1(x_test.view(x_test.shape[0],-1))).detach().numpy()

# Then, we evaluate the output of the second layer of the network for that mini-batch
activations_2 = my_MLP_lessparams.relu(my_MLP_lessparams.output2(torch.tensor(activations_1))).detach().numpy()

fig, ax = plt.subplots(nrows=2, ncols=2,figsize=(16, 8))
im = ax[0,0].matshow(activations_1)
ax[0,0].set_title('Activations in the first layer\n')

ax[0,1].matshow(activations_2)
ax[0,1].set_title('Activations in the second layer\n')

ax[1,0].plot(np.var(activations_1,0))
ax[1,0].set_title('Activation variance in the first layer\n')
ax[1,0].grid()

ax[1,1].plot(np.var(activations_2,0))
ax[1,1].set_title('Activation variance in the second layer\n')
ax[1,1].grid()

active_layer1 = np.sum(np.var(activations_1,0)<=0.1)
active_layer2 = np.sum(np.var(activations_2,0)<=0.1)
print("In the first layer, there are {0:d} hidden units that are unactive".format(active_layer1))

print("In the second layer, there are {0:d} hidden units that are unactive".format(active_layer2))_____no_output_____# now we compute the accuracies of the model for both training and testing sets
# with the newly trained model
my_MLP.accuracy(trainloader)
my_MLP.accuracy(testloader)_____no_output_____

### Part IV. Saving and restoring the model

Finally, we will show you how to save and load models (i.e. values of the parameters) with PyTorch. This is important because you'll often want to load previously trained models to use in making predictions or to continue training on new data.

As you can imagine, it's impractical to train a network every time you need to use it. Instead, we can save trained networks then load them later to train more or use them for predictions.

The parameters for PyTorch networks are stored in a model's `state_dict`. We can see the state dict contains the weight and bias matrices for each of our layers._____no_output_____

print("Our model: \n\n", my_MLP, '\n')
print("The state dict keys: \n\n", my_MLP.state_dict().keys())_____no_output_____

The simplest thing to do is simply save the state dict with `torch.save`. For example, we can save it to a file `'checkpoint.pth'`._____no_output_____

torch.save(my_MLP.state_dict(), 'checkpoint.pth')_____no_output_____

Then we can load the state dict with `torch.load`._____no_output_____

state_dict = torch.load('checkpoint.pth')
print(state_dict.keys())_____no_output_____

And to load the state dict in to the network, you do `my_MLP.load_state_dict(state_dict)`._____no_output_____

my_MLP.load_state_dict(state_dict)_____no_output_____

**Important:** `load_state_dict` will raise an error if the architecture of the network is different from the one saved in the pth file. For example, if we define the following model._____no_output_____

my_MLP2 = MLP_extended(dimx=784,hidden1=256,hidden2=128,nlabels=10,epochs=10,lr=1e-3)_____no_output_____

which differs from `my_MLP` in the dimension of the hidden layers, we will get an error if we call the method  `load_state_dict(state_dict)`.

> **Exercise:** Check that you get an error when trying to initialize my_MLP2 from `state_dict` using the method `load_state_dict`_____no_output_____

# this will raise an error
'''
my_MLP2.load_state_dict(state_dict)
'''
# the architechture of the created model must be the same as the one of the stored one_____no_output_____

### Creating a new method in the class, that predicts a single digit image, and outputs the predicted value along with its probability

I will now add a new method that predicts a single image and outputs the result, this is for visualization purposes only, since I was interested in seeing what the outcome probability of each image is._____no_output_____

x_image, y_label = next(iter(testloader))

logprobs = my_LR.forward(x_image[0].view(x_image[0].shape[0], -1))

# get the top probability and class per image
top_p, top_class = logprobs.topk(1, dim=1)

print(f'The most probable digit for this image is: {top_class.item()}')
plt.imshow(x_image[0].view([28,28]), cmap='Greys_r')The most probable digit for this image is: 1

Here I will create an extended class and add this new method that predicts a single image. This is simply a simplification of the overall batch-classifier that was used before._____no_output_____

# adding a method in the class to output the most porbable digit of a single image
class MLP_extended_2(MLP_extended):
    def __init__(self, dimx, hidden1, hidden2, nlabels, epochs=100,lr=0.001):

        super().__init__(dimx, hidden1, hidden2, nlabels, epochs=100,lr=0.001)

        self.lr = lr #Learning Rate

        self.optim = optim.Adam(self.parameters(), self.lr)

        self.epochs = epochs

        self.criterion = nn.NLLLoss()

        # A list to store the loss evolution along training
        self.loss_during_training = []

    def predict(self, image):

        logprobs = self.forward(image[0].view(image[0].shape[0], -1))
        # get the top probability and class per image
        top_p, top_class = logprobs.topk(1, dim=1)
        print('The loaded image is the following: ')
        plt.imshow(image[0].view([28,28]), cmap='Greys_r')
        print(f'The most probable digit for this image is: {top_class.item()} with probablity: {math.e**(top_p.item())}')

_____no_output_____other_MLP = MLP_extended_2(dimx, 124, 64, 10, 5)
other_MLP.train(trainloader)Training loss after 0 epochs: 0.392014
Training loss after 1 epochs: 0.188357
Training loss after 2 epochs: 0.137305
Training loss after 3 epochs: 0.112669
Training loss after 4 epochs: 0.094480
x_image, y_label = next(iter(testloader))_____no_output_____other_MLP.predict(x_image)The loaded image is the following:
The most probable digit for this image is: 8 with probablity: 0.9999710329823779


# Notebook from theVelopr/algorithm
Path: python_algorithm/pyPractice/p081to090.ipynb

### 081 별 표현식
기본적으로 데이터 언패킹은 좌변의 변수와 우변 데이터 개수가 같아야 합니다. 하지만 star expression을 사용하면 변수의 개수가 달라도 데이터 언패킹을 할 수 있습니다. 튜플에 저장된 데이터 중에서 앞에 있는 두 개의 데이터만 필요할 경우 나머지 데이터의 언패킹 코드를 작성할 필요가 없습니다.

>
 >>a, b, *c = (0, 1, 2, 3, 4, 5)

>>a

0

>>b

1

>>c

[2, 3, 4, 5]

다음과 같이 10개의 값이 저장된 scores 리스트가 있을 때, start expression을 사용하여 좌측 8개의 값을 valid_score 변수에 바인딩하여라.

scores = [8.8, 8.9, 8.7, 9.2, 9.3, 9.7, 9.9, 9.5, 7.8, 9.4]_____no_output_____

scores = [8.8, 8.9, 8.7, 9.2, 9.3, 9.7, 9.9, 9.5, 7.8, 9.4]
*valid_score, _, _= scores
print(valid_score)[8.8, 8.9, 8.7, 9.2, 9.3, 9.7, 9.9, 9.5]

### 082
다음과 같이 10개의 값이 저장된 scores 리스트가 있을 때, start expression을 사용하여 우측 8개의 값을 valid_score 변수에 바인딩하여라.

scores = [8.8, 8.9, 8.7, 9.2, 9.3, 9.7, 9.9, 9.5, 7.8, 9.4]_____no_output_____

scores = [8.8, 8.9, 8.7, 9.2, 9.3, 9.7, 9.9, 9.5, 7.8, 9.4]
a, b, *valid_score = scores
print(valid_score)[8.7, 9.2, 9.3, 9.7, 9.9, 9.5, 7.8, 9.4]

### 083
다음과 같이 10개의 값이 저장된 scores 리스트가 있을 때, start expression을 사용하여 가운데 있는 8개의 값을 valid_score 변수에 바인딩하여라.

scores = [8.8, 8.9, 8.7, 9.2, 9.3, 9.7, 9.9, 9.5, 7.8, 9.4]_____no_output_____

scores = [8.8, 8.9, 8.7, 9.2, 9.3, 9.7, 9.9, 9.5, 7.8, 9.4]
a, *valid_score, b = scores
print(valid_score)[8.9, 8.7, 9.2, 9.3, 9.7, 9.9, 9.5, 7.8]

### 084 비어있는 딕셔너리
temp 이름의 비어있는 딕셔너리를 만들라._____no_output_____

temp = { }_____no_output_____

### 085
다음 아이스크림 이름과 희망 가격을 딕셔너리로 구성하라.

이름	희망 가격

메로나	1000

폴라포	1200

빵빠레	1800_____no_output_____

icecream = {"메로나" : 1000, "폴라포" : 1200, "빵빠레" : 1800}

print(type(icecream))<class 'dict'>

### 086 정보추가
085 번의 딕셔너리에 아래 아이스크림 가격정보를 추가하라.

이름	희망 가격

죠스바	1200

월드콘	1500_____no_output_____

icecream = {"메로나" : 1000, "폴라포" : 1200, "빵빠레" : 1800}
icecream["죠스바"] = 1200
icecream["월드콘"] = 1500

print (icecream){'메로나': 1000, '폴라포': 1200, '빵빠레': 1800, '죠스바': 1200, '월드콘': 1500}

### 087 정보출력
다음 딕셔너리를 사용하여 메로나 가격을 출력하라.

ice = {'메로나': 1000,
       '폴로포': 1200,
       '빵빠레': 1800,
       '죠스바': 1200,
       '월드콘': 1500}
       _____no_output_____

ice = {'메로나': 1000, '폴로포': 1200, '빵빠레': 1800, '죠스바': 1200, '월드콘': 1500}

print("메로나 가격: ", ice["메로나"])메로나 가격:  1000

### 088 정보수정
다음 딕셔너리에서 메로나의 가격을 1300으로 수정하라.

ice = {'메로나': 1000, '폴로포': 1200, '빵빠레': 1800, '죠스바': 1200, '월드콘': 1500}_____no_output_____

ice = {'메로나': 1000, '폴로포': 1200, '빵빠레': 1800, '죠스바': 1200, '월드콘': 1500}

ice["메로나"] = 1300

print (ice){'메로나': 1300, '폴로포': 1200, '빵빠레': 1800, '죠스바': 1200, '월드콘': 1500}

### 089 정보삭제
다음 딕셔너리에서 메로나를 삭제하라.

ice = {'메로나': 1000, '폴로포': 1200, '빵빠레': 1800, '죠스바': 1200, '월드콘': 1500}
_____no_output_____

ice = {'메로나': 1000, '폴로포': 1200, '빵빠레': 1800, '죠스바': 1200, '월드콘': 1500}

del ice["메로나"]

print (ice){'폴로포': 1200, '빵빠레': 1800, '죠스바': 1200, '월드콘': 1500}


# Notebook from diogoflduarte/ssm
Path: notebooks/7 Variatonal Laplace EM for SLDS Tutorial.ipynb

import autograd.numpy as np
import autograd.numpy.random as npr
npr.seed(0)

import matplotlib.pyplot as plt
%matplotlib inline

import seaborn as sns
sns.set_style("white")
sns.set_context("talk")

color_names = ["windows blue",
               "red",
               "amber",
               "faded green",
               "dusty purple",
               "orange",
               "clay",
               "pink",
               "greyish",
               "mint",
               "light cyan",
               "steel blue",
               "forest green",
               "pastel purple",
               "salmon",
               "dark brown"]

colors = sns.xkcd_palette(color_names)

import ssm
from ssm.util import random_rotation, find_permutation_____no_output_____# Set the parameters of the SLDS
T = 1000    # number of time bins
K = 5       # number of discrete states
D = 2       # number of latent dimensions
N = 10      # number of observed dimensions_____no_output_____# Make an SLDS with the true parameters
true_slds = ssm.SLDS(N, K, D, emissions="gaussian_orthog")
for k in range(K):
    true_slds.dynamics.As[k] = .95 * random_rotation(D, theta=(k+1) * np.pi/20)
z, x, y = true_slds.sample(T)

# Mask off some data
mask = npr.rand(T, N) < 0.9
y_masked = y * mask_____no_output_____

# Variational Laplace-EM

We will fit a switching linear dynamical system (SLDS) to the observed data $y$. Our approach combines variational inference and a Laplace approximation. The objective we maximize is the evidence lower bound (ELBO)

$$ \mathcal{L}_q(\theta) = \mathbb{E}_{q(z) q(x)}[\log p(z,x,y|\theta)] - \mathbb{E}_{q(z)}[\log q(z)] - \mathbb{E}_{q(x)}[\log q(x)] $$

where $\theta$ are the model parameters, $z$ are the discrete latent variables, and $x$ are the continuous latent variables. We introduced an approximate posterior that factorizes over the discrete and continuous latent variables, $p(z, x \, | \, \theta, y) \approx q(z) \, q(x)$. This form of the posterior corresponds to the `variational_posterior="structured_meanfield"` argument in the `fit` method.

The variational Laplace-EM inference method performs three updates at each iteration:
1. Update $q(z)$ using the optimal coordinate ascent variational inference update
2. Update $q(x)$ using a Laplace approximation at the most likely latent path (note that this step is not guaranteed to increase the ELBO)
3. Update $\theta$ by optimizing the ELBO with respect to the model parameters

You can initialize a variational posterior and fit the model by running the function
```
# Fit the model using Laplace-EM with a structured variational posterior
elbos, q_lem = slds.fit(data, inputs, masks, method="laplace_em",
                        variational_posterior="structured_meanfield",
                        initialize=False, num_iters=100)
```
The output variables are the values of the objective at each iteration `elbos` and the variational posterior object `q_lem`. If you have already initialized a variational posterior object, you can pass that object as the argument of the `variational_posterior` parameter instead. Here, the `initialize` parameter was set to `False`, which assumes we have already initialized the model. Additionally, note that in `fit` you can also pass in keyword arguments for initialization of the variational posterior. _____no_output_____### Hyperparameters
There are a number of hyperparameters that can be used when fitting with `laplace_em`. These include generic hyperparameters:
- `num_iters` - the number of iterations to run (default = 100)
- `learning` - optimize the model parameters when True (default = True)

Discrete state hyperparameters:
- `num_samples` - number of Monte Carlo samples (default = 1) used for evaluating expectations with respect to $q(x)$ in the update for $q(z)$

Continuous state hyperparameters:
- `continuous_optimizer`, `continuous_tolerance`, `continuous_maxiter` specify parameters of the optimization for finding the most likely latent path in the continuous latent update. We recommend using the default optimizer Newton's method (`newton`) to compute the most likely latent path. However, Hessian-free optimization is supported with `lbfgs`. The tolerance and maxiter parameters can be adjusted for the user's requirements.

Model parameter update hyperparameters:
- `alpha` - parameter in $[0,1)$ with default $0.5$ that determines how greedy we are in updating the model parameters at each iteration. This is only used in conjunction with `parameters_update="mstep"`.
- `parameters_update` - the model parameter updates are implemented via an m-step given a single sample from $q(x)$ (default, `"mstep"`) or whether using SGD with samples from $q(x)$ (`"sgd"`).
- `emission_optimizer` - the optimizer used to update parameters in the m-step. This defaults to the `adam` optimizer when using the `"sgd"` parameter updates.
- `emission_optimizer_maxiter` - the maximum number of iterations for the inner loop of optimizing the emissions parameters. _____no_output_____### Exploring the effect of the `alpha` hyperparameter

In the `"mstep"` parameters update, we optimize the joint log-likelihood conditioned on a single sample from $q(x)$ to get a point estimate of the parameters $\theta^\star$. The new parameters at iteration $t$ are then given by

$$ \theta^{(t)} = (1 - \alpha) \, \theta^\star + \alpha \, \theta^{(t-1)}. $$

If `alpha=0.0` we set the parameters to $\theta^\star$, whereas for nonzero `alpha` the parameters are partially updated towards $\theta^\star$. The default is `alpha=0.5`. We have found that performance varies for different values of `alpha` and that the performance can depend on the specific problem. We will demonstrate below how `alpha` can affect inference by sweeping over five different values of `alpha` in model fits below._____no_output_____

alphas = [0.0, 0.25, 0.5, 0.75, 0.9]
results = {}
for alpha in alphas:
    print("Fitting SLDS with Laplace-EM, alpha = {}".format(alpha))

    # Create the model and initialize its parameters
    slds = ssm.SLDS(N, K, D, emissions="gaussian_orthog")
    slds.initialize(y_masked, masks=mask)

    # Fit the model using Laplace-EM with a structured variational posterior
    q_lem_elbos, q_lem = slds.fit(y_masked, masks=mask, method="laplace_em",
                               variational_posterior="structured_meanfield",
                               initialize=False, num_iters=100, alpha=alpha)

    # Get the posterior mean of the continuous states
    q_lem_x = q_lem.mean_continuous_states[0]

    # Find the permutation that matches the true and inferred states
    slds.permute(find_permutation(z, slds.most_likely_states(q_lem_x, y)))
    q_lem_z = slds.most_likely_states(q_lem_x, y)

    # Smooth the data under the variational posterior
    q_lem_y = slds.smooth(q_lem_x, y)

    results[(alpha)] = (slds, q_lem, q_lem_elbos, q_lem_x, q_lem_z, q_lem_y)Fitting SLDS with Laplace-EM, alpha = 0.0
Initializing with an ARHMM using 25 steps of EM.
# Plot the ELBOs
q_elbos = []
for alpha in alphas:
    _,_,q_lem_elbos,_,_,_ = results[(alpha)]
    q_elbos += [q_lem_elbos]
    plt.plot(q_lem_elbos[1:], label="alpha = {}".format(alpha), alpha=0.8)
plt.xlabel("Iteration")
plt.ylabel("ELBO")
q_max = np.array(q_elbos).max()
plt.ylim([q_max-0.1*np.abs(q_max),q_max+0.01*np.abs(q_max)])
plt.legend()_____no_output_____# Plot the true and inferred states
xlim = (0, 1000)

zs = []
for alpha in alphas:
    (_, _, _, _, q_lem_z, _) = results[(alpha)]
    zs += [q_lem_z]

plt.figure(figsize=(16,4))
plt.imshow(np.row_stack((z, zs)), aspect="auto", interpolation="none")
plt.plot(xlim, [0.5, 0.5], '-k', lw=2)
plt.yticks([0,1,2,3,4,5],["$z_{\\mathrm{true}}$", *alphas])
plt.ylabel("alpha")
plt.xlim(xlim)_____no_output_____plt.figure(figsize=(8,4))
for d in range(D):
    plt.plot(x[:,d] + 4 * d, '-', color='k', label="True" if d==0 else None)
    for i, alpha in enumerate(alphas):
        _,_,_,q_lem_x,_,_ = results[(alpha)]
        plt.plot(q_lem_x[:,d] + 4 * d, '-',  color=colors[i+1], label=alpha if d == 0 else None, alpha=0.75)
plt.ylabel("$x$")
plt.xlim(xlim)
plt.legend()_____no_output_____# Plot the smoothed observations
plt.figure(figsize=(12,8))
for n in range(N):
    plt.plot(y[:, n] + 6 * n, '-', color='k', label="True" if n == 0 else None)
    for i, alpha in enumerate(alphas):
        _,_,_,_,_,q_lem_y = results[(alpha)]
        plt.plot(q_lem_y[:,n] + 6 * n, '-',  color=colors[i+1], label=alpha if n == 0 else None, alpha=0.75)
plt.legend()
plt.xlabel("time")
plt.xlim(xlim)_____no_output_____