# Notebook from dhrithideshpande/review-technology-dictates-algorithms
Path: notebooks/Supplemental.ipynb

<code>
import pandas as pd
raw_data=pd.read_csv('../raw_data/Tool_Info.csv')
clean_data=pd.read_csv('../summary_data/table1_final.csv')
raw_data = raw_data.loc[:,['aligner', 'num_citations', 'wrapper_of']]
data = pd.merge(clean_data, raw_data, on='aligner', how='outer')
data.loc[data['aligner'] == 'PatternHunter', 'num_citations'] = 954
data.loc[data['aligner'] == 'LAMSA', 'num_citations'] = 14
data.loc[data['aligner'] == 'conLSH', 'num_citations'] = 1

data['age']=2020-data['year_of_pub']+1
data['citation_per_year']=data['num_citations']/data['age']
data.loc[data.loc[:,'aligner'] == 'HISAT2', 'citation_per_year'] = 95/2

# Creating new columns that combine rarer types
data.loc[data.loc[:,'indexing'] == 'hashing', 'index'] = 'Hashing'
data.loc[data.loc[:,'indexing'] == 'BWT-FM', 'index'] = 'BWT-FM'
data.loc[data.loc[:,'indexing'] == 'BWT-FM + hashing', 'index'] = 'BWT-FM & Hashing'
data.loc[data.loc[:,'indexing'] == 'suffix array', 'index'] = 'Other Suffix'
data.loc[data.loc[:,'indexing'] == 'BWT-ST', 'index'] = 'Other Suffix'
data.loc[data.loc[:,'indexing'] == 'suffix tree', 'index'] = 'Other Suffix'
data.loc[data.loc[:,'indexing'] == 'BWT', 'index'] = 'Other Suffix'

data.loc[data.loc[:,'pairwise_alignment'] == 'SW', 'pairwise'] = 'Smith-Waterman'
data.loc[data.loc[:,'pairwise_alignment'] == 'HD', 'pairwise'] = 'Hamming Distance'
data.loc[data.loc[:,'pairwise_alignment'] == 'NW', 'pairwise'] = 'Needleman-Wunsch'
data.loc[data.loc[:,'pairwise_alignment'] == 'Non-DP Heuristic', 'pairwise'] = 'Non-DP Heuristic'
data.loc[data.loc[:,'pairwise_alignment'] == 'Non-DP heuristic', 'pairwise'] = 'Non-DP Heuristic'
data.loc[data.loc[:,'pairwise_alignment'] == 'SW & NW', 'pairwise'] = 'Multiple Methods'
data.loc[data.loc[:,'pairwise_alignment'] == 'SW and NW', 'pairwise'] = 'Multiple Methods'
data.loc[data.loc[:,'pairwise_alignment'] == 'SW & Semi-Global', 'pairwise'] = 'Multiple Methods'
data.loc[data.loc[:,'pairwise_alignment'] == 'SW and Semi-Global', 'pairwise'] = 'Multiple Methods'
data.loc[data.loc[:,'pairwise_alignment'] == 'Rabin-Karp Algorithm', 'pairwise'] = 'Other DP'
data.loc[data.loc[:,'pairwise_alignment'] == 'Landau-Vishkin', 'pairwise'] = 'Other DP'
data.loc[data.loc[:,'pairwise_alignment'] == 'Sparse DP', 'pairwise'] = 'Other DP'
data.loc[data.loc[:,'pairwise_alignment'] == 'Banded Myers Bit Vector', 'pairwise'] = 'Other DP'
data.loc[data.loc[:,'pairwise_alignment'] == 'Myers Bit-Vector', 'pairwise'] = 'Other DP'
data.loc[data.loc[:,'pairwise_alignment'] == 'Semi-Global', 'pairwise'] = 'Other DP'
data.loc[data.loc[:,'pairwise_alignment'] == 'Semi-global', 'pairwise'] = 'Other DP'

data.loc[data.loc[:, 'wrapper'] == 'Y', 'wrap'] = 'Yes'
data.loc[data.loc[:, 'wrapper'] == 'N', 'wrap'] = 'No'
data.loc[data['aligner'] == 'LAMSA', 'wrapper_of'] = 'GEM'

data.loc[data.loc[:,'fix_length_seed'] == 'Y', 'seeds_fixed'] = 'Yes'
data.loc[data.loc[:,'fix_length_seed'] == 'N', 'seeds_fixed'] = 'No'

data.loc[data.loc[:,'seed_chaining'] == 'Y', 'seeds_chained'] = 'Yes'
data.loc[data.loc[:,'seed_chaining'] == 'N', 'seeds_chained'] = 'No'

data.loc[data.loc[:,'spaced_seed'] == 'Y', 'seeds_spaced'] = 'Yes'
data.loc[data.loc[:,'spaced_seed'] == 'N', 'seeds_spaced'] = 'No'

data_year = pd.DataFrame(data.loc[:,['year_of_pub', 'aligner', 'application', 'index', 'pairwise',
                              'seeds_fixed', 'seeds_spaced', 'seeds_chained', 'wrap', 'wrapper_of', 'citation_per_year']])

data_year.loc[data_year['wrap'] == 'Yes', :]_____no_output_____rt = pd.read_csv('../raw_data/compare.tools.runtime.hours.csv')
mem = pd.read_csv('../raw_data/compare.tools.memory.csv')_____no_output_____rt.loc[:,'tools_lower'] = rt.loc[:,'Tools'].str.lower()
data_year.loc[:,'aligner_lower'] = data_year.loc[:,'aligner'].str.lower()
rtyear = pd.merge(rt, data_year, left_on='tools_lower', right_on='aligner_lower', how='inner')
rtyear.loc[:,'average_runtime'] = rtyear.iloc[:,1:11].mean(axis=1)
rtyear.loc[:,'std_dev'] = rtyear.iloc[:,1:11].std(axis=1)

# Imputing the one missing value
rtyear.loc[3,'ERR162843'] = rtyear.loc[3,'average_runtime']

# Only compare tools with DNA application
rtyear = rtyear.loc[rtyear.loc[:,'application'] != 'RNA-Seq'].reset_index(drop=True)

# Bowtie2 can be run in global or local, our data was generated using Needleman-Wunsch (Global)
rtyear.loc[rtyear.loc[:,'Tools'] == 'Bowtie2','pairwise'] = 'Needleman-Wunsch'
rtyear.loc[rtyear.loc[:,'Tools'] == 'last','pairwise'] = 'Smith-Waterman'
rtyear_summ = rtyear.loc[:,['aligner', 'year_of_pub', 'average_runtime', 'std_dev', 'application',
                            'index', 'pairwise', 'seeds_fixed', 'seeds_spaced', 'seeds_chained', 'citation_per_year']]
rtyear_summ.loc[:,'Aligner'] = rtyear_summ.loc[:,'aligner']
#rtyear_summ.sort_values(by=['year_of_pub'], inplace=True)
for i in range(0,11):
    if rtyear.loc[i,'year_of_pub'] < 2013:
        rtyear.loc[i,'year_split'] = 'Before 2013'
    else:
        rtyear.loc[i,'year_split'] = '2013 and later'
rtyear_melt = pd.melt(rtyear, id_vars = ['aligner', 'year_split', 'application', 'index', 'pairwise'],
        value_vars = ['ERR009309', 'ERR013127', 'ERR013138', 'ERR045708', 'ERR050158', 'ERR162843', 'ERR181410',
                     'ERR183377', 'SRR061640', 'SRR360549'], value_name = 'runtime')
#rtyear.to_csv(path_or_buf='../summary_data/runtime_comparison_with_tool_info.csv')
rtyear_summ_____no_output_____mem.loc[:,'tools_lower'] = mem['Tools'].str.lower()
memyear = pd.merge(mem, data_year, left_on='tools_lower', right_on='aligner_lower', how='inner')
memyear.loc[:,'average_memory'] = memyear.iloc[:,1:11].mean(axis=1)
memyear = memyear.loc[memyear.loc[:,'application'] != 'RNA-Seq'].reset_index(drop=True)
memyear.loc[memyear.loc[:,'Tools'] == 'Bowtie2','pairwise'] = 'Needleman-Wunsch'
memyear.loc[memyear.loc[:,'Tools'] == 'last','pairwise'] = 'Smith-Waterman'

memyear_summ = memyear.loc[:,['aligner', 'year_of_pub', 'average_memory', 'application', 'index', 'citation_per_year',
                             'pairwise','seeds_fixed','seeds_spaced', 'seeds_chained',]]
memyear_summ.loc[:,'Aligner'] = memyear_summ.loc[:,'aligner']
#memyear_summ.sort_values(by=['year_of_pub'], inplace=True)
for i in range(0,11):
    memyear.loc[i,'year_split'] = 'Before 2013' if memyear.loc[i,'year_of_pub'] < 2013 else '2013 and later'

memyear_melt = pd.melt(memyear, id_vars = ['aligner', 'year_split', 'application', 'index', 'pairwise'],
        value_vars = ['ERR009309', 'ERR013127', 'ERR013138', 'ERR045708', 'ERR050158', 'ERR162843', 'ERR181410',
                     'ERR183377', 'SRR061640', 'SRR360549'], value_name = 'memory')
#memyear.to_csv(path_or_buf='../summary_data/memory_comparison_with_tool_info.csv')
memyear_summ_____no_output_____idx_rt = pd.read_csv('../raw_data/compare.tools.index.csv')

data_year.loc[:,'aligner_lower'] = data_year.loc[:,'aligner'].str.lower()
idx_rt.loc[:,'tools_lower'] = idx_rt.loc[:,'Tools'].str.lower()

rtidx = pd.merge(idx_rt, data_year, left_on='tools_lower',
                 right_on='aligner_lower', how='inner').loc[:,['aligner', 'application', 'year_of_pub',
                                                               'Runtime','Virtual Mem (G)', 'index']]
rtidx = rtidx.loc[rtidx.loc[:,'application'].str.contains('DNA'),:]
rtidx = rtidx.loc[rtidx.loc[:,'aligner'] != 'MOSAIK']
rtidx = rtidx.dropna().reset_index(drop=True)
t = pd.DatetimeIndex(rtidx.loc[:,'Runtime'])
rtidx.loc[:,'Runtime (hours)'] = (t.hour*60 + t.minute)/60
rtidx_____no_output_____import seaborn as sns
import matplotlib.pyplot as plt
import scipy as scp
import numpy as np

tool_clrs = ['teal', 'palegreen', 'purple', 'deepskyblue', 'magenta', 'maroon',
        'darkorange', 'limegreen', 'turquoise', 'blue']

cite_clrs = ['teal', 'purple', 'palegreen', 'deepskyblue', 'magenta', 'maroon', 'khaki',
        'darkorange', 'limegreen', 'turquoise', 'blue']

index_clrs = ['darkorange', 'mediumorchid', 'cornflowerblue', 'navy']

local_clrs = ['orange', 'deepskyblue', 'chocolate', 'gold', 'royalblue', 'lightseagreen']

order_by_year = ["segemehl", "Bowtie", "BWA", "GSNAP", "SMALT", "LAST", "SNAP", "Bowtie2", "Subread", "HISAT2"]

cite_order_by_year = ["RMAP", "Bowtie", "BWA", "GSNAP", "SMALT", "LAST", "SNAP", "Bowtie2", "Subread",
                 "HISAT2", "minimap2"]_____no_output_____plt.figure(figsize=(8,5))

medians = rtidx.groupby('index').median()
upper = rtidx.groupby('index').quantile(q=.75)
lower = rtidx.groupby('index').quantile(q=.25)

sns.set_style('white')
sns.set_context('talk')
#ax = sns.boxplot(data = rtidx, x = 'index', y = 'Runtime (hours)', whis=0,
#                 order = ['Hashing', 'BWT-FM', 'Other Suffix'], color="1", fliersize = 0)
ax = sns.stripplot(data= rtidx, x = 'index', y = 'Runtime (hours)', order=['Hashing','BWT-FM','Other Suffix'],
                   palette=tool_clrs, hue="aligner", hue_order=order_by_year, size=8, jitter=.3, alpha=.75)
ax.set(xlabel = 'Indexing Algorithm', ylabel = 'CPU Time (hours)')
leg = plt.legend(title='Aligner', loc='center left', bbox_to_anchor=(1, .5), frameon=False)

#plt.hlines(upper.loc[upper.index == 'Hashing','Runtime (hours)'], -.05, .05, color="dimgray", linestyle='solid')
#plt.hlines(lower.loc[lower.index == 'Hashing','Runtime (hours)'], -.05, .05, color="dimgray", linestyle='solid')
#plt.vlines(0, lower.loc[lower.index == 'Hashing','Runtime (hours)'], upper.loc[upper.index == 'Hashing','Runtime (hours)'],
#          color="dimgray", linestyle='solid')
plt.hlines(medians.loc[medians.index == 'Hashing','Runtime (hours)'], -.4, .4, color="dimgray", linestyle='solid')
#plt.hlines(upper.loc[upper.index == 'BWT-FM','Runtime (hours)'], .95, 1.05, color="dimgray", linestyle='solid')
#plt.hlines(lower.loc[lower.index == 'BWT-FM','Runtime (hours)'], .95, 1.05, color="dimgray", linestyle='solid')
#plt.vlines(1, lower.loc[lower.index == 'BWT-FM','Runtime (hours)'], upper.loc[upper.index == 'BWT-FM','Runtime (hours)'],
#           color="dimgray", linestyle='solid')
plt.hlines(medians.loc[medians.index == 'BWT-FM','Runtime (hours)'], 0.6, 1.4, color="dimgray", linestyle='solid')
#plt.hlines(upper.loc[upper.index == 'Other Suffix','Runtime (hours)'], 1.95, 2.05, color="dimgray", linestyle='solid')
#plt.hlines(lower.loc[lower.index == 'Other Suffix','Runtime (hours)'], 1.95, 2.05, color="dimgray", linestyle='solid')
#plt.vlines(2, lower.loc[lower.index == 'Other Suffix','Runtime (hours)'], upper.loc[upper.index == 'Other Suffix',
#                                                                                'Runtime (hours)'],
#           color="dimgray", linestyle='solid')
plt.hlines(medians.loc[medians.index == 'Other Suffix','Runtime (hours)'], 1.6, 2.4, color="dimgray", linestyle='solid')

plt.tight_layout()
sns.despine()

plt.savefig('../figures/FigureS1_a.png')
plt.savefig('../figures/FigureS1_a.pdf')_____no_output_____plt.figure(figsize=(8,5))

sns.set_style('white')
sns.set_context('talk')

medians = rtidx.groupby('index').median()
upper = rtidx.groupby('index').quantile(q=.75)
lower = rtidx.groupby('index').quantile(q=.25)

#ax = sns.boxplot(data = rtidx, x = 'index', y = 'Virtual Mem (G)', whis=0,
#                 order = ['Hashing', 'BWT-FM', 'Other Suffix'], color="1", fliersize = 0)
ax = sns.stripplot(data= rtidx, x = 'index', y = 'Virtual Mem (G)', order=['Hashing','BWT-FM','Other Suffix'],
                   palette=tool_clrs, hue="aligner", hue_order=order_by_year, size=8, jitter=.3, alpha=.75)
ax.set(xlabel = 'Indexing Algorithm', ylabel = 'Memory (GB)')
leg = plt.legend(title='Aligner', loc='center left', bbox_to_anchor=(1, .5), frameon=False)

#plt.hlines(upper.loc[upper.index == 'Hashing','Virtual Mem (G)'], -.05, .05, color="dimgray", linestyle='solid')
#plt.hlines(lower.loc[lower.index == 'Hashing','Virtual Mem (G)'], -.05, .05, color="dimgray", linestyle='solid')
#plt.vlines(0, lower.loc[lower.index == 'Hashing','Virtual Mem (G)'], upper.loc[upper.index == 'Hashing','Virtual Mem (G)'],
#           color="dimgray", linestyle='solid')
plt.hlines(medians.loc[medians.index == 'Hashing','Virtual Mem (G)'], -.4, .4, color="dimgray", linestyle='solid')
#plt.hlines(upper.loc[upper.index == 'BWT-FM','Virtual Mem (G)'], .95, 1.05, color="dimgray", linestyle='solid')
#plt.hlines(lower.loc[lower.index == 'BWT-FM','Virtual Mem (G)'], .95, 1.05, color="dimgray", linestyle='solid')
#plt.vlines(1, lower.loc[lower.index == 'BWT-FM','Virtual Mem (G)'], upper.loc[upper.index == 'BWT-FM','Virtual Mem (G)'],
#           color="dimgray", linestyle='solid')
plt.hlines(medians.loc[medians.index == 'BWT-FM','Virtual Mem (G)'], 0.6, 1.4, color="dimgray", linestyle='solid')
#plt.hlines(upper.loc[upper.index == 'Other Suffix','Virtual Mem (G)'], 1.95, 2.05, color="dimgray", linestyle='solid')
#plt.hlines(lower.loc[lower.index == 'Other Suffix','Virtual Mem (G)'], 1.95, 2.05, color="dimgray", linestyle='solid')
#plt.vlines(2, lower.loc[lower.index == 'Other Suffix','Virtual Mem (G)'], upper.loc[upper.index == 'Other Suffix','Virtual Mem (G)'],
#           color="dimgray", linestyle='solid')
plt.hlines(medians.loc[medians.index == 'Other Suffix','Virtual Mem (G)'], 1.6, 2.4, color="dimgray", linestyle='solid')

plt.tight_layout()
sns.despine()

plt.savefig('../figures/FigureS1_b.png')
plt.savefig('../figures/FigureS1_b.pdf')_____no_output_____rna_tools = pd.DataFrame(data_year.loc[data_year.application.str.contains('RNA'),:])
rna_tools_____no_output_____plt.figure(figsize=(11,4))

sns.set_style('white')
sns.set_context('talk')

idx_group = rna_tools.groupby('index').count().sort_values('aligner', ascending=True)
patches, texts = plt.pie(idx_group.aligner, colors=index_clrs[2::-1])
pct = 100.*idx_group.aligner/idx_group.aligner.sum()
labels = ['{0} - {1} ({2:2.2f} %)'.format(i,j,k) for i,j,k in zip(idx_group.index, idx_group.aligner, pct)]
plt.legend(patches, labels, loc='center left', bbox_to_anchor=(.9, .5), frameon=False)
#plt.show()
plt.tight_layout()
sns.despine()

plt.savefig('../figures/FigureS2_0.png')
plt.savefig('../figures/FigureS2_0.pdf')_____no_output_____plt.figure(figsize=(8,4))

idx = range(min(rna_tools['year_of_pub']), max(rna_tools['year_of_pub'])+1)
by_year = rna_tools.groupby(['year_of_pub','index']).count()
by_year = by_year.xs('aligner', axis=1)
by_year = by_year.unstack(level = -1, fill_value=0)
by_year = by_year.cumsum()
by_year = by_year.reindex(idx, method='ffill')
#by_year.loc[:,'Total'] = by_year.sum(axis=1)
x = by_year.index
by_year.reset_index()
year_melt = pd.melt(by_year.reset_index(), id_vars=['year_of_pub'], var_name='Indexing Algorithm',
                    value_name='Number of Tools')
year_melt = year_melt.rename(columns={'year_of_pub':'Year of Publication'})

sns.set_style('white')
sns.set_context('talk')

g = sns.FacetGrid(year_melt, row='Indexing Algorithm', aspect=4, hue='Indexing Algorithm',
                  row_order=['Hashing', 'BWT-FM', 'Other Suffix'],
                  palette=index_clrs, hue_order=['Hashing', 'BWT-FM', 'Other Suffix'])
g = g.map(plt.bar, 'Year of Publication', 'Number of Tools', linewidth=0)
plt.xticks(range(min(rna_tools['year_of_pub']), max(rna_tools['year_of_pub'])+2, 2))
plt.ylim(0,12)
plt.yticks(range(0, 13, 4))
sns.despine()

plt.tight_layout()
plt.savefig('../figures/FigureS2_1.png')
plt.savefig('../figures/FigureS2_1.pdf')_____no_output_____rna_wrappers = rna_tools[rna_tools.wrap == 'Yes']
rna_nonwrappers = rna_tools[rna_tools.wrap == 'No']

rna_wrappers_____no_output_____plt.figure(figsize=(6,6))

sns.set_style('white')
sns.set_context('talk')
idx_group = rna_tools.groupby('wrap').count().sort_values('aligner', ascending=True)
ax = plt.bar(idx_group.index, idx_group.aligner, color=['darkslategray', 'coral'])
pct = 100.*idx_group.aligner/idx_group.aligner.sum()

plt.title("Number of RNA-Seq Tools Classified as Wrappers")
plt.yticks(range(0,16,5))
#plt.show()

sns.despine()
plt.tight_layout()
plt.savefig('../figures/FigureS3_0.png')
plt.savefig('../figures/FigureS3_0.pdf')_____no_output_____plt.figure(figsize=(6,4))

sns.set_style('white')
sns.set_context('talk')

bowtie_tools = rna_wrappers.loc[rna_wrappers.wrapper_of.str.contains('Bowtie'),:]
num_b = len(bowtie_tools)
num_o = len(rna_wrappers) - num_b
ax = plt.bar(['Bowtie Wrappers', 'Other Wrappers'], [num_b, num_o], color=['darkslategray', 'coral'])

#plt.title("Number of RNA-Seq Tools Wrapping Bowtie")
#plt.show()

sns.despine()
plt.tight_layout()
plt.savefig('../figures/FigureS3_1.png')
plt.savefig('../figures/FigureS3_1.pdf')_____no_output_____plt.figure(figsize=(11,4))

sns.set_style('white')
sns.set_context('talk')
idx_group = rna_nonwrappers.groupby('index').count().sort_values('aligner', ascending=True)
patches, texts = plt.pie(idx_group.aligner, colors=index_clrs[2::-1])
pct = 100.*idx_group.aligner/idx_group.aligner.sum()
labels = ['{0} - {1} ({2:2.2f} %)'.format(i,j,k) for i,j,k in zip(idx_group.index, idx_group.aligner, pct)]
plt.legend(patches, labels, loc='center left', bbox_to_anchor=(.9, .5), frameon=False)
#plt.show()
plt.tight_layout()
sns.despine()
plt.savefig('../figures/FigureS4_0.png')
plt.savefig('../figures/FigureS4_0.pdf')_____no_output_____plt.figure(figsize=(8,4))

idx = range(min(rna_nonwrappers['year_of_pub']), max(rna_nonwrappers['year_of_pub'])+1)
by_year = rna_nonwrappers.groupby(['year_of_pub','index']).count()
by_year = by_year.xs('aligner', axis=1)
by_year = by_year.unstack(level = -1, fill_value=0)
by_year = by_year.cumsum()
by_year = by_year.reindex(idx, method='ffill')
#by_year.loc[:,'Total'] = by_year.sum(axis=1)
x = by_year.index
by_year.reset_index()
year_melt = pd.melt(by_year.reset_index(), id_vars=['year_of_pub'], var_name='Indexing Algorithm',
                    value_name='Number of Tools')
year_melt = year_melt.rename(columns={'year_of_pub':'Year of Publication'})

sns.set_style('white')
sns.set_context('talk')

g = sns.FacetGrid(year_melt, row='Indexing Algorithm', aspect=4, hue='Indexing Algorithm',
                  row_order=['Hashing', 'BWT-FM', 'Other Suffix'],
                  palette=index_clrs, hue_order=['Hashing', 'BWT-FM', 'Other Suffix'])
g = g.map(plt.bar, 'Year of Publication', 'Number of Tools', linewidth=0)
plt.xticks(range(min(rna_nonwrappers['year_of_pub']), max(rna_nonwrappers['year_of_pub'])+2, 2))
plt.ylim(0,10)
plt.yticks(range(0,11,2))
sns.despine()

plt.tight_layout()
plt.savefig('../figures/FigureS4_1.png')
plt.savefig('../figures/FigureS4_1.pdf')_____no_output_____bs_tools = pd.DataFrame(data_year.loc[data_year.application.str.contains('BS'),:])
bs_tools_____no_output_____plt.figure(figsize=(11,4))

sns.set_style('white')
sns.set_context('talk')

idx_group = bs_tools.groupby('index').count().sort_values('aligner', ascending=True)
patches, texts = plt.pie(idx_group.aligner, colors=index_clrs[::-1])
pct = 100.*idx_group.aligner/idx_group.aligner.sum()
labels = ['{0} - {1} ({2:2.2f} %)'.format(i,j,k) for i,j,k in zip(idx_group.index, idx_group.aligner, pct)]
plt.legend(patches, labels, loc='center left', bbox_to_anchor=(.9, .5), frameon=False)
#plt.show()
plt.tight_layout()
sns.despine()

plt.savefig('../figures/FigureS5_0.png')
plt.savefig('../figures/FigureS5_0.pdf')_____no_output_____plt.figure(figsize=(8,4))

idx = range(min(bs_tools['year_of_pub']), max(bs_tools['year_of_pub'])+1)
by_year = bs_tools.groupby(['year_of_pub','index']).count()
by_year = by_year.xs('aligner', axis=1)
by_year = by_year.unstack(level = -1, fill_value=0)
by_year = by_year.cumsum()
by_year = by_year.reindex(idx, method='ffill')
#by_year.loc[:,'Total'] = by_year.sum(axis=1)
x = by_year.index
by_year.reset_index()
year_melt = pd.melt(by_year.reset_index(), id_vars=['year_of_pub'], var_name='Indexing Algorithm',
                    value_name='Number of Tools')
year_melt = year_melt.rename(columns={'year_of_pub':'Year of Publication'})

sns.set_style('white')
sns.set_context('talk')

g = sns.FacetGrid(year_melt, row='Indexing Algorithm', aspect=4, hue='Indexing Algorithm',
                  row_order=['Hashing', 'BWT-FM', 'Other Suffix', 'BWT-FM & Hashing'],
                  palette=index_clrs, hue_order=['Hashing', 'BWT-FM', 'Other Suffix', 'BWT-FM & Hashing'])
g = g.map(plt.bar, 'Year of Publication', 'Number of Tools', linewidth=0)
plt.xticks(range(min(bs_tools['year_of_pub']), max(bs_tools['year_of_pub'])+2, 2))
plt.ylim(0,6)
sns.despine()

plt.tight_layout()
plt.savefig('../figures/FigureS5_1.png')
plt.savefig('../figures/FigureS5_1.pdf')_____no_output_____bs_wrappers = bs_tools[bs_tools.wrap == 'Yes']
bs_nonwrappers = bs_tools[bs_tools.wrap == 'No']

bs_wrappers_____no_output_____plt.figure(figsize=(6,6))

sns.set_style('white')
sns.set_context('talk')
idx_group = bs_tools.groupby('wrap').count().sort_values('aligner', ascending=True)
ax = plt.bar(idx_group.index, idx_group.aligner, color=['lightgreen', 'olive'])
pct = 100.*idx_group.aligner/idx_group.aligner.sum()

plt.title("Number of BS-Seq Tools Classified as Wrappers")
#plt.show()

sns.despine()
plt.tight_layout()
plt.savefig('../figures/FigureS6_0.png')
plt.savefig('../figures/FigureS6_0.pdf')_____no_output_____plt.figure(figsize=(6,4))

sns.set_style('white')
sns.set_context('talk')

bowtie_tools = bs_wrappers.loc[bs_wrappers.wrapper_of.str.contains('Bowtie'),:]
num_b = len(bowtie_tools)
num_o = len(bs_wrappers) - num_b
ax = plt.bar(['Bowtie Wrappers', 'Other Wrappers'], [num_b, num_o], color=['lightgreen', 'olive'])

#plt.title("Number of RNA-Seq Tools Classified as Wrappers")
#plt.show()

sns.despine()
plt.tight_layout()

plt.savefig('../figures/FigureS6_1.png')
plt.savefig('../figures/FigureS6_1.pdf')_____no_output_____plt.figure(figsize=(11,4))

sns.set_style('white')
sns.set_context('talk')
idx_group = bs_nonwrappers.groupby('index').count().sort_values('aligner', ascending=True)
patches, texts = plt.pie(idx_group.aligner, colors=index_clrs[::-1])
pct = 100.*idx_group.aligner/idx_group.aligner.sum()
labels = ['{0} - {1} ({2:2.2f} %)'.format(i,j,k) for i,j,k in zip(idx_group.index, idx_group.aligner, pct)]
plt.legend(patches, labels, loc='center left', bbox_to_anchor=(.9, .5), frameon=False)
#plt.show()
plt.tight_layout()
sns.despine()
plt.savefig('../figures/FigureS7_0.png')
plt.savefig('../figures/FigureS7_0.pdf')_____no_output_____plt.figure(figsize=(6,3))

idx = range(min(bs_nonwrappers['year_of_pub']), max(bs_nonwrappers['year_of_pub'])+1)
by_year = bs_nonwrappers.groupby(['year_of_pub','index']).count()
by_year = by_year.xs('aligner', axis=1)
by_year = by_year.unstack(level = -1, fill_value=0)
by_year = by_year.cumsum()
by_year = by_year.reindex(idx, method='ffill')
#by_year.loc[:,'Total'] = by_year.sum(axis=1)
x = by_year.index
by_year.reset_index()
year_melt = pd.melt(by_year.reset_index(), id_vars=['year_of_pub'], var_name='Indexing Algorithm',
                    value_name='Number of Tools')
year_melt = year_melt.rename(columns={'year_of_pub':'Year of Publication'})

sns.set_style('white')
sns.set_context('talk')

g = sns.FacetGrid(year_melt, row='Indexing Algorithm', aspect=4, hue='Indexing Algorithm',
                  row_order=['Hashing', 'BWT-FM', 'Other Suffix', 'BWT-FM & Hashing'],
                  palette=index_clrs, hue_order=['Hashing', 'BWT-FM', 'Other Suffix', 'BWT-FM & Hashing'])
g = g.map(plt.bar, 'Year of Publication', 'Number of Tools', linewidth=0)
plt.xticks(range(min(bs_tools['year_of_pub']), max(bs_tools['year_of_pub'])+2, 2))
sns.despine()

plt.tight_layout()
plt.savefig('../figures/FigureS7_1.png')
plt.savefig('../figures/FigureS7_1.pdf')_____no_output_____plt.figure(figsize=(8,6))

sns.set_style('white')
sns.set_context('talk')

ax = sns.scatterplot(data = rtyear_summ, x = 'citation_per_year',
                     y = 'average_runtime', hue='Aligner', palette=cite_clrs, hue_order=cite_order_by_year)

ax.set(xlabel = 'Citations per Year', ylabel = 'Average CPU Time (Hours)')
ax.legend(loc='center left', bbox_to_anchor=(1, 0.5), ncol=1, frameon=False)
sns.despine()
#plt.show()

plt.tight_layout()
plt.savefig('../figures/FigureS8_a.png')
plt.savefig('../figures/FigureS8_a.pdf')_____no_output_____plt.figure(figsize=(8,6))

sns.set_style('white')
sns.set_context('talk')

ax = sns.scatterplot(data = memyear_summ, x = 'citation_per_year',
                     y = 'average_memory', hue='Aligner', palette=cite_clrs, hue_order=cite_order_by_year)

ax.set(xlabel = 'Citations per Year', ylabel = 'Average Memory (GB)')
ax.legend(loc='center left', bbox_to_anchor=(1, 0.5), ncol=1, frameon=False)
sns.despine()
#plt.show()

plt.tight_layout()
plt.savefig('../figures/FigureS8_b.png')
plt.savefig('../figures/FigureS8_b.pdf')_____no_output_____
</code>


# Notebook from fnauman/ds-blog
Path: _notebooks/2020-10-04-gaussian-processes.ipynb

# Gaussian processes
> Non-parametric probabilistic models

- toc: true
- hide: false
- branch: master
- search_exclude: false
- badges: true
- comments: true
- categories: [probabilistic models, machine learning]_____no_output_____## Introduction_____no_output_____Gaussian processes (GPs) are an example of non-parametric Bayesian models. Non-parametric means the model has theoretically infinite parameters and does not rely on a specific functional form of the parameters. To specify a Gaussian process, one requires two things:
1. Mean vector.
2. Covariance matrix/Kernel: Relation of data points (samples) with one another.

The kernel or the covariance matrix represents the relation between the sample points. The off-diagonal elements represent dependence of one observation to another. A commonly used kernel is the Radial Basis Function,
$$
    k(x, x') = \exp \left( - \frac{||x - x'||^2}{2\ell^2}\right)
$$
where $\ell$ represents the smoothing length.

**Difference between GPs and Naive Bayes**:

Naive Bayes takes the 'weight-space' approach similar to Ordinary Least Squares, $f(x) = w^T x + b$: the goal is to find the coefficients of the basis (polynomial, for example) $w^T$ such that the negative log likelihood is minimized. GPs take a 'function-space' approach where the goal is to find the optimal $g(x) = w^T * x$: the advantage being that an explicit basis (polynomial or otherwise) is not required anymore.

**Advantages**:

1. No need for iid assumption: For most algorithms, the samples are assumed to be identically independentally distributed. Sample interdependence differs from the assumption of 'collinearity' that refers to correlation among features. Regularization like Lasso/Ridge can get rid of collinearity among features but does not take get rid of mutually dependence in samples. Non-parametric Bayesian models address this problem by using a kernel function that quantifies interdependence of sample points. Example: time series data where adjacent samples can be strongly dependent on one another.

2. Confidence intervals: GPs provide uncertainty estimates as opposed to deterministic algorithms.

3. Fewer hyperparameters: The kernel has a fewer hyper parameters as opposed to a big neural network with billions of parameters to tune.

4. Applications: GPs are particularly suited for:
 - Time series modeling (Kalman filters - do not require information about all evolution only the most recent update?)
 - Probabilistic numerics (Hening: Model deterministic equations numerically with uncertainty estimates)
 - Uncertainty quantification (Probabilistic methods, in general, are much better at this compared to deterministic algorithms).

Gaussian Processes are not easy to interpret since they do not use explicit features. The computational complexity of inverting the covariance matrix inversion is $n^3$ but this has been addressed by more recent approaches.

References:
1. [Self-contained discussion with python code](https://github.com/fonnesbeck/Bayes_Computing_Course/blob/master/notebooks/Section5_1-Gaussian_Processes.ipynb)
2. [scikit-learn](https://scikit-learn.org/stable/modules/gaussian_process.html#gp-kernels)
3. [Application to dynamical systems](https://www.youtube.com/watch?list=PLFfvLE9TGnegjHFetV-zjPztaM_1UQk9B&v=BepKfyGbOM4&feature=emb_logo&ab_channel=WilWard)
4. [GP summer school](https://www.youtube.com/watch?v=U85XFCt3Lak&ab_channel=MLSSAfrica)
5. [pyro](https://pyro.ai/examples/gp.html)
6. [GPy](https://github.com/SheffieldML/GPy)_____no_output_____## Example application to candy dataset_____no_output_____

# hide
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt_____no_output_____

**Get data**

[Kaggle](https://www.kaggle.com/rtatman/us-candy-production-by-month)

or

[here](https://github.com/fnauman/timeseries/blob/master/candydata/candy_production.csv)_____no_output_____

# hide
!wget https://github.com/fnauman/timeseries/blob/master/candydata/candy_production.csv -O candy_production.csv_____no_output_____# hide
df = pd.read_csv('candy_production.csv')
df.columns = ['date', 'sales']
df.head()_____no_output_____# hide
X = np.atleast_2d(np.linspace(1, df.shape[0], df.shape[0])).T
y = df['sales'].values

X.shape, y.shape_____no_output_____from sklearn.gaussian_process import GaussianProcessRegressor
from sklearn.gaussian_process.kernels import RBF

gp = GaussianProcessRegressor(kernel=RBF())
gp.fit(X, y)
y_pred, sigma = gp.predict(X, return_std=True)_____no_output_____# hide_input
plt.figure()

plt.plot(X[-200:], y[-200:], 'r.', markersize=10, label='Observations')
plt.plot(X[-200:], y_pred[-200:], 'b-', label='Prediction')
plt.fill(np.concatenate([X[-200:], X[:-201:-1]]),
         np.concatenate([y_pred[-200:] - 1.9600 * sigma[-200:],
                        (y_pred[-200:] + 1.9600 * sigma[-200:])[::-1]]),
         alpha=.5, fc='b', ec='None', label='95% confidence interval')
plt.xlabel('$x$')
plt.ylabel('$f(x)$')_____no_output_____


# Notebook from AnonymousSDD/muAugment
Path: muAugment.ipynb

# **M**odel **U**ncertainty-based Data **Augment**ation (muAugment)

 | https://supaerodatascience.github.io/deep-learning/_____no_output_____## Preliminary work for colab_____no_output_____**This notebook was written in google colab, so it is recommended that you run it in colab as well.**_____no_output_____

_____no_output_____Before starting to work on the notebook, make sure you `change the Runtime type` to **GPU**, in the `Tool` drop menu._____no_output_____In colab, please execute first the following cells, to retrieve the GitHub repository content. _____no_output_____

!git clone https://github.com/AnonymousSDD/muAugment/fatal: destination path 'muAugment' already exists and is not an empty directory.

## Preliminary Imports_____no_output_____

# !pip install matplotlib_____no_output_____# !pip install torch torchvision_____no_output_____import torch
import torchvision
import numpy as np
%matplotlib inline
import matplotlib.pyplot as plt_____no_output_____from torch import nn, optim
import torch.nn.functional as F
from torch.utils.data import DataLoader, Dataset
from torchvision import datasets, transforms
from torchvision.datasets import FashionMNIST_____no_output_____device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
print(device)cuda:0

## Overview of Data Augmentation_____no_output_____Modern machine learning models, such as deep neural networks, may have billions of parameters and, consequently, require massive labeled training datasets, which are often not available. In order to avoid the **problem of data scarcity** in such models, data augmentation has become the standard technique used in nearly every state-of-the-art model in applications such as **image** and **text classification**.

> **Data augmentation refers to the technique of artificially expanding labelled training datasets by generating new data through transformation functions.**

_____no_output_____Data augmentation schemes often rely on the composition of a set of simple transformation functions (TFs) such as rotation and flip.

"Label-invariant transformations." [torchvision.transforms docs](https://pytorch.org/vision/stable/transforms.html#transforms-on-pil-image-only)

As was briefly discussed in the [computer vision class](https://github.com/SupaeroDataScience/deep-learning/blob/main/vision/1_hands_on.ipynb), when chosen carefully, data augmentation schemes tuned by human experts can improve model performance. However, such heuristic strategies in practice can cause large variances in end model performance, and may not produce parameterizations and compositions needed for state-of-the-art models. In addition, they are extremely laborious._____no_output_____### Automated Data Augmentation Schemes_____no_output_____Instead of performing manual search, automated data augmentation approaches hold promise to search for more powerful parameterizations and compositions of transformations.

The biggest difficulty with automating data augmentation is how to search over the space of transformations. This can be prohibitively expensive due to the large number of transformation functions in the search space.

> **How can we design algorithms that explore the space of transformation functions efficiently and effectively, and find augmentation strategies that can outperform human-designed heuristics?**

The folklore wisdom behind data augmentation is that adding more labeled data improves generalization, i.e. the performance of the trained model on unseen test data. However, even for simpler models, **it is not well-understood how training on augmented data affects the learning process, the parameters, and the decision surface of the resulting model**._____no_output_____
    Extra information on the Adversarial AutoAugment Scheme previsouly discussed in class (click to expand)

One of the current state-of-the-art algorithms in terms of performance is [Adversarial AutoAugment](https://openreview.net/pdf?id=ByxdUySKvS), which makes use of [GANs](https://proceedings.neurips.cc/paper/2014/file/5ca3e9b122f61f8f06494c97b1afccf3-Paper.pdf), already presented in a previous [class](https://github.com/SupaeroDataScience/deep-learning/tree/main/GAN), to generate new data, rather than using the traditional heuristic transformations presented above.

Li et al. "Adversarial AutoAugment training framework (Zhang et al. 2019) is formulated as an adversarial min-max game." [Automating the Art of Data Augmentation.](https://hazyresearch.stanford.edu/blog/2020-02-26-data-augmentation-part2) 2020.

Although proven effective, this technique is still computationally expensive. Additionally, despite its rapid progress, this technique does not allow for a theoretical understanding of the benefits of a given transformation.

_____no_output_____## How should we think of the effects of applying a transformation?

### Intuition for Linear Transformations_____no_output_____Suppose we are given $n$ training data points $x_1,...,x_n \in \mathbb{R}^p$ as $X \in \mathbb{R}^{n\times p}$ with labels $Y\in \mathbb{R}^n$.

> Suppose that the labels $Y$ obey the true linear model under ground parameters $\beta \in \mathbb{R}^p$, $$Y = X \beta + \epsilon,$$ where $\epsilon \in \mathbb{R}^n$  denotes i.d.d. random noise with mean zero and variance $\sigma²$.

Importantly, we assume that $p>n$, hence the span of the training data does not contain the entire space of $\mathbb{R}^p$.

Let's suppose we have an estimator $\hat{\beta}$ for the linear model $\beta \in \mathbb{R}^p$. The error of that given estimator is

> $$e(\hat{\beta}) = \underbrace{\lVert \underset{
  \epsilon}{\mathbb{E}}[\hat{\beta}]-\beta\rVert^2}_{bias} + \underbrace{\lVert\hat{\beta} - \underset{\epsilon}{\mathbb{E}}[\beta] \rVert^2}_{variance}$$

where the bias part, intuitively, measures the intrinsic error of the model after taking into account the randomness which is present in $\hat{\beta}$.

In the next section, we will try to demonstrate how label-invariant transformations can **reduce the estimation error**.
_____no_output_____### Label-Invariant Transformations_____no_output_____For a matrix $F \in \mathbb{R}^{p\times p}$, we say that $F$ is a label-invariant transformation over $\chi \subseteq \mathbb{R}^p $ for $\beta \in \mathbb{R}^p$ if $$x^\top\beta = (Fx)^\top\beta, \quad \text{ for any } x \in \chi.$$

> In simpler words, a label-invariant transformation will not alter the label $y$ of a given data point $x$._____no_output_____**But what is the effect of such a transformation?**

Given a training data point $(x,y)$, let $(x^{aug},y^{aug})$ denote the augmented data where $y^{aug} = y$ and $x^{aug} = Fx$.

**Note**: In order to be able to present the next result, let's consider adding the augmented data point $(z,y^{aug})$, where $z = P^\bot_X Fx$, meaning $z$ is not $x^{aug}$, but the projection of $x^{aug}$ onto $P^\bot_X = Id_p -P_X$, which denotes the projection operator which is ortogonal to the projection matrix onto the row of $X$ ($P_X$). In such a case, $y^{aug} = y - Diag[(X^\top)^†Fx]Y$.
> An intuiton to understand why we chose to use the projection $P^\bot_X a^{aug}$ instead of the augmented data is to think about the idea of "adding new information". Remember: we assume that $p>n$, hence the subspace over which we can make an accurate estimation does not contain the entire space of $\mathbb{R}^p$. When we add a data point belonging to a space ortogonal to the one we know, we expand the subspace over which we can make an accurate estimation, by adding a direction corresponding to $P^\bot_X a^{aug}$.

Suppose the estimator $\hat{\beta}$ used to infer labels is a ridge estimator with a penalty parameter $\lambda$, given by

$$\hat{\beta}(X,Y) = (X^\top X + n \lambda Id)^{-1}X^\top Y, $$

where, just to recap, $X$ denotes the training data and $Y$ the training labels.

Considering $e(\hat{\beta})$ and $e(\hat{\beta}^F)$ has the errors of the estimator before and after adding the augmented data point $(z,y^{aug})$ to $(X,Y)$, it is possible to demonstrate that,

$$ 0 \leq e(\hat{\beta}) - e(\hat{\beta}^F) - (2+o(1))\dfrac{\langle z,\beta \rangle²}{\lambda n} \leq \dfrac{poly(\gamma/\lambda)}{n²}, $$ where $poly(\gamma/\lambda)$ denotes a polynomial of $\gamma/\lambda$.

This powerful result, which we will not explain in class but can be found in the [muAugment paper](https://hazyresearch.stanford.edu/blog/2020-02-26-data-augmentation-part2), shows that

* **the reduction of the estimation error**, $e(\hat{\beta}) - e(\hat{\beta}^F)$, **scales with  the correlation between the new signal and the true model**, $\langle z,\beta \rangle²$.

In other words, by adding $P^\bot_X Fx$, we reduce the **estimation error** of the ridge estimator at a rate proportional to $\langle z,\beta \rangle²$.

Finally, we know that the larger the correlation $\langle z,\beta \rangle²$, the higher the loss of $(x^{aug},y^{aug})$ would be under $\hat{\beta}$. With this information, we can extrapolate the following:
* **the reduction of the estimation error**, $e(\hat{\beta}) - e(\hat{\beta}^F)$, **scales with  the loss of $(x^{aug},y^{aug})$ under $\hat{\beta}$**, given by $l_{\hat{\beta}}(x^{aug},y^{aug})$._____no_output_____> **In an intuitive sense, an augmented data point with a small loss means the model has already learned how to predict that type of data well, so if trained on it further, the model will only pick up incidental, possibly spurious patterns — overfitting. Conversely,  an augmented data point with a large loss means the model has not learned the general mapping between the type of data and its target yet, so we need to train more on those kinds of data points.**
_____no_output_____Additional results regarding **label-mixing transformations** were obtained in the [muAugment paper](https://hazyresearch.stanford.edu/blog/2020-02-26-data-augmentation-part2). These results will not be discussed in the class._____no_output_____## Uncertainty-based Sampling Scheme_____no_output_____In order to take advantage of the last result presented, the **muAugment** algorithm was developped. The algorithm is as follows:

* In a first step, for each data point, **C** compositions of **L** linear transformations are randomly sampled and fed to the learning model (in this example a neural network).

* In a second step, the **S** transformed samples with the highest losses are picked for training the model and a backpropagation is performed using those samples.

> **The intuition behind the sampling scheme is that these transformed samples that have the largest losses should also provide the most information.**

**The model learns more generalizable patterns, because the algorithm assures extra fitting on the "hard" augmentations while skipping the easy ones.**

Senwu. "Uncertainty-based random Sampling Scheme for Data Augmentation. Each transformation function is randomly sampled from a pre-defined set of operations." [Dauphin](https://github.com/senwu/dauphin) 2020._____no_output_____
    Comparison to Adversarial Autoaugment (click to expand)

The idea behing this sampling scheme is conceptually similar to [Adversarial Autoaugment](https://openreview.net/pdf?id=ByxdUySKvS). However, while in the case of Adversarial Autoaugment, an additional adversarial network is used to generate augmented samples with large losses, in the current case, the model uses the training network itself to generate augmented samples.

_____no_output_____Our goal today is to implement the **muAugment** algorithm and evaluate its performance._____no_output_____### The Dataset: FashionMNIST_____no_output_____The dataset we will use for this application is the FashionMNIST dataset.  We'll download this dataset and make batching data loaders._____no_output_____

batch_size = 4
n_images = 10 if (batch_size>10) else batch_size

# data must be normalized between -1 and 1
transform = transforms.Compose([transforms.ToTensor(),
                                transforms.Normalize((0.5,), (0.5,))])

full_trainset = FashionMNIST(root='../data', train=True, download=True, transform=transform)
trainset, full_validset = torch.utils.data.random_split(full_trainset, (10000, 50000)) # 10000 images for the training set
validset, _ = torch.utils.data.random_split(full_validset, (1000, 49000)) # 1000 images for the validation set

trainloader = DataLoader(trainset, batch_size=64, shuffle=True, num_workers=2)
validloader = DataLoader(validset, batch_size=64, shuffle=True, num_workers=2)

testset = FashionMNIST(root='../data', train=False, download=True, transform=transform)
testloader = DataLoader(testset, batch_size=64, shuffle=True)_____no_output_____

We can verify the normalization of our data._____no_output_____

images,labels = next(iter(trainloader))
images.min(),images.max()_____no_output_____

Let's look at some example images from the FashionMNIST set._____no_output_____

# get the first batch of images and labels
labels_text = ["T-shirt/top", "Trouser", "Pullover", "Dress", "Coat", "Sandal", "Shirt", "Sneaker", "Bag", "Ankle boot"]

plt.figure(figsize=(n_images,4))
for i in range(n_images):
    l = labels[i].numpy()
    plt.subplot(2, n_images/2, i+1)
    plt.title('%d: %s' % (l, labels_text[l]))
    plt.imshow(images[i].numpy()[0], cmap='Greys')
    plt.axis('off')_____no_output_____

### The Model_____no_output_____As mentioned above, the advantage of the **muAugment** algorithm is that it uses the learning model to automate data augmentation. The goal is to generate data which will improve our training model.

In today's example, we wish to learn to classify images into 10 possible labels:_____no_output_____

labels_text_____no_output_____

In order to do this, the training model we will use is a convolutional neural network, presented during a [previous class](https://github.com/SupaeroDataScience/deep-learning/blob/main/deep/PyTorch%20Ignite.ipynb)._____no_output_____

class CNN(nn.Module):

    def __init__(self):
        super(CNN, self).__init__()

        self.convlayer1 = nn.Sequential(
            nn.Conv2d(1, 32, 3,padding=1),
            nn.BatchNorm2d(32),
            nn.ReLU(),
            nn.MaxPool2d(kernel_size=2, stride=2)
        )

        self.convlayer2 = nn.Sequential(
            nn.Conv2d(32,64,3),
            nn.BatchNorm2d(64),
            nn.ReLU(),
            nn.MaxPool2d(2)
        )

        self.fc1 = nn.Linear(64*6*6,600)
        self.drop = nn.Dropout2d(0.25)
        self.fc2 = nn.Linear(600, 120)
        self.fc3 = nn.Linear(120, 10)

    def forward(self, x):
        x = self.convlayer1(x)
        x = self.convlayer2(x)
        x = x.view(-1,64*6*6)
        x = self.fc1(x)
        x = self.drop(x)
        x = self.fc2(x)
        x = self.fc3(x)

        return F.log_softmax(x,dim=1)_____no_output_____

### Training_____no_output_____In order to train the model, we must first create it and define the loss function and optimizer._____no_output_____

#creating model for original data
model_original = CNN()
# creating model for augmented data
model = CNN()
# moving models to gpu if available
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
model.to(device)
model_original.to(device)_____no_output_____

By using the parameter `weight_decay` in our optimizer we are applying a similar penalty parameter to the one in ridge regression._____no_output_____

lr = 0.01 # learning rate
# defining optimizer and loss for original model
optimizer_original = torch.optim.SGD(model_original.parameters(), lr=lr, weight_decay=0.0001, momentum=0.9)
criterion_original = nn.CrossEntropyLoss()
# defining optimizer and loss for augmented model
optimizer = torch.optim.SGD(model.parameters(), lr=lr, weight_decay=0.0001, momentum=0.9)
criterion = nn.CrossEntropyLoss()_____no_output_____

In a typical training phase, each batch of images would be treated in the following loop:
```python
for epoch in range(max_epochs):
  for batch in trainloader:
    # zero the parameter gradients
    optimizer.zero_grad()
    # get inputs and labels from batch
    inputs, labels = batch
    # forward + backward + optimize
    outputs = model(inputs)
    loss = criterion(outputs, labels)
    loss.backward()
    optimizer.step()
```
In order to perform data augmentation with pre-defined transforms, it would suffice to declare the transforms while generating the data loader and the loop would remain unchanged._____no_output_____However, because we don't wish to train the model without evaluating the performance of each transform, this loop is going to change._____no_output_____### The Random Sampling Scheme (hands-on exercises)_____no_output_____As mentioned above, the goal is to implement the following algorithm:

Wu et al. "Uncertainty-based random Sampling Algorithm." [On the Generalization Effects of Linear Transformations in Data Augmentation](https://arxiv.org/pdf/2005.00695.pdf) 2020._____no_output_____Today, to simplify our work, we will not use default transformations. We will also only consider label-invariant transformations._____no_output_____In our implementation, lets consider the following required arguments:
*  **L** (int): Number of **linear** transformations uniformly sampled for each composition
* **C** (int): Number of **compositions** placed on each image
* **S** (int): Number of **selected** compositions for each image_____no_output_____> **In a first exercise, let's attempt to code the lines 4 and 5 of the algorithm. Complete the function `compute_composed_data` which takes as input a `transform_list` similar to the one presented bellow, the arguments `L` and `C` described above and the images `xb` and labels `yb` of a batch and returns 2 tensors `C_images` and `C_targets` which contain the images xb$^{\mathbf{aug}}$ and labels yb$^{\mathbf{aug}}$ of the augmented data.**
```python
transform_list = [transforms.RandomAutocontrast(p=p),
                  transforms.ColorJitter(brightness=MAGN/30),
                  transforms.ColorJitter(contrast=MAGN/30),
                  transforms.RandomInvert(p=p),
                  transforms.RandomRotation(degrees=MAGN*3),
                  transforms.RandomAdjustSharpness(0.18*MAGN+0.1, p=p),
                  transforms.RandomAffine(degrees=0, shear=MAGN/30),
                  transforms.RandomSolarize(MAGN*8, p=p),
                  transforms.RandomAffine(degrees=(0,0),
                  translate=(MAGN/30,0),shear=(0,0)),
                  transforms.RandomAffine(degrees=(0,0),
                  translate=(0,MAGN/30),shear=(0,0)),
                  ]
``` _____no_output_____

# the load command only works on jupyter notebook
# %load solutions/compute_composed_data.py
def compute_composed_data(transform_list,L, C, xb,yb):
    BS,N_CHANNELS,HEIGHT,WIDTH = xb.shape

    C_images = torch.zeros(C, BS, N_CHANNELS, HEIGHT, WIDTH, device=device)
    C_targets = torch.zeros(C, BS, device=device, dtype=torch.long)

    for c in range(C):
        # create a list of L linear transforms randomly sampled from the transform_list

        # create a composition of transforms from the list sampled above. Use nn.Sequential instead of transforms.Compose in order to script the transformations

        # apply the composition to the original images xb

        # update tensors C_images and C_targets

    return C_images, C_targets_____no_output_____# the cat command works on google colab
# %cat muAugment/solutions/compute_composed_data.py_____no_output_____

Now that we have implemented the data augmentation part, we can attempt to code the content of the main loop of the algorithm.

**Remember**: the idea is to feed the transformed batches to the model without updating it and compare the losses obtained for each batch. Since you do not want to call `python loss.backward()`, you can disable gradient calculation in your function by using `python @torch.no_grad()`._____no_output_____> **In a second exercise, complete the function `compute_selected_data` that takes as inputs the learning `model`, the `loss` function,  the tensors `C_images` and `C_targets` and the argument `S` and returns the seleted transformed images (`S_images`) and labels (`S_labels`).**_____no_output_____

# the load command only works on jupyter notebook
# %load solutions/compute_selected_data.py

#disable gradient calculation

def compute_selected_data(model, loss, C_images, C_targets, S):
    C, BS, N_CHANNELS, HEIGHT, WIDTH = C_images.shape

    # create a list of predictions 'pred' by applying the model to the augmented batches contained in C_images

    # create a list of losses by applying the loss function to the predictions and labels C_targets
    # convert the list to a loss tensor 'loss_tensor' through the function torch.stack

    # select the S indices 'S_idxs' of the loss_tensor with the highest value. You may use the function torch.topk

    # select the S images 'S_images' from C_images with the highest losses

    # convert the tensor 'S_images' so that it passes from shape [S, BS, N_CHANNELS, HEIGHT, WIDTH] to shape
    # [S*BS, N_CHANNELS, HEIGHT, WIDTH]. You may use the function torch.view

    # select the S labels 'S_targets' from C_targets corresponding to the highest losses

    # convert the tensor 'S_targets' so that it passes from shape [S, BS] to shape
    # [S*BS]. You may use the function torch.view

    return S_images, S_targets_____no_output_____# the cat command works on google colab
# %cat muAugment/solutions/compute_selected_data.py_____no_output_____

We have created two functions which give us the augmented data we wish to use in the training phase of our model._____no_output_____### Back to Training (hands-on exercise)

_____no_output_____Let's consider the following arguments for the algorithm:_____no_output_____

# algorithm arguments
L = 3 # number of linear transformations sampled for each composition
C = 4 # number of compositions placed on each image.
S = 2 # number of selected compositions for each image_____no_output_____

Let's consider the following list of linear transformations, similar to the ones used in the original paper:_____no_output_____

MAGN = 4  # (int) Magnitude of augmentation applied. Ranges from [0, 10] with 10 being the max magnitude.
# function of list of linear transformations
def transform_list(MAGN,p):
    return [transforms.RandomAutocontrast(p=p),
            transforms.ColorJitter(brightness=MAGN/30),
            transforms.ColorJitter(contrast=MAGN/30),
            transforms.RandomInvert(p=p),
            transforms.RandomRotation(degrees=MAGN*3),
            transforms.RandomAdjustSharpness(0.18*MAGN+0.1, p=p),
            transforms.RandomAffine(degrees=0, shear=MAGN/30),
            transforms.RandomSolarize(MAGN, p=p),
            transforms.RandomAffine(degrees=(0,0), translate=(MAGN/30,0),shear=(0,0)),
            transforms.RandomAffine(degrees=(0,0), translate=(0,MAGN/30),shear=(0,0)),
            ]_____no_output_____

The following three code boxes were adapted from the tutorial on pytorch done in [class](https://github.com/SupaeroDataScience/deep-learning/blob/main/deep/Deep%20Learning.ipynb). _____no_output_____In order to compare validation and training losses, we will calculate the validation losses and accuracy at each epoch._____no_output_____

def validation(model,criterion):
    correct_pred = 0
    total_pred = 0
    valid_loss = 0
    with torch.no_grad():
        for data in validloader:
            images, labels = data
            images = images.to(device)
            labels = labels.to(device)
            outputs = model(images)
            loss = criterion(outputs, labels)
            valid_loss += loss.item()

            # calculate predictions
            predictions=[]
            for i in range(outputs.shape[0]):
                ps = torch.exp(outputs[i])
                predictions.append(torch.argmax(ps))
            # collect the correct predictions
            for label, prediction in zip(labels, predictions):
                if label == prediction:
                    correct_pred += 1
                total_pred += 1
        accuracy = 100 * (correct_pred / total_pred)

    return valid_loss, accuracy_____no_output_____def plot_train_val(train, valid, title, label1 = 'Training', label2 = 'Validation'):
    fig, ax1 = plt.subplots()
    color = 'tab:red'
    ax1.set_ylabel(label1, color=color)
    ax1.plot(train, color=color)
    ax2 = ax1.twinx()
    color = 'tab:blue'
    ax2.set_ylabel(label2, color=color)
    ax2.plot(valid, color=color)
    fig.tight_layout()
    plt.title(title)_____no_output_____

In order to avoid overfitting, we will implement early stopping._____no_output_____

class EarlyStopping:

    def __init__(self, patience=5, delta=0):
        self.patience = patience
        self.counter = 0
        self.best_score = None
        self.delta = delta
        self.early_stop = False

    def step(self, val_loss):
        score = -val_loss
        if self.best_score is None:
            self.best_score = score
        elif score < self.best_score + self.delta:
            self.counter += 1
            print('EarlyStopping counter: %d / %d' % (self.counter, self.patience))
            if self.counter >= self.patience:
                self.early_stop = True
        else:
            self.best_score = score
            self.counter = 0_____no_output_____

It is time to implement the algorithm in the training loop!

_____no_output_____> **In the final exercise, take the almost complete code of the training loop presented bellow (adapted from the [pytorch class](https://github.com/SupaeroDataScience/deep-learning/blob/main/deep/Deep%20Learning.ipynb)) and change it, so that the algorithm is implemented.**_____no_output_____

# the load command only works on jupyter notebook
# %load solutions/train.py
def train(model,criterion,optimizer, earlystopping=True,max_epochs=30,patience=2, augment=False):
    train_history = []
    valid_history = []
    accuracy_history = []
    estop = EarlyStopping(patience=patience)
    for epoch in range(max_epochs):
        train_loss = 0.0
        for i, data in enumerate(trainloader, 0):
            if augment:
                # generate transform list
                p = np.random.random() # probability of each transformation occurring
                transforms = transform_list(MAGN,p)
                # get the inputs; data is a list of [inputs, labels]
                xb,yb = data
                xb = xb.to(device)
                yb = yb.to(device)
                # generate the tensors 'C_images' and 'C_targets'

                # generated the augmented data = [inputs,labels]

            else:
                # get the inputs; data is a list of [inputs, labels]
                inputs,labels = data
                inputs = inputs.to(device)
                labels = labels.to(device)
            # zero the parameter gradients
            optimizer.zero_grad()
            # forward + backward + optimize
            outputs = model(inputs)
            loss = criterion(outputs, labels)
            loss.backward()
            optimizer.step()
            train_loss += loss.item()
        valid_loss, accuracy = validation(model,criterion)
        train_history.append(train_loss)
        valid_history.append(valid_loss)
        accuracy_history.append(accuracy)
        print('Epoch %02d: train loss %0.5f, validation loss %0.5f, accuracy %3.1f ' % (epoch, train_loss, valid_loss, accuracy))
        estop.step(valid_loss)
        if earlystopping and estop.early_stop:
            break
    return train_history, valid_history, accuracy_history_____no_output_____# the cat command works on google colab
# %cat muAugment/solutions/train.py_____no_output_____

We did it! Let's train our models: one without and one with augmented data._____no_output_____

max_epochs = 50
patience = 6 #early stopping parameter, we are choosing a high value to check a little longer the evolution of the validation model

print("\n Training for the original dataset...\n")
train_history_original, valid_history_original, accuracy_history_original = train(model_original,criterion_original,optimizer_original,max_epochs=max_epochs,patience=patience)
print("\n Training for the augmented dataset...\n")
train_history, valid_history, accuracy_history = train(model,criterion,optimizer,max_epochs=max_epochs,patience=patience,augment=True)
 Training for the original dataset...

Epoch 00: train loss 107.40087, validation loss 9.54017, accuracy 78.7
Epoch 01: train loss 65.31258, validation loss 7.11776, accuracy 82.7
Epoch 02: train loss 53.65551, validation loss 7.14902, accuracy 82.9
EarlyStopping counter: 1 / 6
Epoch 03: train loss 48.46113, validation loss 5.79279, accuracy 85.4
Epoch 04: train loss 42.72672, validation loss 6.20212, accuracy 85.8
EarlyStopping counter: 1 / 6
Epoch 05: train loss 41.72654, validation loss 5.90067, accuracy 86.0
EarlyStopping counter: 2 / 6
Epoch 06: train loss 36.24185, validation loss 6.38821, accuracy 86.3
EarlyStopping counter: 3 / 6
Epoch 07: train loss 30.89906, validation loss 5.40398, accuracy 87.0
Epoch 08: train loss 31.40080, validation loss 6.17783, accuracy 86.4
EarlyStopping counter: 1 / 6
Epoch 09: train loss 27.44940, validation loss 5.63667, accuracy 88.4
EarlyStopping counter: 2 / 6
Epoch 10: train loss 23.74384, validation loss 5.67578, accuracy 88.2
EarlyStopping counter: 3 / 6
Epoch 11: train loss 23.53422, validation loss 7.25856, accuracy 85.5
EarlyStopping counter: 4 / 6
Epoch 12: train loss 22.91191, validation loss 6.22305, accuracy 87.4
EarlyStopping counter: 5 / 6
Epoch 13: train loss 19.23445, validation loss 5.87766, accuracy 88.3
EarlyStopping counter: 6 / 6

 Training for the augmented dataset...

Epoch 00: train loss 200.17301, validation loss 11.53903, accuracy 73.3
Epoch 01: train loss 138.08817, validation loss 9.07099, accuracy 77.6
Epoch 02: train loss 129.78504, validation loss 7.45362, accuracy 82.3
Epoch 03: train loss 121.72741, validation loss 9.19193, accuracy 78.6
EarlyStopping counter: 1 / 6
Epoch 04: train loss 114.61139, validation loss 7.20788, accuracy 82.8
Epoch 05: train loss 112.52326, validation loss 9.27048, accuracy 77.5
EarlyStopping counter: 1 / 6
Epoch 06: train loss 110.64931, validation loss 7.62662, accuracy 81.8
EarlyStopping counter: 2 / 6
Epoch 07: train loss 100.62967, validation loss 6.44072, accuracy 84.2
Epoch 08: train loss 99.47019, validation loss 6.58639, accuracy 84.1
EarlyStopping counter: 1 / 6
Epoch 09: train loss 97.43430, validation loss 8.97916, accuracy 79.4
EarlyStopping counter: 2 / 6
Epoch 10: train loss 96.43212, validation loss 6.22396, accuracy 84.2
Epoch 11: train loss 92.57581, validation loss 6.54693, accuracy 84.5
EarlyStopping counter: 1 / 6
Epoch 12: train loss 90.88633, validation loss 6.33956, accuracy 84.3
EarlyStopping counter: 2 / 6
Epoch 13: train loss 89.47278, validation loss 6.64320, accuracy 83.5
EarlyStopping counter: 3 / 6
Epoch 14: train loss 91.09531, validation loss 6.44105, accuracy 83.5
EarlyStopping counter: 4 / 6
Epoch 15: train loss 89.41556, validation loss 6.03098, accuracy 86.1
Epoch 16: train loss 88.40112, validation loss 6.55470, accuracy 84.6
EarlyStopping counter: 1 / 6
Epoch 17: train loss 88.57243, validation loss 6.22672, accuracy 85.8
EarlyStopping counter: 2 / 6
Epoch 18: train loss 86.86290, validation loss 6.51018, accuracy 83.2
EarlyStopping counter: 3 / 6
Epoch 19: train loss 82.87512, validation loss 5.78412, accuracy 85.5
Epoch 20: train loss 79.14971, validation loss 5.53806, accuracy 86.7
Epoch 21: train loss 83.67297, validation loss 5.34317, accuracy 86.9
Epoch 22: train loss 80.67956, validation loss 5.86662, accuracy 86.1
EarlyStopping counter: 1 / 6
Epoch 23: train loss 77.71820, validation loss 5.52212, accuracy 86.6
EarlyStopping counter: 2 / 6
Epoch 24: train loss 77.07045, validation loss 5.18545, accuracy 87.7
Epoch 25: train loss 80.60019, validation loss 5.10139, accuracy 87.9
Epoch 26: train loss 80.49870, validation loss 6.36130, accuracy 86.2
EarlyStopping counter: 1 / 6
Epoch 27: train loss 77.92610, validation loss 5.44655, accuracy 86.0
EarlyStopping counter: 2 / 6
Epoch 28: train loss 76.61362, validation loss 5.20318, accuracy 86.9
EarlyStopping counter: 3 / 6
Epoch 29: train loss 74.06104, validation loss 5.32655, accuracy 86.1
EarlyStopping counter: 4 / 6
Epoch 30: train loss 73.67970, validation loss 5.54085, accuracy 87.8
EarlyStopping counter: 5 / 6
Epoch 31: train loss 76.51041, validation loss 5.12992, accuracy 88.1
EarlyStopping counter: 6 / 6

#### Plotting the Training and Validation Loss_____no_output_____Now that we trained both models, we can compare how the loss of training and validation evolves in both cases._____no_output_____

plot_train_val(train_history_original, valid_history_original,"Original Data")
plot_train_val(train_history, valid_history,"Augmented Data")_____no_output_____

Most times you can see that the training loss tends to be higher at the beginning while using the augmented data and, even, ocasionally, augment from epoch to epoch i.e. training is less stable. This is consistent with the fact that the augmented data is more difficult to predict. However, because the model with augmented data does not excessively train the data points it already knows, the model also suffers less from overfitting._____no_output_____We can also compare accuracy between models._____no_output_____

plot_train_val(accuracy_history, accuracy_history_original,"Accuracy",label1='Augmented',label2='Original')_____no_output_____

#### Verifying models with Testing Dataset_____no_output_____Finally, let's check the results by applying our model to the test dataset._____no_output_____

# put model in evaluation mode
model.eval()
# moving model to cpu for inference
model.to('cpu')

# creating arrays to save predictions
y_true = []
y_pred = []
images_ = []
# disable all gradients things
with torch.no_grad():
        for data in iter(testloader):
            images, labels = data
            outputs = model(images)
            for i in range(outputs.shape[0]):
                images_.append(images[i].unsqueeze(0))
                ps = torch.exp(outputs[i])
                y_pred.append(np.argmax(ps))
                y_true.append(labels[i].item())
_____no_output_____

Firstly, let's examine the confusion matrix._____no_output_____

from sklearn.metrics import ConfusionMatrixDisplay, confusion_matrix

print("Confusion matrix")
cm = confusion_matrix(y_true, y_pred)

disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=labels_text)

fig, ax = plt.subplots(figsize=(10,10))
disp.plot(ax=ax)
plt.show()Confusion matrix

We can also plot some of the results of the test dataset._____no_output_____

# plotting the results
fig = plt.figure(figsize=(n_images+5,4))
for i in range(n_images):
    ax = fig.add_subplot(2, n_images/2, i+1, xticks=[], yticks=[])
    ax.imshow(images_[i].resize_(1, 28, 28).numpy().squeeze())
    ax.set_title("{} ({})".format(labels_text[y_pred[i]], labels_text[y_true[i]]),
                 color=("green" if y_pred[i]==y_true[i] else "red"))_____no_output_____

Not bad! But could you spot a problem with the algorithm? **Here's a tip**: we were very conservative when choosing the parameters of our algorithm, namely the number of compositions generated **C**, number of compositions  selected **S**, the number of linear transforms per composition **L** and the magnitude **M** of the chosen transforms.

> **What do you think would happen if we increased those values?** If you have time, you can try to play with the values._____no_output_____### Shortcomings of the Algorithm_____no_output_____It is possible that the transforms applied on an image are so severe that the image becomes indistinguishable, losing its target information. In such a case, we would end up feeding the model pure noise. However, pure noise yields a high loss when fed into the model, so using **MuAugment** selects for those unrecognizable images if they are created. There’s no simple solution for this issue other than to choose appropriate hyperparameters so as to reduce the generation of inscrutable images, so it’s a good idea to keep the number of transforms in a composition **C** under 4 and the magnitude of each transform **M** under 6._____no_output_____As a heuristic, larger models and datasets require more regularization and would accordingly perform better with a greater magnitude **M**. This is because bigger models are more prone to overfit and lengthier datasets have a higher signal-to-noise ratio which should be reduced to an optimal point._____no_output_____


# Notebook from klarman-cell-observatory/pegasus
Path: notebooks/pegasus_analysis.ipynb

# Pegasus Tutorial

Author: [Yiming Yang](https://github.com/yihming)

Date: 2021-06-24

Notebook Source: [pegasus_analysis.ipynb](https://raw.githubusercontent.com/klarman-cell-observatory/pegasus/master/notebooks/pegasus_analysis.ipynb)_____no_output_____

import pegasus as pg_____no_output_____

## Count Matrix File

For this tutorial, we provide a count matrix dataset on Human Bone Marrow with 8 donors stored in zarr format (with file extension ".zarr.zip").

You can download the data at https://storage.googleapis.com/terra-featured-workspaces/Cumulus/MantonBM_nonmix_subset.zarr.zip.

This file is achieved by aggregating gene-count matrices of the 8 10X channels using PegasusIO, and further filtering out cells with fewer than $100$ genes expressed. Please see [here](https://pegasusio.readthedocs.io/en/latest/_static/tutorials/pegasusio_tutorial.html#Case-5:-Data-aggregation-with-filtering) for how to do it interactively.

Now load the file using pegasus `read_input` function:_____no_output_____

data = pg.read_input("MantonBM_nonmix_subset.zarr.zip")
data_____no_output_____

The count matrix is managed as a UnimodalData object defined in [PegasusIO](https://pegasusio.readthedocs.io) module, and users can manipulate the data from top level via MultimodalData structure, which can contain multiple UnimodalData objects as members.

For this example, as show above, `data` is a MultimodalData object, with only one UnimodalData member of key `"GRCh38-rna"`, which is its default UnimodalData. Any operation on `data` will be applied to this default UnimodalData object.

UnimodalData has the following structure:

It has 6 major parts:
* Raw count matrix: `data.X`, a [Scipy sparse matrix](https://docs.scipy.org/doc/scipy/reference/generated/scipy.sparse.csr_matrix.html) (sometimes can be a dense matrix), with rows the cell barcodes, columns the genes/features:_____no_output_____

data.X_____no_output_____

This dataset contains $48,219$ barcodes and $36,601$ genes.

* Cell barcode attributes: `data.obs`, a [Pandas data frame](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.html) with barcode as the index. For now, there is only one attribute `"Channel"` referring to the donor from which the cell comes from:_____no_output_____

data.obs.head()_____no_output_____data.obs['Channel'].value_counts()_____no_output_____

* Gene attributes: `data.var`, also a Pandas data frame, with gene name as the index. For now, it only has one attribute `"gene_ids"` referring to the unique gene ID in the experiment:_____no_output_____

data.var.head()_____no_output_____

* Unstructured information: `data.uns`, a Python [hashed dictionary](https://docs.python.org/3/library/collections.html#collections.OrderedDict). It usually stores information not restricted to barcodes or features, but about the whole dataset, such as its genome reference and modality type:_____no_output_____

data.uns['genome']_____no_output_____data.uns['modality']_____no_output_____

* Finally, embedding attributes on cell barcodes: `data.obsm`; as well as on genes, `data.varm`. We'll see it in later sections._____no_output_____## Preprocessing

### Filtration

The first step in preprocessing is to perform the quality control analysis, and remove cells and genes of low quality.

We can generate QC metrics using the following method with default settings:_____no_output_____

pg.qc_metrics(data, min_genes=500, max_genes=6000, mito_prefix='MT-', percent_mito=10)_____no_output_____

The metrics considered are:
* **Number of genes**: Keep cells with $500 \leq \text{# Genes} < 6000$. **(Notice that starting from v1.4.0, Pegasus doesn't filter cells by number of genes threshold by default.)**
* **Number of UMIs**: Don't filter cells due to UMI bounds *(Default)*;
* **Percent of Mitochondrial genes**: Look for mitochondrial genes with name prefix `MT-` **(Notice that starting from v1.4.0, you'll need to manually specify this prefix.)**, and keep cells with percent $< 10\%$.

For details on customizing your own thresholds, see [documentation](https://pegasus.readthedocs.io/en/stable/api/pegasus.qc_metrics.html).

Numeric summaries on filtration on cell barcodes and genes can be achieved by `get_filter_stats` method:_____no_output_____

df_qc = pg.get_filter_stats(data)
df_qc_____no_output_____

The results is a Pandas data frame on samples.

You can also check the QC stats via plots. Below is on number of genes:_____no_output_____

pg.qcviolin(data, plot_type='gene', dpi=100)_____no_output_____

Then on number of UMIs:_____no_output_____

pg.qcviolin(data, plot_type='count', dpi=100)_____no_output_____

On number of percentage of mitochondrial genes:_____no_output_____

pg.qcviolin(data, plot_type='mito', dpi=100)_____no_output_____

Now filter cells based on QC metrics set in `qc_metrics`:_____no_output_____

pg.filter_data(data)_____no_output_____

You can see that $35,465$ cells ($73.55\%$) are kept.

Moreover, for genes, only those with no cell expression are removed. After that, we identify **robust** genes for downstream analysis:_____no_output_____

pg.identify_robust_genes(data)_____no_output_____

The metric is the following:
* Gene is expressed in at least $0.05\%$ of cells, i.e. among every 6000 cells, there are at least 3 cells expressing this gene.

Please see [its documentation](https://pegasus.readthedocs.io/en/stable/api/pegasus.identify_robust_genes.html) for details.

As a result, $25,653$ ($70.09\%$) genes are kept. Among them, $17,516$ are robust.

We can now view the cells of each sample after filtration:_____no_output_____

data.obs['Channel'].value_counts()_____no_output_____

### Normalization and Logarithmic Transformation

After filtration, we need to first normalize the distribution of counts w.r.t. each cell to have the same sum (default is $10^5$, see [documentation](https://pegasus.readthedocs.io/en/stable/api/pegasus.log_norm.html)), and then transform into logarithmic space by $log(x + 1)$ to avoid number explosion:_____no_output_____

pg.log_norm(data)_____no_output_____

For the downstream analysis, we may need to make a copy of the count matrix, in case of coming back to this step and redo the analysis:_____no_output_____

data_trial = data.copy()_____no_output_____

### Highly Variable Gene Selection

**Highly Variable Genes (HVG)** are more likely to convey information discriminating different cell types and states.
Thus, rather than considering all genes, people usually focus on selected HVGs for downstream analyses.

You need to set `consider_batch` flag to consider or not consider batch effect. At this step, set it to `False`:_____no_output_____

pg.highly_variable_features(data_trial, consider_batch=False)_____no_output_____

By default, we select 2000 HVGs using the pegasus selection method. Alternative, you can also choose the traditional method that both *Seurat* and *SCANPY* use, by setting `flavor='Seurat'`. See [documentation](https://pegasus.readthedocs.io/en/stable/api/pegasus.highly_variable_features.html) for details.

We can view HVGs by ranking them from top:_____no_output_____

data_trial.var.loc[data_trial.var['highly_variable_features']].sort_values(by='hvf_rank')_____no_output_____

We can also view HVGs in a scatterplot:_____no_output_____

pg.hvfplot(data_trial, dpi=200)_____no_output_____

In this plot, each point stands for one gene. Blue points are selected to be HVGs, which account for the majority of variation of the dataset._____no_output_____### Principal Component Analysis

To reduce the dimension of data, Principal Component Analysis (PCA) is widely used. Briefly speaking, PCA transforms the data from original dimensions into a new set of Principal Components (PC) of a much smaller size. In the transformed data, dimension is reduced, while PCs still cover a majority of the variation of data. Moreover, the new dimensions (i.e. PCs) are independent with each other.

`pegasus` uses the following method to perform PCA:_____no_output_____

pg.pca(data_trial)_____no_output_____

By default, `pca` uses:
* Before PCA, scale the data to standard Normal distribution $N(0, 1)$, and truncate them with max value $10$;
* Number of PCs to compute: 50;
* Apply PCA only to highly variable features.

See [its documentation](https://pegasus.readthedocs.io/en/stable/api/pegasus.pca.html) for customization.

To explain the meaning of PCs, let's look at the first PC (denoted as $PC_1$), which covers the most of variation:_____no_output_____

coord_pc1 = data_trial.uns['PCs'][:, 0]
coord_pc1_____no_output_____

This is an array of 2000 elements, each of which is a coefficient corresponding to one HVG.

With the HVGs as the following:_____no_output_____

data_trial.var.loc[data_trial.var['highly_variable_features']].index.values_____no_output_____

$PC_1$ is computed by

\begin{equation*}
PC_1 = \text{coord_pc1}[0] \cdot \text{HES4} + \text{coord_pc1}[1] \cdot \text{ISG15} + \text{coord_pc1}[2] \cdot \text{TNFRSF18} + \cdots + \text{coord_pc1}[1997] \cdot \text{RPS4Y2} + \text{coord_pc1}[1998] \cdot \text{MT-CO1} + \text{coord_pc1}[1999] \cdot \text{MT-CO3}
\end{equation*}

Therefore, all the 50 PCs are the linear combinations of the 2000 HVGs.

The calculated PCA count matrix is stored in the `obsm` field, which is the first embedding object we have _____no_output_____

data_trial.obsm['X_pca'].shape_____no_output_____

For each of the $35,465$ cells, its count is now w.r.t. 50 PCs, instead of 2000 HVGs._____no_output_____## Nearest Neighbors

All the downstream analysis, including clustering and visualization, needs to construct a k-Nearest-Neighbor (kNN) graph on cells. We can build such a graph using `neighbors` method:_____no_output_____

pg.neighbors(data_trial)_____no_output_____

It uses the default setting:
* For each cell, calculate its 100 nearest neighbors;
* Use PCA matrix for calculation;
* Use L2 distance as the metric;
* Use [hnswlib](https://github.com/nmslib/hnswlib) search algorithm to calculate the approximated nearest neighbors in a really short time.

See [its documentation](https://pegasus.readthedocs.io/en/stable/api/pegasus.neighbors.html) for customization.

Below is the result:_____no_output_____

print(f"Get {data_trial.uns['pca_knn_indices'].shape[1]} nearest neighbors (excluding itself) for each cell.")
data_trial.uns['pca_knn_indices']_____no_output_____data_trial.uns['pca_knn_distances']_____no_output_____

Each row corresponds to one cell, listing its neighbors (not including itself) from nearest to farthest. `data_trial.uns['pca_knn_indices']` stores their indices, and `data_trial.uns['pca_knn_distances']` stores distances._____no_output_____## Clustering and Visualization

Now we are ready to cluster the data for cell type detection. `pegasus` provides 4 clustering algorithms to use:
* `louvain`: Louvain algorithm, using [louvain](https://github.com/vtraag/louvain-igraph) package.
* `leiden`: Leiden algorithm, using [leidenalg](https://github.com/vtraag/leidenalg) package.
* `spectral_louvain`: Spectral Louvain algorithm, which requires Diffusion Map.
* `spectral_leiden`: Spectral Leiden algorithm, which requires Diffusion Map.

See [this documentation](https://pegasus.readthedocs.io/en/stable/api/index.html#cluster-algorithms) for details.

In this tutorial, we use the Louvain algorithm:_____no_output_____

pg.louvain(data_trial)_____no_output_____

As a result, Louvain algorithm finds 19 clusters:_____no_output_____

data_trial.obs['louvain_labels'].value_counts()_____no_output_____

We can check each cluster's composition regarding donors via a composition plot:_____no_output_____

pg.compo_plot(data_trial, 'louvain_labels', 'Channel')_____no_output_____

However, we can see a clear batch effect in the plot: e.g. Cluster 11 and 14 have most cells from Donor 3.

We can see it more clearly in its FIt-SNE plot (a visualization algorithm which we will talk about later):_____no_output_____

pg.tsne(data_trial)_____no_output_____pg.scatter(data_trial, attrs=['louvain_labels', 'Channel'], basis='tsne')_____no_output_____

## Batch Correction

Batch effect occurs when data samples are generated in different conditions, such as date, weather, lab setting, equipment, etc. Unless informed that all the samples were generated under the similar condition, people may suspect presumable batch effects if they see a visualization graph with samples kind-of isolated from each other.

For this dataset, we need the batch correction step to reduce such a batch effect, which is observed in the plot above.

In this tutorial, we use [Harmony](https://www.nature.com/articles/s41592-019-0619-0) algorithm for batch correction. It requires redo HVG selection, calculate new PCA coordinates, and apply the correction:_____no_output_____

pg.highly_variable_features(data, consider_batch=True)
pg.pca(data)
pca_key = pg.run_harmony(data)_____no_output_____

The corrected PCA coordinates are stored in `data.obsm`:_____no_output_____

data.obsm['X_pca_harmony'].shape_____no_output_____

`pca_key` is the representation key returned by `run_harmony` function, which is equivalent to string `"pca_harmony"`. In the following sections, you can use either `pca_key` or `"pca_harmony"` to specify `rep` parameter in Pegasus functions whenever applicable._____no_output_____## Repeat Previous Steps on the Corrected Data

As the count matrix is changed by batch correction, we need to recalculate nearest neighbors and perform clustering. Don't forget to use the corrected PCA coordinates as the representation:_____no_output_____

pg.neighbors(data, rep=pca_key)
pg.louvain(data, rep=pca_key)_____no_output_____

Let's check the composition plot now:_____no_output_____

pg.compo_plot(data, 'louvain_labels', 'Channel')_____no_output_____

If everything goes properly, you should be able to see that no cluster has a dominant donor cells. Also notice that Louvain algorithm on the corrected data finds 16 clusters, instead of the original 19 ones.

Also, FIt-SNE plot is different:_____no_output_____

pg.tsne(data, rep=pca_key)_____no_output_____pg.scatter(data, attrs=['louvain_labels', 'Channel'], basis='tsne')_____no_output_____

You can see that the right-hand-side plot has a much better mixture of cells from different donors._____no_output_____## Visualization_____no_output_____### tSNE Plot

In previous sections, we have seen data visualization using FIt-SNE. FIt-SNE is a fast implementation on tSNE algorithm, and Pegasus uses it for the tSNE embedding calculation. [See details](https://pegasus.readthedocs.io/en/stable/api/pegasus.tsne.html)_____no_output_____### UMAP Plot

Besides tSNE, `pegasus` also provides UMAP plotting methods:

* `umap`: UMAP plot, using [umap-learn](https://github.com/lmcinnes/umap) package. [See details](https://pegasus.readthedocs.io/en/stable/api/pegasus.umap.html)
* `net_umap`: Approximated UMAP plot with DNN model based speed up. [See details](https://pegasus.readthedocs.io/en/stable/api/pegasus.net_umap.html)

Below is the UMAP plot of the data using `umap` method:_____no_output_____

pg.umap(data, rep=pca_key)_____no_output_____pg.scatter(data, attrs=['louvain_labels', 'Channel'], basis='umap')_____no_output_____

## Differential Expression Analysis

With the clusters ready, we can now perform Differential Expression (DE) Analysis. DE analysis is to discover cluster-specific marker genes. For each cluster, it compares cells within the cluster with all the others, then finds genes significantly highly expressed (up-regulated) and lowly expressed (down-regulated) for the cluster._____no_output_____Now use `de_analysis` method to run DE analysis. We use Louvain result here. _____no_output_____

pg.de_analysis(data, cluster='louvain_labels')_____no_output_____

By default, DE analysis runs Mann-Whitney U (MWU) test.

Alternatively, you can also run the follow tests by setting their corresponding parameters to be `True`:
* `fisher`: Fisher’s exact test.
* `t`: Welch’s T test.

DE analysis result is stored with key `"de_res"` (by default) in `varm` field of data. See [documentation](https://pegasus.readthedocs.io/en/stable/api/pegasus.de_analysis.html) for more details.

To load the result in a human-readable format, use `markers` method:_____no_output_____

marker_dict = pg.markers(data)_____no_output_____

By default, `markers`:
* Sort genes by Area under ROC curve (AUROC) in descending order;
* Use $\alpha = 0.05$ significance level on q-values for inference.

See [documentation](https://pegasus.readthedocs.io/en/stable/api/pegasus.markers.html) for customizing these parameters.

Let's see the up-regulated genes for Cluster 1, and rank them in descending order with respect to log fold change:_____no_output_____

marker_dict['1']['up'].sort_values(by='log2FC', ascending=False)_____no_output_____

Among them, **TRAC** worth notification. It is a critical marker for T cells._____no_output_____We can also use Volcano plot to see the DE result. Below is such a plot w.r.t. Cluster 1 with MWU test results (by default):_____no_output_____

pg.volcano(data, cluster_id = '1', dpi=200)_____no_output_____

The plot above uses the default thresholds: log fold change at $1$ (i.e. fold change at $2$), and q-value at $0.05$. Each point stands for a gene. Red ones are significant marker genes: those at right-hand side are up-regulated genes for Cluster 1, while those at left-hand side are down-regulated genes.

We can see that gene **TRAC** is the second to rightmost point, which is a significant up-regulated gene for Cluster 1.

To store a specific DE analysis result to file, you can `write_results_to_excel` methods in `pegasus`:_____no_output_____

pg.write_results_to_excel(marker_dict, "MantonBM_subset.de.xlsx")_____no_output_____

## Cell Type Annotation

After done with DE analysis, we can use the test result to annotate the clusters._____no_output_____

celltype_dict = pg.infer_cell_types(data, markers = 'human_immune')
cluster_names = pg.infer_cluster_names(celltype_dict)_____no_output_____

`infer_cell_types` has 2 critical parameters to set:
* `markers`: Either `'human_immune'`, `'mouse_immune'`, `'human_brain'`, `'mouse_brain'`, `'human_lung'`, or a user-specified marker dictionary.
* `de_test`: Decide which DE analysis test to be used for cell type inference. It can be either `'t'`, `'fisher'`, or `'mwu'`. Its default is `'mwu'`.

`infer_cluster_names` by default uses `threshold = 0.5` to filter out candidate cell types of scores lower than 0.5.

See [documentation](https://pegasus.readthedocs.io/en/stable/api/pegasus.infer_cell_types.html) for details.

Below is the cell type annotation report for Cluster 1:_____no_output_____

celltype_dict['1']_____no_output_____

The report has a list of predicted cell types along with their scores and support genes for users to decide.

Next, substitute the inferred cluster names in data using `annotate` function:_____no_output_____

pg.annotate(data, name='anno', based_on='louvain_labels', anno_dict=cluster_names)
data.obs['anno'].value_counts()_____no_output_____

So the cluster-specific cell type information is stored in `data.obs['anno']`.

The `anno_dict` can be either a list or a dictionary. If provided a list (which is the case here), Pegasus will match cell types with cluster labels in the same order. Alternatively, you can create an annotation dictionary with keys being cluster labels and cell types being values.

In practice, users may want to manually create this annotation structure by reading the report in `celltype_dict`. In this tutorial, we'll just use the output of `infer_cluster_names` function for demonstration.

Now plot the data with cell types:_____no_output_____

pg.scatter(data, attrs='anno', basis='tsne', dpi=100)_____no_output_____pg.scatter(data, attrs='anno', basis='umap', legend_loc='on data', dpi=150)_____no_output_____

## Raw Count vs Log-norm Count

Now let's check the count matrix:_____no_output_____

data_____no_output_____

You can see that besides `X`, there is another matrix `raw.X` generated for this analysis. As the key name indicates, `raw.X` stores the raw count matrix, which is the one after loading from the original Zarr file; while `X` stores the log-normalized counts.

`data` currently binds to matrix `X`. To use the raw count instead, type:_____no_output_____

data.select_matrix('raw.X')
data_____no_output_____

Now `data` binds to raw counts.

We still need log-normalized counts for the following sections, so reset the default count matrix:_____no_output_____

data.select_matrix('X')_____no_output_____

## Cell Development Trajectory and Diffusion Map

Alternative, pegasus provides cell development trajectory plots using Force-directed Layout (FLE) algorithm:

* `pg.fle`: FLE plot, using Force-Atlas 2 algorithm in [forceatlas2-python](https://github.com/klarman-cell-observatory/forceatlas2-python) package. [See details](https://pegasus.readthedocs.io/en/stable/api/pegasus.fle.html)
* `pg.net_fle`: Approximated FLE plot with DNN model based speed up. [See details](https://pegasus.readthedocs.io/en/stable/api/pegasus.net_fle.html)

Moreover, calculation of FLE plots is on Diffusion Map of the data, rather than directly on data points, in order to achieve a better efficiency. Thus, we need to first compute the diffusion map structure:_____no_output_____

pg.diffmap(data, rep=pca_key)_____no_output_____

By default, diffmap method uses:

* Number of Diffusion Components = 100
* Compute diffusion map from PCA matrix.

In this tutorial, we should use the corrected PCA matrix, which is specified in `pca_key`. The resulting diffusion map is in `data.obsm` with key `"X_diffmap"`:_____no_output_____

data.obsm['X_diffmap'].shape_____no_output_____

Now we are ready to calculate the pseudo-temporal trajectories of cell development. We use `fle` here:_____no_output_____

pg.fle(data)_____no_output_____

And show FLE plot regarding cell type annotations:_____no_output_____

pg.scatter(data, attrs='anno', basis='fle')_____no_output_____

## Save Result to File

Use `write_output` function to save analysis result `data` to file:_____no_output_____

pg.write_output(data, "result.zarr.zip")_____no_output_____

It's stored in `zarr` format, because this is the default file format in Pegasus.

Alternatively, you can also save it in `h5ad`, `mtx`, or `loom` format.  See [its documentation](https://pegasus.readthedocs.io/en/stable/api/pegasus.write_output.html) for instructions._____no_output_____## Read More...

* Read [Plotting Tutorial](https://pegasus.readthedocs.io/en/stable/_static/tutorials/plotting_tutorial.html) for more plotting functions provided by Pegasus.

* Read [Pegasus API](https://pegasus.readthedocs.io/en/stable/api/index.html) documentation for details on Pegasus functions._____no_output_____


# Notebook from trevor48j/YELP_Worth_It
Path: web_scraping.ipynb

<code>
# Import dependencies
import pandas as pd
import re
import time
import pymongo
from bs4 import BeautifulSoup
from requests import get
from random import randint
from pymongo import MongoClient_____no_output_____# --- EXTRACT ---

# Set 'restaurant_list' to store information scraped from source website
restaurant_list = []

# Set a for loop to go through the Yelp website from page 1 through 10 using BeautifulSoup to extract HTML code and pull information related to Resturant Name, Cuisine Style, Price, and Rating.
url = "https://www.yelp.com/search?find_desc=Restaurants&find_loc=West%20Hollywood%2C%20CA&start="
for page in range(0,110,10):
    response = get(url+str(page))
    html = response.text
    soup = BeautifulSoup(html,"html.parser")

    rl = re.compile("list*")
    restaurant_ul = soup.find_all('ul', class_=rl)
    for r_ul in restaurant_ul:
        border = re.compile("border*")
        restaurant_li = r_ul.find_all('li', class_=border)

        for restaurant in restaurant_li:
            container = re.compile("container*")
            aux = restaurant.find_all('div', class_=container)

            if len(aux) == 3:
                restaurant_dict = {}

                name = re.compile("businessNam*")
                restaurant_name = restaurant.find_all('div', class_=name)
                try:
                    restaurant_dict['name'] = restaurant_name[0].find('a').text
                except:
                    restaurant_dict['name'] = None

                stars = re.compile("i-stars*")
                restaurant_stars = restaurant.find_all('div', class_=stars)
                try:
                    restaurant_dict['stars'] = restaurant_stars[0]["aria-label"]
                except:
                    restaurant_dict['stars'] = None

                price = re.compile("priceRange*")
                restaurant_price = restaurant.find_all('span', class_=price)
                try:
                    restaurant_dict['price'] = restaurant_price[0].text
                except:
                    restaurant_dict['price'] = None

                cuisine = re.compile("text__09f24__2NHRu css-1hx6l2b*")
                restaurant_cuisine = restaurant.find_all('p', class_=cuisine)
                try:
                    restaurant_dict['cuisine'] = restaurant_cuisine[0].text
                except:
                    restaurant_dict['cuisine'] = None

                restaurant_list.append(restaurant_dict)

    time.sleep(randint(1,5))

# Displays the extracted list
restaurant_list_____no_output_____# ---TRANSFORM ---

# Create a dataframe from the extracted list using Pandas
restaurant_df = pd.DataFrame(restaurant_list)
restaurant_df_____no_output_____# Convert the names in the data frame
result_df = restaurant_df[["name","stars","price","cuisine"]]

# Drop 'star rating' from the rating column to be able to manipulate the data
rating = result_df["stars"].replace("star rating","", regex=True)
result_df["star rating"] = rating

# Replace the '$' with integers to manipulate the data
price_1 = result_df["price"].replace("$",1)
result_df["price"] = price_1

price_2 = result_df["price"].replace("$$", 2)
result_df["price"] = price_2

price_3 = result_df["price"].replace("$$$",3)
result_df["price"] = price_3

price_4 = result_df["price"].replace("$$$$",4)
result_df["price"] = price_4

# Convert the rating column to float to manipulate data
result_df['rating'] = result_df['star rating'].astype(float)

# Create a new dataframe with the cleaned data
result_clean_df = result_df[["name", "cuisine", "price", "rating"]]

# Store cleaned dataframe as a csv for visualisations and analysis
result_clean_df.to_csv("Resources/result_clean.csv")

# Display play cleaned dataframe
result_clean_df_____no_output_____# --- LOAD ---

# Making a connection with MongoClient
client = MongoClient("mongodb://localhost:27017/")

# Create Database
db = client["YELP_Worth_It"]

# Create Collection
collection = db["CSV Data"]

# Convert dataframe into dictionary
results_dict = result_clean_df.to_dict('index')
results_dict_____no_output_____# Insert data into MongoDB
for result in results_dict.values():
    collection.insert_one(result)_____no_output_____
</code>