# Notebook from mukamel-lab/SingleCellFusion_EnhancerPaper
Path: archives/old/correlation_analysis_celllevel-v3-example-cases.ipynb

import sys
sys.path.insert(0, "/cndd/fangming/CEMBA/snmcseq_dev")

from multiprocessing import Pool,cpu_count
from functools import partial

from scipy import sparse
from scipy import stats
import importlib
from __init__ import *
from __init__jupyterlab import *
import snmcseq_utils
importlib.reload(snmcseq_utils)

import pickle
import datetime
from scipy import optimize_____no_output_____def turn_cluster_labels_to_knn(cluster_labels, uniq_labels):
    """
    """

    clst_idx = snmcseq_utils.get_index_from_array(uniq_labels, cluster_labels)
    m, n = len(clst_idx), len(uniq_labels)
    _i = np.arange(m)
    _j = clst_idx
    _data = [1]*m
    knn = sparse.coo_matrix((_data, (_i, _j)), shape=(m, n))
    return knn

_____no_output_____def row_dot_product_norm_by_numcol(X_zscore, Y_zscore, x_idx, y_idx,
                                   chunksize=100000, verbose_level=100000):
    """compute (X_zscore[x_idx]*Y_zscore[y_idx]).mean(axis=1)
    correlation values given matched x_idx and y_idx...
    """
    ti = time.time()

    assert len(x_idx) == len(y_idx)
    num_pairs = len(x_idx)
    corrs = []
    for pair_idx in snmcseq_utils.chunks(np.arange(num_pairs), chunksize):
        if verbose_level and pair_idx[0] % verbose_level == 0:
            print(pair_idx[0], time.time()-ti)

        _res = (X_zscore[x_idx[pair_idx]]*Y_zscore[y_idx[pair_idx]]).mean(axis=1)
        corrs.append(_res)
    corrs = np.hstack(corrs)
    return corrs _____no_output_____def compute_enh_gene_corrs(gc_rna, ec_mccg,
    enhancer_gene_to_eval,
    common_genes, common_enhancers,
    output_file='', **kwargs,
    ):
    """
    gc_rna: enh-by-cell RNA matrix
    ec_mccg: enh-by-cell mC matrix
    enhancer_gene_to_eval: a table with enhancer-gene pairs need to be ...

    common_genes: orders gene in the mat gc_rna
    common_enhancers: orders of enh in the mat ec_mccg (ec_mccg.index.values)

    """
    common_enhancers = ec_mccg.index.values

    gc_rna_zscore = stats.zscore(np.array(gc_rna), axis=1, ddof=0, nan_policy='propagate')
    ec_mccg_zscore = stats.zscore(np.array(ec_mccg), axis=1, ddof=0, nan_policy='propagate')

    # correlate e-g according to a e-g table
    gene_idx = snmcseq_utils.get_index_from_array(common_genes, enhancer_gene_to_eval['gene'])
    enh_idx = snmcseq_utils.get_index_from_array(common_enhancers, enhancer_gene_to_eval['ens']) # be careful here!
    to_correlate = ~np.logical_or(gene_idx==-1, enh_idx==-1)

    gene_idx = gene_idx[to_correlate]
    enh_idx = enh_idx[to_correlate]

    # corr
    corrs = row_dot_product_norm_by_numcol(gc_rna_zscore, ec_mccg_zscore, gene_idx, enh_idx, **kwargs)

    # corr shuffled cells
    corrs_shuffled_cells = row_dot_product_norm_by_numcol(
        gc_rna_zscore[:,np.random.permutation(gc_rna_zscore.shape[1])],
        ec_mccg_zscore,
        gene_idx, enh_idx, **kwargs)

    # corr shuffled genes (break up the pairs)
    gene_idx_uniq = np.unique(gene_idx)
    shuff_genes = {
        gene: gene_shuff for gene, gene_shuff in
            zip(gene_idx_uniq, gene_idx_uniq[np.random.permutation(len(gene_idx_uniq))])
        }
    gene_idx_shuff = np.array([shuff_genes[gene] for gene in gene_idx])

    corrs_shuffled = row_dot_product_norm_by_numcol(
        gc_rna_zscore,
        ec_mccg_zscore,
        gene_idx_shuff, enh_idx, **kwargs)

    # save corrs
    if output_file:
        with open(output_file, 'wb') as fh:
            pickle.dump((to_correlate, corrs, corrs_shuffled, corrs_shuffled_cells), fh)

    return to_correlate, corrs, corrs_shuffled, corrs_shuffled_cells_____no_output_____

# setup - cluster level correlation_____no_output_____

today = datetime.date.today()
print(today)

output_corr = '/cndd2/fangming/projects/scf_enhancers/results/{}_corrs_test1.pkl'.format(today)
output_figures = "/cndd2/fangming/projects/scf_enhancers/results/figures/{}_{{}}.pdf".format(today)
# output_to_correlate = "/cndd2/fangming/projects/scf_enhancers/results/{}_corrs_idx_test1.pkl".format(today)
# output_figure1 = '/cndd2/fangming/projects/scf_enhancers/results/figures/{}_cell_level_mc_rna_corr_v1.pdf'.format(today)
# output_figure2 = '/cndd2/fangming/projects/scf_enhancers/results/figures/{}_cell_level_mc_rna_corr_v2.pdf'.format(today)2020-09-15

# Load data _____no_output_____

# enhancer-gene linkage
ti = time.time()

input_enh_gene_table = '/cndd2/fangming/projects/scf_enhancers/results/200521_to_evals.tsv'
enhancer_gene_to_eval = pd.read_csv(input_enh_gene_table, sep='\t')

print(enhancer_gene_to_eval.shape)
print(time.time()-ti)
enhancer_gene_to_eval.head()(2589994, 18)
9.28837251663208
# input_bundle
f = '/cndd2/fangming/projects/scf_enhancers/data/organized_cell_level/version_july29'
bundle_fnames = (
    'cell_smarter_cells.txt',
    'cell_snmcseq_gene.txt',
    'gene_smarter_cells.txt',
    'enh_snmcseq_gene.tsv',

    'mat_smarter_cells.npz',
    'mat_mcg_snmcseq_gene.npz',
    'mat_cg_snmcseq_gene.npz',
    'knn_smarter_cells_snmcseq_gene.npz',
    'knn_smarter_cells.npz',
)

ti = time.time()
with snmcseq_utils.cd(f):
    bundle = []
    for fname in bundle_fnames:
        #  save all as pickle file
        with open(fname, "rb") as fh:
            item = pickle.load(fh)
        bundle.append(item)
        print(type(item), item.shape, fname, time.time()-ti)

(common_rna_cells, common_mc_cells,
 common_genes, common_enhancer_regions,
 X, Y_mcg, Y_cg, knn_xy, knn_xx,
) = bundle

print(time.time()-ti)<class 'numpy.ndarray'> (6244,) cell_smarter_cells.txt 0.06375741958618164
<class 'numpy.ndarray'> (9364,) cell_snmcseq_gene.txt 0.10713577270507812
<class 'numpy.ndarray'> (32324,) gene_smarter_cells.txt 0.1126861572265625
<class 'pandas.core.frame.DataFrame'> (233514, 3) enh_snmcseq_gene.tsv 0.17925381660461426
<class 'scipy.sparse.csc.csc_matrix'> (32324, 6244) mat_smarter_cells.npz 5.082681179046631
<class 'scipy.sparse.csc.csc_matrix'> (233514, 9364) mat_mcg_snmcseq_gene.npz 19.14578413963318
<class 'scipy.sparse.csc.csc_matrix'> (233514, 9364) mat_cg_snmcseq_gene.npz 42.06675314903259
<class 'scipy.sparse.csc.csc_matrix'> (6244, 9364) knn_smarter_cells_snmcseq_gene.npz 42.15261173248291
<class 'scipy.sparse.csc.csc_matrix'> (6244, 6244) knn_smarter_cells.npz 42.2011137008667
42.20182538032532
f = '/cndd/fangming/CEMBA/data/MOp_all/results_final/miniatlas_fig4_scf_clusterings.tsv'
df_clst = pd.read_csv(f, sep='\t').set_index('sample')
print(df_clst.shape)
df_clst.head()(408885, 10)

# Compute metacell level signal _____no_output_____

# # construct kNN matrix (different kNN matrices give different results)

# cluster level
# pull all cells in a cluster
# kNN
# kNN within clusters

## method 1 ### cluster level
uniq_labels = np.intersect1d(
    df_clst.reindex(common_rna_cells)['joint_cluster_round2'].unique(),
    df_clst.reindex(common_mc_cells)['joint_cluster_round2'].unique(),
)
knn_xz = turn_cluster_labels_to_knn(df_clst.reindex(common_rna_cells)['joint_cluster_round2'].values,
                                    uniq_labels,
                                   )
knn_xz = knn_xz.dot(sparse.diags(np.ravel(1.0/knn_xz.sum(axis=0))))

knn_yz = turn_cluster_labels_to_knn(df_clst.reindex(common_mc_cells)['joint_cluster_round2'].values,
                                    uniq_labels,
                                   )

print(knn_xz.shape, knn_yz.shape)
## end method 1 ###

# ### method 2 ###
# uniq_labels = np.intersect1d(
#     df_clst.reindex(common_rna_cells)['joint_cluster_round2'].unique(),
#     df_clst.reindex(common_mc_cells)['joint_cluster_round2'].unique(),
# )
# knn_xz = turn_cluster_labels_to_knn(
#     df_clst.reindex(common_rna_cells)['joint_cluster_round2'].values,
#     uniq_labels,
# )
# knn_yz = turn_cluster_labels_to_knn(
#     df_clst.reindex(common_mc_cells)['joint_cluster_round2'].values,
#     uniq_labels,
# )

# # duplicate cols (uniq_labels -> rna_cell_clst_label_idx)
# rna_cell_clst_label_idx = snmcseq_utils.get_index_from_array(
#     uniq_labels,
#     df_clst.reindex(common_rna_cells)['joint_cluster_round2'].values,
# )
# knn_xz = knn_xz.tocsc()[:,rna_cell_clst_label_idx]
# knn_yz = knn_yz.tocsc()[:,rna_cell_clst_label_idx]

# # normalization
# knn_xz = knn_xz.dot(sparse.diags(np.ravel(1.0/knn_xz.sum(axis=0))))

# print(knn_xz.shape, knn_yz.shape)
# ### method 2 ###

# ### method 3 ###
# # files
# input_dir = '/cndd2/fangming/projects/miniatlas/results/'
# name_tag = 'mop_2mods_200807_ka50_knn50'
# mod_x = 'smarter_cells'
# mod_y = 'snmcseq_gene'
# i_sub = 0

# input_knn_xy = 'knn_across_{}_{}_{}.npz.{}.npz'.format(name_tag, mod_x, mod_y, i_sub)
# input_knn_xx = 'knn_within_{}_{}.npz.{}.npz'.format(name_tag, mod_x, i_sub)
# input_knn_cells_xaxis = 'cells_{}_{}.npy.{}.npy'.format(mod_x, name_tag, i_sub)
# input_knn_cells_yaxis = 'cells_{}_{}.npy.{}.npy'.format(mod_y, name_tag, i_sub)

# with snmcseq_utils.cd(input_dir):

#     # knn_xy, knn_xx
#     knn_xy = sparse.load_npz(input_knn_xy)
#     knn_xx = sparse.load_npz(input_knn_xx)

#     # import their axes
#     cell_cell_knn_xaxis = np.load(input_knn_cells_xaxis, allow_pickle=True)
#     cell_cell_knn_yaxis = np.load(input_knn_cells_yaxis, allow_pickle=True)

# # new cells
# common_rna_cells_updated = np.intersect1d(common_rna_cells, cell_cell_knn_xaxis)
# common_mc_cells_updated = np.intersect1d(common_mc_cells, cell_cell_knn_yaxis)

# # make sure the original matrices have the correct index
# x_idx = snmcseq_utils.get_index_from_array(common_rna_cells, common_rna_cells_updated)
# y_idx = snmcseq_utils.get_index_from_array(common_mc_cells, common_mc_cells_updated)
# X = X.tocsc()[:, x_idx]
# Y_cg = Y_cg.tocsc()[:, y_idx]
# Y_mcg = Y_mcg.tocsc()[:, y_idx]

# # make sure knn_xy, knn_xx have the right cell index
# cell_idx_xaxis = snmcseq_utils.get_index_from_array(cell_cell_knn_xaxis, common_rna_cells_updated)
# cell_idx_yaxis = snmcseq_utils.get_index_from_array(cell_cell_knn_yaxis, common_mc_cells_updated)
# knn_xy = knn_xy.tocsr()[cell_idx_xaxis,:].tocsc()[:,cell_idx_yaxis] # x-by-y
# knn_xx = knn_xx.tocsr()[cell_idx_xaxis,:].tocsc()[:,cell_idx_xaxis] # x-by-x

# # ...
# knn_xz = knn_xx.T
# knn_yz = knn_xy.T
# print(knn_xz.shape, knn_yz.shape)
# ### end method 3 ###

# ### method 4  not yet useful!!!!!  !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!###
# # files
# input_dir = '/cndd2/fangming/projects/miniatlas/results/'
# name_tag = 'mop_2mods_200807_ka50_knn50'
# mod_x = 'smarter_cells'
# mod_y = 'snmcseq_gene'
# i_sub = 0

# input_knn_xy = 'knn_across_{}_{}_{}.npz.{}.npz'.format(name_tag, mod_x, mod_y, i_sub)
# input_knn_xx = 'knn_within_{}_{}.npz.{}.npz'.format(name_tag, mod_x, i_sub)
# input_knn_cells_xaxis = 'cells_{}_{}.npy.{}.npy'.format(mod_x, name_tag, i_sub)
# input_knn_cells_yaxis = 'cells_{}_{}.npy.{}.npy'.format(mod_y, name_tag, i_sub)

# with snmcseq_utils.cd(input_dir):

#     # knn_xy, knn_xx
#     knn_xy = sparse.load_npz(input_knn_xy)
#     knn_xx = sparse.load_npz(input_knn_xx)

#     # import their axes
#     cell_cell_knn_xaxis = np.load(input_knn_cells_xaxis, allow_pickle=True)
#     cell_cell_knn_yaxis = np.load(input_knn_cells_yaxis, allow_pickle=True)

# # new cells
# common_rna_cells_updated = np.intersect1d(common_rna_cells, cell_cell_knn_xaxis)
# common_mc_cells_updated = np.intersect1d(common_mc_cells, cell_cell_knn_yaxis)

# # make sure the original matrices have the correct index
# x_idx = snmcseq_utils.get_index_from_array(common_rna_cells, common_rna_cells_updated)
# y_idx = snmcseq_utils.get_index_from_array(common_mc_cells, common_mc_cells_updated)
# X = X.tocsc()[:, x_idx]
# Y_cg = Y_cg.tocsc()[:, y_idx]
# Y_mcg = Y_mcg.tocsc()[:, y_idx]

# # make sure knn_xy, knn_xx have the right cell index
# cell_idx_xaxis = snmcseq_utils.get_index_from_array(cell_cell_knn_xaxis, common_rna_cells_updated)
# cell_idx_yaxis = snmcseq_utils.get_index_from_array(cell_cell_knn_yaxis, common_mc_cells_updated)
# knn_xy = knn_xy.tocsr()[cell_idx_xaxis,:].tocsc()[:,cell_idx_yaxis] # x-by-y
# knn_xx = knn_xx.tocsr()[cell_idx_xaxis,:].tocsc()[:,cell_idx_xaxis] # x-by-x

# # ...
# knn_xz = knn_xx.T
# knn_yz = knn_xy.T
# print(knn_xz.shape, knn_yz.shape)
# ### end method 4 ###

(6244, 29) (9364, 29)
ti = time.time()

# gene by metacell
gc_rna = X.dot(knn_xz).todense()

# enhancer by metacell
ec_cg = Y_cg.dot(knn_yz).todense()
ec_mcg = Y_mcg.dot(knn_yz).todense()

print(time.time()-ti)

print(gc_rna.shape, ec_cg.shape, ec_mcg.shape)3.5419692993164062
(32324, 29) (233514, 29) (233514, 29)
# get mcc
ti = time.time()
ec_mccg =  snmcseq_utils.get_mcc_lite_v4(
                                pd.DataFrame(ec_cg).astype(np.float32),
                                pd.DataFrame(ec_mcg).astype(np.float32),
                                base_call_cutoff=5, sufficient_coverage_fraction=0.8, fillna=True)
print(ec_mccg.shape)

print(time.time()-ti)(143526, 29)
0.31096959114074707

# Correlate enhancer and gene _____no_output_____

ti = time.time()

(to_correlate, corrs, corrs_shuffled, corrs_shuffled_cells) = compute_enh_gene_corrs(
    gc_rna, ec_mccg,
    enhancer_gene_to_eval,
    common_genes, ec_mccg.index.values,
    output_file=output_corr, chunksize=100000, verbose_level=0,
    )

print(time.time() - ti)/cndd2/fangming/venvs/routine/lib/python3.8/site-packages/scipy/stats/stats.py:2500: RuntimeWarning: invalid value encountered in true_divide
  return (a - mns) / sstd

# plotting results_____no_output_____

# enhancer-gene linkage table (needed to interpret the results)
enhancer_gene_to_eval = pd.read_csv(input_enh_gene_table, sep='\t')
print(enhancer_gene_to_eval.shape)

# results
with open(output_corr, 'rb') as fh:
    to_correlate, corrs, corrs_shuffled, corrs_shuffled_cells = pickle.load(fh)
print(to_correlate.shape, corrs.shape, corrs_shuffled.shape, corrs_shuffled_cells.shape)(2589994, 18)
(2589994,) (2316805,) (2316805,) (2316805,)
# some basic stats
dists = enhancer_gene_to_eval.loc[to_correlate, 'dist'].values
uniq_enh = enhancer_gene_to_eval.loc[to_correlate, 'ens'].unique()
uniq_gene = enhancer_gene_to_eval.loc[to_correlate, 'gene'].unique()

print(np.min(dists), np.max(dists))
print(len(uniq_enh), len(uniq_gene))2067 999999
134532 18963
# plotting setup

num_bins = 200
bin_min, bin_max = -1, 0.5
fdr_threshold = 0.2
bins = np.linspace(bin_min, bin_max, num_bins)

tracks = {
    'pairs (<100kb)': corrs[dists<1e5],
    'pairs (<500kb)': corrs[dists<5e5],
    'pairs (<1Mb)': corrs,
    'shuffled pairs': corrs_shuffled,
    'shuffled cells': corrs_shuffled_cells,
    }

colors = snmcseq_utils.get_grad_colors(len(tracks), cmap='Blues_r')
track_colors = {
    'pairs (<100kb)': colors[0],
    'pairs (<500kb)': colors[1],
    'pairs (<1Mb)': colors[2],
    'shuffled pairs': 'gray',
    'shuffled cells': 'black',
    }

config = {
    'kde': False,
    "hist_kws": {
                'histtype': 'step',
#                 'edgecolor': 'none',
                'alpha': 1,
                'density': True,
                },
}
_____no_output_____
track_pdf_ratios = {}
track_cdfs = {}
track_fdrs = {}

track_r_thresholds = {}
track_num_sigs = {}
track_frac_sigs = {}

track_r_thresholds_cells = {}
track_num_sigs_cells = {}
track_frac_sigs_cells = {}

hist_shuff, _ = np.histogram(corrs_shuffled, bins=bins, normed=True)
cdf_shuff = np.cumsum(hist_shuff)
hist_shuff_cells, _ = np.histogram(corrs_shuffled_cells, bins=bins, normed=True)
cdf_shuff_cells = np.cumsum(hist_shuff_cells)
for label, track in tracks.items():
    # dist
    hist, _ = np.histogram(track, bins=bins, normed=True)
    cdf = np.cumsum(hist)
    # fdr
    fdr = cdf_shuff/cdf
    fdr_cells = cdf_shuff_cells/cdf
    # record
    track_pdf_ratios[label] = hist/hist_shuff
    track_cdfs[label] = cdf
    track_fdrs[label] = fdr

    if label.startswith("pairs"):
        # get r_threshold
        # remove nan
        isnan = np.isnan(fdr)
        _y = fdr[~isnan]
        _x = bins[1:][~isnan]

        # find r threshold
        try:
            f = lambda _x_func: np.interp(_x_func, _x, _y) - fdr_threshold
            r_threshold = optimize.root_scalar(f, bracket=(-1, 0)).root
        except:
            r_threshold = 0

        # num and frac significant
        num_sig = np.sum(track < r_threshold)
        frac_sig = num_sig/len(track)

        # record
        track_r_thresholds[label] = r_threshold
        track_num_sigs[label] = num_sig
        track_frac_sigs[label] = frac_sig

        ## shuff cells
        # get r_threshold
        # remove nan
        isnan = np.isnan(fdr_cells)
        _y = fdr_cells[~isnan]
        _x = bins[1:][~isnan]

        # find r threshold
        try:
            f = lambda _x_func: np.interp(_x_func, _x, _y) - fdr_threshold
            r_threshold = optimize.root_scalar(f, bracket=(-1, 0)).root
        except:
            r_threshold = 0

        # num and frac significant
        num_sig = np.sum(track < r_threshold)
        frac_sig = num_sig/len(track)

        # record
        track_r_thresholds_cells[label] = r_threshold
        track_num_sigs_cells[label] = num_sig
        track_frac_sigs_cells[label] = frac_sig<ipython-input-16-b36d64745046>:13: VisibleDeprecationWarning: Passing `normed=True` on non-uniform bins has always been broken, and computes neither the probability density function nor the probability mass function. The result is only correct if the bins are uniform, when density=True will produce the same result anyway. The argument will be removed in a future version of numpy.
  hist_shuff, _ = np.histogram(corrs_shuffled, bins=bins, normed=True)
<ipython-input-16-b36d64745046>:15: VisibleDeprecationWarning: Passing `normed=True` on non-uniform bins has always been broken, and computes neither the probability density function nor the probability mass function. The result is only correct if the bins are uniform, when density=True will produce the same result anyway. The argument will be removed in a future version of numpy.
  hist_shuff_cells, _ = np.histogram(corrs_shuffled_cells, bins=bins, normed=True)
<ipython-input-16-b36d64745046>:19: VisibleDeprecationWarning: Passing `normed=True` on non-uniform bins has always been broken, and computes neither the probability density function nor the probability mass function. The result is only correct if the bins are uniform, when density=True will produce the same result anyway. The argument will be removed in a future version of numpy.
  hist, _ = np.histogram(track, bins=bins, normed=True)
<ipython-input-16-b36d64745046>:22: RuntimeWarning: invalid value encountered in true_divide
  fdr = cdf_shuff/cdf
<ipython-input-16-b36d64745046>:23: RuntimeWarning: invalid value encountered in true_divide
  fdr_cells = cdf_shuff_cells/cdf
<ipython-input-16-b36d64745046>:25: RuntimeWarning: divide by zero encountered in true_divide
  track_pdf_ratios[label] = hist/hist_shuff
<ipython-input-16-b36d64745046>:25: RuntimeWarning: invalid value encountered in true_divide
  track_pdf_ratios[label] = hist/hist_shuff
<ipython-input-16-b36d64745046>:22: RuntimeWarning: divide by zero encountered in true_divide
  fdr = cdf_shuff/cdf
# figure 2
fig, axs = plt.subplots(3, 1, figsize=(4*2,4*3), sharex=True)

ax = axs[0]
for label, track in tracks.items():
    sns.distplot(track, bins=bins, ax=ax, label=label, color=track_colors[label], **config)
ax.legend(bbox_to_anchor=(1,1), loc='upper left')
ax.set_ylabel('fraction of pairs')
# ax.set_ylim([0, 6])

ax = axs[1]
for label, track in track_pdf_ratios.items():
    ax.plot(bins[1:], track, label="data/shuffled", color=track_colors[label])
ax.set_yscale('log')
ax.hlines(1, bins[0], bins[-1], linestyle='--', color='gray')
ax.set_ylabel('ratio (data/shuffled)')
ax.set_ylim([0.5, 50])

ax = axs[2]
for label, track in track_fdrs.items():
    ax.plot(bins[1:], track, color=track_colors[label])
    if label.startswith('pairs'):
        ax.scatter(track_r_thresholds[label], fdr_threshold, color=track_colors[label])
ax.hlines(fdr_threshold, bins[0], bins[-1], color='gray', linestyle='--', zorder=0)

ax.set_yscale('log')
ax.set_ylabel('FDR')
ax.set_xlabel('Pearson r')
ax.set_ylim([1e-3, 2])

snmcseq_utils.savefig(fig, output_figures.format("corr_dist_plot"))
plt.show()_____no_output_____# figure 2
fig, axs = plt.subplots(4, 1, figsize=(4*2,4*4), sharex=True)
# fig.suptitle("k, ka = ({}, {})\n Num. sig. pairs: {}".format(k, ka, track_num_sigs), y=0.88)
fig.suptitle("29 cell types \n Num. sig. pairs: {}".format(track_num_sigs), y=0.88)

axs[0].axis('off')
ax = fig.add_subplot(912)
for label, track in tracks.items():
    sns.distplot(track, bins=bins, ax=ax, label=label, color=track_colors[label], **config)
ax.set_ylim([20, 35])
sns.despine(bottom=True, top=False, right=False, ax=ax)

ax = axs[1]
for label, track in tracks.items():
    sns.distplot(track, bins=bins, ax=ax, label=label, color=track_colors[label], **config)
ax.legend(bbox_to_anchor=(1,1), loc='upper left')
ax.set_ylabel('fraction of pairs')
ax.set_ylim([0, 6])
sns.despine(top=True, right=False, ax=ax)

ax = axs[2]
for label, track in track_pdf_ratios.items():
    ax.plot(bins[1:], track, label="data/shuffled", color=track_colors[label])
ax.set_yscale('log')
ax.hlines(1, bins[0], bins[-1], linestyle='--', color='gray')
ax.set_ylabel('ratio (data/shuffled)')
ax.set_ylim([0.5, 50])

ax = axs[3]
for label, track in track_fdrs.items():
    ax.plot(bins[1:], track, color=track_colors[label])
    if label.startswith('pairs'):
        ax.scatter(track_r_thresholds[label], fdr_threshold, color=track_colors[label])
ax.hlines(fdr_threshold, bins[0], bins[-1], color='gray', linestyle='--', zorder=0)

ax.set_yscale('log')
ax.set_ylabel('FDR')
ax.set_xlabel('Pearson r')
ax.set_ylim([1e-3, 2])

snmcseq_utils.savefig(fig, output_figures.format('corr_dist_plot_v2'))
plt.show()_____no_output_____

# Num sig. linked pairs vs num sig. correlated pairs_____no_output_____

# input: to_correlate, corrs, corrs_shuffled, corrs_shuffled_cells
# paras: distance_threshold

def get_r_threshold(bins, fdr, fdr_threshold, r_min, r_max):
    """given fdr function (bins, fdr) and fdr_threshold
    get r threshold
    """
    # get r_threshold
    # remove nan
    isnan = np.isnan(fdr)
    _y = fdr[~isnan]
    _x = bins[1:][~isnan]

    # find r threshold
    f = lambda _x_func: np.interp(_x_func, _x, _y) - fdr_threshold
    if f(r_min)*f(r_max) < 0:
        sol = optimize.root_scalar(f, bracket=(r_min, r_max))
        if sol:
            r_threshold = sol.root

        else:
            print("failed to detect r_threshold:")
            r_threshold = np.nan

    else:
        print("failed to detect r_threshold:")
        r_threshold = np.nan

    return r_threshold

def get_significance_stats(
    enhancer_gene_to_eval,
    to_correlate, corrs, corrs_shuffled, corrs_shuffled_cells,
    distance_threshold=1e5, fdr_threshold=0.2,
    positive_side=False,
    ):
    """assuming significant negative correlation
    """
    r_min, r_max = -1, 0

    # align all to negative side
    if positive_side:
        corrs *= (-1)
        corrs_shuffled *= (-1)
        corrs_shuffled_cells *= (-1)

    # dists
    dists = enhancer_gene_to_eval.loc[to_correlate, 'dist'].values

    ## sig numbers
    label_cond = dists < distance_threshold
    track = corrs[label_cond]

    ### total numbers with the condition
    num_total_pairs = len(enhancer_gene_to_eval[to_correlate][label_cond])
    num_total_genes = len(enhancer_gene_to_eval[to_correlate][label_cond]['gene'].unique())
    num_total_enhs = len(enhancer_gene_to_eval[to_correlate][label_cond]['ens'].unique())

    # hist_shuff
    hist_shuff, _ = np.histogram(corrs_shuffled, bins=bins, density=True)
    cdf_shuff = np.cumsum(hist_shuff)
    # hist_shuff_cells
    hist_shuff_cells, _ = np.histogram(corrs_shuffled_cells, bins=bins, density=True)
    cdf_shuff_cells = np.cumsum(hist_shuff_cells)
    # hist
    hist, _ = np.histogram(track, bins=bins, density=True)
    cdf = np.cumsum(hist)

    # fdr
    fdr_linked = cdf_shuff/(cdf+1e-7)
    fdr_correlated = cdf_shuff_cells/(cdf+1e-7)

    # get r_threshold
    r_threshold_linked = get_r_threshold(bins, fdr_linked, fdr_threshold, r_min, r_max)
    r_threshold_correlated = get_r_threshold(bins, fdr_correlated, fdr_threshold, r_min, r_max)

    # stats
    if ~np.isnan(r_threshold_linked):
        # all sig info
        linked_table = enhancer_gene_to_eval[to_correlate][label_cond][track<r_threshold_linked]
        # num_sig
        num_linked_pairs = len(linked_table)
        num_linked_genes = len(linked_table['gene'].unique())
        num_linked_enhs = len(linked_table['ens'].unique())
    else:
        num_linked_pairs, num_linked_genes, num_linked_enhs = np.nan, np.nan, np.nan

    if ~np.isnan(r_threshold_correlated):
        # all sig info
        correlated_table = enhancer_gene_to_eval[to_correlate][label_cond][track<r_threshold_correlated]
        # num_sig
        num_correlated_pairs = len(correlated_table)
        num_correlated_genes = len(correlated_table['gene'].unique())
        num_correlated_enhs = len(correlated_table['ens'].unique())
    else:
        num_correlated_pairs, num_correlated_genes, num_correlated_enhs = np.nan, np.nan, np.nan

    return {
            'dist_th': distance_threshold,
            'num_total_pairs': num_total_pairs,
            'num_total_genes': num_total_genes,
            'num_total_enhs': num_total_enhs,

            'r_th_linked': r_threshold_linked,
            'num_linked_pairs': num_linked_pairs,
            'num_linked_genes': num_linked_genes,
            'num_linked_enhs': num_linked_enhs,

            'r_th_correlated': r_threshold_correlated,
            'num_correlated_pairs': num_correlated_pairs,
            'num_correlated_genes': num_correlated_genes,
            'num_correlated_enhs': num_correlated_enhs,
           }_____no_output_____distance_threshold = 1e5
res = get_significance_stats(
            enhancer_gene_to_eval,
            to_correlate, corrs, corrs_shuffled, corrs_shuffled_cells,
            distance_threshold=distance_threshold,
            positive_side=False,
            )
res_____no_output_____distance_threshold = 1e5
res = get_significance_stats(
            enhancer_gene_to_eval,
            to_correlate, corrs, corrs_shuffled, corrs_shuffled_cells,
            distance_threshold=distance_threshold,
            positive_side=False,
            )
res_____no_output_____dist_ths = np.hstack([
    np.linspace(1e5, 1e6, 10),
    np.logspace(6, 7, 10),
])

print(dist_ths)
res_all = []
for dist_th in dist_ths:
    res = get_significance_stats(
                enhancer_gene_to_eval,
                to_correlate, corrs, corrs_shuffled, corrs_shuffled_cells,
                distance_threshold=dist_th,
                positive_side=False,
                )
    res_all.append(res)
res_all = pd.DataFrame(res_all)

print(res_all.shape)
res_all.head()

    [  100000.           200000.           300000.           400000.
   500000.           600000.           700000.           800000.
   900000.          1000000.          1000000.          1291549.66501488
  1668100.53720006  2154434.69003188  2782559.40220713  3593813.66380463
  4641588.83361278  5994842.50318941  7742636.82681128 10000000.        ]
(20, 12)

# problems:
- can't go > 1e-6; points >1e-6 are artificial, because no pairs >= 1e6 distance are recorded
- number and fraction have some inconsistencies
       - need to employ local FDR
- add more yaxis
    - (fraction of true positives)
_____no_output_____

to_plot = res_all[res_all['dist_th']<=1e6]

ys = [
    to_plot['num_linked_pairs']/to_plot['num_total_pairs'],
    to_plot['num_correlated_pairs']/to_plot['num_total_pairs'],

    to_plot['num_linked_pairs'],
    to_plot['num_correlated_pairs'],
]

titles = [
    'Linked',
    'Correlated',
]

fig, axs = plt.subplots(2, 2, figsize=(4*2, 4*2))
for i, ax in enumerate(axs.flat):
    ax.plot(to_plot['dist_th'], ys[i],
            '-o', markersize=5)
    ax.set_xlim(xmin=0)
    ax.set_ylim(ymin=0)
    ax.xaxis.set_major_formatter(mtick.EngFormatter())

    if i == 0:
        ax.set_ylabel('Fraction of pairs')
    if i < 2:
        ax.set_title(titles[i])
    if i == 2:
        ax.set_ylabel('Number of pairs')
    if i >= 2:
        ax.set_xlabel('Enhancer-TSS distance')
        ax.yaxis.set_major_formatter(mtick.EngFormatter())

fig.subplots_adjust(wspace=0.3, hspace=0.2)
plt.show()_____no_output_____


# Notebook from chisomloius/iLearnPy-
Path: iLearn/py/practice/1-basic/notes/2.Data_Structures.ipynb

![logo.png](image/logo.png)
# Data Types

### You can access this notebook on:
[colab](https://colab.com/py/), [github](https://github.com/chisomloius/iLearnPy/), [kaggle](https://kaggle.com/chisomloius/ilearnPy/), [medium](https://medium.com/@chisomloius/ilearnPy/), [web](https://chisomloius.github.io), [zindi](https://zindi.com/@chisomloius/ilearnPy/)_____no_output_____# Table of Contents

### Click on the links to go directly to specific sections on  the notebook.

1. [Import Dependencies](#dependencies)

2. [Tuples](#tuples)

3. [List](#list)

4. [Sets](#sets)

5. [Dictionaries](#dictionaries)

6. [After Thoughts](#after-thoughts)

7. [Assignment Link](#assignment-link)

8. [About Author](#about)

9. [More Info](#more-info)

Estimated time needed: 50 min

----
_____no_output_____### Python - Let's get you writing some Python Code now!

Welcome! This notebook will teach you about the tuples, list, sets and dictionaries in the Python Programming Language. By the end of this lab, you'll know their basic operations in Python, including indexing, slicing, sorting, updating, deleting and so on.
_____no_output_____

### Import Dependencies

_____no_output_____

%%expect_exception TypeError_____no_output_____

### Tuples

_____no_output_____
Welcome! This notebook will teach you about the tuples in the Python Programming Language. By the end of this lab, you'll know the basics tuple operations in Python, including indexing, slicing and sorting.
 _____no_output_____#### About this Dataset_____no_output_____#### What is in a Tuple_____no_output_____In Python, there are different data types: string, integer and float. These data types can all be contained in a tuple as follows:_____no_output__________no_output_____Now, let us create your first tuple with string, integer and float._____no_output_____

# Create your first tuple

tuple1 = ("disco", 10, 1.2)
tuple1_____no_output_____

The type of variable is a **tuple**. _____no_output_____

# Print the type of the tuple you created

type(tuple1)_____no_output_____

#### Indexing Tuples_____no_output_____ Each element of a tuple can be accessed via an index. The following table represents the relationship between the index and the items in the tuple. Each element can be obtained by the name of the tuple followed by a square bracket with the index number:_____no_output__________no_output_____We can print out each value in the tuple:_____no_output_____

# Print the variable on each index

print(tuple1[0])
print(tuple1[1])
print(tuple1[2])disco
10
1.2

We can print out the **type** of each value in the tuple:
_____no_output_____

# Print the type of value on each index

print(type(tuple1[0]))
print(type(tuple1[1]))
print(type(tuple1[2]))<class 'str'>
<class 'int'>
<class 'float'>

We can also use negative indexing. We use the same table above with corresponding negative values:_____no_output__________no_output_____We can obtain the last element as follows (this time we will not use the print statement to display the values):_____no_output_____

# Use negative index to get the value of the last element
tuple1[-1]_____no_output_____

We can display the next two elements as follows:_____no_output_____

# Use negative index to get the value of the second last element
tuple1[-2]_____no_output_____# Use negative index to get the value of the third last element
tuple1[-3]_____no_output_____

#### Concatenate Tuples_____no_output_____We can concatenate or combine tuples by using the **+** sign:_____no_output_____

# Concatenate two tuples

tuple2 = tuple1 + ("hard rock", 10)
tuple2_____no_output_____

We can slice tuples obtaining multiple values as demonstrated by the figure below:_____no_output__________no_output_____#### Slicing Tuples_____no_output_____We can slice tuples, obtaining new tuples with the corresponding elements: _____no_output_____

# Slice from index 0 to index 2
tuple2[0:3]_____no_output_____

We can obtain the last two elements of the tuple:_____no_output_____

# Slice from index 3 to index 4
tuple2[3:5]_____no_output_____

We can obtain the length of a tuple using the length command: _____no_output_____

# Get the length of tuple
len(tuple2)_____no_output_____

This figure shows the number of elements:_____no_output__________no_output_____#### Sorting Tuples_____no_output_____ Consider the following tuple:_____no_output_____

# A sample tuple
Ratings = (0, 9, 6, 5, 10, 8, 9, 6, 2, 78547, 7402, 90, 1234, 10, 1)_____no_output_____

We can sort the values in a tuple and save it to a new tuple: _____no_output_____

# Sort the tuple

RatingsSorted = sorted(Ratings)
RatingsSorted_____no_output_____

##### Nested Tuple_____no_output_____A tuple can contain another tuple as well as other more complex data types. This process is called 'nesting'. Consider the following tuple with several elements: _____no_output_____

# Create a nest tuple

NestedT =(1, 2, ("pop", "rock") ,(3,4),("disco",(1,2)))_____no_output_____

Each element in the tuple including other tuples can be obtained via an index as shown in the figure:_____no_output__________no_output_____

# Print element on each index

print("Element 0 of Tuple: ", NestedT[0])
print("Element 1 of Tuple: ", NestedT[1])
print("Element 2 of Tuple: ", NestedT[2])
print("Element 3 of Tuple: ", NestedT[3])
print("Element 4 of Tuple: ", NestedT[4])Element 0 of Tuple:  1
Element 1 of Tuple:  2
Element 2 of Tuple:  ('pop', 'rock')
Element 3 of Tuple:  (3, 4)
Element 4 of Tuple:  ('disco', (1, 2))

We can use the second index to access other tuples as demonstrated in the figure:_____no_output__________no_output_____ We can access the nested tuples :_____no_output_____

# Print element on each index, including nest indexes

print("Element 2, 0 of Tuple: ",   NestedT[2][0])
print("Element 2, 1 of Tuple: ",   NestedT[2][1])
print("Element 3, 0 of Tuple: ",   NestedT[3][0])
print("Element 3, 1 of Tuple: ",   NestedT[3][1])
print("Element 4, 0 of Tuple: ",   NestedT[4][0])
print("Element 4, 1 of Tuple: ",   NestedT[4][1])Element 2, 0 of Tuple:  pop
Element 2, 1 of Tuple:  rock
Element 3, 0 of Tuple:  3
Element 3, 1 of Tuple:  4
Element 4, 0 of Tuple:  disco
Element 4, 1 of Tuple:  (1, 2)

We can access strings in the second nested tuples using a third index:_____no_output_____

#print the first element in the first nested tuples

NestedT [2][0][0]_____no_output_____# Print the first element in the second nested tuples

NestedT[2][1][0]_____no_output_____# Print the second element in the second nested tuples

NestedT[2][1][1]_____no_output_____

 We can use a tree to visualise the process. Each new index corresponds to a deeper level in the tree:_____no_output__________no_output_____Similarly, we can access elements nested deeper in the tree with a fourth index:_____no_output_____

# Print the first element in the second nested tuples

NestedT[4][1][0]_____no_output_____# Print the second element in the second nested tuples

NestedT[4][1][1]_____no_output_____# Print the second element in the first nested tuples of the fourth index
NestedT [4][0][3]_____no_output_____

The following figure shows the relationship of the tree and the element NestedT[4][1][1]:_____no_output__________no_output_____While we can retrieve data through indexing (because a `tuple` is ordered), we cannot modify it (because a `tuple` is immutable)._____no_output_____

tuple3 = ('Dylan', 26, 167.6, True)
print(tuple3)('Dylan', 26, 167.6, True)

While we can retrieve data through indexing (because a `tuple` is ordered), we cannot modify it (because a `tuple` is immutable)._____no_output_____

print(tuple3[0])
print(tuple3[1])
print(tuple3[2])Dylan
26
167.6
%%expect_exception TypeError

tuple3[2] = 169.3_____no_output_____

#### Deleting Tuples_____no_output_____

%%expect_exception TypeError

# deletion also fails
del tuple3[-1]UsageError: Cell magic `%%expect_exception` not found.

For clarity enclose tuples with `()`, Python will assume we want a `tuple` if we don't use any symbols to enclose comma separated values as seen below._____no_output_____

tuple4 = 'Jill', 36, 162.3, True #this is a tuple
print(tuple4)
print(type(tuple4))('Jill', 36, 162.3, True)
<class 'tuple'>

This implicit `tuple` comes up most often when working with functions that return multiple outputs. For example, we might have a function that returns the first and last letter of a string._____no_output_____

def first_last(s):
    return s[0], s[-1]

chars = first_last('hello!')
print(chars)('h', '!')

#### Tuple Unpacking_____no_output_____In such cases, we'll sometimes want to store the multiple outputs in separate variables._____no_output_____

first_char, last_char = first_last('hello!')

print(first_char)
print(last_char)h
!

This syntax above is called _unpacking_. We can use it with any `tuple`, whether it was returned by a function or not._____no_output_____

def first_last(s):
    return s[0], s[-1]

first_char, last_char = first_last('computerised!')

print(first_char)
print(last_char)c
!
tuple5 = ('Dylan', 26, 167.6, True)
print(tuple5)

name, age, height, has_dog = tuple5

print(name)
print(age)
print(height)
print(has_dog)('Dylan', 26, 167.6, True)
Dylan
26
167.6
True

Both the Python `list` and `tuple` are ordered and heterogeneous. However, unlike the `list`, the `tuple` is immutable, meaning it cannot be modified after it is created. Therefore, a `list` might be better for representing data that is expected to change over the course of a program, like a to-do list. A `tuple` might be better for representing data that is expected to be fixed, like the responses of an individual subject to a survey.

#### Gotcha

One common mistake people make with immutability and especially with tuples is to assume data structures inside the tuple are immutable because the tuple is immutable.  Lets see an example.

Even though the tuple itself is immutable, we cannot change the exact objects which it contains, those objects themselves can be mutated if they are mutable.  As with anywhere mutability shows up, this requires the programmer to be careful and not assume data has not been modified in some context._____no_output_____

tuple6 = tuple([[], 'a'])
print(tuple6)
tuple6[0].append(1)
print(tuple6)([], 'a')
([1], 'a')
tuple7 = tuple([[], 'a'])
print(tuple7)
tuple7[0].append([23, 90])
print(tuple7)([], 'a')
([[23, 90]], 'a')

### Lists

_____no_output_____
Welcome! This notebook will teach you about the lists in the Python Programming Language. By the end of this lab, you'll know the basics list operations in Python, including indexing, other list operations and copy/clone list.
 _____no_output_____
About the Dataset
_____no_output_____Imagine you received album recommendations from your friends and compiled all of the recommandations into a table, with specific information about each album.

The table has one row for each movie and several columns:

- **artist** - Name of the artist
- **album** - Name of the album
- **released_year** - Year the album was released
- **length_min_sec** - Length of the album (hours,minutes,seconds)
- **genre** - Genre of the album
- **music_recording_sales_millions** - Music recording sales (millions in USD) on [SONG://DATABASE](http://www.song-database.com/)
- **claimed_sales_millions** - Album's claimed sales (millions in USD) on [SONG://DATABASE](http://www.song-database.com/)
- **date_released** - Date on which the album was released
- **soundtrack** - Indicates if the album is the movie soundtrack (Y) or (N)
- **rating_of_friends** - Indicates the rating from your friends from 1 to 10

The dataset can be seen below:

    Artist
    Album
    Released
    Length
    Genre
    Music recording sales (millions)
    Claimed sales (millions)
    Released
    Soundtrack
    Rating (friends)

    Michael Jackson
    Thriller
    1982
    00:42:19
    Pop, rock, R&B
    46
    65
    30-Nov-82

    10.0

    AC/DC
    Back in Black
    1980
    00:42:11
    Hard rock
    26.1
    50
    25-Jul-80

    8.5

    Pink Floyd
    The Dark Side of the Moon
    1973
    00:42:49
    Progressive rock
    24.2
    45
    01-Mar-73

    9.5

    Whitney Houston
    The Bodyguard
    1992
    00:57:44
    Soundtrack/R&B, soul, pop
    26.1
    50
    25-Jul-80
    Y
    7.0

    Meat Loaf
    Bat Out of Hell
    1977
    00:46:33
    Hard rock, progressive rock
    20.6
    43
    21-Oct-77

    7.0

    Eagles
    Their Greatest Hits (1971-1975)
    1976
    00:43:08
    Rock, soft rock, folk rock
    32.2
    42
    17-Feb-76

    9.5

    Bee Gees
    Saturday Night Fever
    1977
    1:15:54
    Disco
    20.6
    40
    15-Nov-77
    Y
    9.0

    Fleetwood Mac
    Rumours
    1977
    00:40:01
    Soft rock
    27.9
    40
    04-Feb-77

    9.5

_____no_output_____
_____no_output_____Below:

|Artist|Album|Relased|Length|Genre|Music recording sales (millions)|Claimed sales (millions)|Released|Soundtrack|Rating (friends)|
|------|-----|-------|------|-----|-------|-------|-----|-----|----|
|Michael Jackson|Thriller|1982|00:42:19|Pop, rock, R&B|46|65|30-Nov-82|b|10.0|
|Michael Jackson|Thriller|1982|00:42:19|Pop, rock, R&B|46|65|30-Nov-82|b|9.0|
|Michael Jackson|Thriller|1982|00:42:19|Pop, rock, R&B|46|65|30-Nov-82|b|10.0|
|Michael Jackson|Thriller|1982|00:42:19|Pop, rock, R&B|46|65|30-Nov-82|b|10.0|_____no_output_____#### Indexing_____no_output_____We are going to take a look at lists in Python. A list is a sequenced collection of different objects such as integers, strings, and other lists as well. The address of each element within a list is called an index. An index is used to access and refer to items within a list._____no_output__________no_output_____ To create a list, type the list within square brackets [ ], with your content inside the parenthesis and separated by commas. Let’s try it!_____no_output_____

# Create a list
L = ["Michael Jackson", 10.1, 1982, True]
L_____no_output_____

We can use negative and regular indexing with a list :_____no_output__________no_output_____

# Print the elements on each index

print('the same element using negative and positive indexing:\n Postive:',L[0],
'\n Negative:' , L[-3] )
print('the same element using negative and positive indexing:\n Postive:',L[1],
'\n Negative:' , L[-2]  )
print('the same element using negative and positive indexing:\n Postive:',L[2],
'\n Negative:' , L[-1]  )the same element using negative and positive indexing:
 Postive: Michael Jackson
 Negative: 10.1
the same element using negative and positive indexing:
 Postive: 10.1
 Negative: 1982
the same element using negative and positive indexing:
 Postive: 1982
 Negative: True
L[2]_____no_output_____

List Content
_____no_output_____Lists can contain strings, floats, and integers. We can nest other lists, and we can also nest tuples and other data structures. The same indexing conventions apply for nesting:
_____no_output_____

# Sample List
["Michael Jackson", 10.1, 1982, [1, 2], ("A", 1)]_____no_output_____

Python `list` are: 'ordered', 'heterogeneous', and 'mutable'. Because it is heterogeneous and mutable, the `list` is very flexible. We need to be careful about the changes we make to a `list`, because they can be very unpredictable. We could break our code or lose data!_____no_output_____#### List Operations: Slicing_____no_output_____ We can also perform slicing in lists. For example, if we want the last two elements, we use the following command:_____no_output_____

# Sample List
List = ["Michael Jackson", 10.1, 1982, "MJ" ,1]
List_____no_output_____

_____no_output_____

# List slicing
List[3:5]_____no_output_____

#### Extend Method
We can use the method extend to add new elements to the list:_____no_output_____

# Use extend to add elements to list
List = ["Michael Jackson", 10.2]
List.extend(['pop', 10])
List_____no_output_____List = ["Michael Jackson", 10.2]
List.extend([90])
List_____no_output_____

Another similar method is extendend. If we apply append instead of extend, we add one element to the list:_____no_output_____

# Use append to add elements to list
List = [ "Michael Jackson", 10.2]
List.append(['pop', 10])
List_____no_output_____

 Each time we apply a method, the list changes. If we apply extend we add two new elements to the list. The list L is then modified by adding two new elements:_____no_output_____

# Use extend to add elements to list
List = [ "Michael Jackson", 10.2]
List.extend(['pop', 10])
List_____no_output_____

If we append the list  ['a','b'] we have one new element consisting of a nested list:_____no_output_____

# Use append to add elements to list
List.append(['a','b'])
List_____no_output_____

As lists are mutable, we can change them. For example, we can change the first element as follows:_____no_output_____

# Change the element based on the index
List2 = ["disco", 10, 1.2]
print('Before change:', List2)
List2[0] = 'hard rock'
print('After change:', List2)Before change: ['disco', 10, 1.2]
After change: ['hard rock', 10, 1.2]

#### Deleting Elements in List
We can also delete an element of a list using the del command:_____no_output_____

# Delete the element based on the index

print('Before change:', List2)
del(List2[0])
print('After change:', List2)Before change: ['hard rock', 10, 1.2]
After change: [10, 1.2]

We can convert a string to a list using split.  For example, the method split translates every group of characters separated by a space into an element in a list:_____no_output_____

# Split the string, default is by space

'hard rock'.split()_____no_output_____

We can use the split function to separate strings on a specific character. We pass the character we would like to split on into the argument, which in this case is a comma.  The result is a list, and each element corresponds to a set of characters that have been separated by a comma: _____no_output_____

# Split the string by comma

'A,B,C,D'.split(",")_____no_output_____

Copy and Clone List
_____no_output_____When we set one variable List4 equal to List3; both List3 and List4 are referencing the same list in memory:_____no_output_____

# Copy (copy by reference) the list A
List3= ["hard rock", 10, 1.2]
List4 = List3
print('List3:', List3)
print('List4:', List4)List3: ['hard rock', 10, 1.2]
List4: ['hard rock', 10, 1.2]

_____no_output_____Initially, the value of the first element in List4 is set as hard rock. If we change the first element in List3 to banana, we get an unexpected side effect.  As List3 and List4 are referencing the same list, if we change list List3, then list List4 also changes. If we check the first element of List4 we get banana instead of hard rock:_____no_output_____

# Examine the copy by reference
print('List3[0]:', List3[0])
List3[0] = "banana"
print('List4[0]:', List4[0])_____no_output_____

This is demonstrated in the following figure: _____no_output__________no_output_____You can clone list  **List3** by using  the following syntax:_____no_output_____

# Clone (clone by value) the list A
List4 = List3[:]
List4_____no_output_____List4[0] = "cherry"
List4_____no_output_____List3_____no_output_____

 Variable **B** references a new copy or clone of the original list; this is demonstrated in the following figure:_____no_output__________no_output_____Now if you change A, B will not change: _____no_output_____

print('List4[0]:', List4[0])
List3[0] = "hard rock"
print('List4[0]:', List4[0])List4[0]: cherry
List4[0]: cherry

We see we successfully dealt with both a shorter list  so far.

Collections (or [**containers**](https://stackoverflow.com/questions/11575925/what-exactly-are-containers-in-python-and-what-are-all-the-python-container) as they're known in Python) can be very useful for tackling complex data problems.
Python provides several types of containers, which we will explore. Each one has different properties and structure that make them useful for specific tasks. Later in the course, we'll also introduce powerful, highly-structured containers that Python users have invented and shared with the Python community.

In the context of data science, we'll often call a collection of data that logically belongs together a **data set**, and the type of variable we use to store it in Python a **data structure**. These terms are meant to emphasize the relationships between the individual pieces of information that creates their meaning as a whole._____no_output_____#### Complex List_____no_output_____

grocery_a = 'chicken'
grocery_b = 'onions'
grocery_c = 'rice'
grocery_d = 'peppers'
grocery_e = 'bananas'

grocery_list = ['chicken', 'onions', 'rice', 'peppers', 'bananas']
print(grocery_list)

grocery_list = [grocery_a, grocery_b, grocery_c, grocery_d, grocery_e]
print(grocery_list)['chicken', 'onions', 'rice', 'peppers', 'bananas']
['chicken', 'onions', 'rice', 'peppers', 'bananas']

So far we've worked with `list`s of strings (and `range`), but we're not limited to only that data type._____no_output_____

int_list = [2, 6, 3049, 18, 37]
float_list = [3.7, 8.2, 178.245, 63.1]
mixed_list = [26, False, 'some words', 1.264]

print(int_list)
print(float_list)
print(mixed_list)[2, 6, 3049, 18, 37]
[3.7, 8.2, 178.245, 63.1]
[26, False, 'some words', 1.264]

We can store any `type` of data in a `list`. We can even put a `list` inside of a `list`._____no_output_____

list_of_lists = [['a', 'list', 'of', 'words'], [1, 5, 209], [True, True, False]]
print(list_of_lists)[['a', 'list', 'of', 'words'], [1, 5, 209], [True, True, False]]

There are very few restrictions on how we structure a list or what we put in it. This can lead to a very complicated nested structure._____no_output_____

confusing_list = [[23, 73, 50], 'some words', 12.308, [[False, True], 'more words']]
print(confusing_list)[[23, 73, 50], 'some words', 12.308, [[False, True], 'more words']]
confusing_list[0]_____no_output_____confusing_list[0][2]_____no_output_____confusing_list[3]_____no_output_____

Again, the Python `list` is _heterogeneous_ because it can hold a collection of mixed objects. This is one of the major defining properties of the Python `list`.

You may have also noticed that when we put data into a `list` in particular order, it stays in that order when we `print` or use the `list` in a `for` loop. Because `list` preserves order, we say it is _ordered_. We can use this property to retrieve particular items from a `list` based on their position (or **index**) in the list._____no_output_____

#again let's use the list
grocery_a = 'chicken'
grocery_b = 'onions'
grocery_c = 'rice'
grocery_d = 'peppers'
grocery_e = 'bananas'

grocery_list = ['chicken', 'onions', 'rice', 'peppers', 'bananas']
print(grocery_list)

grocery_list = [grocery_a, grocery_b, grocery_c, grocery_d, grocery_e]
print(grocery_list)['chicken', 'onions', 'rice', 'peppers', 'bananas']
['chicken', 'onions', 'rice', 'peppers', 'bananas']

Printing `grocery_list[2]` returned the third item in the list: 'rice'. Why did it return the third item if we asked for the item at index 2? Python `list`s are _zero-indexed_._____no_output_____

print(grocery_list[0])
print(grocery_list[1])
print(grocery_list[2])chicken
onions
rice

'again' only the last line will print. We can also retrieve a _slice_ of items from a list._____no_output_____

grocery_list[0]
grocery_list[1]
grocery_list[2]_____no_output_____# hint of the list is given as ; ['chicken', 'onions', 'rice', 'peppers', 'bananas']
print(grocery_list[1:4]) #prints the 2th to the element before the 5th element
print(grocery_list[3:]) #prints the 4th to the last element
print(grocery_list[:3]) #print from 1st to the element before the 4th element['onions', 'rice', 'peppers']
['peppers', 'bananas']
['chicken', 'onions', 'rice']

Python also has a negative indexing syntax, allowing us to access the list from the end instead of the beginning. The last element is indexed by -1._____no_output_____

print(grocery_list[-1])
print(grocery_list[-3:])bananas
['rice', 'peppers', 'bananas']

We can also slice a list using a step-size other than 1. For instance, we can slice every other item of the list, or even reverse the list by making negative steps._____no_output_____

print(grocery_list[::2])
print(grocery_list[4:1:-1])
print(grocery_list[4:1:-2])['chicken', 'rice', 'bananas']
['bananas', 'peppers', 'rice']
['bananas', 'rice']

We can of course also retrieve information from a list by using a `for` loop._____no_output_____

for item in grocery_list:
    print('we have ... %s' % item)we have ... chicken
we have ... onions
we have ... rice
we have ... peppers
we have ... bananas

While we'll usuaally use the syntax `for item in list`, sometimes we will combine a `for` loop with indexing. The `range` function (which we used in the last notebook) is useful for this. For example, we can pick out every other item in the list._____no_output_____

for i in range(0, len(grocery_list), 2):
    print(i, grocery_list[i])0 chicken
2 rice
4 bananas

The `range` function returns a sequence of integers between the first and second argument, using the third argument as the step size. Notice that the upper bound (i.e. second argument) is not included in the output._____no_output_____

print(range(0, 10, 3)) # starts at 0 and counts by 3 and exclude now
print(range(104, 100, -1)) # starts counting backwward from 104 to 101
print(range(5)) # starts at 0 and counts by 1 by defaultrange(0, 10, 3)
range(104, 100, -1)
range(0, 5)

We can also use indexing/slicing to replace items in the list._____no_output_____

grocery_list = ['chicken', 'onions', 'rice', 'peppers', 'bananas']
print(grocery_list)
grocery_list[-1] = 'oranges' # replace bananas with oranges
print(grocery_list)
grocery_list[1:3] = ['carrots', 'couscous'] #replace onions and rice with carrots and couscous
print(grocery_list)['chicken', 'onions', 'rice', 'peppers', 'bananas']
['chicken', 'onions', 'rice', 'peppers', 'oranges']
['chicken', 'carrots', 'couscous', 'peppers', 'oranges']

## modifying list
Since we can modify lists after they are created, we call them _mutable_ (the modifications are called _mutations_). Some Python data types are _immutable_, meaning once they are created they cannot be changed. We'll explore this further as we introduce more data types.

Another way we can mutate a `list` is to `append` new items._____no_output_____

grocery_list = ['chicken', 'onions', 'rice', 'peppers', 'bananas']
print(grocery_list)
grocery_list.append('squash')
print(grocery_list)
grocery_list.append(['bread', 'salt'])
print(grocery_list) # what happened?['chicken', 'onions', 'rice', 'peppers', 'bananas']
['chicken', 'onions', 'rice', 'peppers', 'bananas', 'squash']
['chicken', 'onions', 'rice', 'peppers', 'bananas', 'squash', ['bread', 'salt']]

Since lists can contain lists, we have to be careful about adding multiple items to our list. Instead of `append`, we might want to use `extend`._____no_output_____

grocery_list = ['chicken', 'onions', 'rice', 'peppers', 'bananas', 'squash']
print(grocery_list)
grocery_list.extend(['bread', 'salt'])
print(grocery_list)['chicken', 'onions', 'rice', 'peppers', 'bananas', 'squash']
['chicken', 'onions', 'rice', 'peppers', 'bananas', 'squash', 'bread', 'salt']

## Sorting List
Another mutation we can make to a list is to sort it._____no_output_____

grocery_list.sort()
print(grocery_list)['bananas', 'bread', 'chicken', 'onions', 'peppers', 'rice', 'salt', 'squash']

### Sets

_____no_output_____A Python `set` is also similar to a `list`, except it is unordered. It can store heterogeneous data and it is mutable, but what does it mean to be unordered? The simplest explanation is simply to look at an example. We can create a set by enclosing our data with curly brackets `{}`._____no_output_____
Set Content
_____no_output_____A set is a unique collection of objects in Python. You can denote a set with a curly bracket {}. Python will automatically remove duplicate items:
A Python `set` is also similar to a `list`, except it is unordered. It can store heterogeneous data and it is mutable, but what does it mean to be unordered? The simplest explanation is simply to look at an example. We can create a set by enclosing our data with curly brackets `{}`._____no_output_____

# Create a set
set1 = {"pop", "rock", "soul", "hard rock", "rock", "R&B", "rock", "disco"}
set1_____no_output_____

The process of mapping is illustrated in the figure:_____no_output__________no_output_____ You can also  create a set from a list as follows:_____no_output_____

# Convert list to set

album_list = [ "Michael Jackson", "Thriller", 1982, "00:42:19", \
              "Pop, Rock, R&B", 46.0, 65, "30-Nov-82", None, 10.0]
album_set = set(album_list)
album_set_____no_output_____

Now let us create a set of  genres:_____no_output_____

# Convert list to set

music_genres = set(["pop", "pop", "rock", "folk rock", "hard rock", "soul", \
                    "progressive rock", "soft rock", "R&B", "disco"])
music_genres_____no_output_____

Set Operations
 _____no_output_____Let us go over set operations, as these can be used to change the set. Consider the set A:_____no_output_____

# Sample set

A = set(["Thriller", "Back in Black", "AC/DC"])
A_____no_output_____

 We can add an element to a set using the add() method: _____no_output_____

# Add element to set

A.add("NSYNC")
A_____no_output_____

 If we add the same element twice, nothing will happen as there can be no duplicates in a set:
_____no_output_____

# Try to add duplicate element to the set

A.add("NSYNC")
A_____no_output_____

 We can remove an item from a set using the remove method:_____no_output_____

# Remove the element from set

A.remove("NSYNC")
A_____no_output_____

 We can verify if an element is in the set using the in command:_____no_output_____

# Verify if the element is in the set

"AC/DC" in A_____no_output_____

Sets Logic Operations
_____no_output_____Remember that with sets you can check the difference between sets, as well as the symmetric difference, intersection, and union:_____no_output_____ Consider the following two sets:_____no_output_____

# Sample Sets

album_set1 = set(["Thriller", 'AC/DC', 'Back in Black'])
album_set2 = set([ "AC/DC", "Back in Black", "The Dark Side of the Moon"])_____no_output_____

_____no_output_____

# Print two sets

album_set1, album_set2_____no_output_____

As both sets contain AC/DC and Back in Black we represent these common elements with the intersection of two circles._____no_output__________no_output_____You can find the intersect of two sets as follow using &:_____no_output_____

# Find the intersections

intersection = album_set1 & album_set2
intersection_____no_output_____

You can find all the elements that are only contained in album_set1 using the difference method:_____no_output_____

# Find the difference in set1 but not set2

album_set1.difference(album_set2)  _____no_output_____album_set2.difference(album_set1) _____no_output_____

You only need to consider elements in album_set1; all the elements in album_set2, including the intersection, are not included._____no_output__________no_output_____The elements in album_set2 but not in album_set1 is given by:_____no_output_____

album_set2.difference(album_set1)  _____no_output_____

_____no_output_____You can also find the intersection of album_list1 and album_list2, using the intersection method:_____no_output_____The union corresponds to all the elements in both sets, which is represented by coloring both circles:_____no_output__________no_output_____ The union is given by:_____no_output_____

# Find the union of two sets

album_set1.union(album_set2)_____no_output_____

And you can check if a set is a superset or subset of another set, respectively, like this:_____no_output_____

# Check if superset

set(album_set1).issuperset(album_set2)   _____no_output_____# Check if subset

set(album_set2).issubset(album_set1)     _____no_output_____

Here is an example where issubset() and issuperset() return true:_____no_output_____

# Check if subset
set({"Back in Black", "AC/DC"}).issubset(album_set1) _____no_output_____# Check if superset
album_set1.issuperset({"Back in Black", "AC/DC"})   _____no_output_____

Consider the list A = [1, 2, 2, 1] and set B = set([1, 2, 2, 1]), does sum(A) = sum(B) _____no_output_____

# Write your code below and press Shift+Enter to execute
A = [1, 2, 2, 1]
B = set([1, 2, 2, 1])
print("the sum of A is:", sum(A))
print("the sum of B is:", sum(B))the sum of A is: 6
the sum of B is: 3

Create a new set album_set3 that is the union of album_set1 and album_set2:_____no_output_____

# Write your code below and press Shift+Enter to execute

album_set1 = set(["Thriller", 'AC/DC', 'Back in Black'])
album_set2 = set([ "AC/DC", "Back in Black", "The Dark Side of the Moon"])_____no_output_____album_set3 = album_set1.union(album_set2)
album_set3_____no_output_____

Find out if album_set1 is a subset of album_set3:_____no_output_____

# Write your code below and press Shift+Enter to execute
album_set1.issubset({"Back in Black", "AC/DC"}) _____no_output_____

#### More set methods
However, we can still add and delete items from a set._____no_output_____

print(example_set)
print(example_set.pop())
print(example_set)

example_set.add('True')
print(example_set)
example_set.update([58.1, 'brown'])
print(example_set){'Dylan', True, 26, 167.6}
Dylan
{True, 26, 167.6}
{True, 167.6, 'True', 26}
{True, 58.1, 167.6, 'True', 'brown', 26}
# Adding an element
example_set.add(10)
example_set_____no_output_____example_set.add(10)
example_set_____no_output_____

The `add` method of a `set` works similarly to the `append` method of a `list`. The `update` method of a `set` works similarly to the `extend` method of a `list`.

_**Why is `set` useful?**_

It seems strange that we might want an _unordered_ data structure. We can't access or modify the data through indexing. How does giving up order benefit us? The answer is that it gives us flexibility about how the data is stored in memory, and that flexibility can make data retrieval much faster.

Imagine we have ten boxes and ten piles of money. We put the ten piles of money in the ten boxes. Now say we want to find the box that has \$5.37 in it. We don't know which box this is, so we start with the first box and check. If it isn't in the first box, we move on to the second box. We keep checking boxes until we find it. This might take awhile.

![hash_illustration.png](attachment:992d43fb-a799-4ec0-a275-abe1c062f888.png)

Now imagine we have the same ten piles of money, but we have 31 boxes. Instead of putting each pile of money into the boxes in order, instead put each pile into a box based on the amount of money in the pile. First we multiply the amount of money by 100, and then take modulus division by 31. This gives the number of the box we should put the pile of money in._____no_output_____

piles = [2.83, 8.23, 9.38, 10.23, 25.58, 0.42, 5.37, 28.10, 32.14, 7.31]_____no_output_____def hash_function(x):
    return int(x * 100 % 31)

[hash_function(pile) for pile in piles]_____no_output_____def hash_function(x):
    return int(x * 100 % 31)

[hash_function(pile) for pile in piles]_____no_output_____#Now say we want to find the box with \$5.37 in it. We don't have to search through box after box. We can compute:
print(int(5.37 * 100 % 31))_____no_output_____

![hash_illustration.png](attachment:5c826384-08e0-473b-ac2e-37c8a7a21fda.png)

Box number 10 contains the \$5.37 pile.

This technique of assigning boxes (i.e. memory) based on the object it contains is called **hashing**. It makes searching for data very fast (as we've illustrated), but at the cost of increase memory allocation (we needed more boxes). It also means that we cannot assign an order to the objects as they are stored in memory.

Hashing also puts two major restrictions on the `set`. First of all, objects in a `set` must be immutable. If an object were to change, its position in memory would no longer correspond with its **hash**. Secondly, the objects in a `set` must be unique. Identical objects end up with the same hash. Since we can't store multiple objects in the same chunk of memory, we simply discard any duplicates.

This second restriction means we can use a `set` to easily determine the unique objects in a `list` or `tuple`._____no_output_____suppose we have 50 boxes with with we want to put in 10 piles of money, we can use the code below_____no_output_____

def hash_function(x):
    return int(x * 100 % 50)

[hash_function(pile) for pile in piles]_____no_output_____(print (int(7.3 *100 % 50)))_____no_output_____

Unique function for set is hashing as displayed below:_____no_output_____

new_list = 'my string'
hash(new_list)_____no_output_____print(set([23, 609, 348, 10, 5, 23, 340, 82]))
print(set(('a', 'b', 'q', 'c', 'c', 'd', 'r', 'a'))){609, 5, 10, 82, 340, 23, 348}
{'q', 'b', 'c', 'a', 'd', 'r'}
student_a_courses = {'history', 'english', 'biology', 'theatre'}
student_b_courses = {'biology', 'english', 'mathematics', 'computer science'}

print(student_a_courses.intersection(student_b_courses))
print(student_a_courses.union(student_b_courses))
print(student_a_courses.difference(student_b_courses))
print(student_b_courses.difference(student_a_courses))
print(student_a_courses.symmetric_difference(student_b_courses)){'biology', 'english'}
{'theatre', 'mathematics', 'history', 'english', 'biology', 'computer science'}
{'theatre', 'history'}
{'computer science', 'mathematics'}
{'theatre', 'computer science', 'mathematics', 'history'}

### Dictionaries

_____no_output_____
What are Dictionaries?
_____no_output__________no_output_____An example of a Dictionary Dict:_____no_output_____

# Create the dictionary

Dict = {"key1": 1, "key2": "2", "key3": [3, 3, 3], "key4": (4, 4, 4), ('key5'): 5, (0, 1): 6}
Dict_____no_output_____

The keys can be strings:_____no_output_____Keys can also be any immutable object such as a tuple: _____no_output_____ Each key is separated from its value by a colon ":".  Commas separate the items, and the whole dictionary is enclosed in curly braces. An empty dictionary without any items is written with just two curly braces, like this  "{}"._____no_output_____

# Create a sample dictionary

release_year_dict = {"Thriller": "1982", "Back in Black": "1980", \
                    "The Dark Side of the Moon": "1973", "The Bodyguard": "1992", \
                    "Bat Out of Hell": "1977", "Their Greatest Hits (1971-1975)": "1976", \
                    "Saturday Night Fever": "1977", "Rumours": "1977"}
release_year_dict_____no_output_____

In summary, like a list, a dictionary holds a sequence of elements. Each element is represented by a key and its corresponding value. Dictionaries are created with two curly braces containing keys and values separated by a colon. For every key, there can only be one single value, however,  multiple keys can hold the same value. Keys can only be strings, numbers, or tuples, but values can be any data type._____no_output_____It is helpful to visualize the dictionary as a table, as in the following image. The first column represents the keys, the second column represents the values._____no_output__________no_output_____
Keys
_____no_output_____You can retrieve the values based on the names:_____no_output_____

# Get value by keys
release_year_dict['Thriller'] _____no_output_____

This corresponds to:
_____no_output__________no_output_____Similarly for The Bodyguard_____no_output_____

# Get value by key

release_year_dict['The Bodyguard']_____no_output_____

_____no_output_____Now let you retrieve the keys of the dictionary using the method release_year_dict():_____no_output_____

# Get all the keys in dictionary

release_year_dict.keys()_____no_output_____

You can retrieve the values using the method  values():_____no_output_____

# Get all the values in dictionary

release_year_dict.values() _____no_output_____

We can add an entry:_____no_output_____

# Append value with key into dictionary

release_year_dict['Graduation'] = '2007'
release_year_dict_____no_output_____

We can delete an entry:   _____no_output_____

# Delete entries by key

del(release_year_dict['Thriller'])
del(release_year_dict['Graduation'])
release_year_dict_____no_output_____

 We can verify if an element is in the dictionary: _____no_output_____

# Verify the key is in the dictionary

'The Bodyguard' in release_year_dict_____no_output_____

We can o understand the Python `dict`, let's start again with the Python `list`._____no_output_____

me = ['chisom', 25, 175.6, 60.1, 'black', 'grey', True]_____no_output_____

This `list` describes me: my name, my age, my height (in centimeters), my weight (in kilograms), my hair color, my eye color, and whether or not I have a dog. We know we can access this information individually by index._____no_output_____

print('My name is %s' % me[0])
print('I have %s hair' % me[4])My name is chisom
I have black hair

It would be easy to get mixed up about which data is which (for example, which `'brown'` is hair color and which is eye color?), or where I should find it (will age always be at index 1?).

A better solution would be a data structure where we could index using meaningful values. For example instead of using `me[0]` to recover `Dylan`, I could use `me['name']`. Instead of hair color being `me[4]`, it could be `me['hair']`. This feature is the central characteristic of the Python `dict`._____no_output_____

me_dict = {'name': 'Chisom', 'age': 25, 'height': 170.1, 'weight': 60.1,
           'hair': 'black', 'eyes': 'black', 'has dog': True}

print('My name is %s' % me_dict['name'])
print('I have %s hair' % me_dict['hair'])
print('I will be %d years old next month' % me_dict['age'])My name is Chisom
I have black hair
I will be 25 years old next month
me_dict.values()_____no_output_____me_dict.keys()_____no_output_____

Instead of calling `'name'` and '`hair'` the index, we call them **keys**. Each key is associated with a **value** in a **key-value pair**. We can see the key-value pairs in the `{}` syntax used for creating a `dict`.
Each key-value pair is separated by a comma, and within a pair the key and value are separated by a colon `:`._____no_output_____### Exercises
1. When might a `dict` be more useful than a `list`?
2. Compare the flexibility of a `dict` which contains other `dict` object to that of a multi dimensional array._____no_output_____Answers
1. A 'dict' is more useful when want to reference each elemebt in a list and hence for easy reference we use a 'dict' rather than a list.

2.

|Dictionary|Multi-dimensional array|
|---|---|
|they are very effective in xyz| they are not effective in xyz|
_____no_output_____### `zip`

The `zip` function can be very handy for creating a `dict`. Let's go back to the `list` we made before that contained all the values describing me. We'll make a second `list` containing all the keys we would want for putting these values in a dictionary_____no_output_____

value_list = me
key_list = ['name', 'age', 'height', 'weight', 'hair', 'eyes', 'has dog']

print(value_list)
print(key_list)['chisom', 25, 175.6, 60.1, 'black', 'grey', True]
['name', 'age', 'height', 'weight', 'hair', 'eyes', 'has dog']

Currently we have two lists: one of values and one of keys. They have no relationship to each other within Python, but we can see that they belong together logically. How do we combine them in Python? By using the `zip` function._____no_output_____

key_value_pairs = list(zip(key_list, value_list))
print(key_value_pairs)[('name', 'chisom'), ('age', 25), ('height', 175.6), ('weight', 60.1), ('hair', 'black'), ('eyes', 'grey'), ('has dog', True)]

We now have a list of tuples. We interpret the first element of each tuple as a key, and the second element as a value. We can turn this list of tuples directly into a `dict`._____no_output_____

me_dict = dict(key_value_pairs)
print(me_dict){'name': 'chisom', 'age': 25, 'height': 175.6, 'weight': 60.1, 'hair': 'black', 'eyes': 'grey', 'has dog': True}

You may have noticed that even though our list of tuples began with `('name', 'Dylan')`, when we printed `me_dict` it started with `'eyes': 'brown'`. If you guessed this means that a `dict` is unordered, you are correct! The keys are hashed to assign key-value pairs to memory. Therefore, keys must be immutable and unique, similar to the elements of a `set`. However, values don't have these restrictions._____no_output_____### Some ways we can't write a 'dict'_____no_output_____

%%expect_exception TypeError

# this doesn't work
invalid_dict = {[1, 5]: 'a', 5: 23}UsageError: Cell magic `%%expect_exception` not found.

The `dict` is also mutable. We can add new key-value pairs by simple assignment._____no_output_____

print(me_dict)
me_dict['favorite book'] =  'The Little Prince'
print(me_dict)_____no_output_____

We can also use `update`, similar to the way we used it for a `set`, except now with key-value pairs._____no_output_____

print(me_dict)
me_dict.update({'favorite color': 'orange', 'siblings': 3, 'nieces/nephews': 75})
print(me_dict){'name': 'chisom', 'age': 25, 'height': 175.6, 'weight': 60.1, 'hair': 'black', 'eyes': 'grey', 'has dog': True}
{'name': 'chisom', 'age': 25, 'height': 175.6, 'weight': 60.1, 'hair': 'black', 'eyes': 'grey', 'has dog': True, 'favorite color': 'orange', 'siblings': 3, 'nieces/nephews': 75}

We can replace or delete key-value pairs from the `dict`._____no_output_____

del me_dict['favorite book']
print(me_dict)_____no_output_____print(me_dict.pop('siblings'))
print(me_dict)_____no_output_____

Because the `dict` uses hashing, searching it is very fast (as with `set`). Sometimes dictionaries are referred to as **lookup tables** or **hash tables**. It is incredibly useful for referencing data through meaningful keys. While the data in a `dict` is unordered, it remains organized by the keys. We can retrieve a list of the keys and values directly, or as key-value pairs, using the appropriate methods of `dict`._____no_output_____

print(me_dict.keys())
print(me_dict.values())
print(me_dict.items())_____no_output_____

## Switching data structures
Each of the containers we've introduced has different properties and characteristics. Sometimes we will want to change one data structure into another to take advantage of these differences. We've already seen some methods for transforming a `dict` into a `list` of `tuple`s or vice versa. We can easily transform between `list`, `tuple`, and `set`.m_____no_output_____

example_list = ['a', 'b', 23, 10, True, 'a', 10]
example_tuple = tuple(example_list)
example_set = set(example_tuple)
example_list = list(example_set)

print(example_tuple)
print(example_set)
print(example_list) # note we lost the duplicates because of set('a', 'b', 23, 10, True, 'a', 10)
{'a', True, 10, 'b', 23}
['a', True, 10, 'b', 23]

## Search

We discussed the idea of searching for data in our data structures when describing what makes `set` (and `dict`) so special. What does search look like in Python? We search for data using the keyword `in`._____no_output_____

print(example_list)
print('a' in example_list)
print('c' in example_list)['a', True, 10, 'b', 23]
True
False

When dealing with a `dict`, we can search keys, but not values._____no_output_____

print(me_dict)
print('hair' in me_dict)
print('has cat' in me_dict)
print('brown' in me_dict){'name': 'chisom', 'age': 25, 'height': 175.6, 'weight': 60.1, 'hair': 'black', 'eyes': 'grey', 'has dog': True}
True
False
False

Searching for keys is important in dictionaries so that we don't accidentally try to retrieve a key-value pair that doesn't exist._____no_output_____

me_dict.update({'has cat':'ukra'})_____no_output_____if 'has dog' in me_dict:
    print('Has dog: %s' % me_dict['has dog'])
else:
    print(None)

if 'has cat' in me_dict:
    print('Has cat: %s' % me_dict['has cat'])
else:
    print(None)Has dog: True
Has cat: ukra
# can use get method for same results

print('Has dog: %s' % me_dict.get('has dog'))
print('Has cat: %s' % me_dict.get('has cat'))_____no_output_____

## Sorting

Since a `tuple` is immutable, can we sort it? Or is that a mutation? What would it mean to sort a `set` or a `dict`, which has no order?

Out of the data structures we've studied so far, only `list` has a `sort` method. However, Python also has a `sorted` function, which will create a sorted `list` out of other data structures. By default `sorted` applied to a `dict` makes a `list` of sorted keys. We must use the `items` method if we want our output to be key-value pairs._____no_output_____

print(sorted(map(str, example_tuple)))
print(sorted(map(str, example_set)))
print(sorted(me_dict.items()))
print(sorted(me_dict))['10', '10', '23', 'True', 'a', 'a', 'b']
['10', '23', 'True', 'a', 'b']
[('age', 25), ('eyes', 'grey'), ('favorite color', 'orange'), ('hair', 'black'), ('has cat', 'ukra'), ('has dog', True), ('height', 175.6), ('name', 'chisom'), ('nieces/nephews', 75), ('siblings', 3), ('weight', 60.1)]
['age', 'eyes', 'favorite color', 'hair', 'has cat', 'has dog', 'height', 'name', 'nieces/nephews', 'siblings', 'weight']

## Iteration

As we've seen in some examples already, it will often be useful to iterate through a data structure, whether to execute some task based on the information contained or to transform or analyze a data set. We will most often use `for` loops to iterate over data structures. With a `list`, `tuple`, or `set` the elements of the container are returned one after another. With a `dict` things are a little more complicated: do we want to iterate over keys, values, or key-value pairs?_____no_output_____

# by default we iterate over keys of a dict
for f in me_dict:
    print(f)_____no_output_____# to iterate over values...
for y in me_dict.values():
    print(y)_____no_output_____# or to iterate over key-value pairs...
for k, v in me_dict.items():
    print('%s:%s' % (k, v))_____no_output_____

Notice we used `tuple` unpacking in the `for` loop in the last example!_____no_output_____

### After Thoughts ??

_____no_output_____What do you make of this rather simplistic structures. They are the backbone of learning python._____no_output_____

### Assignment Link

_____no_output_____Now we would try out some practical examples with what we have learnt so far ! Let us try out this [notebook](https://typei.com)_____no_output_____

### About this Instructor:

 ChisomLoius is very passionate about Data Analysis and Machine Learning and does lot of free lance teaching and learning. Holding a B.Eng. in Petroleum Engineering, my focused is leveraging the knowledge of Data Science and Machine Learning to help build solutions in Education and High Tech Security. I currently work as a Petrochemist.

_____no_output_____

### More Info

_____no_output_____

 Visit our website, or further enquire more information via our email.

Copyright © 2022 TechOrigin. This notebook and its source code are released under the terms of the MIT License.
_____no_output_____