# Notebook from barakcohenlab/CRX-Information-Content
Path: 07_figures.ipynb

# Generate figures
This notebook will generate all main and supplemental figures that are not already generated by notebook 4._____no_output_____

import os
import sys
import pickle

import matplotlib as mpl
import matplotlib.patches as mpatches
import matplotlib.pyplot as plt
from mpl_toolkits.axes_grid1 import make_axes_locatable
import numpy as np
import pandas as pd
from scipy import stats
import logomaker
from pybedtools import BedTool
from IPython.display import display

sys.path.insert(0, "utils")
from utils import fasta_seq_parse_manip, gkmsvm, modeling, plot_utils, predicted_occupancy, sequence_annotation_processing

data_dir = os.path.join("Data")
seq_bins_dir = os.path.join(data_dir, "ActivityBins")
models_dir = os.path.join("Models", "StrongEnhancerVsSilencer")
figures_dir = os.path.join("Figures")
all_seqs = fasta_seq_parse_manip.read_fasta(os.path.join(data_dir, "library1And2.fasta"))
# Drop scrambled sequences
all_seqs = all_seqs[~(all_seqs.index.str.contains("scr"))]_____no_output_____plot_utils.set_manuscript_params()_____no_output_____

## Load in all the data_____no_output_____

# Mapping activity class to a color
color_mapping = {
    "Silencer": "#e31a1c",
    "Inactive": "#33a02c",
    "Weak enhancer": "#a6cee3",
    "Strong enhancer": "#1f78b4",
    np.nan: "grey"
}
color_mapping = pd.Series(color_mapping)

# Sort order for the four activity bins
class_sort_order = ["Silencer", "Inactive", "Weak enhancer", "Strong enhancer"]

# MPRA measurements
activity_df = pd.read_csv(os.path.join(data_dir, "wildtypeMutantPolylinkerActivityAnnotated.txt"), sep="\t", index_col=0)
activity_df["group_name_WT"] = sequence_annotation_processing.to_categorical(activity_df["group_name_WT"])
activity_df["group_name_MUT"] = sequence_annotation_processing.to_categorical(activity_df["group_name_MUT"])

# TF occupancy metrics, also separate out the WT sequences
occupancy_df = pd.read_csv(os.path.join(data_dir, "predictedOccupancies.txt"), sep="\t", index_col=0)
wt_occupancy_df = occupancy_df[occupancy_df.index.str.contains("WT$")].copy()
wt_occupancy_df = sequence_annotation_processing.remove_mutations_from_seq_id(wt_occupancy_df)
wt_occupancy_df = wt_occupancy_df.loc[activity_df.index]
n_tfs = len(wt_occupancy_df.columns)

# Entropy and related metrics, also separate out WT and MUT
entropy_df = pd.read_csv(os.path.join(data_dir, "entropyDiversityOccupancy.txt"), sep="\t", index_col=0)
wt_entropy_df = entropy_df[entropy_df.index.str.contains("WT$")].copy()
wt_entropy_df = sequence_annotation_processing.remove_mutations_from_seq_id(wt_entropy_df)
wt_entropy_df = wt_entropy_df.loc[activity_df.index]

mut_entropy_df = entropy_df[entropy_df.index.str.contains("MUT")].copy()
mut_entropy_df = sequence_annotation_processing.remove_mutations_from_seq_id(mut_entropy_df)
mut_entropy_df = mut_entropy_df.loc[activity_df.index]

# Occupancy logistic regression (need this to sort by feature importance)
occ_clf = pickle.load(open(os.path.join(models_dir, "logit_model.pkl"), "rb"))

# Axis ticks for Rho assay
rho_ticks = np.arange(-10, 7, 2)

# xaxis for ROC/PR curves
modeling_xaxis = np.linspace(0, 1, 100)

# Model performance metrics with a helper function
def read_metrics(filename, input_dir=models_dir):
    return pd.read_csv(os.path.join(input_dir, filename), sep="\t", header=None)

# Cross-validated metrics
svm_tpr_cv = read_metrics("svmTprCv.txt")
svm_prec_cv = read_metrics("svmPrecCv.txt")
crx_occ_tpr_cv = read_metrics("crxTprCv.txt")
crx_occ_prec_cv = read_metrics("crxPrecCv.txt")
entropy_tpr_cv = read_metrics("entropyTprCv.txt")
entropy_prec_cv = read_metrics("entropyPrecCv.txt")
occ_tpr_cv = read_metrics("occupancyTprCv.txt")
occ_prec_cv = read_metrics("occupancyPrecCv.txt")

# White 2013 test set
white_svm_tpr = read_metrics("whiteSvmTpr.txt").squeeze()
white_svm_prec = read_metrics("whiteSvmPrec.txt").squeeze()
white_occ_tpr = read_metrics("whiteOccTpr.txt").squeeze()
white_occ_prec = read_metrics("whiteOccPrec.txt").squeeze()

# Random PWMs background
random_occ_tprs = read_metrics("randomFeaturesMeanTprs.txt")
random_occ_precs = read_metrics("randomFeaturesMeanPrecs.txt")

# Strong enhancer vs. inactive for info content
inactive_entropy_tpr_cv = read_metrics("entropyTprCv.txt", input_dir=os.path.join("Models", "StrongEnhancerVsInactive"))
inactive_entropy_prec_cv = read_metrics("entropyPrecCv.txt", input_dir=os.path.join("Models", "StrongEnhancerVsInactive"))

# PWMs
pwms = predicted_occupancy.read_pwm_files(os.path.join("Data", "Downloaded", "Pwm", "photoreceptorAndEnrichedMotifs.meme"))
pwms = pwms.rename(lambda x: x.split("_")[0])
motif_len = pwms.apply(len)
mu = 9
ewms = pwms.apply(predicted_occupancy.ewm_from_letter_prob).apply(predicted_occupancy.ewm_to_dict)

# Reverse compliment RAX for display purposes
rax = pwms["RAX"].copy()
rax = rax[::-1].reset_index(drop=True)
rax_rc = rax.copy()
rax_rc["A"] = rax["T"]
rax_rc["C"] = rax["G"]
rax_rc["G"] = rax["C"]
rax_rc["T"] = rax["A"]
pwms["RAX"] = rax_rc_____no_output_____

## Figure 1: Activity of putative cis-regulatory sequences with CRX motifs._____no_output_____

# Can only plot points that were detected in DNA
activity_measured_wt_df = activity_df[activity_df["expression_log2_WT"].notna()]
print("Frequency of each activity bin in WT sequences:")
display(activity_measured_wt_df["group_name_WT"].value_counts(normalize=True, dropna=False, sort=False))

# Count frequency of activity bins for CRX bound/unbound
crx_bound_grouper = activity_df.groupby("crx_bound")
chip_activity_bin_freqs = crx_bound_grouper["group_name_WT"].value_counts().unstack()
chip_activity_bin_freqs = chip_activity_bin_freqs[class_sort_order].rename(index=lambda x: "ChIP-seq" if x else "ATAC-seq")

# Different ways to format group names
chip_group_names_with_n = [f"{i}\nn={j.sum()}" for i, j in chip_activity_bin_freqs.iterrows()]
chip_group_names_with_n_oneline = [" ".join(i.split()) for i in chip_group_names_with_n]
chip_group_names = chip_activity_bin_freqs.index.values
chip_group_count = [j.sum() for i, j in chip_activity_bin_freqs.iterrows()]

# Display the data behind Fig 1b
print("Frequency of activity bins vs. CRX binding status:")
display(chip_activity_bin_freqs)

# Test if CRX binding and inactive status is independent
chip_group_inactive_counts = crx_bound_grouper["group_name_WT"].apply(lambda x: (x == "Inactive").value_counts()).unstack()
oddsratio, pval = stats.fisher_exact(chip_group_inactive_counts)
# Take inverse of odds ratio to match language of manuscript and be more intuitive to the reader
print(f"ChIP-seq status is independent of if a sequence is inactive, Fisher's exact test p={pval:.0e}, odds ratio={1/oddsratio:.2f}")

# Same for strong enhancer
chip_group_inactive_counts = crx_bound_grouper["group_name_WT"].apply(lambda x: (x == "Strong enhancer").value_counts()).unstack()
oddsratio, pval = stats.fisher_exact(chip_group_inactive_counts)
# Take inverse of odds ratio to match language of manuscript and be more intuitive to the reader
print(f"ChIP-seq status is independent of if a sequence is inactive, Fisher's exact test p={pval:.0e}, odds ratio={oddsratio:.2f}")

# Row-normalize the counts
chip_activity_bin_freqs = chip_activity_bin_freqs.div(chip_activity_bin_freqs.sum(axis=1), axis=0)
display(chip_activity_bin_freqs)

# Setup for some downstream stuff
wt_activity_grouper = activity_df.groupby("group_name_WT")
wt_activity_names_oneline = ["Silencer", "Inactive", "Weak enh.", "Strong enh."]
wt_activity_count = [len(j) for i, j in wt_activity_grouper]

# Predicted CRX occupancy vs. WT group
wt_occupancy_grouper = wt_occupancy_df.groupby(activity_df["group_name_WT"])
wt_occupancy_grouper_crx = wt_occupancy_grouper["CRX"]
print("Predicted CRX occupancies:")
display(wt_occupancy_grouper_crx.describe())

# Statistics for differences in CRX occupancy between groups
ustat, pval = stats.mannwhitneyu(wt_occupancy_grouper_crx.get_group("Strong enhancer"), wt_occupancy_grouper_crx.get_group("Inactive"), alternative="two-sided")
print(f"Strong enhancers and inactive sequences have the same CRX occupancy, Mann-Whitney U test p = {pval:.0e} U = {ustat:.2f}")
ustat, pval = stats.mannwhitneyu(wt_occupancy_grouper_crx.get_group("Silencer"), wt_occupancy_grouper_crx.get_group("Inactive"), alternative="two-sided")
print(f"Silencers and inactive sequences have the same CRX occupancy, Mann-Whitney U test p = {pval:.0e}, U = {ustat:.2f}")Frequency of each activity bin in WT sequences:
# Setup the figure
gs_kw = dict(width_ratios=[1, 3])
fig, ax_list = plt.subplots(nrows=2, ncols=2, figsize=(6, 8), gridspec_kw=gs_kw)
gs = ax_list[0, 0].get_gridspec()
for ax in ax_list[0, :]:
    ax.remove()

axbig = fig.add_subplot(gs[0, :])
ax = axbig

# 1a: Volcano plot
fig = plot_utils.volcano_plot(activity_measured_wt_df, "expression_log2_WT", "expression_qvalue_WT",
                             activity_measured_wt_df["plot_color_WT"], xaxis_label="log2 Enhancer Activity/Rho",
                             yaxis_label="-log10 FDR", xline=-np.log10(0.05), yline=[-1, 1],
                             xticks=rho_ticks[1:], figax=(fig, ax))
ax.set_yticks(np.arange(5))
plot_utils.add_letter(ax, -0.125, 1, "a")

# 1b: CRX binding status vs. activity classes
ax = ax_list[1, 0]
fig = plot_utils.stacked_bar_plots(chip_activity_bin_freqs, "Fraction of group", chip_group_names, color_mapping, figax=(fig, ax), vert=True)
ax.set_yticks(np.linspace(0, 1, 6))
plot_utils.rotate_ticks(ax.get_xticklabels())

# Add ticks above to show the n
ax_twin = ax.twiny()
ax_twin.set_xticks(ax.get_xticks())
ax_twin.set_xlim(ax.get_xlim())
ax_twin.set_xticklabels(chip_group_count, fontsize=10, rotation=45)
plot_utils.add_letter(ax, -0.7, 1.03, "b")

# 1c: Predicted CRX occupancy of different groups
ax = ax_list[1, 1]
fig = plot_utils.violin_plot_groupby(wt_occupancy_grouper_crx, "Predicted CRX occupancy", class_names=wt_activity_names_oneline, class_colors=color_mapping, figax=(fig, ax))
ax.set_yticks(np.linspace(0, 8, 5))
plot_utils.rotate_ticks(ax.get_xticklabels())

# Add ticks above to show the n
ax_twin = ax.twiny()
ax_twin.set_xticks(ax.get_xticks())
ax_twin.set_xlim(ax.get_xlim())
ax_twin.set_xticklabels(wt_activity_count, fontsize=10, rotation=45)
plot_utils.add_letter(ax, -0.2, 1.03, "c")

plot_utils.save_fig(fig, os.path.join(figures_dir, "figure1"), timestamp=False)_____no_output_____

## Figure 2: Strong enhancers contain a diverse array of motifs_____no_output_____### Panels a and b
Also Supplemental Figure 3: Precision recall curve for strong enhancer vs. silencer classifiers_____no_output_____

fig, ax_list = plot_utils.setup_multiplot(2, sharex=False, sharey=False)
# Separate figure handle for the PR curves
fig_pr, ax_pr = plt.subplots()

# 2a and supplemental figure 3: ROC and PR curves with SVM, TF occupancies, CRX occupancy
model_data = [ # (TPR, precision, name, color)
    (svm_tpr_cv, svm_prec_cv, "SVM", "black"),
    (occ_tpr_cv, occ_prec_cv, f"{n_tfs} TFs", "#E69B04"),
    (crx_occ_tpr_cv, crx_occ_prec_cv, "CRX", "#009980")
]

model_tprs, model_precs, model_names, model_colors = zip(*model_data)
prc_chance = activity_df["group_name_WT"].str.contains("Strong").sum() / activity_df["group_name_WT"].str.contains("Strong|Silencer").sum()

# Generate figures
_, model_aurocs, model_aurocs_std, model_auprs, model_auprs_std = plot_utils.roc_pr_curves(
    modeling_xaxis, model_tprs, model_precs, model_names, model_colors=model_colors,
    prc_chance=prc_chance, figax=([fig, fig_pr], [ax_list[0], ax_pr])
)
ax_list[0].set_xticks(np.linspace(0, 1, 6))
plot_utils.add_letter(ax_list[0], -0.25, 1.03, "a")

# Display model metrics
print("Model metrics:")
for name, auroc, auroc_std, aupr, aupr_std in zip(model_names, model_aurocs, model_aurocs_std, model_auprs, model_auprs_std):
    print(f"{name}\tAUROC={auroc:.3f}+/-{auroc_std:.3f}\tAUPR={aupr:.3f}+/-{aupr_std:.3f}")

# Calculate total predicted occupancy of each class
wt_entropy_grouper = wt_entropy_df.groupby(activity_df["group_name_WT"])
print("Total predicted occupancy of all TFs in each group:")
display(wt_entropy_grouper["total_occupancy"].describe())

# 2b: Total predicted occupancy of each class
ax = ax_list[1]
fig = plot_utils.violin_plot_groupby(wt_entropy_grouper["total_occupancy"], "Total predicted TF occupancy", class_names=wt_activity_names_oneline, class_colors=color_mapping, figax=(fig, ax))
plot_utils.rotate_ticks(ax.get_xticklabels())
plot_utils.add_letter(ax, -0.25, 1.03, "b")

# Add ticks above to show the n
ax_twin = ax.twiny()
ax_twin.set_xticks(ax.get_xticks())
ax_twin.set_xlim(ax.get_xlim())
ax_twin.set_xticklabels(wt_activity_count, fontsize=10, rotation=45)

plot_utils.save_fig(fig, os.path.join(figures_dir, "figure2ab"), timestamp=False, tight_pad=0)
plot_utils.save_fig(fig_pr, os.path.join(figures_dir, "supplementalFigure3"), timestamp=False)Model metrics:
SVM	AUROC=0.781+/-0.013	AUPR=0.812+/-0.020
8 TFs	AUROC=0.698+/-0.036	AUPR=0.745+/-0.032
CRX	AUROC=0.548+/-0.023	AUPR=0.571+/-0.020
Total predicted occupancy of all TFs in each group:

### Panel c_____no_output_____

# Calculate motif frequency in each class
occupied_cutoff = 0.5
motif_freq_df = wt_occupancy_grouper.apply(lambda x: (x > occupied_cutoff).sum() / len(x))
# Sort by the feature importance in the logistic model
feature_importance = occ_clf.coef_[0]
feature_order = feature_importance.argsort()
motif_freq_df = motif_freq_df.iloc[:, feature_order]

# Make the fig
fig, ax_list = plt.subplots(nrows=8, ncols=2, figsize=(6, 4), gridspec_kw=dict(width_ratios=[1, 2]))
gs = ax_list[0, 0].get_gridspec()
for ax in ax_list[:, 1]:
    ax.remove()

axbig = fig.add_subplot(gs[:, 1])

ax = axbig
vmax = 0.25
thresh = vmax / 2
motif_freq_no_crx_df = motif_freq_df.drop(columns="CRX")
heatmap = ax.imshow(motif_freq_no_crx_df.T, aspect="auto", vmin=0, vmax=vmax, cmap="Reds")
ax.set_xticks(np.arange(len(wt_activity_names_oneline)))
ax.set_xticklabels(wt_activity_names_oneline, rotation=90)
ax.set_yticks(np.arange(len(motif_freq_no_crx_df.columns)))
ax.set_yticklabels(motif_freq_no_crx_df.columns)
plot_utils.annotate_heatmap(ax, motif_freq_no_crx_df, thresh)

# Add the logos
for cax, tf in zip(ax_list[1:, 0], motif_freq_no_crx_df.columns):
    pwm = logomaker.transform_matrix(pwms[tf], from_type="probability", to_type="information")
    logomaker.Logo(pwm, ax=cax, color_scheme="colorblind_safe", show_spines=False)
    # Right-align the logos
    cax.set_xlim(left=motif_len[tf] - motif_len.max() - 0.5)
    cax.set_ylim(top=2)
    cax.set_xticks([])
    cax.set_yticks([])

# Add a colorbar
divider = make_axes_locatable(ax)
cax = divider.append_axes("right", size="5%", pad="2%")
colorbar = fig.colorbar(heatmap, cax=cax, label="Frequency of motif")
ticks = cax.get_yticks()
ticks = [f"{i:.2f}" for i in ticks]
ticks[-1] = r"$\geq$" + ticks[-1]
cax.set_yticklabels(ticks)

# Add CRX
cax = divider.append_axes("top", size="14%", pad="2%")
heatmap = cax.imshow(motif_freq_df["CRX"].to_frame().T, aspect="auto", vmin=0, vmax=vmax, cmap="Reds")
cax.xaxis.tick_top()
cax.set_xticks(ax.get_xticks())
cax.set_xlim(ax.get_xlim())
cax.set_xticklabels(wt_activity_count, fontsize=10, rotation=45)
cax.set_yticks([0])
cax.set_yticklabels(["CRX"])
plot_utils.annotate_heatmap(cax, motif_freq_df["CRX"].to_frame(), thresh)

# Add CRX logo
cax = ax_list[0, 0]
pwm = logomaker.transform_matrix(pwms["CRX"], from_type="probability", to_type="information")
logomaker.Logo(pwm, ax=cax, color_scheme="colorblind_safe", show_spines=False)
# Right-align the logos
cax.set_xlim(left=motif_len[tf] - motif_len.max() - 0.5)
cax.set_ylim(top=2)
cax.set_xticks([])
cax.set_yticks([])

plot_utils.add_letter(cax, 0, 1.03, "c")
plot_utils.save_fig(fig, os.path.join(figures_dir, "figure2c"), timestamp=False, tight_pad=0)in validate_matrix(): Row sums in df are not close to 1. Reormalizing rows...

### Panels d-f_____no_output_____

# Setup figure
fig, ax_list = plt.subplots(nrows=2, ncols=2, figsize=(8, 4), gridspec_kw=dict(height_ratios=[3, 2]))
ax2d = ax_list[0, 0]
ax2f = ax_list[1, 0]
for ax in ax_list[:, 1]:
    ax.remove()

ax2e = fig.add_subplot(ax2d.get_gridspec()[:, 1])

# Calculate co-occurrance of motifs in strong enhancers
strong_enh_coocc_df = wt_occupancy_grouper.get_group("Strong enhancer")[["RAX", "NRL", "MAZ", "NDF1", "RORB"]]
strong_enh_coocc_df = (strong_enh_coocc_df > occupied_cutoff).astype(int)
strong_enh_coocc_df = strong_enh_coocc_df.T.dot(strong_enh_coocc_df) / len(strong_enh_coocc_df)
# Fill in lower triangle with the expected values
for row in range(len(strong_enh_coocc_df)):
    for col in range(row + 1, len(strong_enh_coocc_df)):
        strong_enh_coocc_df.iloc[row, col] = strong_enh_coocc_df.iloc[row, row] * strong_enh_coocc_df.iloc[col, col]

# 2d: Make the heatmap
ax = ax2d
vmax = 0.25
thresh = vmax / 2
heatmap = ax.imshow(strong_enh_coocc_df, aspect="auto", cmap="Reds", vmax=vmax, vmin=0)
ax.set_title("Strong enhancers")
ax.set_xticks(np.arange(len(strong_enh_coocc_df.columns)))
ax.set_xticklabels(strong_enh_coocc_df.columns)
ax.set_yticks(np.arange(len(strong_enh_coocc_df.columns)))
ax.set_yticklabels(strong_enh_coocc_df.columns)
plot_utils.annotate_heatmap(ax, strong_enh_coocc_df, thresh, adjust_lower_triangle=True)

# Add colorbar
divider = make_axes_locatable(ax)
cax = divider.append_axes("right", size="5%", pad="2%")
colorbar = fig.colorbar(heatmap, cax=cax, label="Freq. motifs\nco-occur", ticks=[0, round(thresh, 2), vmax])
plot_utils.add_letter(ax, -0.25, 1.03, "d")

# Calculate activity classes for different binding combos
binding_combos_activity_freq = activity_measured_wt_df.groupby("binding_group")["group_name_WT"].value_counts().unstack()
binding_combos_activity_freq = binding_combos_activity_freq[class_sort_order]
# Ignore cases where there is NRL or MEF2D but not CRX
binding_combos_activity_freq = binding_combos_activity_freq.loc[["No binding", "CRX only", "CRX+NRL", "CRX+MEF2D", "All three"]]
binding_combos_activity_freq = binding_combos_activity_freq.astype(int)

# Generate names then normalize data
binding_combos_names = binding_combos_activity_freq.index.values
binding_combos_count = [j.sum() for i, j in binding_combos_activity_freq.iterrows()]
binding_combos_activity_freq = binding_combos_activity_freq.div(binding_combos_activity_freq.sum(axis=1), axis=0)
display(binding_combos_activity_freq)

# 2e: make plot
ax = ax2e
fig = plot_utils.stacked_bar_plots(binding_combos_activity_freq, "Fraction of group", binding_combos_names, color_mapping, figax=(fig, ax), vert=True)
ax.set_yticks(np.linspace(0, 1, 6))
plot_utils.rotate_ticks(ax.get_xticklabels())

# Add the n
ax_twin = ax.twiny()
ax_twin.set_xticks(ax.get_xticks())
ax_twin.set_xlim(ax.get_xlim())
ax_twin.set_xticklabels(binding_combos_count, fontsize=10, rotation=45)
plot_utils.add_letter(ax, -0.25, 1.03, "e")

# Frequency each class is bound by each TF
group_bound_freqs = activity_measured_wt_df.groupby("group_name_WT")[["crx_bound", "nrl_bound", "mef2d_bound"]].apply(lambda x: x.sum() / len(x))
group_bound_freqs.columns = group_bound_freqs.columns.str.split("_").str[0].str.upper()

# 2f: Make heatmakt
vmax = 1
thresh = vmax / 2
ax = ax2f
heatmap = ax.imshow(group_bound_freqs.T, aspect="auto", cmap="Reds", vmax=vmax, vmin=0)
ax.set_xticks(np.arange(len(wt_activity_names_oneline)))
ax.set_xticklabels(wt_activity_names_oneline, rotation=90)
ax.set_yticks(np.arange(len(group_bound_freqs.columns)))
ax.set_yticklabels(group_bound_freqs.columns)
plot_utils.annotate_heatmap(ax, group_bound_freqs, thresh)

# Add colorbar
divider = make_axes_locatable(ax)
cax = divider.append_axes("right", size="5%", pad="2%")
colorbar = fig.colorbar(heatmap, cax=cax, label="Fraction\nbound")
plot_utils.add_letter(ax, -0.25, 1.03, "f")

# Add ticks above to show the n
ax_twin = ax.twiny()
ax_twin.set_axes_locator(ax.get_axes_locator())
ax_twin.set_xticks(ax.get_xticks())
ax_twin.set_xlim(ax.get_xlim())
ax_twin.set_xticklabels(wt_activity_count, fontsize=10, rotation=45)

plot_utils.save_fig(fig, os.path.join(figures_dir, "figure2def"), timestamp=False, tight_pad=0)_____no_output_____

## Supplemental figure 5: Additional validation of the 8 TF predicted occupancy logistic regression model_____no_output_____

# Setup figure
fig, ax_list = plot_utils.setup_multiplot(4, n_cols=2, sharex=False, sharey=False)
ax_list = ax_list.flatten()

# Plot White 2013 test set
_, white_aurocs, _, white_auprs, _ = plot_utils.roc_pr_curves(
    modeling_xaxis, [white_svm_tpr, white_occ_tpr], [white_svm_prec, white_occ_prec],
    model_names[:2], model_colors=model_colors[:2], prc_chance=white_svm_prec.iloc[-1],
    figax=([fig, fig], ax_list[:2])
)

plot_utils.add_letter(ax_list[0], -0.15, 1.03, "a")
plot_utils.add_letter(ax_list[1], -0.15, 1.03, "b")

# Display model performance
print("Model performance on White 2013 test set:")
print(f"{model_names[0]}\tAUROC = {white_aurocs[0]:.3f}\tAUPR = {white_auprs[0]:.3f}")
print(f"{model_names[1]}\tAUROC = {white_aurocs[1]:.3f}\tAUPR = {white_auprs[1]:.3f}")

# Plot performance of random background relative to the true features
niter_rand = len(random_occ_tprs)
rand_tpr_plotting = [[j] for i, j in random_occ_tprs.iterrows()] + [occ_tpr_cv]
rand_prec_plotting = [[j] for i, j in random_occ_precs.iterrows()] + [occ_prec_cv]
rand_names = [""]  * niter_rand + ["True features"]
rand_colors = ["#8080801A"] * niter_rand + ["#E69B04"]

_, background_aurocs, _, background_auprs, _ = plot_utils.roc_pr_curves(
    modeling_xaxis, rand_tpr_plotting, rand_prec_plotting, rand_names, model_colors=rand_colors,
    prc_chance=prc_chance, figax=([fig, fig], ax_list[2:])
)

plot_utils.add_letter(ax_list[2], -0.15, 1.03, "c")
plot_utils.add_letter(ax_list[3], -0.15, 1.03, "d")

# KS test, null hypothesis: random AUROCs and AUPRs are normally distributed
# One-tailed Z-test that the real data is drawn from this distribution
for data, name in zip([background_aurocs, background_auprs], ["AUROC", "AUPR"]):
    real, rand = data[niter_rand], data[:niter_rand]
    dstat, pval = stats.kstest(stats.zscore(rand), "norm")
    print(f"{name}s of random features are normally distributed, KS test p = {pval:.2f}, D = {dstat:.2f}")
    zscore = (real - np.mean(rand)) / np.std(rand)
    pval = stats.norm.cdf(-np.abs(zscore))
    print(f"Probability that the {name} of the real features is drawn from the background distribution, one-tailed Z-test p = {pval:2f}")

plot_utils.save_fig(fig, os.path.join(figures_dir, "supplementalFigure5"), timestamp=False)Model performance on White 2013 test set:
SVM	AUROC = 0.800	AUPR = 0.821
8 TFs	AUROC = 0.662	AUPR = 0.714
AUROCs of random features are normally distributed, KS test p = 0.80, D = 0.06
Probability that the AUROC of the real features is drawn from the background distribution, one-tailed Z-test p = 0.000393
AUPRs of random features are normally distributed, KS test p = 0.81, D = 0.06
Probability that the AUPR of the real features is drawn from the background distribution, one-tailed Z-test p = 0.000796

## Figure 3: Information content classifies strong enhancers
Also supplemental figure 6: Precision recall curve of logistic regression classifier using information content_____no_output_____

# Setup figures
fig, ax_list = plot_utils.setup_multiplot(2, sharex=False, sharey=False)
fig_pr, ax_pr = plt.subplots()

# 3a: violin plot of information content
print("Information content for each class:")
display(wt_entropy_grouper["entropy"].describe())

ax = ax_list[0]
fig = plot_utils.violin_plot_groupby(wt_entropy_grouper["entropy"], "Information content", class_names=wt_activity_names_oneline, class_colors=color_mapping, figax=(fig, ax))
plot_utils.rotate_ticks(ax.get_xticklabels())
ax.set_yticks(np.arange(0, wt_entropy_df["entropy"].max() + 1, 2))
plot_utils.add_letter(ax, -0.2, 1.03, "a")

# Add ticks above to show the n
ax_twin = ax.twiny()
ax_twin.set_xticks(ax.get_xticks())
ax_twin.set_xlim(ax.get_xlim())
ax_twin.set_xticklabels(wt_activity_count, fontsize=10, rotation=45)

# Statistics for differences in information content
ustat, pval = stats.mannwhitneyu(wt_entropy_grouper["entropy"].get_group("Strong enhancer"), wt_entropy_grouper["entropy"].get_group("Silencer"), alternative="two-sided")
print(f"Strong enhancers and silencers have the same information content, Mann-Whitney U test p = {pval:.0e} U = {ustat:.2f}")
ustat, pval = stats.mannwhitneyu(wt_entropy_grouper["entropy"].get_group("Strong enhancer"), wt_entropy_grouper["entropy"].get_group("Inactive"), alternative="two-sided")
print(f"Strong enhancers and inactive sequences have the same information content, Mann-Whitney U test p = {pval:.0e}, U = {ustat:.2f}")

# 3b: ROC and PR curves with information content vs. two classes
model_data = [
    (entropy_tpr_cv, entropy_prec_cv, "Strong vs.\nsilencer", "#E69B04"),
    (inactive_entropy_tpr_cv, inactive_entropy_prec_cv, "Strong vs.\ninactive", plot_utils.set_color(1))
]

model_tprs, model_precs, model_names, model_colors = zip(*model_data)
ax = ax_list[1]

# Plot the models
_, model_aurocs, model_aurocs_std, model_auprs, model_auprs_std = plot_utils.roc_pr_curves(
    modeling_xaxis, model_tprs, model_precs, model_names, model_colors=model_colors,
    figax=([fig, fig_pr], [ax, ax_pr])
)
ax.set_xticks(np.linspace(0, 1, 6))
plot_utils.add_letter(ax, -0.2, 1.03, "b")

# Display model metrics
print("Model metrics:")
for name, auroc, auroc_std, aupr, aupr_std in zip(model_names, model_aurocs, model_aurocs_std, model_auprs, model_auprs_std):
    print(f"{name}\tAUROC={auroc:.3f}+/-{auroc_std:.3f}\tAUPR={aupr:.3f}+/-{aupr_std:.3f}")

plot_utils.save_fig(fig, os.path.join(figures_dir, "figure3"), timestamp=False, tight_pad=0)
plot_utils.save_fig(fig_pr, os.path.join(figures_dir, "supplementalFigure6"), timestamp=False)Information content for each class:

## Figure 4: Sequence features of autonomous and non-autonomous strong enhancers_____no_output_____

# Keep sequences where the WT and Polylinker were both measured
poly_measured_mask = activity_df[["expression_log2_WT", "expression_log2_POLY"]].notna().all(axis=1)
activity_poly_df = activity_df[poly_measured_mask]
wt_occupancy_poly_df = wt_occupancy_df[poly_measured_mask]
wt_entropy_poly_df = wt_entropy_df[poly_measured_mask]

# Setup the figure
fig, ax_list = plot_utils.setup_multiplot(4, sharex=False, sharey=False)
ax_list = ax_list.flatten()

# 4a: scatterplot of Rho vs. Polylinker
ax = ax_list[0]
print("Correlation between WT activity with Rho vs. Polylinker:")
fig, ax = plot_utils.scatter_with_corr(activity_poly_df["expression_log2_WT"], activity_poly_df["expression_log2_POLY"], "log2 Enhancer Activity/Rho", "log2 Autonomous Activity", colors=activity_poly_df["plot_color_WT"], xticks=rho_ticks, figax=(fig, ax))
ax.axhline(0, color="k", linestyle="--")
plot_utils.add_letter(ax, -0.2, 1.03, "a")

# Display some numbers for the manuscript
print("Fraction of autonomous sequences belonging to each activity class:")
display(activity_poly_df.loc[activity_poly_df["autonomous_activity"], "group_name_WT"].value_counts(normalize=True))

print("Fraction of each activity class that has autonomous activity:")
display(activity_poly_df.groupby("group_name_WT")["autonomous_activity"].apply(lambda x: x.sum() / len(x)))

# Information content of strong autonomous vs. non-autonomous
# Set up grouping
strong_enh_poly_mask = activity_poly_df["group_name_WT"].str.contains("Strong")
strong_enh_poly_mask = strong_enh_poly_mask & strong_enh_poly_mask.notna()
autonomous_occ_grouper = wt_occupancy_poly_df[strong_enh_poly_mask].groupby(activity_poly_df.loc[strong_enh_poly_mask, "autonomous_activity"])
autonomous_entropy_grouper = wt_entropy_poly_df[strong_enh_poly_mask].groupby(activity_poly_df.loc[strong_enh_poly_mask, "autonomous_activity"])

# Set up for plotting
strong_color = color_mapping["Strong enhancer"]
autonomous_names = ["Non-autonomous ", " Autonomous"]
autonomous_counts = [len(i) for i in autonomous_occ_grouper.groups.values()]

# Do stats for difference in IC
print("Information content of autonomous and non-autonomous strong enhancers:")
display(autonomous_entropy_grouper["entropy"].describe())
ustat, pval = stats.mannwhitneyu(*[j for i, j in autonomous_entropy_grouper["entropy"]], alternative="two-sided")
print(f"Autonomous and non-autonomous strong enhancers have the same information content, Mann-Whitney U test p={pval:.0e}, U={ustat:.2f}")

# 4b: Make the plot
ax = ax_list[1]
fig = plot_utils.violin_plot_groupby(autonomous_entropy_grouper["entropy"], "Information content", class_names=autonomous_names, class_colors=[strong_color]*2, figax=(fig, ax))
ax.set_xlabel("Strong enhancers")
# Add ticks for the n
ax_twin = ax.twiny()
ax_twin.set_xticks(ax.get_xticks())
ax_twin.set_xlim(ax.get_xlim())
ax_twin.set_xticklabels(autonomous_counts, fontsize=10, rotation=45)
plot_utils.add_letter(ax, -0.2, 1.03, "b")

# Differences in CRX occupancy
print("Predicted CRX occupancy of autonomous and non-autonomous strong enhancers:")
display(autonomous_occ_grouper["CRX"].describe())
ustat, pval = stats.mannwhitneyu(*[j for i, j in autonomous_occ_grouper["CRX"]], alternative="two-sided")
print(f"Autonomous and non-autonomous strong enhancers have the same predicted CRX occupancy, Mann-Whitney U test p={pval:.0e}, U={ustat:.2f}")

# 4c
ax = ax_list[2]
fig = plot_utils.violin_plot_groupby(autonomous_occ_grouper["CRX"], "Predicted CRX occupancy", class_names=autonomous_names, class_colors=[strong_color]*2, figax=(fig, ax))
ax.set_xlabel("Strong enhancers")
ax.set_yticks(np.arange(8))
# Add ticks for the n
ax_twin = ax.twiny()
ax_twin.set_xticks(ax.get_xticks())
ax_twin.set_xlim(ax.get_xlim())
ax_twin.set_xticklabels(autonomous_counts, fontsize=10, rotation=45)
plot_utils.add_letter(ax, -0.2, 1.03, "c")

# Differences in motif frequencies
autonomous_motif_freq_df = autonomous_occ_grouper.apply(lambda x: (x > occupied_cutoff).sum() / len(x))
# Sort by the feature importance in the logistic model
autonomous_motif_freq_df = autonomous_motif_freq_df.iloc[:, feature_order]

# 4d: Make heatmakt, but put CRX separate
ax = ax_list[3]
autonomous_motif_freq_no_crx_df = autonomous_motif_freq_df.drop(columns="CRX")
vmax = 0.25
thresh = vmax / 2
heatmap = ax.imshow(autonomous_motif_freq_no_crx_df.T, aspect="auto", cmap="Reds", vmax=vmax, vmin=0)
ax.set_xlabel("Strong enhancers")
ax.set_xticks(np.arange(len(autonomous_motif_freq_no_crx_df)))
ax.set_xticklabels(autonomous_names)
ax.set_yticks(np.arange(len(autonomous_motif_freq_no_crx_df.columns)))
ax.set_yticklabels(autonomous_motif_freq_no_crx_df.columns)
plot_utils.annotate_heatmap(ax, autonomous_motif_freq_no_crx_df, thresh)

# Add colorbar
divider = make_axes_locatable(ax)
cax = divider.append_axes("right", size="5%", pad="2%")
colorbar = fig.colorbar(heatmap, cax=cax, label="Frequency of motif")
ticks = cax.get_yticks()
ticks = [f"{i:.2f}" for i in ticks]
ticks[-1] = r"$\geq$" + ticks[-1]
cax.set_yticklabels(ticks)

# Add CRX
cax = divider.append_axes("top", size="14%", pad="2%")
heatmap = cax.imshow(autonomous_motif_freq_df["CRX"].to_frame().T, aspect="auto", cmap="Reds", vmax=vmax, vmin=0)
cax.set_xticks([])
cax.set_yticks([0])
cax.set_yticklabels(["CRX"])
plot_utils.annotate_heatmap(cax, autonomous_motif_freq_df["CRX"].to_frame(), thresh)
plot_utils.add_letter(cax, -0.2, 1.03, "d")

# Add ticks for the n
cax.xaxis.tick_top()
cax.set_xticks(ax.get_xticks())
cax.set_xlim(ax.get_xlim())
cax.set_xticklabels(autonomous_counts, fontsize=10, rotation=45)

# Test relationship between NRL binding and strong enhancer autonomous activity
print("Strong enhancers with autonomous and non-autonomous activity vs. NRL bound and unbound:")
nrl_chip_vs_autonomous = activity_poly_df[strong_enh_poly_mask].groupby("autonomous_activity")["nrl_bound"].value_counts().unstack()
display(nrl_chip_vs_autonomous)
oddsratio, pval = stats.fisher_exact(nrl_chip_vs_autonomous)
print(f"Fisher's exact test that NRL binding and strong enhancer autonomous activity are independent, p={pval:.0e}, odds ratio={oddsratio:.1f}")

plot_utils.save_fig(fig, os.path.join(figures_dir, "figure4"), timestamp=False)Correlation between WT activity with Rho vs. Polylinker:
PCC = 0.338
SCC = 0.359
n = 4751
Fraction of autonomous sequences belonging to each activity class:

## Figure 5: Independence of TF motifs in strong enhancers_____no_output_____

# Helper functions to visualize sequence
def hex_to_rgb(hexcode):
    return tuple(int(hexcode[i:i+2], 16) / 255 for i in (1, 3, 5))

strong_color_rgb = hex_to_rgb(strong_color)
weak_color_rgb = hex_to_rgb(color_mapping["Weak enhancer"])
crx_color = mpl.colors.to_rgb("orange")
other_tf_color = mpl.colors.to_rgb("red")

def visualize_sequence(seq_id, ax, title, below_text, basecolor):
    seq_occupancy_df = predicted_occupancy.total_landscape(all_seqs[seq_id], ewms, mu)
    visual = np.full(((len(seq_occupancy_df), 3)), basecolor) # (number of positions, RGB values)
    text_mapping = [] # (name of TF, center position of motif)
    # Loop over each TF, identify motifs, and fill in the representation with the predicted occupancy for the full motif
    for col in seq_occupancy_df:
        tf, orient = col.split("_")
        for motif_start, occ in seq_occupancy_df[col].iteritems():
            if occ > occupied_cutoff:
                motif_end = motif_start + motif_len[tf]
                # Check and make sure all positions of the motif are zeros
                if (visual[motif_start:motif_end] != basecolor).all(axis=1).any():
                    print(f"Error, motif already in the range {motif_start}-{motif_end}! Skipping.")
                else:
                    color = crx_color if tf == "CRX" else other_tf_color
                    visual[motif_start:motif_end] = color
                    text_mapping.append((tf, (motif_start + motif_end) / 2))

    heatmap = ax.imshow(visual[np.newaxis, :], aspect="auto", cmap="Reds")
    ax.set_yticks([])
    # Add text showing which motif is where
    for tf, x in text_mapping:
        ax.text(x, 0, tf, ha="center", va="center", color="white", rotation=90)

    ax.set_title(title)
    ax.set_xlabel(below_text)

    return ax, heatmap_____no_output_____# Setup for sequences where both WT and MUT was measured
wt_mut_mask = activity_df["wt_vs_mut_log2"].notna()
activity_wt_mut_measured_df = activity_df[wt_mut_mask]
wt_occ_mut_measured_df = wt_occupancy_df[wt_mut_mask]
wt_entropy_mut_measured_df = wt_entropy_df[wt_mut_mask]
mut_entropy_measured_df = mut_entropy_df[wt_mut_mask]

# Figure setup
gs_kw = dict(height_ratios=[5, 5, 1, 1])
fig, ax_list = plt.subplots(nrows=4, ncols=2, figsize=(7, 10), gridspec_kw=gs_kw)
gs = ax_list[0, 0].get_gridspec()
for row in [2, 3]:
    for ax in ax_list[row]:
        ax.remove()

axstrong = fig.add_subplot(gs[2, :])
axweak = fig.add_subplot(gs[3, :])

# 5a: Scatter plot of WT and MUT activities
ax = ax_list[0, 0]
print("Correlation between WT and MUT activities:")
fig, ax = plot_utils.scatter_with_corr(activity_wt_mut_measured_df["expression_log2_WT"], activity_wt_mut_measured_df["expression_log2_MUT"],
                                      "log2 WT Activity/Rho", "log2 MUT Activity/Rho", colors=activity_wt_mut_measured_df["plot_color_WT"],
                                      xticks=rho_ticks, yticks=rho_ticks, figax=(fig, ax))
# Plot y = x line
ax.plot(rho_ticks, rho_ticks, color="black", linewidth=1)
# Show cutoffs for different classes
strong_cutoff = activity_df.groupby("group_name_WT")["expression_log2_WT"].get_group("Strong enhancer").min()
for line in [-1, 1, strong_cutoff]:
    ax.axhline(line, color="black", linestyle="--", linewidth=1)

# Add colorbar to show the cutoffs
divider = make_axes_locatable(ax)
color_ax = divider.append_axes("right", size="5%")
color_ax.set_ylim(ax.get_ylim())
color_ax.barh([(-1 - ax.get_ylim()[0]) / 2 + ax.get_ylim()[0], 0, (strong_cutoff - 1) / 2 + 1, (ax.get_ylim()[1] - strong_cutoff) / 2 + strong_cutoff], # Midpoint of the bars
             [1, 1, 1, 1], # Bar height
             [-1 - ax.get_ylim()[0], 2, strong_cutoff - 1, ax.get_ylim()[1] - strong_cutoff], # Bar width
             color=color_mapping)
color_ax.set_xticks([])
color_ax.set_yticks([])
color_ax.set_xlim(right=1)

plot_utils.add_letter(ax, -0.35, 1.03, "a")

# Setup strong enhancer->mutant activity groupings
strong_mask = activity_wt_mut_measured_df["group_name_WT"].str.contains("Strong")
strong_mask = strong_mask & strong_mask.notna()
activity_strong_df = activity_wt_mut_measured_df[strong_mask]

# Group the data based on CRX-dependence (whether or not it stay strong) and name the groups accordingly
stay_strong_mask = activity_strong_df["group_name_MUT"].str.contains("Strong") & activity_strong_df["group_name_MUT"].notna()
wt_occ_strong_grouper = wt_occ_mut_measured_df[strong_mask].groupby(stay_strong_mask)
wt_entropy_strong_grouper = wt_entropy_mut_measured_df[strong_mask].groupby(stay_strong_mask)
strong_mutant_names = wt_entropy_strong_grouper["entropy"].count().rename({False: "High", True: "Low"}).index.values.tolist()
strong_mutant_counts = wt_entropy_strong_grouper["entropy"].count().astype(int).values

# Differences in information content
print("Information content of strong enhancers with different mutant activities:")
display(wt_entropy_strong_grouper["entropy"].describe())

# 5b: Information content
ax = ax_list[0, 1]
fig = plot_utils.violin_plot_groupby(wt_entropy_strong_grouper["entropy"], "Information content", class_names=strong_mutant_names, class_colors=color_mapping[["Weak enhancer", "Strong enhancer"]], figax=(fig, ax))
ax.set_xlabel("CRX-dependence\nStrong enhancers")
ax.set_yticks(np.arange(0, 13, 2))
ax_twin = ax.twiny()
ax_twin.set_xticks(ax.get_xticks())
ax_twin.set_xlim(ax.get_xlim())
ax_twin.set_xticklabels(strong_mutant_counts, fontsize=10, rotation=45)
plot_utils.add_letter(ax, -0.25, 1.03, "b")

# Differences in predicted CRX occupancy of strong enhancers with different CRX-dependences
print("Predicted CRX occupancy of strong enhancers with different mutant activities:")
display(wt_occ_strong_grouper["CRX"].describe())
ustat, pval = stats.mannwhitneyu(*[j for i, j in wt_occ_strong_grouper["CRX"]], alternative="two-sided")
print(f"Strong enhancers that remain strong vs. do not have the same CRX occupancy, Mann-Whitney U test p={pval:.0e}, U={ustat:.2f}")

# 5c: predicted CRX occupancy
ax = ax_list[1, 0]
fig = plot_utils.violin_plot_groupby(wt_occ_strong_grouper["CRX"], "Predicted CRX occupancy", class_names=strong_mutant_names, class_colors=color_mapping[["Weak enhancer", "Strong enhancer"]], figax=(fig, ax))
ax.set_xlabel("CRX-dependence\nStrong enhancers")
ax.set_yticks(np.arange(0, 8, 2))
ax_twin = ax.twiny()
ax_twin.set_xticks(ax.get_xticks())
ax_twin.set_xlim(ax.get_xlim())
ax_twin.set_xticklabels(strong_mutant_counts, fontsize=10, rotation=45)
plot_utils.add_letter(ax, -0.25, 1.03, "c")

# Differences in redisual IC
print("Residual information content of strong enhancers with different mutant activities:")
mut_entropy_strong_grouper = mut_entropy_measured_df[strong_mask].groupby(stay_strong_mask)
display(mut_entropy_strong_grouper["entropy"].describe())
ustat, pval = stats.mannwhitneyu(*[j for i, j in mut_entropy_strong_grouper["entropy"]], alternative="two-sided")
print(f"Strong enhancers that stay strong vs. do not have the same residual information content, Mann-Whitney U test p={pval:.0e}, U={ustat:.2f}")

# 5d: Residual information content
ax = ax_list[1, 1]
fig = plot_utils.violin_plot_groupby(mut_entropy_strong_grouper["entropy"], "Residual information content", class_names=strong_mutant_names, class_colors=color_mapping[["Weak enhancer", "Strong enhancer"]], figax=(fig, ax))
ax.set_xlabel("CRX-dependence\nStrong enhancers")
ax.set_yticks(np.arange(0, 11, 2))
ax_twin = ax.twiny()
ax_twin.set_xticks(ax.get_xticks())
ax_twin.set_xlim(ax.get_xlim())
ax_twin.set_xticklabels(strong_mutant_counts, fontsize=10, rotation=45)
plot_utils.add_letter(ax, -0.25, 1.03, "d")

# 5e Visualize the two depresentative sequences
ax = axstrong
become_weak_example_id = "chr16-43945747-43945911_UPPE"
become_weak_text = become_weak_example_id.split("_")[0] + "\n" + f"{wt_entropy_df.loc[become_weak_example_id, 'entropy']:.1f}" + " bits, " + f"{mut_entropy_df.loc[become_weak_example_id, 'entropy']:.1f}" + " residual bits"
ax, become_weak_visual = visualize_sequence(become_weak_example_id + "_WT", ax, "High CRX-dependence", become_weak_text, weak_color_rgb)
plot_utils.add_letter(ax, -0.05, 1.03, "e")

ax = axweak
stay_strong_example_id = "chr11-114685176-114685340_CPPE"
stay_strong_text = stay_strong_example_id.split("_")[0] + "\n" + f"{wt_entropy_df.loc[stay_strong_example_id, 'entropy']:.1f}" + " bits, " + f"{mut_entropy_df.loc[stay_strong_example_id, 'entropy']:.1f}" + " residual bits"
ax, stay_strong_visual = visualize_sequence(stay_strong_example_id + "_WT", ax, "Low CRX-dependence", stay_strong_text, strong_color_rgb)

plot_utils.save_fig(fig, os.path.join(figures_dir, "figure5"), timestamp=False)Correlation between WT and MUT activities:
PCC = 0.608
SCC = 0.706
n = 4123
Information content of strong enhancers with different mutant activities:


# Notebook from lanmai01/intro-to-python
Path: Class_Contents/Classes/00_intro_00_content.ipynb

**Important**: Click on "*Kernel*" > "*Restart Kernel and Clear All Outputs*" *before* reading this chapter in [JupyterLab ](https://jupyterlab.readthedocs.io/en/stable/)_____no_output_____# An Introduction to Python and Programming

This course is a *thorough* introduction to programming in [Python ](https://www.python.org/).

It teaches the concepts behind and the syntax of the core Python language as defined by the [Python Software Foundation ](https://www.python.org/psf/) in the official [language reference ](https://docs.python.org/3/reference/index.html). Furthermore, it introduces commonly used functionalities from the [standard library ](https://docs.python.org/3/library/index.html) and popular third-party libraries like [numpy ](https://www.numpy.org/), [pandas ](https://pandas.pydata.org/), [matplotlib ](https://matplotlib.org/), and others.

## Prerequisites

This course is suitable for *total beginners*, and there are *no* formal prerequisites. The student only needs to have:

- a *solid* understanding of the **English language**,
- knowledge of **basic mathematics** from high school,
- the ability to **think conceptually** and **reason logically**, and
- the willingness to **invest around time and effort on this course**._____no_output_____## Objective

The **main goal** of this introduction is to **prepare** the student **for further studies** in the "field" of **data science**.

These include but are not limited to topics such as:
- linear algebra
- statistics & econometrics
- data cleaning & wrangling
- data visualization
- data engineering (incl. SQL databases)
- data mining (incl. web scraping)
- feature generation, machine learning, & deep learning
- optimization & (meta-)heuristics
- algorithms & data structures
- quantitative finance (e.g., option valuation)
- quantitative marketing (e.g., customer segmentation)
- quantitative supply chain management (e.g., forecasting)
- management science & decision models
- backend/API/web development (to serve data products to clients)_____no_output_____### Why data science?

The term **[data science ](https://en.wikipedia.org/wiki/Data_science)** is rather vague and does *not* refer to an academic discipline. Instead, the term was popularized by the tech industry, who also coined non-meaningful job titles such as "[rockstar](https://www.quora.com/Why-are-engineers-called-rockstars-and-ninjas)" or "[ninja developers](https://www.quora.com/Why-are-engineers-called-rockstars-and-ninjas)." Most *serious* definitions describe the field as being **multi-disciplinary** *integrating* scientific methods, algorithms, and systems thinking to extract knowledge from structured and unstructured data, *and* also emphasize the importance of **[domain knowledge ](https://en.wikipedia.org/wiki/Domain_knowledge)**.

Recently, this integration aspect feeds back into the academic world. The [MIT](https://www.mit.edu/), for example, created the new [Stephen A. Schwarzman College of Computing](http://computing.mit.edu) for [artificial intelligence ](https://en.wikipedia.org/wiki/Artificial_intelligence) with a 1 billion dollar initial investment where students undergo a "bilingual" curriculum with half the classes in quantitative and method-centric fields - like the ones mentioned above - and the other half in domains such as biology, business, chemistry, politics, (art) history, or linguistics (cf., the [official Q&As](http://computing.mit.edu/faq/) or this [NYT article](https://www.nytimes.com/2018/10/15/technology/mit-college-artificial-intelligence.html)). Their strategists see a future where programming skills are just as naturally embedded into students' curricula as are nowadays subjects like calculus, statistics, or academic writing. Then, programming literacy is not just another "nice to have" skill but a prerequisite, or an enabler, to understanding more advanced topics in the actual domains studied. Top-notch researchers who use programming in their day-to-day lives could then teach students more efficiently in their "language."_____no_output_____## Why Python?

### What is Python?

Here is a brief history of and some background on Python (cf., also this [TechRepublic article](https://www.techrepublic.com/article/python-is-eating-the-world-how-one-developers-side-project-became-the-hottest-programming-language-on-the-planet/) for a more elaborate story):

- [Guido van Rossum ](https://en.wikipedia.org/wiki/Guido_van_Rossum) (Python’s **[Benevolent Dictator for Life ](https://en.wikipedia.org/wiki/Benevolent_dictator_for_life)**) was bored during a week around Christmas 1989 and started Python as a hobby project "that would keep \[him\] occupied" for some days
- the idea was to create a **general-purpose** scripting **language** that would allow fast *prototyping* and would *run on every operating system*
- Python grew through the 90s as van Rossum promoted it via his "Computer Programming for Everybody" initiative that had the *goal to encourage a basic level of coding literacy* as an equal knowledge alongside English literacy and math skills
- to become more independent from its creator, the next major version **Python 2** - released in 2000 and still in heavy use as of today - was **open-source** from the get-go which attracted a *large and global community of programmers* that *contributed* their expertise and best practices in their free time to make Python even better
- **Python 3** resulted from a significant overhaul of the language in 2008 taking into account the *learnings from almost two decades*, streamlining the language, and getting ready for the age of **big data**
- the language is named after the sketch comedy group [Monty Python ](https://en.wikipedia.org/wiki/Monty_Python)

#### Summary

Python is a **general-purpose** programming **language** that allows for *fast development*, is *easy to read*, **open-source**, long-established, unifies the knowledge of *hundreds of thousands of experts* around the world, runs on basically every machine, and can handle the complexities of applications involving **big data**.

### Isn't C a lot faster?

While it is true that a language like C is a lot faster than Python when it comes to *pure* **computation time**, this does not matter in many cases as the *significantly shorter* **development cycles** are the more significant cost factor in a rapidly changing world._____no_output_____### Who uses it?

While ad-hominem arguments are usually not the best kind of reasoning, we briefly look at some examples of who uses Python and leave it up to the reader to decide if this is convincing or not:

- **[Massachusetts Institute of Technology](https://www.mit.edu/)**
  - teaches Python in its [introductory course](https://ocw.mit.edu/courses/electrical-engineering-and-computer-science/6-0001-introduction-to-computer-science-and-programming-in-python-fall-2016/) to computer science independent of the student's major
  - replaced the infamous course on the [Scheme](https://groups.csail.mit.edu/mac/projects/scheme/) language (cf., [source ](https://news.ycombinator.com/item?id=602307))
- **[Google](https://www.google.com/)**
  - used the strategy "Python where we can, C++ where we must" from its early days on to stay flexible in a rapidly changing environment (cf., [source ](https://stackoverflow.com/questions/2560310/heavy-usage-of-python-at-google))
  - the very first web-crawler was written in Java and so difficult to maintain that it was rewritten in Python right away (cf., [source](https://www.amazon.com/Plex-Google-Thinks-Works-Shapes/dp/1416596585/ref=sr_1_1?ie=UTF8&qid=1539101827&sr=8-1&keywords=in+the+plex))
  - Guido van Rossom was hired by Google from 2005 to 2012 to advance the language there
- **[NASA](https://www.nasa.gov/)** open-sources many of its projects, often written in Python and regarding analyses with big data (cf., [source](https://code.nasa.gov/language/python/))
- **[Facebook](https://facebook.com/)** uses Python besides C++ and its legacy PHP (a language for building websites; the "cool kid" from the early 2000s)
- **[Instagram](https://instagram.com/)** operates the largest installation of the popular **web framework  [Django](https://www.djangoproject.com/)** (cf., [source](https://instagram-engineering.com/web-service-efficiency-at-instagram-with-python-4976d078e366))
- **[Spotify](https://spotify.com/)** bases its data science on Python (cf., [source](https://labs.spotify.com/2013/03/20/how-we-use-python-at-spotify/))
- **[Netflix](https://netflix.com/)** also runs its predictive models on Python (cf., [source](https://medium.com/netflix-techblog/python-at-netflix-86b6028b3b3e))
- **[Dropbox](https://dropbox.com/)** "stole" Guido van Rossom from Google to help scale the platform (cf., [source](https://medium.com/dropbox-makers/guido-van-rossum-on-finding-his-way-e018e8b5f6b1))
- **[JPMorgan Chase](https://www.jpmorganchase.com/)** requires new employees to learn Python as part of the onboarding process starting with the 2018 intake (cf., [source](https://www.ft.com/content/4c17d6ce-c8b2-11e8-ba8f-ee390057b8c9?segmentId=a7371401-027d-d8bf-8a7f-2a746e767d56))

As images tell more than words, here are two plots of popular languages' "market shares" based on the number of questions asked on [Stack Overflow ](https://stackoverflow.blog/2017/09/06/incredible-growth-python/), the most relevant platform for answering programming-related questions: As of late 2017, Python surpassed [Java](https://www.java.com/en/), heavily used in big corporates, and [JavaScript](https://developer.mozilla.org/en-US/docs/Web/JavaScript), the "language of the internet" that does everything in web browsers, in popularity. Two blog posts from "technical" people explain this in more depth to the layman: [Stack Overflow ](https://stackoverflow.blog/2017/09/14/python-growing-quickly/) and [DataCamp](https://www.datacamp.com/community/blog/python-scientific-computing-case).

As the graph below shows, neither Google's very own language **[Go](https://golang.org/)** nor **[R](https://www.r-project.org/)**, a domain-specific language in the niche of statistics, can compete with Python's year-to-year growth.

[IEEE Sprectrum](https://spectrum.ieee.org/computing/software/the-top-programming-languages-2019) provides a more recent comparison of programming language's popularity. Even news and media outlets notice the recent popularity of Python: [Economist](https://www.economist.com/graphic-detail/2018/07/26/python-is-becoming-the-worlds-most-popular-coding-language), [Huffington Post](https://www.huffingtonpost.com/entry/why-python-is-the-best-programming-language-with-which_us_59ef8f62e4b04809c05011b9), [TechRepublic](https://www.techrepublic.com/article/why-python-is-so-popular-with-developers-3-reasons-the-language-has-exploded/), and [QZ](https://qz.com/1408660/the-rise-of-python-as-seen-through-a-decade-of-stack-overflow/)._____no_output_____## Anaconda

To "read" this book in the most meaningful way, a working installation of **Python 3.7** or higher is expected.

A popular and beginner-friendly way is to install the [Anaconda Distribution](https://www.anaconda.com/distribution/) that not only ships Python and the standard library but comes pre-packaged with a lot of third-party libraries from the so-called "scientific stack." Just go to the [download](https://www.anaconda.com/download/) page and install the latest version (i.e., *2019-10* with Python 3.7 at the time of this writing) for your operating system.

Then, among others, you will find an entry "Anaconda Navigator" in your start menu like below. Click on it.

A window opens showing you several applications that come with the Anaconda Distribution. Now, click on "JupyterLab."

A new tab in your web browser opens with the website being "localhost" and some number (e.g., 8888). This is the [JupyterLab ](https://jupyterlab.readthedocs.io/en/stable/) application that is used to display and run [Jupyter notebooks ](https://jupyter-notebook.readthedocs.io/en/stable/notebook.html) as described below. On the left, you see the files and folders in your local user folder. This file browser works like any other. In the center, you have several options to launch (i.e., "create") new files.

_____no_output_____## Jupyter Notebooks

The document you are viewing is a so-called [Jupyter notebook ](https://jupyter-notebook.readthedocs.io/en/stable/notebook.html), a file format introduced by the [Jupyter Project ](https://jupyter.org/).

"Jupyter" is an [acronym ](https://en.wikipedia.org/wiki/Acronym) derived from the names of the three major programming languages **[Julia](https://julialang.org/)**, **[Python ](https://www.python.org)**, and **[R](https://www.r-project.org/)**, all of which play significant roles in the world of data science. The Jupyter Project's idea is to serve as an integrating platform such that different programming languages and software packages can be used together within the same project easily.

Furthermore, Jupyter notebooks have become a de-facto standard for communicating and exchanging results in the data science community - both in academia and business - and provide an alternative to terminal-based ways of running Python (e.g., the default [Python interpreter ](https://docs.python.org/3/tutorial/interpreter.html) as shown below or a more advanced interactive version like [IPython ](https://ipython.org/)) or a full-fledged [Integrated Development Environment ](https://en.wikipedia.org/wiki/Integrated_development_environment) (e.g., the commercial [PyCharm](https://www.jetbrains.com/pycharm/) or the free [Spyder ](https://github.com/spyder-ide/spyder) that comes with the Anaconda Distribution).

Jupyter notebooks allow mixing formatted English with Python code in the same document. Text is formatted with the [Markdown ](https://guides.github.com/features/mastering-markdown/) language and mathematical formulas are typeset with [LaTeX](https://www.overleaf.com/learn/latex/Free_online_introduction_to_LaTeX_%28part_1%29). Moreover, we may include pictures, plots, and even videos. Because of these features, the notebooks developed for this book come in a self-contained "tutorial" style enabling students to learn and review the material on their own._____no_output_____### Markdown vs. Code Cells

A Jupyter notebook consists of cells that have a type associated with them. So far, only cells of type "Markdown" have been used, which is the default way to present formatted text.

The cell below is an example of a "Code" cell containing a line of actual Python code: It merely outputs the text "Hello world" when executed. To edit an existing code cell, enter into it with a mouse click. You know that you are "in" a code cell when its frame is highlighted in blue.

Besides this **edit mode**, there is also a so-called **command mode** that you reach by hitting the "Escape" key *after* entering a code cell, which un-highlights the frame. Using the "Enter" and "Escape" keys, you can now switch between the two modes.

To *execute*, or "*run*," a code cell, hold the "Control" key and press "Enter." Note that you do *not* go to the subsequent cell. Alternatively, you can hold the "Shift" key and press "Enter," which executes the cell *and* places your focus on the subsequent cell.

Similarly, a Markdown cell is also in either edit or command mode. For example, double-click on the text you are reading: This puts you into edit mode. Now, you could change the formatting (e.g., make a word printed in *italics* or **bold** with single or double asterisks) and "execute" the cell to render the text as specified.

To change a cell's type, choose either "Code" or "Markdown" in the navigation bar at the top. Alternatively, you can hit either the "Y" or "M" key on your keyboard when in command mode to make the focused cell a code or markdown cell._____no_output_____

print("Hello world")
_____no_output_____

Sometimes, a code cell starts with an exclamation mark `!`. Then, the Jupyter notebook behaves as if the following command were typed directly into a terminal. The cell below asks `python` to show its version number and is *not* Python code but a command in the [Shell ](https://en.wikipedia.org/wiki/Shell_%28computing%29) language. The `!` is useful to execute short commands without leaving a Jupyter notebook._____no_output_____

!python --version_____no_output_____

## Contents

- *Chapter 0*: Introduction
- **Part A: Expressing Logic**
  - *Chapter 1*: Elements of a Program
  - *Chapter 2*: Functions & Modularization
  - *Chapter 3*: Conditionals & Exceptions
  - *Chapter 4*: Recursion & Looping_____no_output_____## How to learn Programming

Do you remember how you first learned to speak in your mother tongue? Probably not.

Your earliest memory as a child should probably be around the age of three or four years old when you could already say simple things and interact with your environment.

Although you did not know any grammar rules yet, other people just understood what you said. At least most of the time.

It is intuitively best to take the very mindset of a small child when learning a new language. And a programming language is no different from that.

This first chapter introduces simplistic examples and we accept them as they are *without* knowing any of the "grammar" rules yet. Then, we analyze them in parts and slowly build up our understanding.

Consequently, if parts of this chapter do not make sense right away, let's not worry too much. Besides introducing the basic elements, it also serves as an outlook for what is to come. So, many terms and concepts used here are deconstructed in great detail in the following chapters._____no_output_____## How to learn Programming

### ABC Rule

**A**lways **b**e **c**oding.

Programming is more than just writing code into a text file. It means reading through parts of the [documentation ](https://docs.python.org/), blogs with best practices, and tutorials, or researching problems on [Stack Overflow ](https://stackoverflow.com/) while trying to implement features in the application at hand. Also, it means using command-line tools to automate some part of the work or manage different versions of a program, for example, with **[git](https://git-scm.com/)**. In short, programming involves a lot of "muscle memory," which can only be built and kept up through near-daily usage.

Further, many aspects of software architecture and best practices can only be understood after having implemented some requirements for the very first time. Coding also means "breaking" things to find out what makes them work in the first place.

Therefore, coding is learned best by just doing it for some time on a daily or at least a regular basis and not right before some task is due, just like learning a "real" language.

### The Maker's Schedule

[Y Combinator ](https://www.ycombinator.com/) co-founder [Paul Graham ](https://en.wikipedia.org/wiki/Paul_Graham_%28programmer%29) wrote a very popular and often cited [article](http://www.paulgraham.com/makersschedule.html) where he divides every person into belonging to one of two groups:

- **Managers**: People that need to organize things and command others (e.g., a "boss" or manager). Their schedule is usually organized by the hour or even 30-minute intervals.
- **Makers**: People that create things (e.g., programmers, artists, or writers). Such people think in half days or full days.

Have you ever wondered why so many tech people work during nights and sleep at "weird" times? The reason is that many programming-related tasks require a "flow" state in one's mind that is hard to achieve when one can get interrupted, even if it is only for one short question. Graham describes that only knowing that one has an appointment in three hours can cause a programmer to not get into a flow state.

As a result, do not set aside a certain amount of time for learning something but rather plan in an *entire evening* or a *rainy Sunday* where you can work on a problem in an *open end* setting. And do not be surprised anymore to hear "I looked at it over the weekend" from a programmer.

### Phase Iteration

When being asked the above question, most programmers answer something that can be classified into one of two broader groups.

**1) Toy Problem, Case Study, or Prototype**: Pick some problem, break it down into smaller sub-problems, and solve them with an end in mind.

**2) Books, Video Tutorials, and Courses**: Research the best book, blog, video, or tutorial for something and work it through from start to end.

The truth is that you need to iterate between these two phases.

Building a prototype always reveals issues no book or tutorial can think of before. Data is never as clean as it should be. An algorithm from a textbook must be adapted to a peculiar aspect of a case study. It is essential to learn to "ship a product" because only then will one have looked at all the aspects.

The major downside of this approach is that one likely learns bad "patterns" overfitted to the case at hand, and one does not get the big picture or mental concepts behind a solution. This gap can be filled in by well-written books: For example, check the Python/programming books offered by [Packt](https://www.packtpub.com/packt/offers/free-learning/) or [O’Reilly](https://www.oreilly.com/)._____no_output_____## Example: Averaging all even Numbers in a List

As our introductory example, we want to calculate the *average* of all *evens* in a **list** of whole numbers: `[7, 11, 8, 5, 3, 12, 2, 6, 9, 10, 1, 4]`.

While we are used to finding an [analytical solution](https://math.stackexchange.com/questions/935405/what-s-the-difference-between-analytical-and-numerical-approaches-to-problems/935446#935446) in math (i.e., derive some equation with "pen and paper"), we solve this task *programmatically* instead.

We start by creating a list called `numbers` that holds all the individual numbers between **brackets** `[` and `]`._____no_output_____

numbers = _____no_output_____

To verify that something happened in our computer's memory, we **reference** `numbers`._____no_output_____

numbers_____no_output_____

So far, so good. Let's see how the desired **computation** could be expressed as a **sequence of instructions** in the next code cell.

Intuitively, the line `for number in numbers` describes a "loop" over all the numbers in the `numbers` list, one at a time.

The `if number % 2 == 0` may look confusing at first sight. Both `%` and `==` must have an unintuitive meaning here. Luckily, the **comment** in the same line after the `#` symbol has the answer: The program does something only for an even `number`.

In particular, it increases `count` by `1` and adds the current `number` onto the [running ](https://en.wikipedia.org/wiki/Running_total) `total`. Both `count` and `number` are **initialized** to `0` and the single `=` symbol reads as "... is *set* equal to ...". It cannot indicate a mathematical equation as, for example, `count` is generally *not* equal to `count + 1`.

Lastly, the `average` is calculated as the ratio of the final **values** of `total` and `count`. Overall, we divide the sum of all even numbers by their count: This is nothing but the definition of an average.

The lines of code "within" the `for` and `if` **statements** are **indented** and aligned with multiples of *four spaces*: This shows immediately how the lines relate to each other._____no_output_____

count = 0  # initialize variables to keep track of the
total = 0  # running total and the count of even numbers

for number in numbers:
    if number % 2 == 0:  # only work with even numbers
        count = count + 1
        total = total + number

average = total / count_____no_output_____

We do not see any **output** yet but obtain the value of `average` by referencing it again._____no_output_____

average_____no_output_____

## Output in a Jupyter Notebook

Only two of the previous four code cells generate an **output** while two remained "silent" (i.e., nothing appears below the cell after running it).

By default, Jupyter notebooks only show the value of the **expression** in the last line of a code cell. And, this output may also be suppressed by ending the last line with a semicolon `;`._____no_output_____

"Hello, World!"
"I am feeling great :-)"_____no_output_____"I am invisible!";_____no_output_____

To see any output other than that, we use the built-in [print() ](https://docs.python.org/3/library/functions.html#print) **function**. Here, the parentheses `()` indicate that we **call** (i.e., "execute") code written somewhere else._____no_output_____

print("Hello, World!")
print("I am feeling great :-)")_____no_output_____

Outside Jupyter notebooks, the semicolon `;` is used as a **separator** between statements that must otherwise be on a line on their own. However, it is *not* considered good practice to use it as it makes code less readable._____no_output_____

print("Hello, World!"); print("I am feeling great :-)")_____no_output_____

### Jupyter Notebook Aspects

#### The Order of Code Cells is arbitrary

We can run the code cells in a Jupyter notebook in *any* arbitrary order.

That means, for example, that a variable defined towards the bottom could accidentally be referenced at the top of the notebook. This happens quickly when we iteratively built a program and go back and forth between cells.

As a good practice, it is recommended to click on "Kernel" > "Restart Kernel and Run All Cells" in the navigation bar once a notebook is finished. That restarts the Python process forgetting all **state** (i.e., all variables) and ensures that the notebook runs top to bottom without any errors the next time it is opened.

#### Notebooks are linear

While this book is built with Jupyter notebooks, it is crucial to understand that "real" programs are almost never "linear" (i.e., top to bottom) sequences of instructions but instead may take many different **flows of execution**.

At the same time, for a beginner's course, it is often easier to code linearly.

In real data science projects, one would probably employ a mixed approach and put reusable code into so-called Python modules (i.e., *.py* files; cf., Chapter 2) and then use Jupyter notebooks to build up a linear report or storyline for an analysis._____no_output_____## HackerRank

HackerRank is a wonderful online platform which contains numerous online coding tests for student to practice their coding skills. Software companies also use HackerRank technical assessment and remote interview solution for hiring developers. Student will see a coding prolem in HackerRank in a form of problem description, sample input and expected output.

The task is writing the code according to problem description so that the code will take the sample input and print out the expected output.

Our course target is completing some (may not all) [HackerRank Python problems](https://www.hackerrank.com/domains/python?filters%5Bsubdomains%5D%5B%5D=py-introduction). In order to do that, please register your account in HackerRank in the link below:
[HackerRank SignUp](
https://www.hackerrank.com/auth/signup?h_l=body_middle_left_button&h_r=sign_up)_____no_output_____## Comments

We use the `#` symbol to write comments in plain English right into the code.

As a good practice, comments should *not* describe *what* happens. This should be evident by reading the code. Otherwise, it is most likely badly written code. Rather, comments should describe *why* something happens.

Comments may be added either at the end of a line of code, by convention separated with two spaces, or on a line on their own._____no_output_____

distance = 891  # in meters
elapsed_time = 93  # in seconds

# Calculate the speed in km/h.
speed = 3.6 * distance / elapsed_time_____no_output_____

But let's think wisely if we need to use a comment.
The second cell is a lot more Pythonic._____no_output_____

seconds = 365 * 24 * 60 * 60  # = seconds in the year_____no_output_____seconds_per_year = 365 * 24 * 60 * 60
_____no_output_____

### Simple `for`-loops

`for`-loops are extremely versatile in Python. That is different from many other programming languages.

As shown in the first example, we can create a `list` like `numbers` and loop over the numbers in it on a one-by-one basis._____no_output_____

numbers = [7, 11, 8, 5, 3, 12, 2, 6, 9, 10, 1, 4]_____no_output_____for number in numbers:
    print(...)_____no_output_____

## Formal vs. Natural Languages

Just like the language of mathematics is good at expressing relationships among numbers and symbols, any programming language is just a formal language that is good at expressing computations.

Formal languages come with their own "grammatical rules" called **syntax**.

### Syntax Errors

If we do not follow the rules, the code cannot be **parsed** correctly, i.e., the program does not even start to run but **raises** a **syntax error** indicated as `SyntaxError` in the output. Computers are very dumb in the sense that the slightest syntax error leads to the machine not understanding our code.

If we were to write an accounting program that adds up currencies, we would, for example, have to model dollar prices as `float` objects as the dollar symbol cannot be understood by Python._____no_output_____

3.99 $ + 10.40 $_____no_output_____

Python requires certain symbols at certain places (e.g., a `:` is missing here)._____no_output_____

for number in numbers
print(number)_____no_output_____

Furthermore, it relies on whitespace (i.e., indentation), unlike many other programming languages. The `IndentationError` below is just a particular type of a `SyntaxError`._____no_output_____## TL;DR

We end each chapter with a summary of the main points (i.e., **TL;DR** = "too long; didn't read").
- program
  - **sequence** of **instructions** that specify how to perform a computation (= a "recipe")
  - a "black box" that processes **inputs** and transforms them into meaningful **outputs** in a *deterministic* way
  - conceptually similar to a mathematical function $f$ that maps some input $x$ to an output $y = f(x)$

- comments
  - **prose** supporting a **human's understanding** of the program
  - ignored by Python

_____no_output_____## Further readings [Miller et al., 2013]:

#### Algorithms

Given a problem, a computer scientist’s goal is to develop an algorithm, a step-by-step list of instructions for solving any instance of the problem that might arise. Algorithms are finite processes that if followed will solve the problem. Algorithms are solutions.

We say that a problem is computable if an algorithm exists for solving it. An alternative
definition for computer science, then, is to say that computer science is the study of problems that are and that are not computable, the study of the existence and the nonexistence of algorithms.

In any case, you will note that the word “computer” did not come up at all. Solutions
are considered independent from the machine.

#### Abstraction

Computer science, as it pertains to the problem-solving process itself, is also the study of abstraction. Abstraction allows us to view the problem and solution in such a way as to separate the so-called logical and physical perspectives.

Consider the automobile that you may have driven to school or work today. As a driver, a user of the car, you have certain interactions that take place in order to utilize the car for its intended purpose. You get in, insert the key, start the car, shift, brake, accelerate, and steer in order to drive. From an abstraction point of view, we can say that you are seeing the logical perspective of the automobile. You are using the functions provided by the car designers for the purpose of transporting you from one location to another. These functions are sometimes also referred to as the interface.

On the other hand, the mechanic who must repair your automobile takes a very different point of view. She not only knows how to drive but must know all of the details necessary to carry out all the functions that we take for granted. She needs to understand how the engine works, how the transmission shifts gears, how temperature is controlled, and so on. This is known as the physical perspective, the details that take place “under the hood.”

The same thing happens when we use computers. Most people use computers to write documents, send and receive email, surf the web, play music, store images, and play games without any knowledge of the details that take place to allow those types of applications to work. They view computers from a logical or user perspective. Computer scientists, programmers, technology support staff, and system administrators take a very different view of the computer. They must know the details of how operating systems work, how network protocols are configured, and how to code various scripts that control function. They must be able to control the low-level details that a user simply assumes.

The common point for both of these examples is that the user of the abstraction, sometimes also called the client, does not need to know the details as long as the user is aware of the way the interface works. This interface is the way we as users communicate with the underlying complexities of the implementation.

Python code:_____no_output_____

import math
math.sqrt(15)
_____no_output_____

This is an example of procedural abstraction. We do not necessarily know how the square root is being calculated, but we know what the function is called and how to use it. If we perform the import correctly, we can assume that the function will provide us with the correct results.
We know that someone implemented a solution to the square root problem but we only
need to know how to use it. This is sometimes referred to as a “black box” view of a process. We simply describe the interface: the name of the function, what is needed (the parameters), and what will be returned. The details are hidden inside.
_____no_output_____#### Why Study Algorithms?

Computer scientists learn by experience. We learn by seeing others solve problems and by solving problems by ourselves. Being exposed to different problem-solving techniques and seeing how different algorithms are designed helps us to take on the next challenging problem that we are given. By considering a number of different algorithms, we can begin to develop pattern recognition so that the next time a similar problem arises, we are better able to solve it.

Algorithms are often quite different from one another. Consider the example of sqrt seen earlier. It is entirely possible that there are many different ways to implement the details to compute the square root function. One algorithm may use many fewer resources than another. One algorithm might take 10 times as long to return the result as the other. We would like to have some way to compare these two solutions. Even though they both work, one is perhaps “better” than the other. We might suggest that one is more efficient or that one simply works faster or uses less memory. As we study algorithms, we can learn analysis techniques that allow us to compare and contrast solutions based solely on their own characteristics, not the characteristics of the program or computer used to implement them.

In the worst case scenario, we may have a problem that is intractable, meaning that there is no algorithm that can solve the problem in a realistic amount of time. It is important to be able to distinguish between those problems that have solutions, those that do not, and those where solutions exist but require too much time or other resources to work reasonably.

Python is a modern, easy-to-learn, object-oriented programming language. It has a powerful set of built-in data types and easy-to-use control constructs.

### References:
1. Miller, Brad, and David Ranum. "Problem solving with algorithms and data structures." (2013)._____no_output_____