# Notebook from martinjzhang/scDRS
Path: experiments/job.ca1_pyramidal/zeisel.ipynb

# Additional results for Zeisel et al. 2015
1. Summary of cell type-disease association at the cell-type level.
2. Further UMAP for other diseases and traits
3. overlay spatial scores on UMAP._____no_output_____

<code>
%load_ext lab_black

import matplotlib

matplotlib.rcParams["pdf.fonttype"] = 42
matplotlib.rcParams["ps.fonttype"] = 42

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from os.path import join
from IPython.display import display, Markdown, Latex
import sys
import seaborn as sns
from scdrs.util import test_gearysc

sys.path.append("/n/home12/khou/holystore/")
import paper_utils

import scanpy as sc

sc.set_figure_params(dpi=150)
from scipy.stats import rankdataThe lab_black extension is already loaded. To reload it, use:
  %reload_ext lab_black
URL_SUPP_TABLE = "https://www.dropbox.com/s/qojbzu5zln33j7f/supp_tables.xlsx?dl=1"_____no_output_____def calc_mc_pval(list_trait, stats="percentile_95"):

    df_obs = adata.obs

    assert stats in ["max", "percentile_99", "percentile_95", "percentile_90"]
    col_ctrl_norm_score = [f"ctrl_norm_score_{i}" for i in range(1000)]

    dict_mc_pval = {}
    for trait in list_trait:
        df_tmp = pd.read_csv(
            f"01_score_cell/score_file/zeisel_2015/{trait}.full_score.gz",
            sep="\t",
            index_col=0,
        )
        df_tmp = pd.merge(
            df_obs[["level1class"]], df_tmp, left_index=True, right_index=True
        )

        if stats == "max":
            stats_func = lambda x: np.max(x)
        elif stats == "percentile_99":
            stats_func = lambda x: np.percentile(x, 99)
        elif stats == "percentile_95":
            stats_func = lambda x: np.percentile(x, 95)
        elif stats == "percentile_90":
            stats_func = lambda x: np.percentile(x, 90)
        else:
            raise NotImplementedError
        df_tmp = df_tmp.groupby("level1class").agg(
            {
                trait: lambda x: stats_func(x)
                for trait in ["norm_score"] + col_ctrl_norm_score
            }
        )

        rank_score = rankdata(df_tmp.values, axis=1)
        mc_pval = pd.Series(
            1 - (rank_score[:, 0] - 1) / rank_score.shape[1], index=df_tmp.index
        )
        dict_mc_pval[trait] = mc_pval
    df_mc_pval = pd.DataFrame(dict_mc_pval).T
    df_mc_pval.columns.name = None
    return df_mc_pval

def celltype_display_name(x):
    if x in dict_celltype_name:
        name = dict_celltype_name[x]
    else:
        name = x
    name += f" ({df_celltype_n_cell[x]})"
    name = name.replace("_", " ")
    name = name[0].upper() + name[1:]
    return name

def trait_display_name(x):
    dict_trait_name = {
        row.Trait_Identifier: row["Trait Name"] for _, row in df_trait_info.iterrows()
    }
    dict_trait_code = {
        row.Trait_Identifier: row["Code"] for _, row in df_trait_info.iterrows()
    }

    if dict_trait_name[x].lower() != dict_trait_code[x].lower():
        return f"{dict_trait_name[x]} ({dict_trait_code[x]})"
    else:
        return dict_trait_name[x]

def asterisk_display(x):
    if x < 0.05:
        return "×"
    else:
        return ""

from matplotlib.patches import Rectangle
import matplotlib.patches as patches

def small_squares(ax, pos, size=1, linewidth=0.8):
    """
    Draw many small squares on ax, given the positions of
    these squares.

    """
    for xy in pos:
        x, y = xy
        margin = (1 - size) / 2
        rect = patches.Rectangle(
            (x + margin, y + margin),
            size,
            size,
            linewidth=linewidth,
            edgecolor="k",
            facecolor="none",
            zorder=20,
        )
        ax.add_patch(rect)_____no_output_____display_trait_list = [
    "PASS_MDD_Howard2019",
    "PASS_Schizophrenia_Pardinas2018",
    "UKB_460K.mental_NEUROTICISM",
    "UKB_460K.cov_SMOKING_STATUS",
    "UKB_460K.cov_EDU_COLLEGE",
    "UKB_460K.body_BMIz",
    "UKB_460K.body_HEIGHTz",
]

df_trait_info = pd.read_excel(
    URL_SUPP_TABLE,
    sheet_name=0,
)_____no_output_____dataset = "zeisel_2015"
processed_dir = "00_prepare_dataset/processed"
adata = sc.read_h5ad(join(processed_dir, f"{dataset}.processed.h5ad"))

df_trs_pval = dict()
for trait in display_trait_list:
    df_trait_score = pd.read_csv(
        join("01_score_cell/score_file", dataset, f"{trait}.score.gz"),
        sep="\t",
        index_col=0,
    )
    df_trs_pval[trait] = df_trait_score["pval"].reindex(adata.obs.index)
df_trs_pval = pd.DataFrame(df_trs_pval)

df_fdr_prop = paper_utils.agg_trs_pval(
    df_obs=adata.obs.copy(),
    df_pval=df_trs_pval,
    stats="fdr_prop",
    groupby="level1class",
    fdr_prop_threshold=0.1,
)

dict_df_gearysc = dict()
for trait in display_trait_list:
    df_score_full = pd.read_csv(
        join("01_score_cell/score_file", dataset, f"{trait}.full_score.gz"),
        sep="\t",
        index_col=0,
    ).reindex(adata.obs.index)
    dict_df_gearysc[trait] = test_gearysc(adata, df_score_full, "level1class")_____no_output_____dict_celltype_display_name = {
    "pyramidal_CA1": "Pyramidal CA1",
    "oligodendrocytes": "Oligodendrocyte",
    "pyramidal_SS": "Pyramidal SS",
    "interneurons": "Interneuron",
    "endothelial-mural": "Endothelial",
    "astrocytes_ependymal": "Astrocyte",
    "microglia": "Microglia",
}_____no_output_____from statsmodels.stats.multitest import multipletests

df_gearysc_fdr = pd.DataFrame(
    {trait: dict_df_gearysc[trait]["pval"] for trait in dict_df_gearysc}
)

# convert to FDR
df_gearysc_fdr = pd.DataFrame(
    multipletests(df_gearysc_fdr.values.flatten(), method="fdr_bh")[1].reshape(
        df_gearysc_fdr.shape
    ),
    index=df_gearysc_fdr.index,
    columns=df_gearysc_fdr.columns,
).T

df_gearysc_fdr.columns = [col.replace(" ", "_") for col in df_gearysc_fdr.columns]_____no_output_____df_ct_pval = calc_mc_pval(display_trait_list)
df_ct_fdr = pd.DataFrame(
    multipletests(df_ct_pval.values.flatten(), method="fdr_bh")[1].reshape(
        df_ct_pval.shape
    ),
    index=df_ct_pval.index,
    columns=df_ct_pval.columns,
)
df_ct_fdr.columns = [c.replace(" ", "_") for c in df_ct_fdr.columns]_____no_output_____df_celltype_n_cell = adata.obs.level1class.value_counts()
df_celltype_n_cell.index = [c.replace(" ", "_") for c in df_celltype_n_cell.index]

df_plot = df_fdr_prop[
    [
        "pyramidal_CA1",
        "pyramidal_SS",
        "interneurons",
        "astrocytes_ependymal",
        "endothelial-mural",
        "microglia",
        "oligodendrocytes",
    ]
].copy()

df_gearysc_plot = df_gearysc_fdr.loc[df_plot.index, df_plot.columns].copy()
df_ct_fdr = df_ct_fdr.loc[df_plot.index, df_plot.columns].copy()
df_gearysc_annot = df_gearysc_plot.applymap(asterisk_display)
df_gearysc_annot[df_ct_fdr > 0.05] = ""

df_plot = df_plot.rename(
    index=trait_display_name,
    columns=lambda x: dict_celltype_display_name[x] + f"({df_celltype_n_cell[x]})",
)

fig, ax = paper_utils.plot_heatmap(
    df_plot,
    squaresize=30,
    heatmap_annot=df_gearysc_annot,
    heatmap_annot_kws={"color": "black", "size": 6},
    heatmap_cbar_kws=dict(
        use_gridspec=False, location="top", fraction=0.03, pad=0.05, drawedges=True
    ),
    heatmap_vmin=0,
    heatmap_vmax=0.2,
    colormap_n_bin=5,
)

small_squares(
    ax,
    pos=[(y, x) for x, y in zip(*np.where(df_ct_fdr < 0.05))],
    size=0.6,
    linewidth=0.5,
)

cb = ax.collections[0].colorbar
cb.ax.tick_params(labelsize=8)

cb.ax.set_title("Prop. of sig. cells (FDR < 0.1)", fontsize=8)
cb.outline.set_edgecolor("black")
cb.outline.set_linewidth(1)

plt.tight_layout()

plt.savefig("results/zeisel_assoc.pdf", bbox_inches="tight")

plt.show()<ipython-input-22-f0a2b829c203>:54: UserWarning: This figure includes Axes that are not compatible with tight_layout, so results might be incorrect.
  plt.tight_layout()
<ipython-input-22-f0a2b829c203>:54: UserWarning: Tight layout not applied. The left and right margins cannot be made large enough to accommodate all axes decorations.
  plt.tight_layout()
from functools import reduce

with pd.ExcelWriter("results/numeric_supp_tables.xlsx") as writer:
    df1 = df_fdr_prop.copy()
    df2 = df_ct_fdr.loc[df1.index, df1.columns].copy()
    df3 = df_gearysc_fdr.loc[df1.index, df1.columns]
    df1 = df1.melt(ignore_index=False).reset_index()
    df2 = df2.melt(ignore_index=False).reset_index()
    df3 = df3.melt(ignore_index=False).reset_index()
    df = reduce(
        lambda left, right: pd.merge(
            left, right, on=["index", "variable"], how="outer"
        ),
        [df1, df2, df3],
    )

    df.columns = [
        "trait",
        "cell-type",
        "prop-sig-cells",
        "ct-assoc-fdr",
        "heterogeneity-fdr",
    ]
    df.to_excel(writer, sheet_name="zeisel", float_format="%.2g", index=False)

    worksheet = writer.sheets["zeisel"]  # pull worksheet object
    for idx, col in enumerate(df):  # loop through all columns
        series = df[col]
        max_len = (
            max(
                (
                    series.astype(str).map(len).max(),  # len of largest item
                    len(str(series.name)),  # len of column name/header
                )
            )
            + 1
        )  # adding a little extra space
        worksheet.set_column(idx, idx, max_len)  # set column width_____no_output_____dataset = "zeisel_2015"
adata_dir = "00_prepare_dataset/processed/"
adata = sc.read_h5ad(join(adata_dir, "zeisel_2015.processed.h5ad"))
adata = adata[adata.obs["level2class"].isin(["CA1Pyr1", "CA1Pyr2"])]

spatial_cols = [
    "spatial_dorsal",
    "spatial_ventral",
    "spatial_proximal",
    "spatial_distal",
    "spatial_deep",
    "spatial_superficial",
]
df_list = []
for col in spatial_cols:
    temp_df = pd.read_csv(
        join("01_score_cell/score_file", dataset, f"{col}.score.gz"),
        sep="\t",
        index_col=0,
    )
    temp_df.columns = ["%s.%s" % (col, x) for x in temp_df.columns]
    df_list.append(temp_df)

df_spatial_score = pd.concat(df_list, axis=1)
df_spatial_score = df_spatial_score[
    [col for col in df_spatial_score if col.endswith(".norm_score")]
].rename(columns=lambda c: c[:-11])

for spatial_col in df_spatial_score.columns:
    adata.obs[spatial_col] = df_spatial_score[spatial_col].reindex(adata.obs.index)Trying to set attribute `.obs` of view, copying.
df_corr = (
    adata.obs[spatial_cols]
    .corr()
    .rename(columns=lambda x: x[8:].capitalize(), index=lambda x: x[8:].capitalize())
)
fig, ax = plt.subplots(dpi=150, figsize=(2.3, 2))
h = sns.heatmap(
    df_corr, center=0, cmap="RdBu", ax=ax, annot=True, fmt=".2f", annot_kws={"size": 6}
)

h.set_xticklabels(h.get_xmajorticklabels(), fontsize=7)
h.set_yticklabels(h.get_ymajorticklabels(), fontsize=7)
ax.set_title("Correlation between\ninferred spatial coordinates", fontsize=9)
plt.savefig("results/spatial_correlation.pdf", bbox_inches="tight")_____no_output_____
</code>


# Notebook from PatrickgHayes/gmm-dnn-for-interpretability
Path: Sampling_From_KDE.ipynb

# DISTRIBUTION STATEMENT A. Approved for public release: distribution unlimited.
#
# This material is based upon work supported by the Assistant Secretary of Defense for Research and
# Engineering under Air Force Contract No. FA8721-05-C-0002 and/or FA8702-15-D-0001. Any opinions,
# findings, conclusions or recommendations expressed in this material are those of the author(s) and
# do not necessarily reflect the views of the Assistant Secretary of Defense for Research and
# Engineering.
#
# © 2018 Massachusetts Institute of Technology.
#
# MIT Proprietary, Subject to FAR52.227-11 Patent Rights - Ownership by the contractor (May 2014)
#
# The software/firmware is provided to you on an As-Is basis
#
# Delivered to the U.S. Government with Unlimited Rights, as defined in DFARS Part 252.227-7013 or
# 7014 (Feb 2014). Notwithstanding any copyright notice, U.S. Government rights in this work are
# defined by DFARS 252.227-7013 or DFARS 252.227-7014 as detailed above. Use of this work other than
# as specifically authorized by the U.S. Government may violate any copyrights that exist in this
# work._____no_output_____from IPython.core.display import display, HTML
display(HTML(""))
import numpy as np
from matplotlib import pyplot as plt
import torch.nn as nn
import torch.nn.functional as F
import torch
import torch.optim as optim
import os
from torch.autograd import Variable
from sklearn.neighbors.kde import KernelDensity
from Plot import plot_loss
from sklearn.model_selection import GridSearchCV
from Datasets import NumberChunks
from Datasets import NumberRings
from Datasets import NumberChunks2D
from Datasets import NumberRings2D
from Datasets import SineWave
import math

from torch.utils.data import DataLoader
from collections import defaultdict
import matplotlib.cm as cm

from mpl_toolkits import mplot3d
from torchvision import datasets, transforms

%matplotlib inline_____no_output_____

## This is a quick demo of the algorithm for sampling when you must sample a point on a manifold_____no_output_____

size=100
x = np.random.randint(-10, 10, size) + np.random.random(size)
y = np.random.normal(0, 1, size)

data = np.stack((x,y), axis=1).reshape(-1, 2)

y = np.random.randint(-10, 10, size) + np.random.random(size)
x = np.random.normal(0, 1, size)

tmp = np.stack((x,y), axis=1).reshape(-1, 2)

data = np.vstack((data, tmp))
labels = (data[:, 0] > 0).astype(int)
#####################################

xs = np.arange(-10, 10, 1)
ys = xs * -1
line = np.stack((xs, ys)).transpose()

plt.scatter(data[idxs.reshape(-1), 0], data[idxs.reshape(-1), 1], marker='o', c='blue')
idxs = np.argwhere(labels == 0)
plt.scatter(data[idxs.reshape(-1), 0], data[idxs.reshape(-1), 1], marker='s', c='green')
plt.scatter(line[:, 0], line[:, 1], marker='.', c='black')
plt.axis('equal')
plt.ylim(-10, 10)
plt.xlim(-10, 10)
plt.show()

orig = np.array([[-10, 10]
                ,[10, -10]])

theta = math.atan((orig[1,1] - orig[0,1]) / (orig[1,0] - orig[0,0]))
r = np.array([[math.cos(theta), -math.sin(theta)]
             ,[math.sin(theta), math.cos(theta)]])

data_r = data @ r
line_r = line @ r

distances = torch.tensor(-1 * abs(data_r[:, 1] - line_r[0, 1])).float()
weights = F.softmax(distances, dim=0).numpy()
star = data[np.random.choice(np.arange(data.shape[0]), p=weights)]

star_r = star @ r
point_on_line_r = np.array([star_r[0], line_r[0, 1]])
sample_r = np.array([np.random.normal(star_r[0], scale=3), line_r[0, 1]])

# rotate it back
r_inv = np.linalg.inv(r)

sample = sample_r @ r_inv
point_on_line = point_on_line_r @ r_inv

idxs = np.argwhere(labels == 1)
plt.scatter(data_r[idxs.reshape(-1), 0], data_r[idxs.reshape(-1), 1], marker='o', c='blue')
idxs = np.argwhere(labels == 0)
plt.scatter(data_r[idxs.reshape(-1), 0], data_r[idxs.reshape(-1), 1], marker='s', c='green')
plt.scatter(line_r[:, 0], line_r[:, 1], marker='.', c='black')
plt.scatter(point_on_line_r[0], point_on_line_r[1], c='red')
plt.scatter(star_r[0], star_r[1], c='red')
plt.scatter(sample_r[0], sample_r[1], c='pink')
plt.axis('equal')
plt.ylim(-10, 10)
plt.xlim(-10, 10)
plt.show()

idxs = np.argwhere(labels == 1)
plt.scatter(data[idxs.reshape(-1), 0], data[idxs.reshape(-1), 1], marker='o', c='blue')
idxs = np.argwhere(labels == 0)
plt.scatter(data[idxs.reshape(-1), 0], data[idxs.reshape(-1), 1], marker='s', c='green')
plt.scatter(line[:, 0], line[:, 1], marker='.', c='black')
plt.scatter(point_on_line[0], point_on_line[1], c='red')
plt.scatter(star[0], star[1], c='red')
plt.scatter(sample[0], sample[1], c='pink')
plt.axis('equal')
plt.ylim(-10, 10)
plt.xlim(-10, 10)
plt.show()_____no_output_____

## Under Constrained Layers

For this network we are only going to worry about under constrained layers so our network will look like this

                        O
                       / \
                      O   O

For any sampled output will (almost) always be able to choose an input that can produce that input _____no_output_____

size=100
x = np.random.randint(-10, 10, size) + np.random.random(size)
y = np.random.normal(0, 1, size)

data = np.stack((x,y), axis=1).reshape(-1, 2)

y = np.random.randint(-10, 10, size) + np.random.random(size)
x = np.random.normal(0, 1, size)

tmp = np.stack((x,y), axis=1).reshape(-1, 2)

data = np.vstack((data, tmp))
# labels = ((data[:, 0] * -1) - data[:, 1] > 0).astype(int) these labels are bad because they have too much
# symmetry
labels = (data[:, 0] > 0).astype(int)_____no_output_____class SimpleNet(nn.Module):

    def __init__(self, in_dim=2, out_dim=1):
        super(SimpleNet, self).__init__()
        self.layer = nn.Linear(in_dim, out_dim)
        self.input_kde = KernelDensity()
        self.output_kde = KernelDensity()
        self.params = {'bandwidth': np.logspace(-1, 1, 20)}
        self.grid = GridSearchCV(KernelDensity(), self.params, cv=5)
        return

    def forward(self, x):
        outputs = self.layer(x)
        return outputs

    def fit_kdes(self, x):
        self.grid.fit(x.detach().numpy())
        self.input_kde = self.grid.best_estimator_

        outputs = self.forward(x)
        self.grid.fit(outputs.detach().numpy())
        self.output_kde = self.grid.best_estimator_
        return

    def sample_lines(self, thr=0, num_lines=10):
        for i in range(num_lines):
            score = self.output_kde.sample()[0]

            # get the equation of the line
            weight_1 = self.layer.weight.detach().numpy()[0, 0]
            weight_2 = self.layer.weight.detach().numpy()[0, 1]
            bias = self.layer.bias.detach().numpy()[0]

            xs = np.arange(-10, 11, 0.1)
            ys = (score - bias - (weight_1 * xs)) / weight_2
            plt.scatter(xs, ys)
        plt.ylim(-20, 20)
        plt.xlim(-20, 20)
        plt.show()
        print("weights " + str(self.layer.weight))
        print("bias " + str(self.layer.bias))

    def sample(self, data, thr=0):
        score = self.output_kde.sample()[0]

        # get the line
        weight_1 = self.layer.weight.detach().numpy()[0, 0]
        weight_2 = self.layer.weight.detach().numpy()[0, 1]
        bias = self.layer.bias.detach().numpy()[0]

        x_1 = 0
        x_2 = 1

        y_1 = (score - bias - (weight_1 * x_1)) / weight_2
        y_2 = (score - bias - (weight_1 * x_2)) / weight_2

        orig = np.array([[x_1, y_1]
                        ,[x_2, y_2]])

        theta = math.atan((orig[1,1] - orig[0,1]) / (orig[1,0] - orig[0,0]))
        r = np.array([[math.cos(theta), -math.sin(theta)]
                     ,[math.sin(theta), math.cos(theta)]])

        # sample from the training points weighted on their distance
        # from the line
        data = data.numpy()
        data_r = data @ r
        lin = np.array([x_1, y_1]) @ r

        distances = torch.tensor(-1 * abs(data_r[:, 1] - lin[1])).float()
        weights = F.softmax(distances, dim=0).numpy()

        star = data[np.random.choice(np.arange(data.shape[0]), p=weights)]

        star_r = star @ r

        sample_r = np.array([np.random.normal(star_r[0], scale=self.input_kde.bandwidth), lin[1]])

        # rotate it back
        r_inv = np.linalg.inv(r)

        sample = sample_r @ r_inv

        return sample

torch.manual_seed(13)
s = SimpleNet()
s.fit_kdes(Variable(torch.tensor(data)).float())
s.sample(torch.tensor(data))
s.sample_lines()_____no_output_____def plot_training_data(data, labels):
    idxs = np.argwhere(labels == 1)
    plt.scatter(data[idxs.reshape(-1), 0], data[idxs.reshape(-1), 1], marker='o')
    idxs = np.argwhere(labels == 0)
    plt.scatter(data[idxs.reshape(-1), 0], data[idxs.reshape(-1), 1], marker='s')
    plt.ylim(-20, 20)
    plt.xlim(-20, 20)
    plt.show()
    return

def sample_and_plot(neural_net, data, num_samples=10000, thr=0):
    samples = np.stack([neural_net.sample(data, thr=thr) for i in range(num_samples)])
    plt.scatter(samples[:, 0], samples[:, 1], marker='x', c='green')
    plt.ylim(-20, 20)
    plt.xlim(-20, 20)
    plt.show()

plot_training_data(data, labels)
sample_and_plot(s, torch.tensor(data))_____no_output_____s_net = SimpleNet()_____no_output_____thr=0
criterion = nn.MSELoss()
optimizer = optim.Adam(s_net.parameters(), lr=0.1)

loss_history = list()

print_every = 5

for epoch in range(30):

    data, labels = Variable(torch.tensor(data).float().view(-1, 2)), Variable(torch.tensor(labels).float().view(-1, 1))

    optimizer.zero_grad()

    outputs = torch.sigmoid(s_net(data))
    loss = criterion(outputs, labels)
    loss.backward()
    optimizer.step()

    loss_history.append(loss.item())
#     print("Loss: " + str(loss.item()))
    if epoch != 0 and epoch % print_every == 0:
        plot_loss(loss_history)

    if epoch != 0 and epoch % print_every == 0:
        s_net.fit_kdes(data)
        plot_training_data(data.detach(), labels.detach())
        sample_and_plot(s_net, data, thr=thr)
        s_net.sample_lines(num_lines=3)
#         s_net.plot_presigs()
#         s_net.presigs = list()
#         break
#         x = input("To continue press enter")
#         if x == "x":
#             break_____no_output_____thr=5
plot_training_data(data.detach(), labels.detach())
sample_and_plot(s_net, data, thr=thr)_____no_output_____

## Over Constrained
This time our network will look like this

        O   O   O
         \ / \ /
          O   O

 So we will have to deal with being over constrained.
 Given an arbitrary output there may not be an input that could produce that output.

 To deal with this we are going to project our sampled point onto the constrained manifold._____no_output_____

def build_tri_sombrero_dataset():
    y = np.random.random(size=(100, 1)) + np.random.randint(0, 1, size=(100, 1)) + 1
    x = np.random.normal(0, 0.1, size=(100, 1))

    tall = np.stack((x,y), axis=1).reshape(-1, 2)

    x =  np.random.random(size=(200, 1)) + np.random.randint(-1, 1, size=(200, 1))
    y =  np.random.normal(0, 0.1, size=(200, 1))

    flat = np.stack((x,y), axis=1).reshape(-1, 2)

    sombrero = np.vstack((tall, flat))

    theta = math.radians(-45)
    r = np.array([[math.cos(theta), -math.sin(theta)]
                     ,[math.sin(theta), math.cos(theta)]])

    right_sombrero = sombrero @ r
    right_sombrero[:, 0] = right_sombrero[:, 0] - 1
    right_sombrero[:, 1] = right_sombrero[:, 1] + 1

    theta = math.radians(45)
    r = np.array([[math.cos(theta), -math.sin(theta)]
                     ,[math.sin(theta), math.cos(theta)]])

    left_sombrero = sombrero @ r
    left_sombrero[:, 0] = left_sombrero[:, 0] + 1
    left_sombrero[:, 1] = left_sombrero[:, 1] + 1

    theta = math.radians(180)
    r = np.array([[math.cos(theta), -math.sin(theta)]
                     ,[math.sin(theta), math.cos(theta)]])

    upsidedown_sombrero = sombrero @ r
    upsidedown_sombrero[:, 1] = upsidedown_sombrero[:, 1]

    data = np.vstack((left_sombrero, right_sombrero, upsidedown_sombrero))
    labels = np.concatenate(([0] * left_sombrero.shape[0], [1] * right_sombrero.shape[0], [2] * upsidedown_sombrero.shape[0]))

    return data, labels

data, labels = build_tri_sombrero_dataset()
idxs = np.argwhere(labels == 2)
plt.scatter(data[idxs.reshape(-1), 0], data[idxs.reshape(-1), 1])
idxs = np.argwhere(labels == 1)
plt.scatter(data[idxs.reshape(-1), 0], data[idxs.reshape(-1), 1])
idxs = np.argwhere(labels == 0)
plt.scatter(data[idxs.reshape(-1), 0], data[idxs.reshape(-1), 1])
plt.ylim(-3, 3)
plt.xlim(-3, 3)
plt.show()_____no_output_____a = KernelDensity()
a.fit(np.array([[0,1]]))
print(a.__dict__.keys())
print(a.tree_.get_arrays())dict_keys(['algorithm', 'bandwidth', 'kernel', 'metric', 'atol', 'rtol', 'breadth_first', 'leaf_size', 'metric_params', 'tree_'])
(array([[0., 1.]]), array([0]), array([(0, 1, 1, 0.)],
      dtype=[('idx_start', '<i8'), ('idx_end', '<i8'), ('is_leaf', '<i8'), ('radius', '<f8')]), array([[[0., 1.]],

       [[0., 1.]]]))
class OverConstrainedNet(nn.Module):

    def __init__(self, in_dim=2, out_dim=3):
        super(OverConstrainedNet, self).__init__()
        self.layer = nn.Linear(in_dim, out_dim)
        self.input_kde = KernelDensity()
        self.output_kde = KernelDensity()
        self.output_kde_elements = None
        self.params = {'bandwidth': np.logspace(-1, 1, 20)}
        self.grid = GridSearchCV(KernelDensity(), self.params, cv=5)
        return

    def forward(self, x):
        outputs = self.layer(x)
        return outputs

    def fit_kdes(self, x):
        self.grid.fit(x.detach().numpy())
        self.input_kde = self.grid.best_estimator_

        outputs = self.forward(x)
        self.output_kde_elements = outputs.detach().numpy()
        self.grid.fit(outputs.detach().numpy())
        self.output_kde = self.grid.best_estimator_
        return

#     def sample_lines(self, thr=0, num_lines=10):
#         for i in range(num_lines):
#             score = self.output_kde.sample()[0]

#             # get the equation of the line
#             weight_1 = self.layer.weight.detach().numpy()[0, 0]
#             weight_2 = self.layer.weight.detach().numpy()[0, 1]
#             bias = self.layer.bias.detach().numpy()[0]

#             xs = np.arange(-10, 11, 0.1)
#             ys = (score - bias - (weight_1 * xs)) / weight_2
#             plt.scatter(xs, ys)
#         plt.ylim(-20, 20)
#         plt.xlim(-20, 20)
#         plt.show()
#         print("weights " + str(self.layer.weight))
#         print("bias " + str(self.layer.bias))

    def sample(self, data, thr=0):
        # get the plane
        points = torch.tensor([[0, 0]
                              ,[0, 1]]).float()

        outputs = self.forward(points)

        theta = math.atan((orig[1,1] - orig[0,1]) / (orig[1,0] - orig[0,0]))
        r = np.array([[math.cos(theta), -math.sin(theta)]
                     ,[math.sin(theta), math.cos(theta)]])

        # sample from the training points weighted on their distance
        # from the line
        data = data.numpy()
        data_r = data @ r
        lin = np.array([x_1, y_1]) @ r

        distances = torch.tensor(-1 * abs(data_r[:, 1] - lin[1])).float()
        weights = F.softmax(distances, dim=0).numpy()

        star = data[np.random.choice(np.arange(data.shape[0]), p=weights)]

        star_r = star @ r

        sample_r = np.array([np.random.normal(star_r[0], scale=self.input_kde.bandwidth), lin[1]])

        # rotate it back
        r_inv = np.linalg.inv(r)

        sample = sample_r @ r_inv

        return sample

over_net = OverConstrainedNet()
a = np.hstack((over_net.layer.weight.detach().numpy(), over_net.layer.bias.detach().numpy().reshape(-1, 1))).transpose()
print(a)

data = np.array([[0, 0, 1]])
for i in np.arange(-10, 10, 0.1):
    lsd = np.arange(-10, 10, 0.1).reshape(-1, 1)
    msd = np.array([i] * lsd.shape[0]).reshape(-1, 1)
    bias = np.array([1] * lsd.shape[0]).reshape(-1, 1)
    unit = np.hstack((msd, lsd, bias))
    data = np.vstack((data, unit))
data[-10:, :]

product = data @ a

ax = plt.axes(projection='3d')
ax.scatter3D(product[:, 0], product[:, 1], product[:, 2], c=product[:, 2], cmap='Greens')
ax.view_init(60, 60)
plt.show()[[ 0.44895715 -0.6306021   0.48155433]
 [-0.5286453  -0.3863557   0.05555707]
 [ 0.6785086   0.508679    0.13819915]]
class KernelNet(nn.Module):

    def __init__(self, in_dim=1, out_dim=3, hid_nodes=20):
        super(KernelNet, self).__init__()
        self.layer1 = nn.Linear(in_dim, hid_nodes)
        self.layer2 = nn.Linear(hid_nodes, hid_nodes)
        self.layer3 = nn.Linear(hid_nodes, out_dim)

        self.kde_layer1 = KernelDensity()
        self.kde_layer2 = KernelDensity()
        self.kde_layer3 = KernelDensity()

        self.params = {'bandwidth': np.logspace(-1, 1, 20)}
        self.grid = GridSearchCV(KernelDensity(), self.params, cv=5)

    def forward(self, x):
        outputs = self.layer1(x)
        outputs = torch.sigmoid(outputs)
        outputs = self.layer2(outputs)
        outputs = torch.sigmoid(outputs)
        outputs = self.layer3(outputs)
        return outputs

    def fit_kdes(self, x):
        self.grid.fit(x.detach().numpy())
        self.kde_layer1 = self.grid.best_estimator_
        outputs = self.layer1(x)
        outputs = torch.sigmoid(outputs)

        self.grid.fit(outputs.detach().numpy())
        self.kde_layer2 = self.grid.best_estimator_
        outputs = self.layer2(outputs)
        outputs = torch.sigmoid(outputs)

        self.grid.fit(outputs.detach().numpy())
        self.kde_layer3 = self.grid.best_estimator_
        # We don't need to compute the last linear layer because we only care
        # about the inputs
        outputs = self.layer3(outputs)

    def domain_check(self, x, threshold=-4):
        log_likelihoods = self.kde_layer1.score_samples(x.detach().numpy())
        log_likelihoods = torch.tensor(log_likelihoods)
#         print("First layer")
#         print(log_likelihoods)
        in_domain = log_likelihoods < threshold
#         print("First layer in domain")
#         print(in_domain)
        outputs = self.layer1(x)
        outputs = torch.sigmoid(outputs)

        log_likelihoods = self.kde_layer2.score_samples(outputs.detach().numpy())
        log_likelihoods = torch.tensor(log_likelihoods)
#         print("Second layer")
#         print(log_likelihoods)
        in_domain = in_domain + (log_likelihoods < threshold)
        outputs = self.layer2(outputs)
        outputs = torch.sigmoid(outputs)

        log_likelihoods = self.kde_layer3.score_samples(outputs.detach().numpy())
        log_likelihoods = torch.tensor(log_likelihoods)
#         print("Third layer")
#         print(log_likelihoods)
        in_domain = in_domain + (log_likelihoods < threshold)
        return in_domain_____no_output_____


# Notebook from abostroem/2020-01-04-aas
Path: python/in_class_notebook.ipynb

1_____no_output_____

shift+enter to execute cell_____no_output_____# Heading 1
## Heading 2
##### Heading

* bullet 1
* bullet 2
    * subbullet 1

$y = \frac{x}{2}$_____no_output_____

1_____no_output_____2_____no_output_____1._____no_output_____1/2_____no_output_____1//2_____no_output_____4//5_____no_output_____5//5_____no_output_____6//5_____no_output_____1+2_____no_output_____2**3_____no_output_____import math_____no_output_____help(math)Help on module math:

NAME
    math

MODULE REFERENCE
    https://docs.python.org/3.7/library/math

    The following documentation is automatically generated from the Python
    source files.  It may be incomplete, incorrect or include features that
    are considered implementation detail and may vary between Python
    implementations.  When in doubt, consult the module reference at the
    location listed above.

DESCRIPTION
    This module is always available.  It provides access to the
    mathematical functions defined by the C standard.

FUNCTIONS
    acos(x, /)
        Return the arc cosine (measured in radians) of x.

    acosh(x, /)
        Return the inverse hyperbolic cosine of x.

    asin(x, /)
        Return the arc sine (measured in radians) of x.

    asinh(x, /)
        Return the inverse hyperbolic sine of x.

    atan(x, /)
        Return the arc tangent (measured in radians) of x.

    atan2(y, x, /)
        Return the arc tangent (measured in radians) of y/x.

        Unlike atan(y/x), the signs of both x and y are considered.

    atanh(x, /)
        Return the inverse hyperbolic tangent of x.

    ceil(x, /)
        Return the ceiling of x as an Integral.

        This is the smallest integer >= x.

    copysign(x, y, /)
        Return a float with the magnitude (absolute value) of x but the sign of y.

        On platforms that support signed zeros, copysign(1.0, -0.0)
        returns -1.0.

    cos(x, /)
        Return the cosine of x (measured in radians).

    cosh(x, /)
        Return the hyperbolic cosine of x.

    degrees(x, /)
        Convert angle x from radians to degrees.

    erf(x, /)
        Error function at x.

    erfc(x, /)
        Complementary error function at x.

    exp(x, /)
        Return e raised to the power of x.

    expm1(x, /)
        Return exp(x)-1.

        This function avoids the loss of precision involved in the direct evaluation of exp(x)-1 for small x.

    fabs(x, /)
        Return the absolute value of the float x.

    factorial(x, /)
        Find x!.

        Raise a ValueError if x is negative or non-integral.

    floor(x, /)
        Return the floor of x as an Integral.

        This is the largest integer <= x.

    fmod(x, y, /)
        Return fmod(x, y), according to platform C.

        x % y may differ.

    frexp(x, /)
        Return the mantissa and exponent of x, as pair (m, e).

        m is a float and e is an int, such that x = m * 2.**e.
        If x is 0, m and e are both 0.  Else 0.5 <= abs(m) < 1.0.

    fsum(seq, /)
        Return an accurate floating point sum of values in the iterable seq.

        Assumes IEEE-754 floating point arithmetic.

    gamma(x, /)
        Gamma function at x.

    gcd(x, y, /)
        greatest common divisor of x and y

    hypot(x, y, /)
        Return the Euclidean distance, sqrt(x*x + y*y).

    isclose(a, b, *, rel_tol=1e-09, abs_tol=0.0)
        Determine whether two floating point numbers are close in value.

          rel_tol
            maximum difference for being considered "close", relative to the
            magnitude of the input values
          abs_tol
            maximum difference for being considered "close", regardless of the
            magnitude of the input values

        Return True if a is close in value to b, and False otherwise.

        For the values to be considered close, the difference between them
        must be smaller than at least one of the tolerances.

        -inf, inf and NaN behave similarly to the IEEE 754 Standard.  That
        is, NaN is not close to anything, even itself.  inf and -inf are
        only close to themselves.

    isfinite(x, /)
        Return True if x is neither an infinity nor a NaN, and False otherwise.

    isinf(x, /)
        Return True if x is a positive or negative infinity, and False otherwise.

    isnan(x, /)
        Return True if x is a NaN (not a number), and False otherwise.

    ldexp(x, i, /)
        Return x * (2**i).

        This is essentially the inverse of frexp().

    lgamma(x, /)
        Natural logarithm of absolute value of Gamma function at x.

    log(...)
        log(x, [base=math.e])
        Return the logarithm of x to the given base.

        If the base not specified, returns the natural logarithm (base e) of x.

    log10(x, /)
        Return the base 10 logarithm of x.

    log1p(x, /)
        Return the natural logarithm of 1+x (base e).

        The result is computed in a way which is accurate for x near zero.

    log2(x, /)
        Return the base 2 logarithm of x.

    modf(x, /)
        Return the fractional and integer parts of x.

        Both results carry the sign of x and are floats.

    pow(x, y, /)
        Return x**y (x to the power of y).

    radians(x, /)
        Convert angle x from degrees to radians.

    remainder(x, y, /)
        Difference between x and the closest integer multiple of y.

        Return x - n*y where n*y is the closest integer multiple of y.
        In the case where x is exactly halfway between two multiples of
        y, the nearest even value of n is used. The result is always exact.

    sin(x, /)
        Return the sine of x (measured in radians).

    sinh(x, /)
        Return the hyperbolic sine of x.

    sqrt(x, /)
        Return the square root of x.

    tan(x, /)
        Return the tangent of x (measured in radians).

    tanh(x, /)
        Return the hyperbolic tangent of x.

    trunc(x, /)
        Truncates the Real x to the nearest Integral toward 0.

        Uses the __trunc__ magic method.

DATA
    e = 2.718281828459045
    inf = inf
    nan = nan
    pi = 3.141592653589793
    tau = 6.283185307179586

FILE
    /Users/bostroem/anaconda/envs/py37/lib/python3.7/lib-dynload/math.cpython-37m-darwin.so

math.cos(3)_____no_output_____dir(math)_____no_output_____math.lgamma?_____no_output_____help(math.lgamma)Help on built-in function lgamma in module math:

lgamma(x, /)
    Natural logarithm of absolute value of Gamma function at x.

'star'_____no_output_____'shooting'+'star'_____no_output_____'star'*3_____no_output_____'star'[0]_____no_output_____'star'[1]_____no_output_____'star'[0:2]_____no_output_____y = 'star'_____no_output_____y[0]_____no_output_____y[1]_____no_output_____y[0:2]_____no_output_____y[-1]_____no_output_____y[-2]_____no_output_____y = 3
z = 10
y+z_____no_output_____y = 'star'
y[-7]_____no_output_____my_list = [1, 2, 3]_____no_output_____print(my_list)[1, 2, 3]
my_list2 = [1, 3., 'a', 'testing']_____no_output_____my_list + my_list2_____no_output_____my_list2*2_____no_output_____my_list2[2]_____no_output_____my_list2[1:3]_____no_output_____import numpy as np_____no_output_____my_array = np.array(my_list)_____no_output_____print(my_array)[1 2 3]
type(my_array)_____no_output_____type(my_list)_____no_output_____np.asarray?_____no_output_____x = np.array([1,2,3])
y = np.array(x)
z = np.asarray(x)_____no_output_____print(y)[1 2 3]
y[1] = 5_____no_output_____print(x, y, z)[1 2 3] [1 5 3] [1 2 3]
x[1] = 10_____no_output_____print(x, y, z)[ 1 10  3] [1 5 3] [ 1 10  3]
np.array()_____no_output_____np.array?_____no_output_____my_array3 = np.array([1,3,6,5,3,9])_____no_output_____my_array3*3_____no_output_____my_list*3_____no_output_____my_array3+5_____no_output_____my_list+5_____no_output_____my_list+[5]_____no_output_____my_array3**5_____no_output_____np.mean?_____no_output_____np.mean(my_array3)_____no_output_____np.mean_____no_output_____my_array3.mean() #method_____no_output_____my_array3.size #attribute_____no_output_____

# Loops_____no_output_____

my_favorite_stars = ['sirius', 'deneb', 'altair']
for istar in my_favorite_stars:
    print('one of my favorite stars is', istar)
print('end loop')one of my favorite stars is sirius
one of my favorite stars is deneb
one of my favorite stars is altair
end loop
constellation = 'cygnus'
if constellation == 'cygnus':
    print('cygnus is my favorite constellation')
else:
    print('{} is not my favorite constellation'.format(constellation))cygnus is my favorite constellation

constellation = 'scorpio'
if constellation == 'cygnus':
    print('cygnus is my favorite constellation')
else:
    print('this constelation, {}, is not my favorite constellation'.format(constellation))this constelation, scorpio, is not my favorite constellation
constellation = 'scorpio'
if constellation == 'cygnus':
    print('cygnus is my favorite constellation')
else:
    print(constellation,' is not my favorite constellation'.format(constellation))scorpio  is not my favorite constellation
constellation = 'cygnus'
if constellation == 'cygnus':
    print('cygnus is my favorite constellation')
else:
    print('{} is not my favorite constellation'.format(constellation))cygnus is my favorite constellation
for istar in my_favorite_stars:
    if len(istar) == 6:
        print(istar)
    else:
        print('length of {} ({}) is too short'.format(istar, len(istar)))sirius
length of deneb (5) is too short
altair
my_list = [1,2,5,7,8, 4, 6]

#find the total
total = 0
for inum in my_list:
    total = total + inum

#calculte mean
mean = total/len(my_list)

#calculate median
sorted_list = sorted(my_list)
median_indx = int(len(my_list)/2)
if len(my_list)%2 == 1:
    median = sorted_list[median_indx]
else:
    median = (sorted_list[median_indx]+sorted_list[median_indx-1])/2

print(mean, median)
print(np.mean(np.array(my_list)), np.median(np.array(my_list)))4.714285714285714 5
4.714285714285714 5.0
def calc_total(list_variable):
    #find the total
    total = 0
    for inum in list_variable:
        total = total + inum
    return total

def calc_mean(my_list):
    '''
    calculate the mean of an array like object
    Inputs:
        my_list: arr
            an array like object to calculate the mean of
    Returns:
        mean: float
            the mean of the input array
    '''
    total = calc_total(my_list)
    #calculte mean
    mean = total/len(my_list)

    return mean

def calc_median(my_list):
    #calculate median
    sorted_list = sorted(my_list)
    median_indx = int(len(my_list)/2)
    if len(my_list)%2 == 1:
        median = sorted_list[median_indx]
    else:
        median = (sorted_list[median_indx]+sorted_list[median_indx-1])/2
    return median
_____no_output_____my_list = [1,2,5,7,8, 4, 6]
#calculate mean
mean = calc_mean(my_list)
#calculate median
median = calc_median(my_list)
print(mean, median)33 5
def convert_to_megaparsecs(dist_pc):
    '''
    Takes a distance in parsecs and converts to megaparsecs
    Inputs:
        dist_pc: float or array like
            distance in parsecs
    Return:
        dist_mpc: float or array like
            distance in megaparsecs
    '''
    dist_mpc = dist_pc/10**6
    return dist_mpc

convert_to_megaparsecs(10**6)_____no_output_____from astropy.table import Table
import astropy.units as u_____no_output_____filename = 'hubble_data.dat'
tbdata = Table.read(filename, format='ascii',
                   names=['galaxy', 'supernova', 'm', 'sig_m',
                          'dist_mod', 'sig_dist_mod',
                         'M', 'sig_M', 'velocity'])_____no_output_____tbdata[0:2]_____no_output_____tbdata['supernova']_____no_output_____from matplotlib import pyplot as plt
%matplotlib inline_____no_output_____plt.plot(tbdata['dist_mod'], tbdata['velocity'], 'o')
plt.xlabel('Distance Modulus')
#plt.ylabel('Velocity (km/s)')
plt.ylabel(r'Velocity ($km\ s^{-1}$)')
plt.title('Almost a hubble diagram')_____no_output_____fit = np.polyfit?_____no_output_____fit = np.polyfit(tbdata['dist_mod'], tbdata['velocity'], 1)_____no_output_____ymodel = np.polyval(fit, np.arange(29.0, 33.0, 0.01))_____no_output_____plt.plot(tbdata['dist_mod'], tbdata['velocity'], 'o')
plt.xlabel('Distance Modulus')
#plt.ylabel('Velocity (km/s)')
plt.ylabel(r'Velocity ($km\ s^{-1}$)')
plt.title('Almost a hubble diagram')
plt.plot(np.arange(29.0, 33.0, 0.01), ymodel)_____no_output_____


# Notebook from stevetjoa/stanford-mir
Path: about_ccrma_workshop.ipynb

[musicinformationretrieval.com](https://musicinformationretrieval.com)_____no_output_____# About the CCRMA Workshop on Music Information Retrieval_____no_output_____<img src="https://s3-us-west-2.amazonaws.com/musicinformationretrieval.com/img/ccrma2018_group.jpg" />

*2018 CCRMA MIR Workshop participants. More photos: [2018](https://photos.app.goo.gl/GDQqCzR3NpuzT8es9) [2015](https://goo.gl/photos/iE27FUUFsuVfpMoA6)*_____no_output_____## Testimonials_____no_output_____"You guys put together a really fantastic workshop."

"I really enjoyed the workshop. It was very well executed and exceeded my expectations in the scope of material covered."

"Well planned and thoughtful."

"My own participation in this course started me down a path that I couldn't have foreseen and landed me several music tech jobs with startups and now an established company. Highly recommend this workshop!"

"It was a fun week. I learned a lot, and met some very cool people. That week helped shape where I am now in a couple of ways."

"I'd take this course again, if my schedule permits. It's a very cool introduction to Information Retrieval and Machine Learning through Music, for sure. Definitely recommended!"

"This was a great introduction to the field of MIR which I am now applying to human studies of the cognitive neuroscience of music."

"This is a fantastic workshop. I'd also love to take it again when I find the time."_____no_output_____## About the Workshop_____no_output_____The Stanford CCRMA Workshop on Music Information Retrieval (MIR) teaches the underlying ideas, approaches, technologies, and practical design of intelligent audio systems using MIR algorithms. It lasts five full days, Monday through Friday. It was founded by [Jay LeBoeuf](https://www.linkedin.com/in/jayleboeuf) (Real Industry, CCRMA consulting professor) in 2008.

The workshop is intended for students, researchers, musicians, and engineers who want to learn more about music information retrieval (MIR). We demonstrate technologies enabled by signal processing and machine learning such as feature extraction, chord estimations, audio similarity clustering, search and retrieval, music synchronization, signal decomposition, rhythm and tempo, and design and evaluation of machine learning systems. Our goal is to introduce these interdisciplinary technologies and complex algorithms in an approachable way.

Knowledge of basic digital audio principles is recommended.  Some experience with any programming language -- whether Python, Matlab, C, C++, Java, Javascript, or other -- is desired.

The workshop consists of lectures, demonstrations, discussions, and interactive lab sessions. Labs allow students to design intelligent audio systems leveraging existing MIR toolboxes, programming environments, and applications._____no_output_____## Instructors_____no_output_____Links redirect to that year's wiki page._____no_output_____- [2008](https://ccrma.stanford.edu/wiki/MIR_workshop_2008): Jay LeBoeuf
- [2009](https://ccrma.stanford.edu/wiki/MIR_workshop_2009): Jay LeBoeuf, Kyogu Lee
- [2010](https://ccrma.stanford.edu/wiki/MIR_workshop_2010): Jay LeBoeuf, Rebecca Fiebrink
- [2011](https://ccrma.stanford.edu/wiki/MIR_workshop_2011): Jay LeBoeuf, Stephen Pope, Leigh Smith, Steve Tjoa
- [2012](https://ccrma.stanford.edu/wiki/MIR_workshop_2012): Jay LeBoeuf, Leigh Smith, Steve Tjoa
- [2013](https://ccrma.stanford.edu/wiki/MIR_workshop_2013): Jay LeBoeuf, Leigh Smith, Steve Tjoa
- [2014](https://ccrma.stanford.edu/wiki/MIR_workshop_2014): Jay LeBoeuf, Leigh Smith, Steve Tjoa
- 2015: Jay LeBoeuf, Leigh Smith, Steve Tjoa
- [2016](https://docs.google.com/document/d/1YpuvCxKHUsqbbqEA_0XB_5-jH3e47v0azO1sn_iVV1k/edit?usp=sharing): Steve Tjoa, Jeff Scott
- 2017: Steve Tjoa, Jeff Scott
- 2018: Steve Tjoa, Meinard Müller, Brian McFee_____no_output_____## Guest Lecturers_____no_output_____- 2011: Rebecca Fiebrink, Doug Eck, George Tzanetakis
- 2012: Oscar Celma, Michael Mandel
- 2013: Ching-Wei Chen, Nick Bryan, Gautham Mysore
- 2014: Stephen Pope, Andreas Ehmann
- 2015: Zafar Rafii, Constantinos Dimitriou, Jeff Scott, Aneesh Kartakavi
- 2016: Doug Eck, Blair Kaneshiro
- 2017: Jonathan Bailey, Stephen Pope_____no_output_____## Alumni_____no_output_____- 2008:
- 2009 [[full list](https://ccrma.stanford.edu/wiki/MIR_workshop_2009/Participants)]: Luke Dahl, Mike Gao, Craig Hanson, Jorge Herrera, Denis Lebel, Sang Won Lee, Gautham Mysore, Jeremy Sawruk, Hwan Shim, Diana Siwiak, Steve Tjoa, Elie Noune, James Hughes, Stefan Tomic, Lisa Lim, Fred Barrett
- 2010:
- 2011: Chris Colatos, Jeff Albert, Kamlesh Lakshminarayanan, Sean Zhang, Eli Stine, David Bird, Gina Collecchia, Dekun Zou, Bill Paseman, John Amuedo
- 2012:
- 2013: Freddie Sanchez, Linda Barnett, Xuchen Yang, Vivek Kumar, Felipe Loáiciga Espeleta, Haoqing (Panda) Geng
- 2014: Krishna Kumar, Owen Campbell, Dan Cartoon, Rob Miller, Davide Fossati, Biagio Gallo, Joel Hunt, Shinobu Yamada, Fredom Luo, Sejin Oh, Phaedon Sinis, Xinyuan Lai, Greg Mertz, Matt Mitchell
- 2015: Eric Raymond, Stelios Andrew Stavroulakis, Richard Mendelsohn, Naithan Bosse, Alessio Bazzica, Karthik Yadati, Martha Larson, Stephen Hartzog, Philip Lee, Jaeyoung Choi, Matthew Gallagher, Yule Wu, Mark Renker, Rohit Ainapure, Eric Tarr, Allen Wu, Aaron Hipple
- 2016: Arjun Kaul, Aditya Joglekar, Jennifer Smith, Trevor Welch, Ruoxi Zhang, Patricia Robinson, Xingxing Yang, David Heid, Gabriel Lamorie, Kaustuv Kanti Ganguli, Zhengyang Ma, Bernard Wang, Thomas Matthias, Alex Kintzer, Yutong Chen
- 2017: Jace Mogill,
Saarish Kareer,
Adi Etsion,
Jiawen Huang,
John Neuharth,
Jung Choi,
Thomas D. Semah,
David Wickland,
Arvind Ramanathan,
Steve Philbert,
Peizhe Gao,
Kimberly Kawczinski,
Samuel Lowe,
Eric Pérez Segura,
Jennifer Smith,
Evan Azevedo,
Mark Goldstein,
Robert Farrimond,
Wayne Chen,
Miguel Saldana,
Shakib Ghassemieh,
Cixian Lu,
Kunwoo Kim,
Han Han,
Oscar Dub
- 2018:
Lindley Cannon,
Kuangyi Lu,
Aaron Kranzler,
Edoardo Bucheli,
Jeongbum Seo,
Ted Moore,
Jan Gryga,
Alexander Tudor,
Kai Xu,
Kaiming Cheng,
Jay Weiler

_____no_output_____


# Notebook from njits23/8dm40-machine-learning
Path: practicals/week_6-Copy1.ipynb

# Preliminaries

## Dataset

In this set of exercises we will use the same dataset as from [week 3](week_3.ipynb).

As before, we provide the data already curated in the following two files:

`RNA_expression_curated.csv`: [148 cell lines , 238 genes]

`drug_response_curated.csv`: [148 cell lines , YM155 drug]

The curated data can be read as `pandas` `DataFrame` in the following way:_____no_output_____

import pandas as pd

gene_expression = pd.read_csv("./data/RNA_expression_curated.csv", sep=',', header=0, index_col=0)
drug_response = pd.read_csv("./data/drug_response_curated.csv", sep=',', header=0, index_col=0)_____no_output_____

The goal of the exercises is to train support vector machine (SVM) and random forests classifiers on this dataset and explore and learn about their hyperparameters. _____no_output_____## Tools

The `scikit-learn` library provides the required tools for support vector machines, as well as for random forest algorithms._____no_output_____

from sklearn import svm
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import train_test_split
from sklearn.datasets.samples_generator import make_blobs, make_circles
from sklearn.metrics import classification_report_____no_output_____

Before proceeding, look up the documentation of the imported functions and read about their basic functionality. Below, we list some important parameters of SVMs and random forests that can be tuned during training._____no_output_____#### Support Vector Machines (SVM)

`C`: error term.

`kernel`: similarity function ('linear', 'poly', 'sigmoid' or 'rbf')

`gamma`: kernel coef. for 'rbf', 'poly' and 'sigmoid' kernels. It can be thought of as the ‘spread’ of the kernel and therefore the decision region.

`degree`: degree for the 'poly' kernel.

`coef0`: independt term in the 'poly' and 'sigmoid' kernels

#### Random Forests

`n_estimators`: number of trees in our random forest.

`max_depth`: maximum number of levels in each decision tree

`max_features`: maximum number of features to consider per split in an individual tree.

`min_sample_leaf`: minimum number of data points per leaf node

`min_samples_split`: minimum number of data points placed in a node before the node is split

`oob_score`: the out-of-bag (OOB) error is the average error for each observation calculated using predictions from the trees that do not contain that observation in their respective bootstrap sample. Set this parameter to true.

`bootstrap`: method for sampling data points (with or without replacement). Set this parameter to true.

`criterion`: function used to measure the quality of the split (e.g. 'entropy' or 'gini')

# Exercises

## Support vector machines

The  `make_blobs` and `make_circles` functions can be used to generate linearly and not linearly separable toy datasets. _____no_output_____

# data generation: linearly separable
X, Y = make_blobs(n_samples=200, centers=2, n_features=2, random_state=1234)
X = pd.DataFrame(X, columns=['x1', 'x2'])

# splitting data into training and test set
X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.33, random_state=333)_____no_output_____

The following code illustrates how to train a linear SVM classifier and plot the data points, the separating hyperplane, the support vectors and the margins that pass through them (considering the training data)_____no_output_____

import numpy as np
import matplotlib.pyplot as plt

%matplotlib inline
# build the model
model = svm.SVC(kernel='linear', random_state=33)
model.fit(X_train, Y_train)

# create plot
fig, ax = plt.subplots()

# get colors from qualitative colormap 'Paired'
cmap = plt.cm.get_cmap('Paired')

# plot data points
ax.scatter(X_train.iloc[Y_train == 1, 0], X_train.iloc[Y_train == 1, 1],
           c=[cmap(11)], label='1')
ax.scatter(X_train.iloc[Y_train == 0, 0], X_train.iloc[Y_train == 0, 1],
           c=[cmap(0)], label='0')
ax.legend(loc='best')

# plot the decision function
# create grid to evaluate model
x1_min, x1_max = X_train.iloc[:, 0].min() - 1, X_train.iloc[:, 0].max() + 1
x2_min, x2_max = X_train.iloc[:, 1].min() - 1, X_train.iloc[:, 1].max() + 1

XX, YY = np.meshgrid(np.arange(x1_min, x1_max, .2),
                     np.arange(x2_min, x2_max, .2))

xy = np.vstack([XX.ravel(), YY.ravel()]).T
Z = model.decision_function(xy).reshape(XX.shape)

# plot decision boundary and margins
ax.contour(XX, YY, Z, colors='k', levels=[-1, 0, 1], alpha=0.5,
           linestyles=['--', '-', '--'])

# Establish the class for each point in the contour
Z = model.predict(xy).reshape(XX.shape)

# Visualization of the contour
ax.contourf(XX, YY, Z, cmap='bwr', alpha=0.3)

# plot support vectors, whose are responsible for building the margins
ax.scatter(model.support_vectors_[:, 0], model.support_vectors_[:, 1], s=100,
           linewidth=1, facecolors='none', edgecolors='k', marker='s')

ax.axis([x1_min, x1_max, x2_min, x2_max])
plt.axis('tight')
plt.title('Linear kernel SVM')
plt.show()_____no_output_____

Train a radial basis function (RBF) SVM classifier with `gamma=0.5` and plot the results in the same way._____no_output_____

# data generation: not linearly separable
X, Y = make_circles(n_samples=200, noise=0.05, random_state=1234)
X = pd.DataFrame(X, columns=['x1', 'x2'])

# splitting data into training and test set
X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.33, random_state=333)_____no_output_____import numpy as np
import matplotlib.pyplot as plt

%matplotlib inline
# build the model
model = svm.SVC(kernel='rbf', gamma=0.5,  random_state=33)
model.fit(X_train, Y_train)

# create plot
fig, ax = plt.subplots()

# get colors from qualitative colormap 'Paired'
cmap = plt.cm.get_cmap('Paired')

# plot data points
ax.scatter(X_train.iloc[Y_train == 1, 0], X_train.iloc[Y_train == 1, 1],
           c=[cmap(11)], label='1')
ax.scatter(X_train.iloc[Y_train == 0, 0], X_train.iloc[Y_train == 0, 1],
           c=[cmap(0)], label='0')
ax.legend(loc='best')

# plot the decision function
# create grid to evaluate model
x1_min, x1_max = X_train.iloc[:, 0].min() - 1, X_train.iloc[:, 0].max() + 1
x2_min, x2_max = X_train.iloc[:, 1].min() - 1, X_train.iloc[:, 1].max() + 1

XX, YY = np.meshgrid(np.arange(x1_min, x1_max, .2),
                     np.arange(x2_min, x2_max, .2))

xy = np.vstack([XX.ravel(), YY.ravel()]).T
Z = model.decision_function(xy).reshape(XX.shape)

# plot decision boundary and margins
ax.contour(XX, YY, Z, colors='k', levels=[-1, 0, 1], alpha=0.5,
           linestyles=['--', '-', '--'])

# Establish the class for each point in the contour
Z = model.predict(xy).reshape(XX.shape)

# Visualization of the contour
ax.contourf(XX, YY, Z, cmap='bwr', alpha=0.3)

# plot support vectors, whose are responsible for building the margins
ax.scatter(model.support_vectors_[:, 0], model.support_vectors_[:, 1], s=100,
           linewidth=1, facecolors='none', edgecolors='k', marker='s')

ax.axis([x1_min, x1_max, x2_min, x2_max])
plt.axis('tight')
plt.title('RBF kernel SVM')
plt.show()_____no_output_____

When should a RBF kernel be used over a linear kernel? Motivate your answer.

Do we need to normalize the data before using a kernel function? Motivate your answer.

_____no_output_____A RBF kernel, which is more complex than the linear kernel, should be used if the data is not linearly seperable. Applying a linear kernel to the data above would not result in a correct performance, because the data is not linearly seperable and a linear line would hence not seperate the classes.

A SVM with standard settings depends on Euclidian distances to calculate the objective function, it is therefore important that each feature dimension is equally as important. This can be achieved by normalizing the provided data, which ensures that each feature dimension is equal in importance when optimizing the boundary. This makes normalization an essential part before using the kernel function. _____no_output_____## Predicting drug response on cell lines from gene expression data with SVMs

Explore the hyper-parameter space of an SVM classifier with cross-validation for the Genomics of Drug Sensitivity in Cancer (GDSC) dataset. The`GridSearchCV` function can be used to specify a grid of parameter values with the `param_grid` parameter.

Calculate the precision of your predictions, and compare your calculations with the results of `classification_report`, which displays many classification metrics._____no_output_____

from sklearn import svm
from sklearn.model_selection import GridSearchCV
from sklearn.preprocessing import binarize
from sklearn.metrics import precision_score, classification_report
import numpy as np
import pandas as pd
import warnings
warnings.filterwarnings("ignore")
from scipy.stats import zscore
from sklearn.model_selection import train_test_split

gene_expression = pd.read_csv("./data/RNA_expression_curated.csv", sep=',', header=0, index_col=0)
drug_response = pd.read_csv("./data/drug_response_curated.csv", sep=',', header=0, index_col=0)
drug_z = binarize(zscore(drug_response))
X_train, X_test, y_train, y_test = train_test_split(gene_expression,drug_z,test_size=0.2)
#print(y_test.shape)

parameters={'kernel':('linear', 'rbf'), 'C':np.linspace(0.001,0.01,num=10), 'gamma':np.linspace(0.0001,1,num=10)}
model=svm.SVC()
clf = GridSearchCV(model, parameters, cv=10, scoring='neg_mean_squared_error')
clf.fit(X_train,y_train)

#Do an evaluation of the model
print('Best estimator: \n')
print(clf.best_estimator_)
print('\n')

best = clf.best_estimator_
pred = best.predict(X_test)
prec = precision_score(y_test,pred)
print('The precision for the best estimator is: '+str(prec))
print(classification_report(y_test,pred))Best estimator:

SVC(C=0.01, cache_size=200, class_weight=None, coef0=0.0,
  decision_function_shape='ovr', degree=3, gamma=0.0001, kernel='linear',
  max_iter=-1, probability=False, random_state=None, shrinking=True,
  tol=0.001, verbose=False)

The precision for the best estimator is: 0.3157894736842105
             precision    recall  f1-score   support

        0.0       0.73      0.38      0.50        21
        1.0       0.32      0.67      0.43         9

avg / total       0.60      0.47      0.48        30

The predictions of the best estimator, determined with a Grid Search with 5-Fold cross-validation and scoring based on the negative mean squared error, have a precision of 0,73. This matches with the displayed precision in the classification report, which also shows that this model is able to achieve a recall of 0.47 and a F1-score of 0.57. This model is thus able to provide significantly reliable positive predictions, given the high precision, but it is not able to find the majority of the real positive classes. The latter can be concluded from the displayed recall in the classification report._____no_output_____## Random forests

Follow the same steps as for SVM. Compare the two algorithms and report which one has better performance.

The random forests classifiers allows to perform feature selection. Evaluate the importance of features extracting the top 50 informative features. A bar plot (`plt.bar()`) can be a useful tool to visualize this.

_____no_output_____

from sklearn import svm
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import GridSearchCV
from sklearn.preprocessing import binarize
from sklearn.metrics import precision_score, classification_report
import numpy as np
import pandas as pd
import warnings
warnings.filterwarnings("ignore")
from scipy.stats import zscore
from sklearn.model_selection import train_test_split
%matplotlib inline
import matplotlib.pyplot as plt

plt.close()

gene_expression = pd.read_csv("./data/RNA_expression_curated.csv", sep=',', header=0, index_col=0)
drug_response = pd.read_csv("./data/drug_response_curated.csv", sep=',', header=0, index_col=0)

#Assign label 0 to responses <0 and 1 to response >0
drug_z = binarize(zscore(drug_response))
#Split the data in training and test sets
X_train, X_test, y_train, y_test = train_test_split(gene_expression,drug_z,test_size=0.2)

#Set parameter grid
parameters={'n_estimators':range(10,100,10),
            'criterion':('entropy', 'gini'),
            'max_depth':[None]+list(range(10,100,10)),
            'max_features':[None,"auto","log2"],
            'min_samples_leaf':[1,2,3],
            'min_samples_split':[2,3,4]
           }

#Perform grid search for the random forest classifier
model=RandomForestClassifier(oob_score=True,bootstrap=True)
clf = GridSearchCV(model, parameters, cv=5, verbose=1)
clf.fit(X_train,y_train)

#Do an evaluation of the model
print('Best estimator: \n')
print(clf.best_estimator_)
print('\n')

#Evaluate and report performance of the best model found
best = clf.best_estimator_
pred = best.predict(X_test)
prec = precision_score(y_test,pred)
print('The precision for the best estimator is: '+str(prec))
print(classification_report(y_test,pred))

#Extract and sort feature importances
importances = best.feature_importances_
indices = np.argsort(importances)[::-1]
importances[indices]

cutoff_idx = 50
xvals = range(0,cutoff_idx)
indices = indices[:cutoff_idx]

#Plot the feature importances
plt.figure(constrained_layout=True,figsize=[20, 10])
plt.title("Feature importances")
plt.bar(xvals, importances[indices], color="r", align="center")
plt.xticks(xvals, indices)
plt.xlim([-1, cutoff_idx])
plt.show()Fitting 5 folds for each of 4860 candidates, totalling 24300 fits

It can clearly be observed that gene 24 is a much more important feature to the classifier than the other features. Throughout multiple runs it was found that the order of the remaining features by importance can vary heavily. _____no_output_____## Biomedical applications

Driven by technological advances, there has recently been a dramatic increase in availability of biomedical data. Machine learning approaches are well suited to take advantage of this data and have been widely applied to many areas of biology.

Example of these applications are genome annotation, biomarker identification, systems biology, genome data analysis, protein  function  prediction, protein  structure prediction, protein localization prediction, identification of protein interactions and drug discovery.

SVM and RF methods are among the most popular machine learning methods applied in bioinformatics or computational biology.

Perform a literature search and find a biomedical study in which SVM or RF is applied to obtain certain insights.
Explain the motivation behind using that specific algorithm in the study.

_____no_output_____V. Lempitsky, M. Verhoek, J. A. Noble, and A. Blake, “Random forest classification for automatic delineation of myocardium in real-time 3D echocardiography,” Lect. Notes Comput. Sci. (including Subser. Lect. Notes Artif. Intell. Lect. Notes Bioinformatics), vol. 5528, pp. 447–456, 2009.

As the title suggests, random forest classification is used for the automatic delineation of myocardium in (real-time) 3D echocardiography. More specifically, the algorithm is used to classify each voxel in the echocardiogram as either myocardium or not myocardium. The main motivation for this algorithm is the 'computational efficiency at both runtime and training time', potentially allowing for real-time classification. Furthermore, a major advantage of random forests with respect to many other classification methods (such as SVMs) is its ability to work with high-dimensional representations. This is also the case for a learning framework called boosting. However, random forests were found to be faster (while achieving similar performance) and were proven to be more robust to overlaps between classes and noise in the training labels (which is particularly relevant for the classification task at hand)._____no_output_____

gene expression
genomics
bioinformatics
computational biology
drug discovery
biology
systems biology


# Notebook from y1zhong/anaerobic-digesters
Path: notebooks/0.2.0-processing-alpha-and-beta.ipynb

# Important: This notebook processes the data with 150nts only!_____no_output_____

cd trimmed-150nts//Users/yoshikivazquezbaeza/Documents/PDF/KnightLaboratory/HastyWater/trimmed-150nts
%matplotlib inline
import pandas as pd, numpy as np, seaborn as sns, matplotlib.pyplot as plt, qiime2 as q2_____no_output_____from biom import load_table, Table
from biom.util import biom_open
from skbio import TreeNode

from qiime2.plugins import diversity, feature_table, metadata, taxa, emperor, diversity

def load_mf(fn, index='#SampleID'):
    _df = pd.read_csv(fn, sep='\t', dtype=str, keep_default_na=False, na_values=[])
    _df.set_index(index, inplace=True)
    return _df_____no_output_____

The tables that we'll deal with are the ones normalized for acinetobacter, see the previous notebooks for details about this._____no_output_____The phylogenetic tree and taxonomic assignments are generated from the representative sequences. Note those processing steps are not shown here as they were executed in our local compute cluster. They can be found in `/home/yovazquezbaeza/research/hasty-water/trimmed-150nts/`.

The commands executed were (roughly):

```bash
qiime deblur denoise-16S --i-demultiplexed-seqs single-end-demux.filtered.qza --p-trim-length 150 --o-representative-sequences rep-seqs-deblur.qza --o-table table-deblur.qza --o-stats deblur-stats.qza

# assign taxonomy greengenes
qiime feature-classifier classify-sklearn --i-classifier gg-13-8-99-nb-classifier.qza --i-reads rep-sequences.qza --o-classification taxonomy.greengenes.qza --p-n-jobs 4

qiime metadata tabulate --m-input-file taxonomy.greengenes.qza --o-visualization taxonomy.greengenes.qzv
```

I also created a fragment-inserted phylogenetic tree based on SEPP and these fragments (?????):

```bash
qiime fragment-insertion sepp-16s-greengenes --i-representative-sequences rep-seqs-deblur.qza --o-tree sepp-tree.rooted.qza --o-placements sepp-placements.qza --p-threads 31
```_____no_output_____# Alpha diversity_____no_output_____

table = q2.Artifact.load('feature-table.even.18813.normalized.qza')
tree = q2.Artifact.load('insertion-tree.qza')_____no_output_____diversity.methods.alpha_phylogenetic(table, tree, 'faith_pd').alpha_diversity.save('faith_pd.qza')

for metric in ['observed_otus', 'shannon']:
    diversity.methods\
    .alpha(table, metric)\
    .alpha_diversity\
    .save('%s.qza' % metric)_____no_output_____

Add the alpha diversity information to the mapping file._____no_output_____

mf = q2.Metadata.load('../mapping-file.tsv').to_dataframe()_____no_output_____for metric in ['observed_otus', 'shannon', 'faith_pd']:
    mf[metric] = q2.Artifact.load(metric + '.qza').view(pd.Series)

mf.to_csv('mapping-file.alpha.tsv', sep='\t')_____no_output_____

Create taxonomic bar charts_____no_output_____

taxonomy = q2.Artifact.load('taxonomy.qza')_____no_output_____taxa.visualizers.barplot(table,
                         taxonomy,
                         q2.Metadata(mf)).visualization.save('taxonomy.barplot.qzv')_____no_output_____

Beta diversity_____no_output_____

diversity.methods.beta_phylogenetic_alt(table=table,
                                        phylogeny=tree,
                                        metric='unweighted_unifrac',
                                        n_jobs=2)\
.distance_matrix.save('unweighted.unifrac.dm.qza')_____no_output_____diversity.methods.beta_phylogenetic_alt(table=table,
                                        phylogeny=tree,
                                        metric='weighted_unifrac',
                                        n_jobs=2)\
.distance_matrix.save('weighted.unifrac.dm.qza')_____no_output_____diversity.methods.pcoa(q2.Artifact.load('unweighted.unifrac.dm.qza')).pcoa.save('unweighted.unifrac.pcoa.qza')
diversity.methods.pcoa(q2.Artifact.load('weighted.unifrac.dm.qza')).pcoa.save('weighted.unifrac.pcoa.qza')/Users/yoshikivazquezbaeza/git_sw/scikit-bio/skbio/stats/ordination/_principal_coordinate_analysis.py:114: RuntimeWarning: The result contains negative eigenvalues. Please compare their magnitude with the magnitude of some of the largest positive eigenvalues. If the negative ones are smaller, it's probably safe to ignore them, but if they are large in magnitude, the results won't be useful. See the Notes section for more details. The smallest eigenvalue is -0.02957537690504521 and the largest is 5.5273874406739525.
  RuntimeWarning
emperor.visualizers.plot(q2.Artifact.load('unweighted.unifrac.pcoa.qza'),
                         q2.Metadata.load('mapping-file.alpha.tsv')).visualization.save('unweighted.unifrac.plot.qzv')

emperor.visualizers.plot(q2.Artifact.load('weighted.unifrac.pcoa.qza'),
                         q2.Metadata.load('mapping-file.alpha.tsv')).visualization.save('weighted.unifrac.plot.qzv')_____no_output_____