# Notebook from vasudev-sharma/course-content
Path: tutorials/W3D1_BayesianDecisions/student/W3D1_Tutorial2.ipynb

_____no_output_____# Tutorial 2: Bayesian inference and decisions with continuous hidden state
**Week 3, Day 1: Bayesian Decisions**

**By Neuromatch Academy**

__Content creators:__ Eric DeWitt, Xaq Pitkow, Saeed Salehi, Ella Batty

__Content reviewers:__ _____no_output_____# Tutorial Objectives

This notebook introduces you to Gaussians and Bayes' rule for continuous distributions, allowing us to model simple put powerful combinations of prior information and new measurements. In this notebook you will work through the same ideas we explored in the binary state tutorial, but you will be introduced to a new problem: finding and guiding Astrocat! You will see this problem again in more complex ways in the following days.

In this notebook, you will:

1. Learn about the Gaussian distribution and its nice properies
2. Explore how we can extend the ideas from the binary hidden tutorial to continuous distributions
3. Explore how different priors can produce more complex posteriors.
4. Explore Loss functions often used in inference and complex utility functions._____no_output_____---
##Setup
Please execute the cells below to initialize the notebook environment._____no_output_____

# imports
import matplotlib.pyplot as plt
import numpy as np
from scipy.stats import multivariate_normal
from scipy.stats import gamma as gamma_distribution
from matplotlib.transforms import Affine2D_____no_output_____#@title Figure Settings
%matplotlib inline
%config InlineBackend.figure_format = 'retina'
import ipywidgets as widgets
from ipywidgets import FloatSlider
from ipywidgets import interact, fixed, HBox, Layout, VBox, interactive, Label
plt.style.use("https://raw.githubusercontent.com/NeuromatchAcademy/course-content/master/nma.mplstyle")

import warnings
warnings.filterwarnings("ignore")_____no_output_____# @title Plotting functions

def plot_gaussian(μ, σ):
    x = np.linspace(-7, 7, 1000, endpoint=True)
    y = gaussian(x, μ, σ)

    plt.figure(figsize=(6, 4))
    plt.plot(x, y, c='blue')
    plt.fill_between(x, y, color='b', alpha=0.2)
    plt.ylabel('$\mathcal{N}(x, \mu, \sigma^2)$')
    plt.xlabel('x')
    plt.yticks([])
    plt.show()

def plot_losses(μ, σ):
    x = np.linspace(-2, 2, 400, endpoint=True)
    y = gaussian(x, μ, σ)
    error = x - μ

    mse_loss = (error)**2
    abs_loss = np.abs(error)
    zero_one_loss = (np.abs(error) >= 0.02).astype(np.float)

    fig, (ax_gaus, ax_error) = plt.subplots(2, 1, figsize=(6, 8))
    ax_gaus.plot(x, y, color='blue', label='true distribution')
    ax_gaus.fill_between(x, y, color='blue', alpha=0.2)
    ax_gaus.set_ylabel('$\\mathcal{N}(x, \\mu, \\sigma^2)$')
    ax_gaus.set_xlabel('x')
    ax_gaus.set_yticks([])
    ax_gaus.legend(loc='upper right')

    ax_error.plot(x, mse_loss, color='c', label='Mean Squared Error', linewidth=3)
    ax_error.plot(x, abs_loss, color='m', label='Absolute Error', linewidth=3)
    ax_error.plot(x, zero_one_loss, color='y', label='Zero-One Loss', linewidth=3)
    ax_error.legend(loc='upper right')
    ax_error.set_xlabel('$\\hat{\\mu}$')
    ax_error.set_ylabel('Error')
    plt.show()

def plot_mvn2d(mu1, mu2, sigma1, sigma2, corr):
    x, y = np.mgrid[-2:2:.02, -2:2:.02]
    cov12 = corr * sigma1 * sigma2
    z = mvn2d(x, y, mu1, mu2, sigma1, sigma2, cov12)

    plt.figure(figsize=(6, 6))
    plt.contourf(x, y, z, cmap='Reds')
    plt.axis("off")
    plt.show()

def plot_marginal(sigma1, sigma2, c_x, c_y, corr):
    mu1, mu2 = 0.0, 0.0
    cov12 = corr * sigma1 * sigma2
    xx, yy = np.mgrid[-2:2:.02, -2:2:.02]
    x, y = xx[:, 0], yy[0]
    p_x = gaussian(x, mu1, sigma1)
    p_y = gaussian(y, mu2, sigma2)
    zz = mvn2d(xx, yy, mu1, mu2, sigma1, sigma2, cov12)

    mu_x_y = mu1+cov12*(c_y-mu2)/sigma2**2
    mu_y_x = mu2+cov12*(c_x-mu1)/sigma1**2
    sigma_x_y = np.sqrt(sigma2**2 - cov12**2/sigma1**2)
    sigma_y_x = np.sqrt(sigma1**2-cov12**2/sigma2**2)
    p_x_y = gaussian(x, mu_x_y, sigma_x_y)
    p_y_x = gaussian(x, mu_y_x, sigma_y_x)

    p_c_y = gaussian(mu_x_y-sigma_x_y, mu_x_y, sigma_x_y)
    p_c_x = gaussian(mu_y_x-sigma_y_x, mu_y_x, sigma_y_x)

    # definitions for the axes
    left, width = 0.1, 0.65
    bottom, height = 0.1, 0.65
    spacing = 0.01

    rect_z = [left, bottom, width, height]
    rect_x = [left, bottom + height + spacing, width, 0.2]
    rect_y = [left + width + spacing, bottom, 0.2, height]

    # start with a square Figure
    fig = plt.figure(figsize=(8, 8))

    ax_z = fig.add_axes(rect_z)
    ax_x = fig.add_axes(rect_x, sharex=ax_z)
    ax_y = fig.add_axes(rect_y, sharey=ax_z)

    ax_z.set_axis_off()
    ax_x.set_axis_off()
    ax_y.set_axis_off()
    ax_x.set_xlim(np.min(x), np.max(x))
    ax_y.set_ylim(np.min(y), np.max(y))

    ax_z.contourf(xx, yy, zz, cmap='Greys')
    ax_z.hlines(c_y, mu_x_y-sigma_x_y, mu_x_y+sigma_x_y, color='c', zorder=9, linewidth=3)
    ax_z.vlines(c_x, mu_y_x-sigma_y_x, mu_y_x+sigma_y_x, color='m', zorder=9, linewidth=3)

    ax_x.plot(x, p_x, label='$p(x)$', c = 'b', linewidth=3)
    ax_x.plot(x, p_x_y, label='$p(x|y = C_y)$', c = 'c', linestyle='dashed', linewidth=3)
    ax_x.hlines(p_c_y, mu_x_y-sigma_x_y, mu_x_y+sigma_x_y, color='c', linestyle='dashed', linewidth=3)

    ax_y.plot(p_y, y, label='$p(y)$', c = 'r', linewidth=3)
    ax_y.plot(p_y_x, y, label='$p(y|x = C_x)$', c = 'm', linestyle='dashed', linewidth=3)
    ax_y.vlines(p_c_x, mu_y_x-sigma_y_x, mu_y_x+sigma_y_x, color='m', linestyle='dashed', linewidth=3)

    ax_x.legend(loc="upper left", frameon=False)
    ax_y.legend(loc="lower right", frameon=False)

    plt.show()

def plot_bayes(mu1, mu2, sigma1, sigma2):
    x = np.linspace(-7, 7, 1000, endpoint=True)
    prior = gaussian(x, mu1, sigma1)
    likelihood = gaussian(x, mu2, sigma2)

    mu_post, sigma_post = product_guassian(mu1, mu2, sigma1, sigma2)
    posterior = gaussian(x, mu_post, sigma_post)

    plt.figure(figsize=(8, 6))
    plt.plot(x, prior, c='b', label='prior')
    plt.fill_between(x, prior, color='b', alpha=0.2)
    plt.plot(x, likelihood, c='r', label='likelihood')
    plt.fill_between(x, likelihood, color='r', alpha=0.2)
    plt.plot(x, posterior, c='k', label='posterior')
    plt.fill_between(x, posterior, color='k', alpha=0.2)
    plt.yticks([])
    plt.legend(loc="upper left")
    plt.ylabel('$\mathcal{N}(x, \mu, \sigma^2)$')
    plt.xlabel('x')
    plt.show()

def plot_information(mu1, sigma1, mu2, sigma2):
    x = np.linspace(-7, 7, 1000, endpoint=True)
    mu3, sigma3 = product_guassian(mu1, mu2, sigma1, sigma2)
    prior = gaussian(x, mu1, sigma1)
    likelihood = gaussian(x, mu2, sigma2)
    posterior = gaussian(x, mu3, sigma3)

    plt.figure(figsize=(8, 6))
    plt.plot(x, prior, c='b', label='Satellite')
    plt.fill_between(x, prior, color='b', alpha=0.2)
    plt.plot(x, likelihood, c='r', label='Space Mouse')
    plt.fill_between(x, likelihood, color='r', alpha=0.2)
    plt.plot(x, posterior, c='k', label='Center')
    plt.fill_between(x, posterior, color='k', alpha=0.2)
    plt.yticks([])
    plt.legend(loc="upper left")
    plt.ylabel('$\mathcal{N}(x, \mu, \sigma^2)$')
    plt.xlabel('x')
    plt.show()

def plot_information_global(mu3, sigma3, mu1, mu2):
    x = np.linspace(-7, 7, 1000, endpoint=True)
    sigma1, sigma2 = reverse_product(mu3, sigma3, mu1, mu2)
    prior = gaussian(x, mu1, sigma1)
    likelihood = gaussian(x, mu2, sigma2)
    posterior = gaussian(x, mu3, sigma3)

    plt.figure(figsize=(8, 6))
    plt.plot(x, prior, c='b', label='Satellite')
    plt.fill_between(x, prior, color='b', alpha=0.2)
    plt.plot(x, likelihood, c='r', label='Space Mouse')
    plt.fill_between(x, likelihood, color='r', alpha=0.2)
    plt.plot(x, posterior, c='k', label='Center')
    plt.fill_between(x, posterior, color='k', alpha=0.2)
    plt.yticks([])
    plt.legend(loc="upper left")
    plt.ylabel('$\mathcal{N}(x, \mu, \sigma^2)$')
    plt.xlabel('x')
    plt.show()

def plot_loss_utility_gaussian(loss_f, mu, sigma, mu_true):
    x = np.linspace(-7, 7, 1000, endpoint=True)
    posterior = gaussian(x, mu, sigma)

    plot_loss_utility(x, posterior, loss_f, mu_true)

def plot_loss_utility_mixture(loss_f, mu1, mu2, sigma1, sigma2, factor, mu_true):
    x = np.linspace(-7, 7, 1000, endpoint=True)
    y_1 = gaussian(x, mu1, sigma1)
    y_2 = gaussian(x, mu2, sigma2)
    posterior = y_1 * factor + y_2 * (1.0 - factor)

    plot_loss_utility(x, posterior, loss_f, mu_true)

def plot_loss_utility(x, posterior, loss_f, mu_true):
    mean, median, mode = calc_mean_mode_median(x, posterior)

    loss = calc_loss_func(loss_f, mu_true, x)

    utility = calc_expected_loss(loss_f, posterior, x)
    min_expected_loss = x[np.argmin(utility)]

    plt.figure(figsize=(12, 8))
    plt.subplot(2, 2, 1)
    plt.title("Probability")
    plt.plot(x, posterior, c='b')
    plt.fill_between(x, posterior, color='b', alpha=0.2)
    plt.yticks([])
    plt.xlabel('x')
    plt.ylabel('$\pi \cdot p(x) + (1-\pi) \cdot p(y)$')
    plt.axvline(mean, ls='dashed', color='red', label='Mean')
    plt.axvline(median, ls='dashdot', color='blue', label='Median')
    plt.axvline(mode, ls='dotted', color='green', label='Mode')
    plt.legend(loc="upper left")

    plt.subplot(2, 2, 2)
    plt.title(loss_f)
    plt.plot(x, loss, c='c', label=loss_f)
    # plt.fill_between(x, loss, color='c', alpha=0.2)
    plt.ylabel('loss')
    # plt.legend(loc="upper left")
    plt.xlabel('x')

    plt.subplot(2, 2, 3)
    plt.title("Expected Loss")
    plt.plot(x, utility, c='y', label='$\mathbb{E}[L]$')
    plt.axvline(min_expected_loss, ls='dashed', color='red', label='$Min~ \mathbb{E}[Loss]$')
    # plt.fill_between(x, utility, color='y', alpha=0.2)
    plt.legend(loc="lower right")
    plt.xlabel('x')
    plt.ylabel('$\mathbb{E}[L]$')

    plt.show()

def plot_loss_utility_bayes(mu1, mu2, sigma1, sigma2, mu_true, loss_f):
    x = np.linspace(-4, 4, 1000, endpoint=True)

    prior = gaussian(x, mu1, sigma1)
    likelihood = gaussian(x, mu2, sigma2)

    mu_post, sigma_post = product_guassian(mu1, mu2, sigma1, sigma2)
    posterior = gaussian(x, mu_post, sigma_post)

    loss = calc_loss_func(loss_f, mu_true, x)

    utility = - calc_expected_loss(loss_f, posterior, x)

    plt.figure(figsize=(18, 5))
    plt.subplot(1, 3, 1)

    plt.title("Posterior distribution")
    plt.plot(x, prior, c='b', label='prior')
    plt.fill_between(x, prior, color='b', alpha=0.2)
    plt.plot(x, likelihood, c='r', label='likelihood')
    plt.fill_between(x, likelihood, color='r', alpha=0.2)
    plt.plot(x, posterior, c='k', label='posterior')
    plt.fill_between(x, posterior, color='k', alpha=0.2)
    plt.yticks([])
    plt.legend(loc="upper left")
    # plt.ylabel('$f(x)$')
    plt.xlabel('x')

    plt.subplot(1, 3, 2)
    plt.title(loss_f)
    plt.plot(x, loss, c='c')
    # plt.fill_between(x, loss, color='c', alpha=0.2)
    plt.ylabel('loss')

    plt.subplot(1, 3, 3)
    plt.title("Expected utility")
    plt.plot(x, utility, c='y', label='utility')
    # plt.fill_between(x, utility, color='y', alpha=0.2)
    plt.legend(loc="upper left")

    plt.show()

def plot_simple_utility_gaussian(mu, sigma, mu_g, mu_c, sigma_g, sigma_c):
    x = np.linspace(-7, 7, 1000, endpoint=True)
    posterior = gaussian(x, mu, sigma)
    gain = gaussian(x, mu_g, sigma_g)
    loss = gaussian(x, mu_c, sigma_c)
    utility = np.multiply(posterior, gain) - np.multiply(posterior, loss)

    plt.figure(figsize=(18, 5))
    plt.subplot(1, 3, 1)
    plt.title("Probability")
    plt.plot(x, posterior, c='b', label='posterior')
    plt.fill_between(x, posterior, color='b', alpha=0.2)
    plt.yticks([])
    # plt.legend(loc="upper left")
    plt.xlabel('x')

    plt.subplot(1, 3, 2)
    plt.title("utility function")
    plt.plot(x, gain, c='m', label='gain')
    # plt.fill_between(x, gain, color='m', alpha=0.2)
    plt.plot(x, -loss, c='c', label='loss')
    # plt.fill_between(x, -loss, color='c', alpha=0.2)
    plt.legend(loc="upper left")

    plt.subplot(1, 3, 3)
    plt.title("expected utility")
    plt.plot(x, utility, c='y', label='utility')
    # plt.fill_between(x, utility, color='y', alpha=0.2)
    plt.legend(loc="upper left")

    plt.show()

def plot_utility_gaussian(mu1, mu2, sigma1, sigma2, mu_g, mu_c, sigma_g, sigma_c, plot_utility_row=True):
    x = np.linspace(-7, 7, 1000, endpoint=True)
    prior = gaussian(x, mu1, sigma1)
    likelihood = gaussian(x, mu2, sigma2)

    mu_post, sigma_post = product_guassian(mu1, mu2, sigma1, sigma2)
    posterior = gaussian(x, mu_post, sigma_post)

    if plot_utility_row:
        gain = gaussian(x, mu_g, sigma_g)
        loss = gaussian(x, mu_c, sigma_c)
        utility = np.multiply(posterior, gain) - np.multiply(posterior, loss)
        plot_bayes_utility_rows(x, prior, likelihood, posterior, gain, loss, utility)
    else:
        plot_bayes_row(x, prior, likelihood, posterior)

    return None

def plot_utility_mixture(mu_m1, mu_m2, sigma_m1, sigma_m2, factor,
                         mu, sigma, mu_g, mu_c, sigma_g, sigma_c, plot_utility_row=True):
    x = np.linspace(-7, 7, 1000, endpoint=True)
    y_1 = gaussian(x, mu_m1, sigma_m1)
    y_2 = gaussian(x, mu_m2, sigma_m2)
    prior = y_1 * factor + y_2 * (1.0 - factor)

    likelihood = gaussian(x, mu, sigma)

    posterior = np.multiply(prior, likelihood)
    posterior = posterior / (posterior.sum() * (x[1] - x[0]))

    if plot_utility_row:
        gain = gaussian(x, mu_g, sigma_g)
        loss = gaussian(x, mu_c, sigma_c)
        utility = np.multiply(posterior, gain) - np.multiply(posterior, loss)
        plot_bayes_utility_rows(x, prior, likelihood, posterior, gain, loss, utility)
    else:
        plot_bayes_row(x, prior, likelihood, posterior)

    return None

def plot_utility_uniform(mu, sigma, mu_g, mu_c, sigma_g, sigma_c, plot_utility_row=True):
    x = np.linspace(-7, 7, 1000, endpoint=True)
    prior = np.ones_like(x) / (x.max() - x.min())
    likelihood = gaussian(x, mu, sigma)

    posterior = likelihood
    # posterior = np.multiply(prior, likelihood)
    # posterior = posterior / (posterior.sum() * (x[1] - x[0]))

    if plot_utility_row:
        gain = gaussian(x, mu_g, sigma_g)
        loss = gaussian(x, mu_c, sigma_c)
        utility = np.multiply(posterior, gain) - np.multiply(posterior, loss)
        plot_bayes_utility_rows(x, prior, likelihood, posterior, gain, loss, utility)
    else:
        plot_bayes_row(x, prior, likelihood, posterior)

    return None

def plot_utility_gamma(alpha, beta, offset, mu, sigma, mu_g, mu_c, sigma_g, sigma_c, plot_utility_row=True):
    x = np.linspace(-7, 7, 1000, endpoint=True)
    prior = gamma_pdf(x-offset, alpha, beta)
    likelihood = gaussian(x, mu, sigma)

    posterior = np.multiply(prior, likelihood)
    posterior = posterior / (posterior.sum() * (x[1] - x[0]))

    if plot_utility_row:
        gain = gaussian(x, mu_g, sigma_g)
        loss = gaussian(x, mu_c, sigma_c)
        utility = np.multiply(posterior, gain) - np.multiply(posterior, loss)
        plot_bayes_utility_rows(x, prior, likelihood, posterior, gain, loss, utility)
    else:
        plot_bayes_row(x, prior, likelihood, posterior)

    return None

def plot_bayes_row(x, prior, likelihood, posterior):

    mean, median, mode = calc_mean_mode_median(x, posterior)

    plt.figure(figsize=(12, 4))
    plt.subplot(1, 2, 1)
    plt.title("Prior and likelihood distribution")
    plt.plot(x, prior, c='b', label='prior')
    plt.fill_between(x, prior, color='b', alpha=0.2)
    plt.plot(x, likelihood, c='r', label='likelihood')
    plt.fill_between(x, likelihood, color='r', alpha=0.2)
    # plt.plot(x, posterior, c='k', label='posterior')
    # plt.fill_between(x, posterior, color='k', alpha=0.2)
    plt.yticks([])
    plt.legend(loc="upper left")
    # plt.ylabel('$f(x)$')
    plt.xlabel('x')

    plt.subplot(1, 2, 2)
    plt.title("Posterior distribution")
    plt.plot(x, posterior, c='k', label='posterior')
    plt.fill_between(x, posterior, color='k', alpha=0.1)
    plt.axvline(mean, ls='dashed', color='red', label='Mean')
    plt.axvline(median, ls='dashdot', color='blue', label='Median')
    plt.axvline(mode, ls='dotted', color='green', label='Mode')
    plt.legend(loc="upper left")
    plt.yticks([])
    plt.xlabel('x')

    plt.show()

def plot_bayes_utility_rows(x, prior, likelihood, posterior, gain, loss, utility):

    mean, median, mode = calc_mean_mode_median(x, posterior)
    max_utility = x[np.argmax(utility)]

    plt.figure(figsize=(12, 8))
    plt.subplot(2, 2, 1)
    plt.title("Prior and likelihood distribution")
    plt.plot(x, prior, c='b', label='prior')
    plt.fill_between(x, prior, color='b', alpha=0.2)
    plt.plot(x, likelihood, c='r', label='likelihood')
    plt.fill_between(x, likelihood, color='r', alpha=0.2)
    # plt.plot(x, posterior, c='k', label='posterior')
    # plt.fill_between(x, posterior, color='k', alpha=0.2)
    plt.yticks([])
    plt.legend(loc="upper left")
    # plt.ylabel('$f(x)$')
    plt.xlabel('x')

    plt.subplot(2, 2, 2)
    plt.title("Posterior distribution")
    plt.plot(x, posterior, c='k', label='posterior')
    plt.fill_between(x, posterior, color='k', alpha=0.1)
    plt.axvline(mean, ls='dashed', color='red', label='Mean')
    plt.axvline(median, ls='dashdot', color='blue', label='Median')
    plt.axvline(mode, ls='dotted', color='green', label='Mode')
    plt.legend(loc="upper left")
    plt.yticks([])
    plt.xlabel('x')

    plt.subplot(2, 2, 3)
    plt.title("utility function")
    plt.plot(x, gain, c='m', label='gain')
    # plt.fill_between(x, gain, color='m', alpha=0.2)
    plt.plot(x, -loss, c='c', label='loss')
    # plt.fill_between(x, -loss, color='c', alpha=0.2)
    plt.legend(loc="upper left")
    plt.xlabel('x')

    plt.subplot(2, 2, 4)
    plt.title("expected utility")
    plt.plot(x, utility, c='y', label='utility')
    # plt.fill_between(x, utility, color='y', alpha=0.2)
    plt.axvline(max_utility, ls='dashed', color='red', label='Max utility')
    plt.legend(loc="upper left")
    plt.xlabel('x')
    plt.ylabel('utility')
    plt.legend(loc="lower right")

    plt.show()

def gaussian_mixture(mu1, mu2, sigma1, sigma2, factor):
    assert 0.0 < factor < 1.0
    x = np.linspace(-7.0, 7.0, 1000, endpoint=True)
    y_1 = gaussian(x, mu1, sigma1)
    y_2 = gaussian(x, mu2, sigma2)
    mixture = y_1 * factor + y_2 * (1.0 - factor)

    plt.figure(figsize=(8, 6))
    plt.plot(x, y_1, c='deepskyblue', label='p(x)', linewidth=3.0)
    plt.fill_between(x, y_1, color='deepskyblue', alpha=0.2)
    plt.plot(x, y_2, c='aquamarine', label='p(y)', linewidth=3.0)
    plt.fill_between(x, y_2, color='aquamarine', alpha=0.2)
    plt.plot(x, mixture, c='b', label='$\pi \cdot p(x) + (1-\pi) \cdot p(y)$',  linewidth=3.0)
    plt.fill_between(x, mixture, color='b', alpha=0.2)
    plt.yticks([])
    plt.legend(loc="upper left")
    # plt.ylabel('$f(x)$')
    plt.xlabel('x')
    plt.show()

def plot_bayes_loss_utility_gaussian(loss_f, mu_true, mu1, mu2, sigma1, sigma2):
    x = np.linspace(-7, 7, 1000, endpoint=True)

    prior = gaussian(x, mu1, sigma1)
    likelihood = gaussian(x, mu2, sigma2)
    mu_post, sigma_post = product_guassian(mu1, mu2, sigma1, sigma2)
    posterior = gaussian(x, mu_post, sigma_post)

    loss = calc_loss_func(loss_f, mu_true, x)

    plot_bayes_loss_utility(x, prior, likelihood, posterior, loss, loss_f)

    return None

def plot_bayes_loss_utility_uniform(loss_f, mu_true, mu, sigma):
    x = np.linspace(-7, 7, 1000, endpoint=True)

    prior = np.ones_like(x) / (x.max() - x.min())
    likelihood = gaussian(x, mu, sigma)
    posterior = likelihood

    loss = calc_loss_func(loss_f, mu_true, x)

    plot_bayes_loss_utility(x, prior, likelihood, posterior, loss, loss_f)

    return None

def plot_bayes_loss_utility_gamma(loss_f, mu_true, alpha, beta, offset, mu, sigma):
    x = np.linspace(-7, 7, 1000, endpoint=True)
    prior = gamma_pdf(x-offset, alpha, beta)
    likelihood = gaussian(x, mu, sigma)
    posterior = np.multiply(prior, likelihood)
    posterior = posterior / (posterior.sum() * (x[1] - x[0]))

    loss = calc_loss_func(loss_f, mu_true, x)

    plot_bayes_loss_utility(x, prior, likelihood, posterior, loss, loss_f)

    return None

def plot_bayes_loss_utility_mixture(loss_f, mu_true, mu_m1, mu_m2, sigma_m1, sigma_m2, factor, mu, sigma):
    x = np.linspace(-7, 7, 1000, endpoint=True)
    y_1 = gaussian(x, mu_m1, sigma_m1)
    y_2 = gaussian(x, mu_m2, sigma_m2)
    prior = y_1 * factor + y_2 * (1.0 - factor)
    likelihood = gaussian(x, mu, sigma)

    posterior = np.multiply(prior, likelihood)
    posterior = posterior / (posterior.sum() * (x[1] - x[0]))

    loss = calc_loss_func(loss_f, mu_true, x)

    plot_bayes_loss_utility(x, prior, likelihood, posterior, loss, loss_f)

    return None

def plot_bayes_loss_utility(x, prior, likelihood, posterior, loss, loss_f):

    mean, median, mode = calc_mean_mode_median(x, posterior)
    expected_loss = calc_expected_loss(loss_f, posterior, x)
    min_expected_loss = x[np.argmin(expected_loss)]

    plt.figure(figsize=(12, 8))

    plt.subplot(2, 2, 1)
    plt.title("Prior and Likelihood")
    plt.plot(x, prior, c='b', label='prior')
    plt.fill_between(x, prior, color='b', alpha=0.2)
    plt.plot(x, likelihood, c='r', label='likelihood')
    plt.fill_between(x, likelihood, color='r', alpha=0.2)
    plt.yticks([])
    plt.legend(loc="upper left")
    plt.xlabel('x')

    plt.subplot(2, 2, 2)
    plt.title("Posterior")
    plt.plot(x, posterior, c='k', label='posterior')
    plt.fill_between(x, posterior, color='k', alpha=0.1)
    plt.axvline(mean, ls='dashed', color='red', label='Mean')
    plt.axvline(median, ls='dashdot', color='blue', label='Median')
    plt.axvline(mode, ls='dotted', color='green', label='Mode')
    plt.legend(loc="upper left")
    plt.yticks([])
    plt.xlabel('x')

    plt.subplot(2, 2, 3)
    plt.title(loss_f)
    plt.plot(x, loss, c='c', label=loss_f)
    # plt.fill_between(x, loss, color='c', alpha=0.2)
    plt.ylabel('loss')
    plt.xlabel('x')

    plt.subplot(2, 2, 4)
    plt.title("expected loss")
    plt.plot(x, expected_loss, c='y', label='$\mathbb{E}[L]$')
    # plt.fill_between(x, expected_loss, color='y', alpha=0.2)
    plt.axvline(min_expected_loss, ls='dashed', color='red', label='$Min~ \mathbb{E}[Loss]$')
    plt.legend(loc="lower right")
    plt.xlabel('x')
    plt.ylabel('$\mathbb{E}[L]$')

    plt.show()

def loss_plot_switcher(what_to_plot):
    if what_to_plot == "Gaussian":
        widget = interact(plot_loss_utility_gaussian,
            loss_f = widgets.Dropdown(
                options=["Mean Squared Error", "Absolute Error", "Zero-One Loss"],
                value="Mean Squared Error", description="Loss: "),
            mu = FloatSlider(min=-4.0, max=4.0, step=0.01, value=-0.5, description="µ_estimate", continuous_update=False),
            sigma = FloatSlider(min=0.1, max=2.0, step=0.01, value=0.5, description="σ_estimate", continuous_update=False),
            mu_true = FloatSlider(min=-3.0, max=3.0, step=0.01, value=0.0, description="µ_true", continuous_update=False))
    elif what_to_plot == "Mixture of Gaussians":
        widget = interact(plot_loss_utility_mixture,
            mu1 = FloatSlider(min=-4.0, max=4.0, step=0.01, value=-0.5, description="µ_est_1", continuous_update=False),
            mu2 = FloatSlider(min=-4.0, max=4.0, step=0.01, value=-0.5, description="µ_est_2", continuous_update=False),
            sigma1 = FloatSlider(min=0.1, max=2.0, step=0.01, value=0.5, description="σ_est_1", continuous_update=False),
            sigma2 = FloatSlider(min=0.1, max=2.0, step=0.01, value=0.5, description="σ_est_2", continuous_update=False),
            factor = FloatSlider(min=0.0, max=1.0, step=0.01, value=0.5, description="π", continuous_update=False),
            mu_true = FloatSlider(min=-3.0, max=3.0, step=0.01, value=0.0, description="µ_true", continuous_update=False),
            loss_f = widgets.Dropdown(
                options=["Mean Squared Error", "Absolute Error", "Zero-One Loss"],
                value="Mean Squared Error", description="Loss: "))

def plot_prior_switcher(what_to_plot):
    if what_to_plot == "Gaussian":
        widget = interact(plot_utility_gaussian,
                  mu1 = FloatSlider(min=-4.0, max=4.0, step=0.01, value=-0.5, description="µ_prior", continuous_update=False),
                  mu2 = FloatSlider(min=-4.0, max=4.0, step=0.01, value=0.5, description="µ_likelihood", continuous_update=False),
                  sigma1 = FloatSlider(min=0.1, max=2.0, step=0.01, value=0.5, description="σ_prior", continuous_update=False),
                  sigma2 = FloatSlider(min=0.1, max=2.0, step=0.01, value=0.5, description="σ_likelihood", continuous_update=False),
                  mu_g = fixed(1.0),
                  mu_c = fixed(-1.0),
                  sigma_g = fixed(0.5),
                  sigma_c = fixed(value=0.5),
                  plot_utility_row=fixed(False))
    elif what_to_plot == "Uniform":
        widget = interact(plot_utility_uniform,
                  mu = FloatSlider(min=-4.0, max=4.0, step=0.01, value=0.5, description="µ_likelihood", continuous_update=False),
                  sigma = FloatSlider(min=0.1, max=2.0, step=0.01, value=0.5, description="σ_likelihood", continuous_update=False),
                  mu_g = fixed(1.0),
                  mu_c = fixed(-1.0),
                  sigma_g = fixed(0.5),
                  sigma_c = fixed(value=0.5),
                  plot_utility_row=fixed(False))
    elif what_to_plot == "Gamma":
        widget = interact(plot_utility_gamma,
                  alpha = FloatSlider(min=1.0, max=10.0, step=0.1, value=2.0, description="α_prior", continuous_update=False),
                  beta = FloatSlider(min=0.5, max=2.0, step=0.01, value=1.0, description="β_prior", continuous_update=False),
                  offset = FloatSlider(min=-6.0, max=2.0, step=0.1, value=0.0, description="offset", continuous_update=False),
                  mu = FloatSlider(min=-4.0, max=4.0, step=0.01, value=0.5, description="µ_likelihood", continuous_update=False),
                  sigma = FloatSlider(min=0.1, max=2.0, step=0.01, value=0.5, description="σ_likelihood", continuous_update=False),
                  mu_g = fixed(1.0),
                  mu_c = fixed(-1.0),
                  sigma_g = fixed(0.5),
                  sigma_c = fixed(value=0.5),
                  plot_utility_row=fixed(False))
    elif what_to_plot == "Mixture of Gaussians":
        widget = interact(plot_utility_mixture,
                  mu_m1 = FloatSlider(min=-4.0, max=4.0, step=0.01, value=-0.5, description="µ_mix_1", continuous_update=False),
                  mu_m2 = FloatSlider(min=-4.0, max=4.0, step=0.01, value=0.5, description="µ_mix_1", continuous_update=False),
                  sigma_m1 = FloatSlider(min=0.1, max=2.0, step=0.01, value=0.5, description="σ_mix_1", continuous_update=False),
                  sigma_m2 = FloatSlider(min=0.1, max=2.0, step=0.01, value=0.5, description="σ_mix_2", continuous_update=False),
                  factor = FloatSlider(min=0.0, max=1.0, step=0.01, value=0.5, description="π", continuous_update=False),
                  mu = FloatSlider(min=-4.0, max=4.0, step=0.01, value=0.5, description="µ_likelihood", continuous_update=False),
                  sigma = FloatSlider(min=0.1, max=2.0, step=0.01, value=0.5, description="σ_likelihood", continuous_update=False),
                  mu_g = fixed(1.0),
                  mu_c = fixed(-1.0),
                  sigma_g = fixed(0.5),
                  sigma_c = fixed(value=0.5),
                  plot_utility_row=fixed(False))

def plot_bayes_loss_utility_switcher(what_to_plot):
    if what_to_plot == "Gaussian":
        widget = interact(plot_bayes_loss_utility_gaussian,
                  mu1 = FloatSlider(min=-4.0, max=4.0, step=0.01, value=-0.5, description="µ_prior", continuous_update=False),
                  mu2 = FloatSlider(min=-4.0, max=4.0, step=0.01, value=0.5, description="µ_likelihood", continuous_update=False),
                  sigma1 = FloatSlider(min=0.1, max=2.0, step=0.01, value=0.5, description="σ_prior", continuous_update=False),
                  sigma2 = FloatSlider(min=0.1, max=2.0, step=0.01, value=0.5, description="σ_likelihood", continuous_update=False),
                  mu_true = FloatSlider(min=-4.0, max=4.0, step=0.01, value=-0.5, description="µ_true", continuous_update=False),
                  loss_f = widgets.Dropdown(
                      options=["Mean Squared Error", "Absolute Error", "Zero-One Loss"],
                      value="Mean Squared Error", description="Loss: "))
    elif what_to_plot == "Uniform":
        widget = interact(plot_bayes_loss_utility_uniform,
                  mu = FloatSlider(min=-4.0, max=4.0, step=0.01, value=0.5, description="µ_likelihood", continuous_update=False),
                  sigma = FloatSlider(min=0.1, max=2.0, step=0.01, value=0.5, description="σ_likelihood", continuous_update=False),
                  mu_true = FloatSlider(min=-4.0, max=4.0, step=0.01, value=-0.5, description="µ_true", continuous_update=False),
                  loss_f = widgets.Dropdown(
                      options=["Mean Squared Error", "Absolute Error", "Zero-One Loss"],
                      value="Mean Squared Error", description="Loss: "))
    elif what_to_plot == "Gamma":
        widget = interact(plot_bayes_loss_utility_gamma,
                  alpha = FloatSlider(min=1.0, max=10.0, step=0.1, value=2.0, description="α_prior", continuous_update=False),
                  beta = FloatSlider(min=0.5, max=2.0, step=0.01, value=1.0, description="β_prior", continuous_update=False),
                  offset = FloatSlider(min=-6.0, max=2.0, step=0.1, value=0.0, description="offset", continuous_update=False),
                  mu = FloatSlider(min=-4.0, max=4.0, step=0.01, value=0.5, description="µ_likelihood", continuous_update=False),
                  sigma = FloatSlider(min=0.1, max=2.0, step=0.01, value=0.5, description="σ_likelihood", continuous_update=False),
                  mu_true = FloatSlider(min=-4.0, max=4.0, step=0.01, value=-0.5, description="µ_true", continuous_update=False),
                  loss_f = widgets.Dropdown(
                      options=["Mean Squared Error", "Absolute Error", "Zero-One Loss"],
                      value="Mean Squared Error", description="Loss: "))
    elif what_to_plot == "Mixture of Gaussians":
        widget = interact(plot_bayes_loss_utility_mixture,
                  mu_m1 = FloatSlider(min=-4.0, max=4.0, step=0.01, value=-0.5, description="µ_mix_1", continuous_update=False),
                  mu_m2 = FloatSlider(min=-4.0, max=4.0, step=0.01, value=0.5, description="µ_mix_1", continuous_update=False),
                  sigma_m1 = FloatSlider(min=0.1, max=2.0, step=0.01, value=0.5, description="σ_mix_1", continuous_update=False),
                  sigma_m2 = FloatSlider(min=0.1, max=2.0, step=0.01, value=0.5, description="σ_mix_2", continuous_update=False),
                  factor = FloatSlider(min=0.0, max=1.0, step=0.01, value=0.5, description="π", continuous_update=False),
                  mu = FloatSlider(min=-4.0, max=4.0, step=0.01, value=0.5, description="µ_likelihood", continuous_update=False),
                  sigma = FloatSlider(min=0.1, max=2.0, step=0.01, value=0.5, description="σ_likelihood", continuous_update=False),
                  mu_true = FloatSlider(min=-4.0, max=4.0, step=0.01, value=-0.5, description="µ_true", continuous_update=False),
                  loss_f = widgets.Dropdown(
                      options=["Mean Squared Error", "Absolute Error", "Zero-One Loss"],
                      value="Mean Squared Error", description="Loss: "))_____no_output_____# @title Helper functions

def gaussian(x, μ, σ):
    return np.exp(-((x - μ) / σ)**2 / 2) / np.sqrt(2 * np.pi * σ**2)

def gamma_pdf(x, α, β):
    return gamma_distribution.pdf(x, a=α, scale=1/β)

def mvn2d(x, y, mu1, mu2, sigma1, sigma2, cov12):
    mvn = multivariate_normal([mu1, mu2], [[sigma1**2, cov12], [cov12, sigma2**2]])
    return mvn.pdf(np.dstack((x, y)))

def product_guassian(mu1, mu2, sigma1, sigma2):
    J_1, J_2 = 1/sigma1**2, 1/sigma2**2
    J_3 = J_1 + J_2
    mu_prod = (J_1*mu1/J_3) + (J_2*mu2/J_3)
    sigma_prod = np.sqrt(1/J_3)
    return mu_prod, sigma_prod

def reverse_product(mu3, sigma3, mu1, mu2):
    J_3 = 1/sigma3**2
    J_1 = J_3 * (mu3 - mu2) / (mu1 - mu2)
    J_2 = J_3 * (mu3 - mu1) / (mu2 - mu1)
    sigma1, sigma2 = 1/np.sqrt(J_1), 1/np.sqrt(J_2)
    return sigma1, sigma2

def calc_mean_mode_median(x, y):
    """

    """
    pdf = y * (x[1] - x[0])
    # Calc mode of an arbitrary function
    mode = x[np.argmax(pdf)]

    # Calc mean of an arbitrary function
    mean = np.multiply(x, pdf).sum()

    # Calc median of an arbitrary function
    cdf = np.cumsum(pdf)
    idx = np.argmin(np.abs(cdf - 0.5))
    median = x[idx]

    return mean, median, mode

def calc_expected_loss(loss_f, posterior, x):
    dx = x[1] - x[0]
    expected_loss = np.zeros_like(x)
    for i in np.arange(x.shape[0]):
        loss = calc_loss_func(loss_f, x[i], x) # or mse or zero_one_loss
        expected_loss[i] = np.sum(loss * posterior) * dx
    return expected_loss

def plot_mixture_prior(x, gaussian1, gaussian2, combined):
    """
    DO NOT EDIT THIS FUNCTION !!!

    Plots a prior made of a mixture of gaussians

    Args:
      x (numpy array of floats):         points at which the likelihood has been evaluated
      gaussian1 (numpy array of floats): normalized probabilities for Gaussian 1 evaluated at each `x`
      gaussian2 (numpy array of floats): normalized probabilities for Gaussian 2 evaluated at each `x`
      posterior (numpy array of floats): normalized probabilities for the posterior evaluated at each `x`

    Returns:
      Nothing
    """
    fig, ax = plt.subplots()
    ax.plot(x, gaussian1, '--b', LineWidth=2, label='Gaussian 1')
    ax.plot(x, gaussian2, '-.b', LineWidth=2, label='Gaussian 2')
    ax.plot(x, combined, '-r', LineWidth=2, label='Gaussian Mixture')
    ax.legend()
    ax.set_ylabel('Probability')
    ax.set_xlabel('Orientation (Degrees)')

def gaussian_mixture(mu1, mu2, sigma1, sigma2, factor):
    assert 0.0 < factor < 1.0
    x = np.linspace(-7.0, 7.0, 1000, endpoint=True)
    y_1 = gaussian(x, mu1, sigma1)
    y_2 = gaussian(x, mu2, sigma2)
    mixture = y_1 * factor + y_2 * (1.0 - factor)

    plt.figure(figsize=(8, 6))
    plt.plot(x, y_1, c='deepskyblue', label='p(x)', linewidth=3.0)
    plt.fill_between(x, y_1, color='deepskyblue', alpha=0.2)
    plt.plot(x, y_2, c='aquamarine', label='p(y)', linewidth=3.0)
    plt.fill_between(x, y_2, color='aquamarine', alpha=0.2)
    plt.plot(x, mixture, c='b', label='$\pi \cdot p(x) + (1-\pi) \cdot p(y)$',  linewidth=3.0)
    plt.fill_between(x, mixture, color='b', alpha=0.2)
    plt.yticks([])
    plt.legend(loc="upper left")
    # plt.ylabel('$f(x)$')
    plt.xlabel('x')
    plt.show()_____no_output_____

---
# Section 1: Astrocat!_____no_output_____

# @title Video 1: Astrocat!
from IPython.display import YouTubeVideo
video = YouTubeVideo(id='D7Z-aTX92Pk', width=854, height=480, fs=1)
print("Video available at https://youtube.com/watch?v=" + video.id)
video_____no_output_____

Remember, in this example, you can think of yourself as a scientist trying to decide where we belive Astrocat is, how to select a point estimate (single guess of location) based on possible errors, and how to account for the uncertainty we have about the location of the satellite and the space mouse. In fact, this is the kind of problem real scientists working to control remote satellites face! However, we can also think of this as what your brain has does when it wants to determine a target to make a movement or hit a tennis ball! A number of classic experiments use this kind of framing to study how *optimal* human decisions or movements are! Some examples are in the further reading document._____no_output_____---
# Section 2: Probability distribution of Astrocat location

_____no_output_____We are going to think first about how Ground Control should estimate his position. We won't consider measurements yet, just how to represent the uncertainty we might have in general. We are now dealing with a continuous distribution - Astrocat's location can be any real number. In the last tutorial, we were dealing with a discrete distribution - the fish were either on one side or the other.

So how do we represent the probability of each possible point (an infinite number) where the Astrocat could be?
The Bayesian approach can be used on any probability distribution. While many variables in the world require representation using complex or unknown (e.g. empirical) distributions, we will be using the Gaussian distributions or extensions of it._____no_output_____## Section 2.1: The Gaussian distribution
_____no_output_____One distribution we will use throughout this tutorial is the **Gaussian distribution**, which is also sometimes called the normal distribution.

This is a special, and commonly used, distribution for a couple reasons. It is actually the focus of one of the most important theorems in statistics: the Central Limit Theorem. This theorem tells us that if you sum a large number of samples of a variable, that sum is normally distributed *no matter what* the original distribution over a variable was. This is a bit too in-depth for us to get into now but check out links in the Bonus for more information. Additionally, Gaussians have some really nice mathematical properties that permit simple closed-form solutions to several important problems. As we will see later in this tutorial, we can extend Gaussians to be even more flexible and well approximate other distributions using mixtures of Gaussians. In short, the Gaussian is probably the most important continous distribution to understand and use.

Gaussians have two parameters. The **mean** $\mu$, which sets the location of its center. Its "scale" or spread is controlled by its **standard deviation** $\sigma$ or its square, the **variance** $\sigma^2$. These can be a bit easy to mix up: make sure you are careful about whether you are referring to/using standard deviation or variance.

The equation for a Gaussian distribution on a variable $x$ is:

$$
\mathcal{N}(\mu,\sigma^2) = \frac{1}{\sqrt{2\pi\sigma^2}}\exp\left(\frac{-(x-\mu)^2}{2\sigma^2}\right)
$$

In our example, $x$ is the location of the Astrocat in one direction. $\mathcal{N}(\mu,\sigma^2)$ is a standard notation to refer to a **N**ormal (Gaussian) distribution. For example, $\mathcal{N}(0, 1)$ denotes a Gaussian distribution with mean 0 and variance 1. The exact form of this equation is not particularly intuitive, but we will see how mean and standard deviation values affect the probability distribution.

We won't implement a Gaussian distribution in code here but please refer to the pre-reqs refresher W0D5 T1 to do this if you need further clarification.
_____no_output_____

# @markdown Execute this cell to enable the function `gaussian`
def gaussian(x, μ, σ):
    return np.exp(-((x - μ) / σ)**2 / 2) / np.sqrt(2 * np.pi * σ**2)_____no_output_____

### Interactive Demo 2.1: Exploring Gaussian parameters:

Let's explore how the parameters of a Gaussian affect the distribution. Play with the demo below by changing the mean $\mu$ and standard deviation $\sigma$.

Discuss the following:

1. What does increasing $\mu$ do? What does increasing $\sigma$ do?
2. If you wanted to know the probability of an event happing at $0$, can you find two different $\mu$ and $\sigma$ values that produce the same probabilty of an event at $0$?
3. How many Gaussian's could produce the same probabilty at $0$?_____no_output_____

# @markdown Execute this cell to enable the widget
widget = interact(plot_gaussian,
                     μ = FloatSlider(min=-4.0, max=4.0, step=0.01, value=0.0, continuous_update=False),
                     σ = FloatSlider(min=0.1, max=2.0, step=0.01, value=0.5, continuous_update=False))_____no_output_____

[*Click for solution*](https://github.com/NeuromatchAcademy/course-content/tree/master//tutorials/W3D1_BayesianDecisions/solutions/W3D1_Tutorial2_Solution_115e84fc.py)

_____no_output_____## Section 2.2: Multiplying Gaussians_____no_output_____

# @title Video 2: Multiplying Gaussians
from IPython.display import YouTubeVideo
video = YouTubeVideo(id='areR25_0FyY', width=854, height=480, fs=1)
print("Video available at https://youtube.com/watch?v=" + video.id)
video_____no_output_____

When we multiply Gaussians, we are not multiplying random variables but the actual underlying distributions. If we multiply two Gaussian distributions, with means $\mu_1$ and $\mu_2$ and standard deviations $\sigma_1$ and $\sigma_2$, we get another Gaussian. The Gaussian resulting from the multiplication will have mean $\mu_3$ and standard deviation $\sigma_3$ where:

$$
\mu_{3} = a\mu_{1} + (1-a)\mu_{2}
$$
$$
\sigma_{3}^{-2} = \sigma_{1}^{-2} + \sigma_{2}^{-2}\\
a = \frac{\sigma_{1}^{-2}}{\sigma_{1}^{-2} + \sigma_{2}^{-2}}
$$

This may look confusing but keep in mind that the information in a Gaussian is the inverse of its variance: $\frac{1}{\sigma^2}$. Basically, when multiplying Gaussians, the mean of the resulting Gaussian is a weighted average of the original means, where the weights are proportional to the amount of information of that Gaussian. The information in the resulting Gaussian is equal to the sum of informations of the original two. We'll dive into this in the next demo._____no_output_____### Interactive Demo 2.2: Multiplying Gaussians

We have implemented the multiplication of two Gaussians for you. Using the following widget, we are going to think about the information and combination of two Gaussians. It In our case, imagine we want to find the middle location between the satellite and the space mouse. This would be the center (average) of the two locations. Because we have uncertainty, we need to weight our uncertainty in thinking about the most likely place.

In this demo, $\mu_{1}$ and $\sigma_{1}$ are the mean and standard deviation of the distribution over satellite location, $\mu_{2}$ and $\sigma_{2}$ are the mean and standard deviation of the distribution over space mouse location, and $\mu_{3}$ and $\sigma_{3}$ are the mean and standard deviation of the distribution over the center location (gained by multiplying the first two).

Questions:

1. What is your uncertainty (how much information) do you have about $\mu_{3}$ with $\mu_{1} = -2, \mu_{2} = 2, \sigma_{1} = \sigma_{2} = 0.5$?
2. What happens to your estimate of $\mu_{3}$ as $\sigma_{2} \to \infty$? (In this case, $\sigma$ only goes to 11... but that should be loud enough.)
3. What is the difference in your estimate of $\mu_{3}$ if $\sigma_{1} = \sigma_{2} = 11$? What has changed from the first example?
4. Set $\mu_{1} = -4, \mu_{2} = 4$ and change the $\sigma$s so that $\mu_{3}$ is close to $2$. How many $\sigma$s will produce the same $\mu_{3}$?
5. Continuing, if you set $\mu_{1} = 0$, what $\sigma$ do you need to change so $\mu_{3} ~= 2$?
6. If $\sigma_{1} = \sigma_{2} = 0.1$, how much information do you have about the average?_____no_output_____

# @markdown Execute this cell to enable the widget
widget = interact(plot_information,
                  mu1 = FloatSlider(min=-5.0, max=-0.51, step=0.01, value=-2.0, description="µ_1",continuous_update=False),
                  mu2 = FloatSlider(min=0.5, max=5.01, step=0.01, value=2.0, description="µ_2",continuous_update=False),
                  sigma1 = FloatSlider(min=0.1, max=11.01, step=0.01, value=1.0, description="σ_1", continuous_update=False),
                  sigma2 = FloatSlider(min=0.1, max=11.01, step=0.01, value=1.0, description="σ_2", continuous_update=False)
                  )_____no_output_____

[*Click for solution*](https://github.com/NeuromatchAcademy/course-content/tree/master//tutorials/W3D1_BayesianDecisions/solutions/W3D1_Tutorial2_Solution_2197032b.py)

_____no_output_____To start thinking about how we might use these concepts directly in systems neuroscience, imagine you want to know how much information is gained combining (averaging) the response of two neurons that represent locations in sensory space (think: how much information is shared by their receptive fields). You would be multiplying Gaussians!_____no_output_____## Section 2.3: Mixtures of Gaussians
_____no_output_____

# @title Video 3: Mixtures of Gaussians
from IPython.display import YouTubeVideo
video = YouTubeVideo(id='5zoRO10urSk', width=854, height=480, fs=1)
print("Video available at https://youtube.com/watch?v=" + video.id)
video_____no_output_____

What if our continuous distribution isn't well described by a single bump? For example, what if the Astrocat is often either in one place or another - a Gaussian distribution would not capture this well! We need a multimodal distribution. Luckily, we can extend Gaussians into a *mixture of Gaussians*, which are more complex distributions.

In a Gaussian mixture distribution, you are essentially adding two or more weighted standard Gaussian distributions (and then normalizing so everything integrates to 1). Each standard Gaussian involved is described, as normal, by its mean and standard deviation. Additional parameters in a mixture of Gaussians are the weights you put on each Gaussian (π). The following demo should help clarify how a mixture of Gaussians relates to the standard Gaussian components. We will not cover the derivation here but you can work it out as a bonus exercise.

Mixture distributions are a common tool in Bayesian modeling and an important tool in general.

_____no_output_____### Interactive Demo 2.3: Exploring Gaussian mixtures

We will examing a mixture of two Gaussians. We will have one weighting parameter, π, that tells us how to weight one of the Gaussians. The other is weighted by 1 - π.

Use the following widget to experiment with the parameters of each Gaussian and the mixing weight ($\pi$) to undersand how the mixture of Gaussian distribution behaves.

Discuss the following questions:

1. What does increasing the weight $\pi$ do to the mixture distribution (dark blue)?
2. How can you make the two bumps of the mixture distribution further apart?
3. Can you make the mixture distribution have just one bump (like a Gaussian)?
4. Any other shapes you can make the mixture distribution resemble other than one nicely rounded bump or two separate bumps?
_____no_output_____

# @markdown Execute this cell to enable the widget
widget = interact(gaussian_mixture,
            mu1 = FloatSlider(min=-4.0, max=4.0, step=0.01, value=1.0, description="µ_1", continuous_update=False),
            mu2 = FloatSlider(min=-4.0, max=4.0, step=0.01, value=-1.0, description="µ_2", continuous_update=False),
            sigma1 = FloatSlider(min=0.1, max=2.0, step=0.01, value=0.5, description="σ_1", continuous_update=False),
            sigma2 = FloatSlider(min=0.1, max=2.0, step=0.01, value=0.5, description="σ_2", continuous_update=False),
            factor = FloatSlider(min=0.1, max=0.9, step=0.01, value=0.5, description="π", continuous_update=False))_____no_output_____

[*Click for solution*](https://github.com/NeuromatchAcademy/course-content/tree/master//tutorials/W3D1_BayesianDecisions/solutions/W3D1_Tutorial2_Solution_201ab7bd.py)

_____no_output_____---
# Section 3: Utility

_____no_output_____

# @title Video 4: Utility
from IPython.display import YouTubeVideo
video = YouTubeVideo(id='I5H7Anh3FXs', width=854, height=480, fs=1)
print("Video available at https://youtube.com/watch?v=" + video.id)
video_____no_output_____

We want to know where Astrocat is. If we were asked to provide the coordinates, for example to display them for Ground Control or to note them in a log, we are not going to provide the whole probability distribution! We will give a single set of coordinates, but we first need to estimate those coordinates. Just like in the last tutorial, this may not be as easy as just what is most likely: we want to know how good or bad it is if we guess a certain location and the Astrocat is in another.

As we have seen, utility represents the gain (or if negative, loss) for us if we take a certain action for a certain value of the hidden state. In our continuous example, we need a function to be able to define the utility with respect to all possible continuous values of the state. Our action here is our guess of the Astrocat location.

We are going to explore this for the Gaussian distribution, where our estimate is $\hat{\mu}$ and the true hidden state we are interested in is $\mu$.

A loss function determines the "cost" (or penalty) of estimating $\hat \mu$ when the true or correct quantity is really $\mu$ (this is essentially the cost of the error between the true hidden state we are interested in: $\mu$ and our estimate: $\hat \mu$). A loss function is equivalent to a negative utility function.
_____no_output_____## Section 3.1: Standard loss functions

There are lots of different possible loss functions. We will focus on three: **mean-squared error** where the loss is the different between truth and estimate squared, **absolute error** where the loss is the absolute difference between truth and estimate, and **Zero-one Loss** where the loss is 1 unless we're exactly right (the estimate equals the truth). We can represent these with the following formulas:

$$
\begin{eqnarray}
\textrm{Mean Squared Error} &=& (\mu - \hat{\mu})^2 \\
\textrm{Absolute Error} &=& \big|\mu - \hat{\mu}\big| \\
\textrm{Zero-One Loss} &=& \begin{cases}
                            0,& \textrm{if } \mu = \hat{\mu} \\
                            1,              & \textrm{otherwise}
                            \end{cases}
\end{eqnarray}
$$

We will now explore how these different loss functions change our expected utility!

Check out the next cell to see the implementation of each loss in the function `calc_loss_func`.
_____no_output_____

# @markdown Execute this cell to enable the function `calc_loss_func`

def calc_loss_func(loss_f, mu_true, x):
    error = x - mu_true
    if loss_f == "Mean Squared Error":
        loss = (error)**2
    elif loss_f == "Absolute Error":
        loss = np.abs(error)
    elif loss_f == "Zero-One Loss":
        loss = (np.abs(error) >= 0.03).astype(np.float)
    return loss_____no_output_____

### Interactive demo 3: Exploring Loss with different distributions

Let's see how our loss functions interact with probability distributions to affect expected utility and consequently, the action we take.

Play with the widget below and discuss the following:

1. With a Gaussian distribution, does the peak of the expected utility ever change position on the x-axis for the three different losses? This peak denotes the action we would choose (the location we would guess) so in other words, would the different choices of loss function affect our action?
2. With a mixture of Gaussian distribution with two bumps, does the peak of the expected loss ever change position on the x-axis for the three different losses?
3.  Find parameters for a mixture of Gaussians that results in the mean, mode, and median all being distinct (not equal to one another). With this distribution, how does the peak of the expected utility correspond to the mean/median/mode of the probability distribution for each of the three losses?
4. When the mixture of Gaussians has two peaks that are exactly the same height, how many modes are there?

_____no_output_____

# @markdown Execute this cell to enable the widget

widget = interact(loss_plot_switcher,
                  what_to_plot = widgets.Dropdown(
                      options=["Gaussian", "Mixture of Gaussians"],
                      value="Gaussian", description="Distribution: "))_____no_output_____

[*Click for solution*](https://github.com/NeuromatchAcademy/course-content/tree/master//tutorials/W3D1_BayesianDecisions/solutions/W3D1_Tutorial2_Solution_909051d1.py)

_____no_output_____You can see that what coordinates you would provide for Astrocat aren't necessarily easy to guess just from the probability distribution. You need the concept of utility/loss and a specific loss function to determine what estimate you should give.

For symetric distributions, you will find that the mean, median and mode are the same. However, for distributions with *skew*, like the Gamma distribution or the Exponential distribution, these will be different. You will be able to explore more distributions as priors below._____no_output_____## Section 3.2: A more complex loss function

The loss functions we just explored were fairly simple and are often used. However, life can be complicated and in this case, Astrocat cares about both being near the space mouse and avoiding the satellite. This means we need a more complex loss function that captures this!

We know that we want to estimate Astrocat to be closer to the mouse, which is safe and desirable, but further away from the satellite, which is dangerous! So, rather than thinking about the *Loss* function, we will consider a generalized utility function that considers gains and losses that *matter* to Astrocat!

In this case, we can assume that depending on our uncertainty about Astrocat's probable location, we may want to 'guess' that Astrocat is close to 'good' parts of space and further from 'bad' parts of space. We will model these utilities as Gaussian gain and loss regions--and we can assume the width of the Gaussian comes from our uncertainty over where the Space Mouse and satellite are.

Let's explore how this works in the next interactive demo.
_____no_output_____### Interactive demo 3.2: Complicated cat costs

Now that we have explored *Loss* functions that can be used to determine both formal *estimators* and our expected loss given our error, we are going to see what happens to our estimates if we use a generalized utility function.

Questions:

1. As you change the $\mu$ of Astrocat, what happens to the expected utility?
2. Can the EU be exactly zero everywhere?
3. Can the EU be zero in a region around Astrocat but positive and negative elsewhere?
4. As our uncertainty about Astrocat's position increases, what happens to the expected utility?_____no_output_____

# @markdown Execute this cell to enable the widget

widget = interact(plot_simple_utility_gaussian,
                  mu = FloatSlider(min=-4.0, max=4.0, step=0.01, value=-0.5, description="µ", continuous_update=False),
                  sigma = FloatSlider(min=0.1, max=2.0, step=0.01, value=0.5, description="σ", continuous_update=False),
                  mu_g = FloatSlider(min=-4.0, max=4.0, step=0.01, value=1.0, description="µ_gain", continuous_update=False),
                  mu_c = FloatSlider(min=-4.0, max=4.0, step=0.01, value=-1.0, description="µ_cost", continuous_update=False),
                  sigma_g = FloatSlider(min=0.1, max=2.0, step=0.01, value=0.5, description="σ_gain", continuous_update=False),
                  sigma_c = FloatSlider(min=0.1, max=2.0, step=0.01, value=0.5, description="σ_cost", continuous_update=False))_____no_output_____

[*Click for solution*](https://github.com/NeuromatchAcademy/course-content/tree/master//tutorials/W3D1_BayesianDecisions/solutions/W3D1_Tutorial2_Solution_043fbb5f.py)

_____no_output_____---
# Section 4: Correlation and marginalization

In this section we will explore a two dimensional Gaussian, often defined as a two-dimension vector of Gaussian random variables. This is, in essence, the joint distribution of two Gaussian random variables.

_____no_output_____## Video 5: Correlation and marginalization_____no_output_____

# @title Video 5: Correlation and marginalization
from IPython.display import YouTubeVideo
video = YouTubeVideo(id='NSDd0kvQtcY', width=854, height=480, fs=1)
print("Video available at https://youtube.com/watch?v=" + video.id)
video_____no_output_____

## Section 4.1: Correlation

If the two variables in a two dimensional Gaussian are independent, looking at one tells us nothing about the other. But what if the the two variables are correlated (covary)?

The covariance of two Gaussians with means $\mu_X$ and $\mu_Y$ and standard deviations $\sigma_X$ and $\sigma_Y$is:

$$
\sigma_{XY} = E[(X-\mu_{X})(Y-\mu_{Y})]
$$

$E$ here denotes the expected value. So the covariance is the expected value of the random variable X minus the mean of the Gaussian distribution on X times the random variable Y minus the mean of the Gaussian distribution on Y.

The correlation is the covariance normalized, so that it goes between -1 (exactly anticorrelated) to 1 (exactly correlated).

$$
\rho_{XY} = \frac{\sigma_{XY}}{\sigma_{X}\sigma_{Y}}
$$

These are key concepts and while we are considering two hidden states (or two random variables), they extend to $N$ dimensional vectors of Gaussian random variables. You will find these used all over computational neuroscience.

_____no_output_____### Interactive demo 4.1: Covarying 2D Gaussian

Let's explore this 2D Gaussian (i.e. joint distribution of two Gaussians).

Use the following widget to think about the following questions:

1. If these variables represent hidden states we care about, what does observing one tell us about the other? How does this depend on the correlation?
2. How does the shape of the distribution change when we change the means? The variances? The correlation?
3. If we want to isolate one or the other hidden state distributions, what do we need to do? (Hint: think about Tutorial 1.)_____no_output_____

# @markdown Execute the cell to enable the widget
widget = interact(plot_mvn2d,
                  mu1 = FloatSlider(min=-1.0, max=1.0, step=0.01, value=0.0, description="µ_1", continuous_update=False),
                  mu2 = FloatSlider(min=-1.0, max=1.0, step=0.01, value=0.0, description="µ_2", continuous_update=False),
                  sigma1 = FloatSlider(min=0.1, max=1.5, step=0.01, value=0.5, description="σ_1", continuous_update=False),
                  sigma2 = FloatSlider(min=0.1, max=1.5, step=0.01, value=0.5, description="σ_2", continuous_update=False),
                  corr = FloatSlider(min=-0.99, max=0.99, step=0.01, value=0.0, description="ρ", continuous_update=False))_____no_output_____

[*Click for solution*](https://github.com/NeuromatchAcademy/course-content/tree/master//tutorials/W3D1_BayesianDecisions/solutions/W3D1_Tutorial2_Solution_e7975bcc.py)

_____no_output_____## Section 4.2: Marginalization and information

We learned in Tutorial 1 that if we want to measure the probability of one or another variable, we need to average over the other. When we extend this to the correlated Gaussians we just played with, marginalization works the same way. Let's say that the two variables reflect Astrocat's position in space (in two dimensions). If we want to get our uncertainty about Astrocat's X or Y position, we need to marginalize.

However, let's imagine we have a measurement from one of the variables, for example X, and we want to understand the uncertainty we have in Y. We no longer want to marginalize because we know X, we don't need to ignore it! Instead, we can calculate the conditional probability $P(Y|X=x)$. You will explore the relationship between these two concepts in the following interactive demo.

But first, let's remember that we can also think about the amount of uncertainty as inversely proportional to the amount of information we have about each variable. This is important, because the joint information is determined by the correlation. For our Bayesian approach, the important intuition is that we can also think about the mutual information between the prior and the likelihood following a measurement.

_____no_output_____### Interactive demo 4.2: Marginalizing 2D Gaussians

Use the following widget to think consider the following questions:

1. When is the marginal distribution the same as the conditional probability distribution? Why?
2. If $\rho$ is large, how much information can we gain (in addition) looking at both variables vs just considering one?
3. If $\rho$ is close to zero, but the variances of the two variables are very different, what happens to the conditional probability compared to the marginals? As $\rho$ changes?_____no_output_____

# @markdown Execute this cell to enable the widget
widget = interact(plot_marginal,
                  sigma1 = FloatSlider(min=0.1, max=1.1, step=0.01, value=0.5, description="σ_x", continuous_update=False),
                  sigma2 = FloatSlider(min=0.1, max=1.1, step=0.01, value=0.5, description="σ_y", continuous_update=False),
                  c_x = FloatSlider(min=-1.0, max=1.0, step=0.01, value=0.0, description="Cx", continuous_update=False),
                  c_y = FloatSlider(min=-1.0, max=1.0, step=0.01, value=0.0, description="Cy", continuous_update=False),
                  corr = FloatSlider(min=-1.0, max=1.0, step=0.01, value=0.0, description="ρ", continuous_update=False))_____no_output_____

[*Click for solution*](https://github.com/NeuromatchAcademy/course-content/tree/master//tutorials/W3D1_BayesianDecisions/solutions/W3D1_Tutorial2_Solution_f5d119b4.py)

_____no_output_____---
# Section 5: Bayes' theorem for continuous distributions

_____no_output_____## Section 5.1: The Gaussian example

Bayes' rule tells us how to combine two sources of information: the prior (e.g., a noisy representation of Ground Control's expectations about where Astrocat is) and the likelihood (e.g., a noisy representation of the Astrocat after taking a measurement), to obtain a posterior distribution (our belief distribution) taking into account both pieces of information. Remember Bayes' rule:

\begin{eqnarray}
\text{Posterior} = \frac{ \text{Likelihood} \times \text{Prior}}{ \text{Normalization constant}}
\end{eqnarray}

We will look at what happens when both the prior and likelihood are Gaussians. In these equations, $\mathcal{N}(\mu,\sigma^2)$ denotes a Gaussian distribution with parameters $\mu$ and $\sigma^2$:
$$
\mathcal{N}(\mu, \sigma) = \frac{1}{\sqrt{2 \pi \sigma^2}} \; \exp \bigg( \frac{-(x-\mu)^2}{2\sigma^2} \bigg)
$$

When both the prior and likelihood are Gaussians, Bayes Rule translates into the following form:

$$
\begin{array}{rcl}
\text{Likelihood} &=& \mathcal{N}(\mu_{likelihood},\sigma_{likelihood}^2) \\
\text{Prior} &=& \mathcal{N}(\mu_{prior},\sigma_{prior}^2) \\
\text{Posterior} &=& \mathcal{N}\left( \frac{\sigma^2_{likelihood}\mu_{prior}+\sigma^2_{prior}\mu_{likelihood}}{\sigma^2_{likelihood}+\sigma^2_{prior}}, \frac{\sigma^2_{likelihood}\sigma^2_{prior}}{\sigma^2_{likelihood}+\sigma^2_{prior}} \right) \\
&\propto& \mathcal{N}(\mu_{likelihood},\sigma_{likelihood}^2) \times \mathcal{N}(\mu_{prior},\sigma_{prior}^2)
\end{array}
$$

We get the parameters of the posterior from multiplying the Gaussians, just as we did in Secton 2.2.

_____no_output_____### Interactive Demo 5.1: Gaussian Bayes
Let's consider the following questions using the following interactive demo:

1. For a Gaussian posterior, explain how the information seems to be combining. (Hint: think about the prior exercises!)
2. What is the difference between the posterior here and the Gaussian that represented the average of two Gaussians in the exercise above?
3. How should we think about the relative weighting of information between the prior and posterior?_____no_output_____

# @markdown Execute this cell to enable the widget
widget = interact(plot_bayes,
                  mu1 = FloatSlider(min=-4.0, max=4.0, step=0.01, value=-0.5, description="µ_prior", continuous_update=False),
                  mu2 = FloatSlider(min=-4.0, max=4.0, step=0.01, value=0.5, description="µ_likelihood", continuous_update=False),
                  sigma1 = FloatSlider(min=0.1, max=2.0, step=0.01, value=0.5, description="σ_prior", continuous_update=False),
                  sigma2 = FloatSlider(min=0.1, max=2.0, step=0.01, value=0.5, description="σ_likelihood", continuous_update=False))_____no_output_____

[*Click for solution*](https://github.com/NeuromatchAcademy/course-content/tree/master//tutorials/W3D1_BayesianDecisions/solutions/W3D1_Tutorial2_Solution_bf0bb545.py)

_____no_output_____## Section 5.2: Exploring priors

What would happen if we had a different prior distribution for Astrocat's location? Bayes' Rule works exactly the same way if our prior is not a Guassian (though the analytical solution may be far more complex or impossible). Let's look at how the posterior behaves if we have a different prior over Astrocat's location.

Consider the following questions:

1. Why does the posterior not look Gaussian when you use a non-Gaussian prior?
2. What does having a flat prior mean?
3. How does the Gamma prior behave differently than the others?
4. From what you know, can you imagine the likelihood being something other than a Gaussian?_____no_output_____

# @markdown Execute this cell to enable the widget
widget = interact(plot_prior_switcher,
                  what_to_plot = widgets.Dropdown(
                      options=["Gaussian", "Mixture of Gaussians", "Uniform", "Gamma"],
                      value="Gaussian", description="Prior: "))_____no_output_____

[*Click for solution*](https://github.com/NeuromatchAcademy/course-content/tree/master//tutorials/W3D1_BayesianDecisions/solutions/W3D1_Tutorial2_Solution_d72e2b3c.py)

_____no_output_____---
# Section 6: Bayesian decisions
_____no_output_____
## Section 6.1: Bayesian estimation on the posterior

Now that we understand that the posterior can be something other than a Gaussian, let's revisit **Loss** functions. In this case, we can see that the posterior can take many forms.

 _____no_output_____### Interactive Demo 6.1: Standard loss functions with various priors

Questions:

1. If we have a bi-modal prior, how do the different loss functions potentially inform us differently about what we learn?
2. Why do the different loss functions behavior differently with respect to the shape of the posterior? When do they produce different expected loss?
3. For the mixture of Gaussians, describe the situations where the expected loss will look different from the Gaussian case.
_____no_output_____

# @markdown Execute this cell to enable the widget
widget = interact(plot_bayes_loss_utility_switcher,
                  what_to_plot = widgets.Dropdown(
                      options=["Gaussian", "Mixture of Gaussians", "Uniform", "Gamma"],
                      value="Gaussian", description="Prior: "))_____no_output_____

[*Click for solution*](https://github.com/NeuromatchAcademy/course-content/tree/master//tutorials/W3D1_BayesianDecisions/solutions/W3D1_Tutorial2_Solution_797c061a.py)

_____no_output_____## Section 6.2: Bayesian decisions

Finally, we can combine everything we have learned so far!

Now, let's imagine we have just received a new measurement of Astrocat's location. We need to think about how we want to decide where Astrocat is, so that we can decide how far to tell Astrocat to move. However, we want to account for the satellite and Space Mouse location in this estimation. If we make an error towards the satellite, it's worse than towards Space Mouse. So, we will use our more complex utility function from Section 3.2.
_____no_output_____### Interactive Demo 6.2: Complicated cat costs with various priors

Questions:

1. If you have a weak prior and likelihood, how much are you relying on the utility function to guide your estimation?
2. If you get a good measurement, that is a likelihood with low variance, how much does this help?
3. Which of the factors are most important in making your decision?_____no_output_____

# @markdown Execute this cell to enable the widget
widget = interact(plot_utility_gaussian,
                  mu1 = FloatSlider(min=-4.0, max=4.0, step=0.01, value=-0.5, description="µ_prior", continuous_update=False),
                  mu2 = FloatSlider(min=-4.0, max=4.0, step=0.01, value=0.5, description="µ_likelihood", continuous_update=False),
                  sigma1 = FloatSlider(min=0.1, max=2.0, step=0.01, value=0.5, description="σ_prior", continuous_update=False),
                  sigma2 = FloatSlider(min=0.1, max=2.0, step=0.01, value=0.5, description="σ_likelihood", continuous_update=False),
                  mu_g = FloatSlider(min=-4.0, max=4.0, step=0.01, value=1.0, description="µ_gain", continuous_update=False),
                  mu_c = FloatSlider(min=-4.0, max=4.0, step=0.01, value=-1.0, description="µ_cost", continuous_update=False),
                  sigma_g = FloatSlider(min=0.1, max=2.0, step=0.01, value=0.5, description="σ_gain", continuous_update=False),
                  sigma_c = FloatSlider(min=0.1, max=2.0, step=0.01, value=0.5, description="σ_cost", continuous_update=False),
                  plot_utility_row=fixed(True))_____no_output_____

[*Click for solution*](https://github.com/NeuromatchAcademy/course-content/tree/master//tutorials/W3D1_BayesianDecisions/solutions/W3D1_Tutorial2_Solution_bcc88699.py)

_____no_output_____---
# Summary

In this tutorial, you extended your exploration of Bayes Rule and the Bayesian approach in the context of finding and choosling a location for Astrocat.

Specifically, we covered:

* The Gaussian distribution and its properties

* That the likelihood is the probability of the measurement given some hidden state

* Information shared between Gaussians (via multiplication of PDFs and via two-dimensional distributions)

* That how the prior and likelihood interact to create the posterior, the probability of the hidden state given a measurement, depends on how they covary

* That utility is the gain from each action and state pair, and the expected utility for an action is the sum of the utility for all state pairs, weighted by the probability of that state happening. You can then choose the action with highest expected utility.
 _____no_output_____


# Notebook from rouseguy/scipy2015_tutorial
Path: notebooks/3. Fitting Regression Models.ipynb

%matplotlib inline
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from scipy.optimize import fmin

np.random.seed(1789)

from IPython.core.display import HTML
def css_styling():
    styles = open("styles/custom.css", "r").read()
    return HTML(styles)
css_styling()_____no_output_____

---

# Regression modeling

A general, primary goal of many statistical data analysis tasks is to relate the influence of one variable on another.

For example:

- how different medical interventions influence the incidence or duration of disease
- how baseball player's performance varies as a function of age.
- [how test scores are correlated with tissue LSD concentration](http://www.lycaeum.org/research/researchpdfs/1748.pdf)_____no_output_____

from io import StringIO

data_string = """
Drugs	Score
0	1.17	78.93
1	2.97	58.20
2	3.26	67.47
3	4.69	37.47
4	5.83	45.65
5	6.00	32.92
6	6.41	29.97
"""

lsd_and_math = pd.read_table(StringIO(data_string), sep='\t', index_col=0)
lsd_and_math_____no_output_____

![lsd](images/_lsd.jpg)

> Taking LSD was a profound experience, one of the most important things in my life --Steve Jobs_____no_output_____

lsd_and_math.plot(x='Drugs', y='Score', style='ro', legend=False, xlim=(0,8))_____no_output_____

We can build a model to characterize the relationship between $X$ and $Y$, recognizing that additional factors other than $X$ (the ones we have measured or are interested in) may influence the response variable $Y$.

- $M(Y|X) = E(Y|X)$
- $M(Y|X) = Pr(Y=1|X)$_____no_output_____In general,

$$M(Y|X) = f(X)$$

for linear regression

$$M(Y|X) = f(X\beta)$$_____no_output_____where $f$ is some function, for example a linear function:

$y_i = \beta_0 + \beta_1 x_{1i} + \ldots + \beta_k x_{ki} + \epsilon_i$

Regression is a **weighted sum** of independent predictors_____no_output_____and $\epsilon_i$ accounts for the difference between the observed response $y_i$ and its prediction from the model $\hat{y_i} = \beta_0 + \beta_1 x_i$. This is sometimes referred to as **process uncertainty**.

Interpretation: coefficients represent the change in Y for a unit increment of the predictor X._____no_output_____Two important regression **assumptions**:

1. normal errors
2. homoscedasticity
_____no_output_____## Parameter estimation

We would like to select $\beta_0, \beta_1$ so that the difference between the predictions and the observations is zero, but this is not usually possible. Instead, we choose a reasonable criterion: ***the smallest sum of the squared differences between $\hat{y}$ and $y$***.

$$R^2 = \sum_i (y_i - [\beta_0 + \beta_1 x_i])^2 = \sum_i \epsilon_i^2 $$

Squaring serves two purposes:

1. to prevent positive and negative values from cancelling each other out
2. to strongly penalize large deviations.

Whether or not the latter is a desired depends on the goals of the analysis.

In other words, we will select the parameters that minimize the squared error of the model._____no_output_____

sum_of_squares = lambda theta, x, y: np.sum((y - theta[0] - theta[1]*x) ** 2)_____no_output_____sum_of_squares([0,1], lsd_and_math.Drugs, lsd_and_math.Score)_____no_output_____x, y = lsd_and_math.T.values
b0, b1 = fmin(sum_of_squares, [0,1], args=(x,y))
b0, b1Optimization terminated successfully.
         Current function value: 253.881329
         Iterations: 97
         Function evaluations: 179
ax = lsd_and_math.plot(x='Drugs', y='Score', style='ro', legend=False, xlim=(0,8))
ax.plot([0,10], [b0, b0+b1*10])_____no_output_____ax = lsd_and_math.plot(x='Drugs', y='Score', style='ro', legend=False, xlim=(0,8), ylim=(20, 90))
ax.plot([0,10], [b0, b0+b1*10])
for xi, yi in zip(x,y):
    ax.plot([xi]*2, [yi, b0+b1*xi], 'k:')_____no_output_____

## Alternative loss functions

Minimizing the sum of squares is not the only criterion we can use; it is just a very popular (and successful) one. For example, we can try to minimize the sum of absolute differences:_____no_output_____

sum_of_absval = lambda theta, x, y: np.sum(np.abs(y - theta[0] - theta[1]*x))_____no_output_____b0, b1 = fmin(sum_of_absval, [0,0], args=(x,y))
print('\nintercept: {0:.2}, slope: {1:.2}'.format(b0,b1))
ax = lsd_and_math.plot(x='Drugs', y='Score', style='ro', legend=False, xlim=(0,8))
ax.plot([0,10], [b0, b0+b1*10])Optimization terminated successfully.
         Current function value: 31.692066
         Iterations: 137
         Function evaluations: 264

intercept: 9e+01, slope: -9.3

We are not restricted to a straight-line regression model; we can represent a curved relationship between our variables by introducing **polynomial** terms. For example, a cubic model:

$y_i = \beta_0 + \beta_1 x_i + \beta_2 x_i^2 + \epsilon_i$

_____no_output_____

sum_squares_quad = lambda theta, x, y: np.sum((y - theta[0] - theta[1]*x - theta[2]*(x**2)) ** 2)_____no_output_____b0,b1,b2 = fmin(sum_squares_quad, [1,1,-1], args=(x,y))
print('\nintercept: {0:.2}, x: {1:.2}, x2: {2:.2}'.format(b0,b1,b2))
ax = lsd_and_math.plot(x='Drugs', y='Score', style='ro', legend=False, xlim=(0,8))
xvals = np.linspace(0, 8, 100)
ax.plot(xvals, b0 + b1*xvals + b2*(xvals**2))Optimization terminated successfully.
         Current function value: 251.093792
         Iterations: 177
         Function evaluations: 319

intercept: 9.2e+01, x: -1.1e+01, x2: 0.24

Although a polynomial model characterizes a nonlinear relationship, it is a linear problem in terms of estimation. That is, the regression model $f(y | x)$ is **linear in the parameters**.

For some data, it may be reasonable to consider polynomials of order>2. For example, consider the relationship between the number of spawning salmon and the number of juveniles recruited into the population the following year; one would expect the relationship to be positive, but not necessarily linear._____no_output_____

sum_squares_cubic = lambda theta, x, y: np.sum((y - theta[0] - theta[1]*x - theta[2]*(x**2)
                                  - theta[3]*(x**3)) ** 2)_____no_output_____salmon = pd.read_table("../data/salmon.dat", delim_whitespace=True, index_col=0)
plt.plot(salmon.spawners, salmon.recruits, 'r.')
b0,b1,b2,b3 = fmin(sum_squares_cubic, [0,1,-1,0], args=(salmon.spawners, salmon.recruits))
xvals = np.arange(500)
plt.plot(xvals, b0 + b1*xvals + b2*(xvals**2) + b3*(xvals**3))Optimization terminated successfully.
         Current function value: 12809.933906
         Iterations: 251
         Function evaluations: 437

## Linear Regression with `scikit-learn`

In practice, we need not fit least squares models by hand because they are implemented generally in packages such as [`scikit-learn`](http://scikit-learn.org/) and [`statsmodels`](https://github.com/statsmodels/statsmodels/). For example, `scikit-learn` package implements least squares models in its `LinearRegression` class:_____no_output_____

from sklearn import linear_model

straight_line = linear_model.LinearRegression()
straight_line.fit(x[:, np.newaxis], y)_____no_output_____straight_line.coef______no_output_____straight_line.intercept______no_output_____plt.plot(x, y, 'ro')
plt.plot(x, straight_line.predict(x[:, np.newaxis]), color='blue',
         linewidth=3)_____no_output_____

For more general regression model building, its helpful to use a tool for describing statistical models, called `patsy`. With `patsy`, it is easy to specify the desired combinations of variables for any particular analysis, using an "R-like" syntax. `patsy` parses the formula string, and uses it to construct the approriate *design matrix* for the model.

For example, the quadratic model specified by hand above can be coded as:_____no_output_____

from patsy import dmatrix

X = dmatrix('salmon.spawners + I(salmon.spawners**2)')_____no_output_____

The `dmatrix` function returns the design matrix, which can be passed directly to the `LinearRegression` fitting method._____no_output_____

poly_line = linear_model.LinearRegression(fit_intercept=False)
poly_line.fit(X, salmon.recruits)_____no_output_____poly_line.coef______no_output_____plt.plot(salmon.spawners, salmon.recruits, 'ro')
frye_range = np.arange(500)
plt.plot(frye_range, poly_line.predict(dmatrix('frye_range + I(frye_range**2)')), color='blue')_____no_output_____

## Generalized linear models

Often our data violates one or more of the linear regression assumptions:

- non-linear
- non-normal error distribution
- heteroskedasticity

this forces us to **generalize** the regression model in order to account for these characteristics.

As a motivating example, consider the Olympic medals data that we compiled earlier in the tutorial.

![relay](images/mascot.png)_____no_output_____

medals = pd.read_csv('../data/medals.csv')
medals.head()_____no_output_____

We expect a positive relationship between population and awarded medals, but the data in their raw form are clearly not amenable to linear regression._____no_output_____

medals.plot(x='population', y='medals', kind='scatter')_____no_output_____

Part of the issue is the scale of the variables. For example, countries' populations span several orders of magnitude. We can correct this by using the logarithm of population, which we have already calculated._____no_output_____

medals.plot(x='log_population', y='medals', kind='scatter')_____no_output_____

This is an improvement, but the relationship is still not adequately modeled by least-squares regression._____no_output_____

linear_medals = linear_model.LinearRegression()
X = medals.log_population[:, np.newaxis]
linear_medals.fit(X, medals.medals)_____no_output_____ax = medals.plot(x='log_population', y='medals', kind='scatter')
ax.plot(medals.log_population, linear_medals.predict(X), color='red',
         linewidth=2)_____no_output_____

This is due to the fact that the response data are **counts**. As a result, they tend to have characteristic properties.

- discrete
- positive
- variance grows with mean

to account for this, we can do two things: (1) model the medal count on the **log scale** and (2) assume **Poisson errors**, rather than normal.

Recall the Poisson distribution from the previous section:

$$p(y)=\frac{e^{-\lambda}\lambda^y}{y!}$$

* $Y=\{0,1,2,\ldots\}$
* $\lambda > 0$

$$E(Y) = \text{Var}(Y) = \lambda$$_____no_output_____So, we will model the logarithm of the expected value as a linear function of our predictors:

$$\log(\lambda) = X\beta$$

In this context, the log function is called a **link function**. This transformation implies the mean of the Poisson is:

$$\lambda = \exp(X\beta)$$

We can plug this into the Poisson likelihood and use maximum likelihood to estimate the regression covariates $\beta$.

$$\log L = \sum_{i=1}^n -\exp(X_i\beta) + Y_i (X_i \beta)- \log(Y_i!)$$

As we have already done, we just need to code the kernel of this likelihood, and optimize!_____no_output_____

# Poisson negative log-likelhood
poisson_loglike = lambda beta, X, y: -(-np.exp(X.dot(beta)) + y*X.dot(beta)).sum()_____no_output_____

Let's use the `assign` method to add a column of ones to the design matrix._____no_output_____

poisson_loglike([0,1], medals[['log_population']].assign(intercept=1), medals.medals)_____no_output_____

We will use Nelder-Mead to minimize the negtive log-likelhood._____no_output_____

b1, b0 = fmin(poisson_loglike, [0,1], args=(medals[['log_population']].assign(intercept=1).values,
                                            medals.medals.values))Optimization terminated successfully.
         Current function value: -1381.299433
         Iterations: 68
         Function evaluations: 131
b0, b1_____no_output_____

The resulting fit looks reasonable._____no_output_____

ax = medals.plot(x='log_population', y='medals', kind='scatter')
xvals = np.arange(12, 22)
ax.plot(xvals, np.exp(b0 + b1*xvals), 'r--')_____no_output_____

### Exercise: Multivariate GLM

Add the OECD indicator variable to the model, and estimate the model coefficients._____no_output_____

# Write your answer here_____no_output_____

### Interactions among variables

Interactions imply that the effect of one covariate $X_1$ on $Y$ depends on the value of another covariate $X_2$.

$$M(Y|X) = \beta_0 + \beta_1 X_1 + \beta_2 X_2 +\beta_3 X_1 X_2$$

the effect of a unit increase in $X_1$:

$$M(Y|X_1+1, X_2) - M(Y|X_1, X_2)$$

$$\begin{align}
&= \beta_0 + \beta_1 (X_1 + 1) + \beta_2 X_2 +\beta_3 (X_1 + 1) X_2
- [\beta_0 + \beta_1 X_1 + \beta_2 X_2 +\beta_3 X_1 X_2] \\
&= \beta_1 + \beta_3 X_2
\end{align}$$_____no_output_____

ax = medals[medals.oecd==1].plot(x='log_population', y='medals', kind='scatter', alpha=0.8)
medals[medals.oecd==0].plot(x='log_population', y='medals', kind='scatter', color='red', alpha=0.5, ax=ax)_____no_output_____

Interaction can be interpreted as:

- $X_1$ interacts with $X_2$
- $X_1$ modifies the effect of $X_2$
- $X_2$ modifies the effect of $X_1$
- $X_1$ and $X_2$ are non-additive or synergistic_____no_output_____Let's construct a model that predicts medal count based on population size and OECD status, as well as the interaction. We can use patsy to set up the design matrix._____no_output_____

y = medals.medals
X = dmatrix('log_population * oecd', data=medals)
X_____no_output_____

Now, fit the model._____no_output_____

interaction_params = fmin(poisson_loglike, [0,1,1,0], args=(X, y))Warning: Maximum number of function evaluations has been exceeded.
interaction_params_____no_output_____

Notice anything odd about these estimates?

The main effect of the OECD effect is negative, which seems counter-intuitive. This is because the variable is interpreted as the OECD effect when the **log-population is zero**. This is not particularly meaningful.

We can improve the interpretability of this parameter by **centering** the log-population variable prior to entering it into the model. This will result in the OECD main effect being interpreted as the marginal effect of being an OECD country for an **average-sized** country. _____no_output_____

y = medals.medals
X = dmatrix('center(log_population) * oecd', data=medals)
X_____no_output_____fmin(poisson_loglike, [0,1,1,0], args=(X, y))Optimization terminated successfully.
         Current function value: -1492.092233
         Iterations: 303
         Function evaluations: 513

## Model Selection

How do we choose among competing models for a given dataset? More parameters are not necessarily better, from the standpoint of model fit. For example, fitting a 6th order polynomial to the LSD example certainly results in an overfit._____no_output_____

def calc_poly(params, data):
        x = np.c_[[data**i for i in range(len(params))]]
        return np.dot(params, x)

x, y = lsd_and_math.T.values

sum_squares_poly = lambda theta, x, y: np.sum((y - calc_poly(theta, x)) ** 2)
betas = fmin(sum_squares_poly, np.zeros(7), args=(x,y), maxiter=1e6)
plt.plot(x, y, 'ro')
xvals = np.linspace(0, max(x), 100)
plt.plot(xvals, calc_poly(betas, xvals))Optimization terminated successfully.
         Current function value: 304.786763
         Iterations: 830
         Function evaluations: 1284

One approach is to use an information-theoretic criterion to select the most appropriate model. For example **Akaike's Information Criterion (AIC)** balances the fit of the model (in terms of the likelihood) with the number of parameters required to achieve that fit. We can easily calculate AIC as:

$$AIC = n \log(\hat{\sigma}^2) + 2p$$

where $p$ is the number of parameters in the model and $\hat{\sigma}^2 = RSS/(n-p-1)$.

Notice that as the number of parameters increase, the residual sum of squares goes down, but the second term (a penalty) increases.

AIC is a metric of **information distance** between a given model and a notional "true" model. Since we don't know the true model, the AIC value itself is not meaningful in an absolute sense, but is useful as a relative measure of model quality.

To apply AIC to model selection, we choose the model that has the **lowest** AIC value._____no_output_____

n = len(x)

aic = lambda rss, p, n: n * np.log(rss/(n-p-1)) + 2*p

RSS1 = sum_of_squares(fmin(sum_of_squares, [0,1], args=(x,y)), x, y)
RSS2 = sum_squares_quad(fmin(sum_squares_quad, [1,1,-1], args=(x,y)), x, y)

print('\nModel 1: {0}\nModel 2: {1}'.format(aic(RSS1, 2, n), aic(RSS2, 3, n)))Optimization terminated successfully.
         Current function value: 253.881329
         Iterations: 97
         Function evaluations: 179
Optimization terminated successfully.
         Current function value: 251.093792
         Iterations: 177
         Function evaluations: 319

Model 1: 33.05400811127588
Model 2: 36.99049978705

Hence, on the basis of "information distance", we would select the 2-parameter (linear) model._____no_output_____### Exercise: Olympic medals model selection

Use AIC to select the best model from the following set of Olympic medal prediction models:

- population only
- population and OECD
- interaction model

For these models, use the alternative form of AIC, which uses the log-likelhood rather than the residual sums-of-squares:

$$AIC = -2 \log(L) + 2p$$_____no_output_____

# Write your answer here_____no_output_____

## Logistic Regression

Fitting a line to the relationship between two variables using the least squares approach is sensible when the variable we are trying to predict is continuous, but what about when the data are dichotomous?

- male/female
- pass/fail
- died/survived

Let's consider the problem of predicting survival in the Titanic disaster, based on our available information. For example, lets say that we want to predict survival as a function of the fare paid for the journey.

![titanic](images/titanic_cube.png)_____no_output_____

titanic = pd.read_excel("../data/titanic.xls", "titanic")
titanic.name_____no_output_____jitter = np.random.normal(scale=0.02, size=len(titanic))

plt.scatter(np.log(titanic.fare), titanic.survived + jitter, alpha=0.3)
plt.yticks([0,1])
plt.ylabel("survived")
plt.xlabel("log(fare)")_____no_output_____

I have added random jitter on the y-axis to help visualize the density of the points, and have plotted fare on the log scale.

Clearly, fitting a line through this data makes little sense, for several reasons. First, for most values of the predictor variable, the line would predict values that are not zero or one. Second, it would seem odd to choose least squares (or similar) as a criterion for selecting the best line._____no_output_____

x = np.log(titanic.fare[titanic.fare>0])
y = titanic.survived[titanic.fare>0]
betas_titanic = fmin(sum_of_squares, [1,1], args=(x,y))Optimization terminated successfully.
         Current function value: 277.621917
         Iterations: 55
         Function evaluations: 103
jitter = np.random.normal(scale=0.02, size=len(titanic))
plt.scatter(np.log(titanic.fare), titanic.survived + jitter, alpha=0.3)
plt.yticks([0,1])
plt.ylabel("survived")
plt.xlabel("log(fare)")
plt.plot([0,7], [betas_titanic[0], betas_titanic[0] + betas_titanic[1]*7.])_____no_output_____

If we look at this data, we can see that for most values of `fare`, there are some individuals that survived and some that did not. However, notice that the cloud of points is denser on the "survived" (`y=1`) side for larger values of fare than on the "died" (`y=0`) side._____no_output_____### Stochastic model

Rather than model the binary outcome explicitly, it makes sense instead to model the *probability* of death or survival in a **stochastic** model. Probabilities are measured on a continuous [0,1] scale, which may be more amenable for prediction using a regression line. We need to consider a different probability model for this exerciese however; let's consider the **Bernoulli** distribution as a generative model for our data:

$$f(y|p) = p^{y} (1-p)^{1-y}$$

  _____no_output_____where $y = \{0,1\}$ and $p \in [0,1]$. So, this model predicts whether $y$ is zero or one as a function of the probability $p$. Notice that when $y=1$, the $1-p$ term disappears, and when $y=0$, the $p$ term disappears.

So, the model we want to fit should look something like this:

$$p_i = \beta_0 + \beta_1 x_i + \epsilon_i$$

_____no_output_____However, since $p$ is constrained to be between zero and one, it is easy to see where a linear (or polynomial) model might predict values outside of this range. As with the Poisson regression, we can modify this model slightly by using a link function to transform the probability to have an unbounded range on a new scale. Specifically, we can use a **logit transformation** as our link function:

$$\text{logit}(p) = \log\left[\frac{p}{1-p}\right] = x$$

_____no_output_____Here's a plot of $p/(1-p)$_____no_output_____

logit = lambda p: np.log(p/(1.-p))
unit_interval = np.linspace(0,1)
plt.plot(unit_interval/(1-unit_interval), unit_interval)
plt.xlabel(r'$p/(1-p)$')
plt.ylabel('p');_____no_output_____

And here's the logit function:_____no_output_____

plt.plot(logit(unit_interval), unit_interval)
plt.xlabel('logit(p)')
plt.ylabel('p');_____no_output_____

The inverse of the logit transformation is:

$$p = \frac{1}{1 + \exp(-x)}$$

_____no_output_____

invlogit = lambda x: 1. / (1 + np.exp(-x))_____no_output_____

So, now our model is:

$$\text{logit}(p_i) = \beta_0 + \beta_1 x_i + \epsilon_i$$

_____no_output_____We can fit this model using maximum likelihood. Our likelihood, again based on the Bernoulli model is:

$$L(y|p) = \prod_{i=1}^n p_i^{y_i} (1-p_i)^{1-y_i}$$

_____no_output_____which, on the log scale is:

$$l(y|p) = \sum_{i=1}^n y_i \log(p_i) + (1-y_i)\log(1-p_i)$$

_____no_output_____We can easily implement this in Python, keeping in mind that `fmin` minimizes, rather than maximizes functions:_____no_output_____

def logistic_like(theta, x, y):

    p = invlogit(theta[0] + theta[1] * x)

    # Return negative of log-likelihood
    return -np.sum(y * np.log(p) + (1-y) * np.log(1 - p))_____no_output_____

Remove null values from variables (a *bad idea*, which we will show later) ..._____no_output_____

x, y = titanic[titanic.fare.notnull()][['fare', 'survived']].values.T_____no_output_____

... and fit the model._____no_output_____

b0, b1 = fmin(logistic_like, [0.5,0], args=(x,y))
b0, b1Optimization terminated successfully.
         Current function value: 827.015955
         Iterations: 47
         Function evaluations: 93
jitter = np.random.normal(scale=0.01, size=len(x))
plt.plot(x, y+jitter, 'r.', alpha=0.3)
plt.yticks([0,.25,.5,.75,1])
xvals = np.linspace(0, 600)
plt.plot(xvals, invlogit(b0+b1*xvals))_____no_output_____

As with our least squares model, we can easily fit logistic regression models in `scikit-learn`, in this case using the `LogisticRegression`._____no_output_____

logistic = linear_model.LogisticRegression()
logistic.fit(x[:, np.newaxis], y)_____no_output_____logistic.coef______no_output_____

### Exercise: multivariate logistic regression

Which other variables might be relevant for predicting the probability of surviving the Titanic? Generalize the model likelihood to include 2 or 3 other covariates from the dataset._____no_output_____

# Write your answer here_____no_output_____


# Notebook from davidwhogg/Euclid
Path: notebooks/self-cal-field.ipynb

<code>
import matplotlib
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline
np.random.seed(42)_____no_output_____# set parameters
Npointings = 60
fov = 0.73 # deg
xlim = (0., 2.) # deg
ylim = (0., 2.) # deg
xmin = xlim[0] + 0.5 * fov # deg
xmax = xlim[1] - 0.5 * fov # deg
ymin = ylim[0] + 0.5 * fov # deg
ymax = ylim[1] - 0.5 * fov # deg_____no_output_____# make a fake set of stars (trust me, it's a good idea)
Nstars = 1024
star_xs = xlim[0] + (xlim[1] - xlim[0]) * np.random.uniform(size=Nstars)
star_ys = ylim[0] + (ylim[1] - ylim[0]) * np.random.uniform(size=Nstars)_____no_output_____# A: simplest plan: 60 pointings; total random
A_xs = xmin + (xmax - xmin) * np.random.uniform(size=60)
A_ys = ymin + (ymax - ymin) * np.random.uniform(size=60)_____no_output_____xp = fov * np.array([-0.5, 0.5, 0.5, -0.5, -0.5])
yp = fov * np.array([-0.5, -0.5, 0.5, 0.5, -0.5])
def plot_one_field(x, y):
    plt.plot(x + xp, y + yp, "r-", alpha=0.75)_____no_output_____def plot_one_survey(name, xs, ys):
    plt.figure(figsize=(5,5))
    plt.scatter(star_xs, star_ys, color="k", marker=".", alpha=0.1)
    for x, y in zip(xs, ys):
        plot_one_field(x, y)
    plt.xlim(xlim)
    plt.ylim(ylim)
    plt.title(name + " field locations")_____no_output_____def plot_star_hits(name, xs, ys):
    plt.figure(figsize=(5,5))
    dxs = (star_xs[:, None] - xs[None, :]).flatten()
    dys = (star_ys[:, None] - ys[None, :]).flatten()
    plt.scatter(dxs, dys, color="k", marker=".", alpha=0.25)
    plt.xlim(-0.5 * fov, 0.5 * fov)
    plt.ylim(-0.5 * fov, 0.5 * fov)
    plt.title(name + " star hits")_____no_output_____plot_one_survey("A", A_xs, A_ys)
plot_star_hits("A", A_xs, A_ys)_____no_output_____# B: More interesting plan: 58 pointings; plaid pattern (good for self-cal)
nx, ny = 5, 4
B_xs = np.outer(np.linspace(xmin, xmax, nx), np.ones(ny)).flatten()
B_ys = np.outer(np.linspace(ymin, ymax, ny), np.ones(nx)).T.flatten()
nx, ny = 4, 5
B_xs = np.append(B_xs, np.outer(np.linspace(xmin, xmax, nx), np.ones(ny)).flatten())
B_ys = np.append(B_ys, np.outer(np.linspace(ymin, ymax, ny), np.ones(nx)).T.flatten())
nx, ny = 6, 3
B_xs = np.append(B_xs, np.outer(np.linspace(xmin, xmax, nx), np.ones(ny)).flatten())
B_ys = np.append(B_ys, np.outer(np.linspace(ymin, ymax, ny), np.ones(nx)).T.flatten())_____no_output_____plot_one_survey("B", B_xs, B_ys)
plot_star_hits("B", B_xs, B_ys)_____no_output_____# C: ultra-uniform plan: 64 pointings, far more uniform than Poisson
nx, ny = 8, 8
C_xs = np.outer(np.linspace(xmin, xmax, nx), np.ones(ny)).flatten()
C_ys = np.outer(np.linspace(ymin, ymax, ny), np.ones(nx)).T.flatten()
C_xs += ((xmin - xmax) / nx) * (0.5 - np.random.uniform(size=C_xs.shape))
C_ys += ((ymin - ymax) / ny) * (0.5 - np.random.uniform(size=C_ys.shape))
C_xs = np.clip(C_xs, xmin, xmax)
C_ys = np.clip(C_ys, ymin, ymax)_____no_output_____plot_one_survey("C", C_xs, C_ys)
plot_star_hits("C", C_xs, C_ys)_____no_output_____
</code>