# Notebook from tp5uiuc/soft_systems_course
Path: lectures/03_cma/code/cmaes.ipynb

# CMAES : Covariance Matrix Adaptation Evolutionary Strategy_____no_output_____Setup code and utility functions to plot and explore_____no_output_____

import numpy as np
import matplotlib.pyplot as plt
import matplotlib.colors as colors
from matplotlib import cm
from mpl_toolkits.mplot3d import axes3d
from numpy.random import multivariate_normal
import copy

%matplotlib inline
%config InlineBackend.figure_format = 'retina'

try:
    import seaborn as sns
    sns.set_style("whitegrid")
    sns.set_context('talk')
    #sns.set(font_scale=1.4)
except ImportError:
    plt.style.use('seaborn-whitegrid')_____no_output_____def range_from_bounds(bounds, resolution):
    (minx,miny),(maxx,maxy) = bounds
    x_range = np.arange(minx, maxx, (maxx-minx)/resolution)
    y_range = np.arange(miny, maxy, (maxy-miny)/resolution)
    return x_range, y_range

def plot_problem_3d(problem, bounds, ax=None, resolution=100.,
                    cmap=cm.viridis_r, rstride=10, cstride=10,
                    linewidth=0.15, alpha=0.65):
    """Plots a given benchmark problem in 3D mesh."""

    x_range, y_range = range_from_bounds(bounds, resolution=resolution)

    X, Y = np.meshgrid(x_range, y_range)
    Z = problem(X,Y)

    if not ax:
        fig = plt.figure(figsize=(11,6))
        ax = fig.gca(projection='3d')

    cset = ax.plot_surface(X, Y, Z, cmap=cmap, rstride=rstride, cstride=cstride, linewidth=linewidth, alpha=alpha)_____no_output_____def plot_problem_contour(problem, bounds, optimum=None,
                          resolution=100., cmap=cm.viridis_r,
                          alpha=0.45, ax=None):
    """Plots a given benchmark problem as a countour."""
    x_range, y_range = range_from_bounds(bounds, resolution=resolution)

    X, Y = np.meshgrid(x_range, y_range)
    Z = problem(X,Y)

    if not ax:
        fig = plt.figure(figsize=(6,6))
        ax = fig.gca()
        ax.set_aspect('equal')
        ax.autoscale(tight=True)

    cset = ax.contourf(X, Y, Z, cmap=cmap, alpha=alpha)

    if optimum:
        ax.plot(optimum[0], optimum[1], 'bx', linewidth=4, markersize=15)_____no_output_____def plot_cov_ellipse(pos, cov, volume=.99, ax=None, fc='lightblue', ec='darkblue', alpha=1, lw=1):
    ''' Plots an ellipse that corresponds to a bivariate normal distribution.
    Adapted from http://www.nhsilbert.net/source/2014/06/bivariate-normal-ellipse-plotting-in-python/'''
    from scipy.stats import chi2
    from matplotlib.patches import Ellipse

    def eigsorted(cov):
        vals, vecs = np.linalg.eigh(cov)
        order = vals.argsort()[::-1]
        return vals[order], vecs[:,order]

    if ax is None:
        ax = plt.gca()

    vals, vecs = eigsorted(cov)
    theta = np.degrees(np.arctan2(*vecs[:,0][::-1]))

    kwrg = {'facecolor':fc, 'edgecolor':ec, 'alpha':alpha, 'linewidth':lw}

    # Width and height are "full" widths, not radius
    width, height = 2 * np.sqrt(chi2.ppf(volume,2)) * np.sqrt(vals)
    ellip = Ellipse(xy=pos, width=width, height=height, angle=theta, **kwrg)
    ax.add_artist(ellip)_____no_output_____

## Test functions

### Why benchmarks (test) functions?

In applied mathematics, [test functions](http://en.wikipedia.org/wiki/Test_functions_for_optimization), also known as artificial landscapes, are useful to evaluate characteristics of optimization algorithms, such as:

* Velocity of convergence.
* Precision.
* Robustness.
* General performance.

### [Bohachevsky benchmark problem](http://benchmarkfcns.xyz/benchmarkfcns/bohachevskyn2fcn.html)

$$\text{minimize } f(\mathbf{x}) = \sum_{i=1}^{N-1}(x_i^2 + 2x_{i+1}^2 - 0.3\cos(3\pi x_i) - 0.4\cos(4\pi x_{i+1}) + 0.7), \mathbf{x}\in \left[-100,100\right]^n,$$

> Optimum in $\mathbf{x}=\mathbf{0}$, $f(\mathbf{x})=0$._____no_output_____

def _shifted_bohachevsky_impl(x, y, shift_x, shift_y):
    return (x-shift_x)**2 + 2.0 * (y-shift_y)**2 - 0.3 * np.cos(3.0 * np.pi * (x - shift_x)) - 0.4 * np.cos(4.0 * np.pi* (y - shift_y)) + 0.7
# def bohachevsky(x,y):
#     return x**2 + 2.0 * y**2 - 0.3 * np.cos(3.0 * np.pi * x) - 0.4 * np.cos(4.0 * np.pi*x) + 0.7_____no_output_____from functools import partial
bohachevsky = partial(_shifted_bohachevsky_impl, shift_x = 0.0, shift_y = 0.0)
shifted_bohachevsky = partial(_shifted_bohachevsky_impl, shift_x = 2.0, shift_y = 2.0)_____no_output_____

### [Griewank benchmark problem](http://benchmarkfcns.xyz/benchmarkfcns/griewankfcn.html)

$$\text{minimize } f(\mathbf{x}) = f(x_1, ..., x_n) = 1 + \sum_{i=1}^{n} \frac{x_i^{2}}{4000} - \prod_{i=1}^{n}\cos\left(\frac{2 \cdot x_i}{\sqrt{i}}\right)$$

> Optimum in $\mathbf{x}=\mathbf{0}$, $f(\mathbf{x})=0$._____no_output_____

def _shifted_griewank_impl(x,y,shift_x, shift_y):
    return 1.0 + ((x-shift_x)**2 + (y-shift_y)**2) / 4000.0 - np.cos(2.0 * (x-shift_x)) * np.cos(2.0 * (y-shift_y) / np.sqrt(2.0))

# def griewank(x,y):
#     return 1.0 + (x**2 + y**2) / 4000.0 - np.cos(2.0 * x) * np.cos(2.0 * y / np.sqrt(2.0))_____no_output_____griewank = partial(_shifted_griewank_impl, shift_x = 0.0, shift_y = 0.0)
shifted_griewank = partial(_shifted_griewank_impl, shift_x = 2.0, shift_y = 2.0)_____no_output_____def _shifted_rastrigin_impl(x,y,shift_x, shift_y):
    return 2 * 10.0 + ((x-shift_x)**2 + (y-shift_y)**2) - 10.0 * np.cos(2.0 * np.pi * (x-shift_x)) - 10.0 * np.cos(2.0 * np.pi * (y-shift_y)) _____no_output_____rastrigin = partial(_shifted_rastrigin_impl, shift_x = 0.0, shift_y = 0.0)
shifted_rastrigin = partial(_shifted_rastrigin_impl, shift_x = 2.0, shift_y = 2.0)_____no_output_____current_problem = bohachevsky_____no_output_____plot_problem_3d(current_problem, ((-10,-10), (10,10)))_____no_output_____

These problems has many local optima._____no_output_____

plot_problem_3d(current_problem, ((-2.5,-2.5), (2.5,2.5)))_____no_output_____ax = plt.figure(figsize=(8, 5)).gca()
plot_problem_contour(current_problem, ((-2.5,-2.5), (2.5,2.5)), optimum=(0,0), ax=ax)
ax.set_aspect('equal')_____no_output_____

## Optimizing test functions using CMA-ES

### CMA-ES features

* Adaptation of the covariance matrix amounts to learning a second order model of the underlying objective function.
* This is similar to the approximation of the inverse Hessian matrix in the Quasi-Newton method in classical optimization.
* In contrast to most classical methods, fewer assumptions on the nature of the underlying objective function are made.
* *Only the ranking between candidate solutions is exploited* for learning the sample distribution and neither derivatives nor even the function values themselves are required by the method.

## Let's code up CMA from scratch!_____no_output_____Here are the steps of CMA in chronological order :
$$
\newcommand{\gv}[1]{\ensuremath{\mbox{\boldmath$ #1 $}}}
\newcommand{\bv}[1]{\ensuremath{\mathbf{#1}}}
\newcommand{\norm}[1]{\left\lVert#1\right\rVert}
\newcommand{\order}[1]{\mathcal O \left( #1 \right)} % order of magnitude
$$

### Initialization
Set $ \mathbf{m} = \mathbf{0}, \mathbf{C} = \mathbf{I}, \sigma = 0.5, \mathbf{p}_c = \mathbf{0}, \mathbf{p}_{\sigma} = \mathbf{0} $

### Sampling
$$ \begin{aligned}
\mathbf{z}_{i} & \sim \mathcal{N}(\mathbf{0}, \mathbf{C}) \\
\mathbf{x}_{i} &= m+\sigma \mathbf{z}_{i}
\end{aligned} $$

### Selection and recombination
Sort the ppopulation by fitness to get $ \mu $  fit individuals
$$ \begin{aligned}
\langle\mathbf{z}\rangle_{w} &= \displaystyle\sum_{i=1}^{\mu} w_{i} \mathbf{z}_{i : \lambda} \\
\mathbf{m} &\longleftarrow \mathbf{m}+\sigma\langle\mathbf{z}\rangle_{w}
\end{aligned} $$

### Step size update
$$ \begin{aligned}
\mathbf{p}_{\sigma} &\longleftarrow\left(1-c_{\sigma}\right)
\mathbf{p}_{\sigma}+\sqrt{1-\left(1-c_{\sigma}\right)^{2}}
\sqrt{\frac{1}{\sum_{i=1}^{\mu} w_{i}^{2}}}
\mathbf{C}^{-\frac{1}{2}}\langle\mathbf{z}\rangle_{w} \\
\sigma &\longleftarrow \sigma
\exp{\left(\frac{c_{\sigma}}{d_{\sigma}}\left(\frac{\left\|p_{\sigma}\right\|}{E\|\mathcal{N}(\mathbf{0},
\mathbf{I})\|}-1\right)\right)} \\
\end{aligned} $$

### Covariance Matrix update
$$ \begin{aligned}
\mathbf{p}_{c} &\longleftarrow \left(1-c_{c}\right)
\mathbf{p}_{c}+\sqrt{1-\left(1-c_{c}\right)^{2}} \sqrt{\frac{1}{\sum_{i=1}^{\mu}
w_{i}^{2}}}\langle\mathbf{z}\rangle_{w} \\
\mathbf{Z} &= \sum_{i=1}^{\mu} w_{i} \mathbf{z}_{i : \lambda} \mathbf{z}_{i :
\lambda}^{T} \\
\mu_{c o v}&=\sqrt{\frac{1}{\sum_{i=1}^{\mu} w_{i}^{2}}} \\
\mathbf{C} &\longleftarrow\left(1-c_{c o v}\right) \mathbf{C}+c_{c o v}
\frac{1}{\mu_{c o v}} \mathbf{p}_{c} \mathbf{p}_{c}^{T}+c_{c o
v}\left(1-\frac{1}{\mu_{c o v}}\right) \mathbf{Z}
\end{aligned} $$

Some considerations:
 - `centroid` and `mean` are interchangeable.
 - `chi_N` is the expectation for the length of a random vector sampled from a multivariate normal distribution with $\mathbf{C} = \mathbf{I}$, and is used in the step-size update above. It can be analytically computed as $ \approx \sqrt{n} \left( 1 - \dfrac{1}{4n} + \dfrac{1}{21n^2} \right)$
 - `mu_eff` $ \mu_{\textrm{eff}} = \left(\displaystyle\sum_{i=1}^{\mu} w_{i}^{2}\right)^{-1} $ is the variance effective selection mass for the mean, as used in the CMA tutorial. Thus $\mu_{\textrm{cov}} = \sqrt{\mu_{\textrm{eff}}}$_____no_output_____

class CMAES:
    """Naive CMA implementation"""

    def __init__(self, initial_mean, sigma, popsize, **kwargs):
        """Please do all the initialization. The reserve space and
        code for collecting the statistics are already provided."""

        # Things that evolve : centroid, sigma, paths etc.
        self.centroid = np.asarray(initial_mean).copy()
        self.sigma = sigma
        self.pc = np.zeros_like(initial_mean)
        self.ps = np.zeros_like(initial_mean)
        self.C = np.eye(initial_mean.shape[0])
        self.B = np.eye(self.C.shape[0])
        self.diagD = np.ones(initial_mean.shape[0])

        # Optimal popsize
        self.popsize = popsize
        self.mu = popsize // 2

        # Update weights later on
        # Constant weight policy
        # self.weights = np.ones((self.mu, )) / self.mu

        # Decreasing weight policy
        self.weights = np.arange(self.mu, 0.0, -1.0)
        self.weights /= np.sum(self.weights)

        # Negative, Positive weight policy
        # unscaled_weights = np.arange(1.0 ,  1.0 + popsize)
        # unscaled_weights = np.log(0.5 * (popsize + 1.0) / unscaled_weights)

        # Utility variables
        self.dim = initial_mean.shape[0]

        # Expectation of a normal distribution
        self.chiN = np.sqrt(self.dim) * (1.0 - 0.25 / self.dim + 1.0/(21.0 * self.dim**2))
        self.mueff = 1.0 / np.linalg.norm(self.weights, 2)**2
        self.generations = 0

        # Options

        # Sigma adaptation
        # cs is short for c_sigma
        self.cs = kwargs.get("cs", (2.0 + self.mueff) / (self.dim + self.mueff + 5.0))
        # ds is short for d_sigma
        self.ds = 1.0 + 2.0 * max(0.0, np.sqrt((self.mueff - 1.0)/ (self.dim + 1.0)) - 1.0) + self.cs

        # Covariance adaptation
        self.cc = kwargs.get("cc", (4.0 + self.mueff/self.dim) / (self.dim + 4.0 + 2.0 * self.mueff/self.dim))
        self.ccov = 0.0
        # If implementing the latest version of CMA according to the tutorial,
        # these parameters can be useful
        self.ccov1 = 2.0 / ((self.dim + 1.3)**2 + self.mueff)
        self.ccovmu = min(1.0 - self.ccov1, 2.0 * (self.mueff - 2.0 + 1.0/self.mueff)/((self.dim + 2.0)**2 + self.mueff))

        # asserts to guide you
        assert self.dim == 2, "We are dealing with a two-dimensional problem only"
        assert self.centroid.shape == (2,), "Centroid shape is incorrect, did you tranpose it by mistake?"
        assert self.sigma > 0.0, "Sigma is not a non-zero positive number!"
        assert self.pc.shape == (2, ), "pc shape is incorrect, did you tranpose it by mistake?"
        assert self.ps.shape == (2, ), "ps shape is incorrect, did you tranpose it by mistake?"
        assert self.C.shape == (2, 2), "C's shape is incorrect, remember C is a matrix!"
        assert type(self.popsize) == int, "Population size not an integer"
        assert self.popsize > 0 , "Population size is negative!"
        assert self.popsize > 2 , "Too little population size, make it >2"

        self.stats_centroids = []
        self.stats_new_centroids = []
        self.stats_covs = []
        self.stats_new_covs = []
        self.stats_offspring = []
        self.stats_offspring_weights = []
        self.stats_ps = []

    def update(self, problem, population):
        """Update the current covariance matrix strategy from the
        *population*.

        :param population: A list of individuals from which to update the
                           parameters.
        """
        # -- store current state of the algorithm
        self.stats_centroids.append(copy.deepcopy(self.centroid))
        self.stats_covs.append(copy.deepcopy(self.C))

        population.sort(key=lambda ind: problem(*ind))
        # population.sort(key=lambda ind: problem(ind[0], ind[1]))
        # population.sort(key=problem)

        # -- store sorted offspring
        self.stats_offspring.append(copy.deepcopy(population))

        old_centroid = self.centroid
        # Note : the following does m <- <x>_w
        # Note : this is equivalent to doing m <- m + sigma * <z>_w
        # as x = m + sigma * z provided the weights sum to 1.0 which it
        # does
        self.centroid = np.dot(self.weights, population[0:self.mu])

        # -- store new centroid
        self.stats_new_centroids.append(copy.deepcopy(self.centroid))

        c_diff = self.centroid - old_centroid

        # Cumulation : update evolution path
        # Equivalent to in-class definition
        self.ps = (1 - self.cs) * self.ps \
             + np.sqrt(self.cs * (2 - self.cs) * self.mueff) / self.sigma \
             * np.dot(self.B, (1. / self.diagD) * np.dot(self.B.T, c_diff))

        # -- store new evol path
        self.stats_ps.append(copy.deepcopy(self.ps))

        hsig = float((np.linalg.norm(self.ps) /
                np.sqrt(1. - (1. - self.cs)**(2. * (self.generations + 1.))) / self.chiN
                < (1.4 + 2. / (self.dim + 1.))))

        self.pc = (1 - self.cc) * self.pc + hsig \
                  * np.sqrt(self.cc * (2 - self.cc) * self.mueff) / self.sigma \
                  * c_diff

        # Update covariance matrix
        artmp = population[0:self.mu] - old_centroid
        self.C = (1 - self.ccov1 - self.ccovmu + (1 - hsig) \
                   * self.ccov1 * self.cc * (2 - self.cc)) * self.C \
                + self.ccov1 * np.outer(self.pc, self.pc) \
                + self.ccovmu * np.dot((self.weights * artmp.T), artmp) \
                / self.sigma**2

        # -- store new covs
        self.stats_new_covs.append(copy.deepcopy(self.C))

        self.sigma *= np.exp((np.linalg.norm(self.ps) / self.chiN - 1.) \
                                * self.cs / self.ds)

        self.diagD, self.B = np.linalg.eigh(self.C)
        indx = np.argsort(self.diagD)

        self.cond = self.diagD[indx[-1]]/self.diagD[indx[0]]

        self.diagD = self.diagD[indx]**0.5
        self.B = self.B[:, indx]
        self.BD = self.B * self.diagD

    def run(self, problem):
        # At the start, clear all stored cache and start a new campaign
        self.reset()

        while self.generations < 40:
            # Sample the population here!
            population = list(multivariate_normal(self.centroid, self.sigma**2 * self.C, self.popsize))

            # Pass the population to update, which computes all new parameters
            self.update(problem, population)
            # print(np.array(population).shape)
            self.generations += 1
        else:
            return population[0]

    def reset(self):
        # Clears everything to rerun the problem
        self.stats_centroids = []
        self.stats_new_centroids = []
        self.stats_covs = []
        self.stats_new_covs = []
        self.stats_offspring = []
        self.stats_offspring_weights = []
        self.stats_ps = []_____no_output_____initial_centroid = np.random.randn(2, )
cma_es = CMAES(initial_centroid, 0.2, 10)_____no_output_____cma_es.run(current_problem)_____no_output_____

### Visualizing CMA-ES progress _____no_output_____First some setup code. This visualizes the progress of CMA based on the data we recorded in the class above and plots it in the objective function manifold._____no_output_____

normalizer = colors.Normalize(vmin=np.min(cma_es.weights), vmax=np.max(cma_es.weights))
sm = cm.ScalarMappable(norm=normalizer, cmap=plt.get_cmap('gray'))_____no_output_____from matplotlib import animation
from IPython.display import HTML_____no_output_____def animate_cma_es(gen):
    ax.cla()
    plot_problem_contour(current_problem, ((-11,-11), (11,11)), optimum=(0,0), ax=ax)

    plot_cov_ellipse(cma_es.stats_centroids[gen], cma_es.stats_covs[gen], volume=0.99, alpha=0.29,
                     fc='red', ec='darkred',
                     ax=ax)
    ax.plot(cma_es.stats_centroids[gen][0], cma_es.stats_centroids[gen][1], 'ro', markeredgecolor = 'none', ms=10)

    plot_cov_ellipse(cma_es.stats_new_centroids[gen], cma_es.stats_new_covs[gen], volume=0.99,
                     alpha=0.29, fc='green', ec='darkgreen', ax=ax)
    ax.plot(cma_es.stats_new_centroids[gen][0], cma_es.stats_new_centroids[gen][1], 'go', markeredgecolor = 'none', ms=10)

    for i in range(gen+1):
        if i == 0:
            ax.plot((0,cma_es.stats_ps[i][0]),
                     (0,cma_es.stats_ps[i][1]), 'b--')
        else:
            ax.plot((cma_es.stats_ps[i-1][0],cma_es.stats_ps[i][0]),
                     (cma_es.stats_ps[i-1][1],cma_es.stats_ps[i][1]),'b--')

    for i,ind in enumerate(cma_es.stats_offspring[gen]):
        if i < len(cma_es.weights):
            color = sm.to_rgba(cma_es.weights[i])
        else:
            color= sm.to_rgba(normalizer.vmin)
        ax.plot(ind[0], ind[1], 'o', color = color, ms=5, markeredgecolor = 'none')

    ax.set_ylim((-10,10))
    ax.set_xlim((-10,10))
    ax.set_title('$generation=$' +str(gen))
    return []_____no_output_____fig = plt.figure(figsize=(10,10))
ax = fig.gca()
anim = animation.FuncAnimation(fig, animate_cma_es, frames=cma_es.generations, interval=300, blit=True)
plt.close()_____no_output_____

In the animation below :
* Current centroid and covariance: **red**.
* Updated centroid and covariance: **green**.
* Sampled individuals: **shades of gray representing their corresponding weight**. (White is best)
* Evolution path: **blue line starting in (0,0)**. _____no_output_____

HTML(anim.to_html5_video())_____no_output_____


# Notebook from mhalabisky/deafrica-sandbox-notebooks
Path: Frequently_used_code/Principal_component_analysis.ipynb

# Principal component analysis for multi-spectral data

* **Products used:**
[s2_l2a](https://explorer.digitalearth.africa/s2_l2a)
_____no_output_____

## Background

Principal Component Analysis (PCA) is a popular technique for dimensionality reduction.
It can be used to explore patterns in high-dimensional data and assist unsupervised learning.

Principal components are a series of linear combinations of the original variables, among which the first principal component accounts for the greatest variance within a dataset.
Each subsequent principal component accounts for the next greatest possible variance and is uncorrelated with the previously defined components.

This technique is useful for understanding Sentinel-2 data as images are captured in 12 spectral bands but only 3 variables can be visualized in a RGB composite.
PCA can also be applied to timeseries data to investigate temporal evolution patterns for different land cover types._____no_output_____## Description

This notebook demonstrates a principal component analysis for Sentinel-2 multi-spectal data.
Following steps are covered:

1. Loading Sentinel-2 multi-spectral data.
2. Applying PCA to transform and visualize data.
***_____no_output_____## Getting started

To run this analysis, run all the cells in the notebook, starting with the "Load packages" cell. _____no_output_____### Load packages
Import Python packages that are used for the analysis._____no_output_____

%matplotlib inline

from sklearn.decomposition import PCA
import datacube

from deafrica_tools.datahandling import load_ard
from deafrica_tools.plotting import rgb
from deafrica_tools.classification import sklearn_flatten, sklearn_unflatten_____no_output_____

### Connect to the datacube

Connect to the datacube so we can access DEAfrica data._____no_output_____

dc = datacube.Datacube(app='pca')_____no_output_____

### Analysis parameters

This section defines the analysis parameters, including

* center lat/lon and analysis window size for the area of interest
* time period to be investigated
* projection and resolution for loading data
* acceptable range of cloud cover percentage for input Sentinel-2 granule (`min_gooddata`)
* spectral bands to be explored

The default location is Betsiboka Estuary, Madagascar.

To limit overall memory usage, if a larger analysis window or higher resolution is desired, the time period should be reduced accordingly._____no_output_____

# Create a query object
lat, lon = -15.92, 46.35
buffer = 0.1

query = {
    'time': ('2020-01', '2020-03'),
    'x': (lon - buffer, lon + buffer),
    'y': (lat + buffer, lat - buffer),
    'output_crs': 'epsg:6933',
    'resolution':(-20,20),
}

# use all non-overlapping 10m and 20m bands
bands = ['blue', 'green', 'red', 'red_edge_1', 'red_edge_2',
         'red_edge_3', 'nir_narrow', 'swir_1', 'swir_2']_____no_output_____

## Loading Sentinel-2 multi-spectral data

Only high probablity cloud is excluded in this example, but this can be modified for a different area.
_____no_output_____

ds = load_ard(dc=dc,
              products=['s2_l2a'],
              measurements=bands,
              min_gooddata=0.05,
              pq_categories_s2=['vegetation', 'snow or ice', 'water', 'bare soils',
                                'unclassified', 'dark area pixels', 'cloud_shadows',
                                'cloud medium probability', 'thin cirrus'],
              group_by='solar_day',
              **query)
Using pixel quality parameters for Sentinel 2
Finding datasets
    s2_l2a
Counting good quality pixels for each time step
Filtering to 18 out of 18 time steps with at least 5.0% good quality pixels
Applying pixel quality/cloud mask
Loading 18 time steps
# visualize data using selected input spectral bands
rgb(ds, bands=['swir_1','nir_narrow','red_edge_1'], index=list(range(len(ds.time))), col_wrap=4)_____no_output_____

## Applying PCA to transform and visualize data

To perform a PCA, data is first transformed into a numpy array that can be used by sklearn._____no_output_____

X = sklearn_flatten(ds)_____no_output_____

A PCA model is generated with 3 principal components and fitted on the data._____no_output_____

pca = PCA(n_components=3)
pca.fit(X)_____no_output_____

We can investigate how much variance is accounted for in each principal component.
In the default example, the first principal component accounts for a much high variance than the next two.

This step can help determine whether more principal components are needed._____no_output_____

print("Relative variance in principal components:", pca.explained_variance_ratio_)_____no_output_____

The input data can now be transformed into this new reference space and rearranged into a xarray dataset compatible with input data._____no_output_____

predict = pca.transform(X)_____no_output_____out = sklearn_unflatten(predict, ds)
out = out.to_dataset(dim=out.dims[0]).transpose('time','y','x')_____no_output_____

### Visualise PCA results_____no_output_____

# Use code comments for low-level documentation of code
rgb(out, bands=[2,1,0], index=list(range(len(out.time))), col_wrap=4)_____no_output_____

***

## Additional information

**License:** The code in this notebook is licensed under the [Apache License, Version 2.0](https://www.apache.org/licenses/LICENSE-2.0).
Digital Earth Africa data is licensed under the [Creative Commons by Attribution 4.0](https://creativecommons.org/licenses/by/4.0/) license.

**Contact:** If you need assistance, please post a question on the [Open Data Cube Slack channel](http://slack.opendatacube.org/) or on the [GIS Stack Exchange](https://gis.stackexchange.com/questions/ask?tags=open-data-cube) using the `open-data-cube` tag (you can view previously asked questions [here](https://gis.stackexchange.com/questions/tagged/open-data-cube)).
If you would like to report an issue with this notebook, you can file one on [Github](https://github.com/digitalearthafrica/deafrica-sandbox-notebooks).

**Compatible datacube version:** _____no_output_____

print(datacube.__version__)_____no_output_____

**Last Tested:**_____no_output_____

from datetime import datetime
datetime.today().strftime('%Y-%m-%d')_____no_output_____


# Notebook from SherylHYX/AUROC-vs-AUPRC
Path: pulsar_star/pulsar_star.ipynb

# Predicting a Pulsar Star_____no_output_____https://www.kaggle.com/pavanraj159/predicting-a-pulsar-star

Data set describes a sample of pulsar candidates collected during the High Time Resolution Universe Survey.

Pulsars are a rare type of Neutron star that produce radio emission detectable here on Earth. They are of considerable scientific interest as probes of space-time, the inter-stellar medium, and states of matter.

As pulsars rotate, their emission beam sweeps across the sky, and when this crosses our line of sight, produces a detectable pattern of broadband radio emission. As pulsars rotate rapidly, this pattern repeats periodically. Thus pulsar search involves looking for periodic radio signals with large radio telescopes.

Each pulsar produces a slightly different emission pattern, which varies slightly with each rotation . Thus a potential signal detection known as a 'candidate', is averaged over many rotations of the pulsar, as determined by the length of an observation. In the absence of additional info, each candidate could potentially describe a real pulsar. However in practice almost all detections are caused by radio frequency interference (RFI) and noise, making legitimate signals hard to find.

Machine learning tools are now being used to automatically label pulsar candidates to facilitate rapid analysis. Classification systems in particular are being widely adopted, which treat the candidate data sets as binary classification problems. Here the legitimate pulsar examples are a minority positive class, and spurious examples the majority negative class.

The data set shared here contains 16,259 spurious examples caused by RFI/noise, and 1,639 real pulsar examples. These examples have all been checked by human annotators.

Each row lists the variables first, and the class label is the final entry. The class labels used are 0 (negative) and 1 (positive)._____no_output_____### Attribute Information:

Each candidate is described by 8 continuous variables, and a single class variable. The first four are simple statistics obtained from the integrated pulse profile (folded profile). This is an array of continuous variables that describe a longitude-resolved version of the signal that has been averaged in both time and frequency . The remaining four variables are similarly obtained from the DM-SNR curve . These are summarised below:

1. Mean of the integrated profile
2. Standard deviation of the integrated profile
3. Excess kurtosis of the integrated profile
4. Skewness of the integrated profile
5. Mean of the DM-SNR curve
6. Standard deviation of the DM-SNR curve
7. Excess kurtosis of the DM-SNR curve
8. Skewness of the DM-SNR curve
9. Class

17,898 total examples. 1,639 positive examples. 16,259 negative examples._____no_output_____

import numpy as np # linear algebra
import pandas as pd # data processing
import warnings
warnings.filterwarnings("ignore")
import matplotlib
import matplotlib.pyplot as plt
import seaborn as sns
%matplotlib inline
sns.set()_____no_output_____

Most of the code was taken from https://www.kaggle.com/efeergun96/prediciting-a-pulsar-star_____no_output_____## Loading the dataset_____no_output_____

DataFrame = pd.read_csv("pulsar_stars.csv")  _____no_output_____

# EDA_____no_output_____

DataFrame.head()    # first 5 rows of whole columns_____no_output_____DataFrame.info() # information about data types and amount of non-null rows of our Dataset<class 'pandas.core.frame.DataFrame'>
RangeIndex: 17898 entries, 0 to 17897
Data columns (total 9 columns):
 Mean of the integrated profile                  17898 non-null float64
 Standard deviation of the integrated profile    17898 non-null float64
 Excess kurtosis of the integrated profile       17898 non-null float64
 Skewness of the integrated profile              17898 non-null float64
 Mean of the DM-SNR curve                        17898 non-null float64
 Standard deviation of the DM-SNR curve          17898 non-null float64
 Excess kurtosis of the DM-SNR curve             17898 non-null float64
 Skewness of the DM-SNR curve                    17898 non-null float64
target_class                                     17898 non-null int64
dtypes: float64(8), int64(1)
memory usage: 1.2 MB

Data types are all numeric and non-null, I don't need to do any transformations or cleaning._____no_output_____

DataFrame.describe()   # statistical information about our data_____no_output_____DataFrame.corr()    # correlation between fields_____no_output_____# PairPlot (each column is compared to others and itself)

sns.pairplot(data=DataFrame,
             palette="husl",
             hue="target_class",
             vars=[" Mean of the integrated profile",
                   " Excess kurtosis of the integrated profile",
                   " Skewness of the integrated profile",
                   " Mean of the DM-SNR curve",
                   " Excess kurtosis of the DM-SNR curve",
                   " Skewness of the DM-SNR curve"])

#plt.suptitle("PairPlot of Data Without Std. Dev. Fields",fontsize=18)

plt.tight_layout()
plt.show()   # pairplot without standard deviaton fields of data_____no_output_____

We can see that our data is quite separable on most of the columns._____no_output_____

# Correlation HeatMap

plt.figure(figsize=(16,12))
sns.heatmap(data=DataFrame.corr(),annot=True,cmap="bone",linewidths=1,fmt=".2f",linecolor="gray")
plt.title("Correlation Map",fontsize=20)
plt.tight_layout()
plt.show()      # lightest and darkest cells are most correlated ones_____no_output_____

Most of our Columns are already related or derived from one or another and we can see it clearly on some cells above._____no_output_____

# ViolinPlot (act as a boxplot but we can see amounts too)

plt.figure(figsize=(16,10))

plt.subplot(2,2,1)
sns.violinplot(data=DataFrame,y=" Mean of the integrated profile",x="target_class")

plt.subplot(2,2,2)
sns.violinplot(data=DataFrame,y=" Mean of the DM-SNR curve",x="target_class")

plt.subplot(2,2,3)
sns.violinplot(data=DataFrame,y=" Standard deviation of the integrated profile",x="target_class")

plt.subplot(2,2,4)
sns.violinplot(data=DataFrame,y=" Standard deviation of the DM-SNR curve",x="target_class")

plt.suptitle("ViolinPlot",fontsize=20)

plt.show()_____no_output_____

We can see that our data has different kind of distributions which is helpful for training our models._____no_output_____## Data PreProcessing_____no_output_____

# Splitting the Feature and Label fields

labels = DataFrame.target_class.values
DataFrame.drop(["target_class"],axis=1,inplace=True)
features = DataFrame.values_____no_output_____# Scaling the Features

from sklearn.preprocessing import MinMaxScaler
scaler = MinMaxScaler(feature_range=(0,1))
features_scaled = scaler.fit_transform(features)/Users/HuongGiangTran/miniconda3/envs/py3iaml/lib/python3.7/site-packages/sklearn/utils/__init__.py:4: DeprecationWarning: Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated, and in 3.8 it will stop working
  from collections import Sequence
# Splitting the Train and the Test rows

from sklearn.model_selection import train_test_split
x_train, x_test, y_train, y_test = train_test_split(features_scaled,labels,random_state = 1, test_size=0.2)_____no_output_____

# Machine Learning Models _____no_output_____### Random Guess_____no_output_____

data_size = DataFrame.shape[0]
is_positive = np.sum(labels == 1)/data_size #probability that y=1
is_negative = np.sum(labels == 0)/data_size

# Accuracy
print("Accuracy is", '%.03f' %is_negative)

# AUROC
print("AUROC is 0.500")

#AUPRC
print("AUPRC is", '%.03f' %is_positive)Accuracy is 0.908
AUROC is 0.500
AUPRC is 0.092
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVC
from sklearn.naive_bayes import GaussianNB
from sklearn.linear_model import LogisticRegression
import sklearn.metrics as metrics
from sklearn.metrics import precision_recall_curve
from sklearn.metrics import auc
from sklearn.metrics import f1_score
from sklearn.metrics import log_loss

dec = DecisionTreeClassifier(random_state=42)
ran = RandomForestClassifier(n_estimators=37,random_state=42,max_leaf_nodes=200,criterion="entropy")
knn = KNeighborsClassifier(n_neighbors=7,weights="distance")
svm = SVC(random_state=42,C=250,gamma=1.6,kernel="poly",probability=True)
naive = GaussianNB()
lg = LogisticRegression(random_state=42,solver="liblinear",C=1.6,penalty="l1")

models = {"Decision tree" : dec,
          "Random forest" : ran,
          "KNN" : knn,
          "SVM" : svm,
          "Naive bayes" : naive,
          "Logistic regression" : lg}
acc_scores = { }
auroc = { }
auprc = { }
logloss = { }
F1 = { }

for key, value in models.items():
    model = value
    model.fit(x_train, y_train)
    acc_scores[key] = model.score(x_test, y_test)
    probs = model.predict_proba(x_test)
    preds = probs[:,1]
    fpr, tpr, threshold = metrics.roc_curve(y_test, preds)
    auroc[key] = metrics.auc(fpr, tpr)
    precision, recall, thresholds = precision_recall_curve(y_test, preds)
    auprc[key] = auc(recall, precision)
    predict = model.predict(x_test)
    logloss[key] = log_loss(y_test, probs)
    F1[key] = f1_score(y_test, predict)/Users/HuongGiangTran/miniconda3/envs/py3iaml/lib/python3.7/site-packages/sklearn/ensemble/weight_boosting.py:29: DeprecationWarning: numpy.core.umath_tests is an internal NumPy module and should not be imported. It will be removed in a future NumPy release.
  from numpy.core.umath_tests import inner1d
#roc
for key, value in models.items():
    probs = value.predict_proba(x_test)
    preds = probs[:,1]
    fpr, tpr, threshold = metrics.roc_curve(y_test, preds)
    roc_auc = metrics.auc(fpr, tpr)

    plt.plot(fpr, tpr, label = '{}'.format(key))

plt.legend()
plt.rcParams["figure.figsize"] = (8,5)
plt.plot([0, 1], [0, 1],'k--')
plt.xlim([-0.04, 1.04])
plt.ylim([-0.04, 1.04])
plt.ylabel('True Positive Rate')
plt.xlabel('False Positive Rate')
plt.savefig('pulsar_star_ROCcurve.png')
plt.show()_____no_output_____#pr
for key, value in models.items():
    probs = value.predict_proba(x_test)
    preds = probs[:,1]

    precision, recall, thresholds = precision_recall_curve(y_test, preds)

    plt.plot(recall, precision, label = '{}'.format(key))

plt.hlines(is_positive, 0, 1, colors = 'k', linestyle = 'dashed')
plt.rcParams["figure.figsize"] = (8,5)
plt.legend()
plt.xlim([-0.04, 1.04])
plt.ylim([-0.04, 1.04])
plt.ylabel('Precision')
plt.xlabel('Recall')
plt.savefig('pulsar_star_PRcurve.png')
plt.show()_____no_output_____

Below is the classifier comparison with five different performance measures:_____no_output_____![Performance Comparison](pulsar_star_table.png)_____no_output_____