# Notebook from Sulles/WindFarmProject
Path: JupyterNotebooks/2_Explore the data_Solution.ipynb

# WIND TURBINES PREDICTIVE MAINTENANCE THROUGH ML: DATA EXPLORATION - SOLUTION (2)_____no_output_____# Load libraries_____no_output_____

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import os
%matplotlib inline
sns.set_style('darkgrid')_____no_output_____

# 1) Load data_____no_output_____

DATA_ROOT = "Training/"
DATA_SIGNAL = "Training_signals.csv"
DATA_CLEAN_ROOT = 'data_clean'
DATA_PROD_CLEAN = 'turbines_df.p'
DATA_MAST_CLEAN = 'mast.p'
DATA_FAILURE = 'Training_failures.csv'_____no_output_____df = pd.read_pickle(os.path.join(DATA_CLEAN_ROOT, DATA_PROD_CLEAN))
mast = pd.read_pickle(os.path.join(DATA_CLEAN_ROOT, DATA_MAST_CLEAN))_____no_output_____turbines = df.index.get_level_values('Turbine_ID').unique()_____no_output_____turbines_____no_output_____

# 2) Produced energy_____no_output_____

df.head()_____no_output_____

## a) Total energy in a year_____no_output_____Here, we want to compare the energy produced by the five turbines in a year.
The energy (in kWh) is defined as power (in kW) * time (in h).
Since we have measureaments of power (*Grd_Prod_Pwr_Avg* [kW]) every 10 minutes (1/6 of an hour),
we can get the energy produced in kWh by dividing by 6._____no_output_____

energy_yearly = df.reset_index(level=0).groupby('Turbine_ID') \
    ['Grd_Prod_Pwr_Avg'].sum() / 6_____no_output_____energy_yearly_____no_output_____energy_yearly.plot(kind='bar', title='Total energy produced (kWh)')
plt.show()_____no_output_____

From this bar plot, wa can note that there are some small differences in the energy produced in a year. For more details, we need to explore the produced energy of each turbine across different months._____no_output_____## b) Energy for each turbine by month_____no_output_____

# Let's group the data by turbine, and then resample by month
energy_monthly = df.reset_index(level=0).groupby(
    'Turbine_ID')['Grd_Prod_Pwr_Avg'].apply(lambda x: (x/6).resample('M').sum())_____no_output_____energy_monthly_____no_output_____energy_monthly = energy_monthly.unstack() / 1000 #Pivot a level of the (necessarily hierarchical) index labels._____no_output_____energy_monthly_____no_output_____

Now we can deduce which turbine produced the biggest amount of energy for each month._____no_output_____

energy_monthly.idxmax()  #Return index of first occurrence of maximum over requested axis._____no_output_____

Since we have two dimensions: turbines and months, we can plot a heatmap to better vizualize how much energy was produced by each turbine in the different months._____no_output_____

fig, ax = plt.subplots(figsize=(15,5))
sns.heatmap(energy_monthly, annot=True, cmap='summer', square=True)
ax.set_title('Energy production in MWh')
ax.set_xticklabels(energy_monthly.columns.month_name())
plt.show()_____no_output_____

We can also use a simple bar chart._____no_output_____

fig, ax = plt.subplots(figsize=(15,5))
energy_monthly.T.plot(kind='bar', ax=ax)  #transpose the data
ax.set_title('Energy production in MWh')
ax.set_xticklabels(energy_monthly.columns.month_name())
plt.show()_____no_output_____

For the first 4 months, we can see that the energy production is almost the same for the turbines. However, we can note that there is a small energy production in March and April. This can be explained by the failure that occured on Turbine T11 in March and also the hydraulic group and generator bearing failures in April for Turbines T06 and T07 respectivey (See below: Section failures). _____no_output_____Let's check the energy produced by the turbines at different hours of the day._____no_output_____

# we'll group the data by turbine and resample per hour
energy_hourly_sum = df.reset_index(level=0).groupby(
    'Turbine_ID')['Grd_Prod_Pwr_Avg'].apply(
    lambda x: (x/6).groupby(x.index.hour).sum())

energy_hourly_sum = energy_hourly_sum.unstack() / 1000

fig, ax = plt.subplots(figsize=(18,4))
energy_hourly_sum.T.plot(kind='bar', ax=ax, zorder=-1)
ax.xaxis.grid(False)  #Configure the grid lines.
ax.set_title('Energy produced at different times of the day \
    for the different turbines')
plt.show()_____no_output_____

We can represent this information using a heatmap as before since we have two dimensions: the different turbines and the hours._____no_output_____

fig, ax = plt.subplots(figsize=(15,4))
sns.heatmap(energy_hourly_sum, ax=ax, square=True, cmap='summer')_____no_output_____

Now, let's represent the evolution of the energy produced by each turbine for each month using a classical lineplot._____no_output_____

df_ = df.reset_index(level=0)
energy_monthly_hourly = df_.groupby(df_.Turbine_ID).Grd_Prod_Pwr_Avg.apply(
    lambda x: x.groupby(x.index.month).apply(
    lambda y: (y/6).groupby(y.index.hour).sum()))
del df_

fig, axes = plt.subplots(figsize=(15,15), nrows=4, ncols=3,
                         sharex=True, sharey=True)

for i in range(12):
    ax = axes.ravel()[i]
    month = i+1
    for turbine in turbines:
        turbine_monthly_hourly_consumption = energy_monthly_hourly \
            .loc[turbine].loc[month]

        sns.lineplot(
            turbine_monthly_hourly_consumption.index,
            turbine_monthly_hourly_consumption.values,
            label=turbine, ax=ax)
        ax.set_xlim(0,23)
        ax.set_title(f'Month {month}')
        ax.set_xlabel('Hour')
        ax.set_ylabel('Energy (kWh)')c:\users\suley\documents\github\windfarmproject\venv\lib\site-packages\seaborn\_decorators.py:36: FutureWarning: Pass the following variables as keyword args: x, y. From version 0.12, the only valid positional argument will be `data`, and passing other arguments without an explicit keyword will result in an error or misinterpretation.
  warnings.warn(
c:\users\suley\documents\github\windfarmproject\venv\lib\site-packages\seaborn\_decorators.py:36: FutureWarning: Pass the following variables as keyword args: x, y. From version 0.12, the only valid positional argument will be `data`, and passing other arguments without an explicit keyword will result in an error or misinterpretation.
  warnings.warn(
c:\users\suley\documents\github\windfarmproject\venv\lib\site-packages\seaborn\_decorators.py:36: FutureWarning: Pass the following variables as keyword args: x, y. From version 0.12, the only valid positional argument will be `data`, and passing other arguments without an explicit keyword will result in an error or misinterpretation.
  warnings.warn(
c:\users\suley\documents\github\windfarmproject\venv\lib\site-packages\seaborn\_decorators.py:36: FutureWarning: Pass the following variables as keyword args: x, y. From version 0.12, the only valid positional argument will be `data`, and passing other arguments without an explicit keyword will result in an error or misinterpretation.
  warnings.warn(
c:\users\suley\documents\github\windfarmproject\venv\lib\site-packages\seaborn\_decorators.py:36: FutureWarning: Pass the following variables as keyword args: x, y. From version 0.12, the only valid positional argument will be `data`, and passing other arguments without an explicit keyword will result in an error or misinterpretation.
  warnings.warn(
c:\users\suley\documents\github\windfarmproject\venv\lib\site-packages\seaborn\_decorators.py:36: FutureWarning: Pass the following variables as keyword args: x, y. From version 0.12, the only valid positional argument will be `data`, and passing other arguments without an explicit keyword will result in an error or misinterpretation.
  warnings.warn(
c:\users\suley\documents\github\windfarmproject\venv\lib\site-packages\seaborn\_decorators.py:36: FutureWarning: Pass the following variables as keyword args: x, y. From version 0.12, the only valid positional argument will be `data`, and passing other arguments without an explicit keyword will result in an error or misinterpretation.
  warnings.warn(
c:\users\suley\documents\github\windfarmproject\venv\lib\site-packages\seaborn\_decorators.py:36: FutureWarning: Pass the following variables as keyword args: x, y. From version 0.12, the only valid positional argument will be `data`, and passing other arguments without an explicit keyword will result in an error or misinterpretation.
  warnings.warn(
c:\users\suley\documents\github\windfarmproject\venv\lib\site-packages\seaborn\_decorators.py:36: FutureWarning: Pass the following variables as keyword args: x, y. From version 0.12, the only valid positional argument will be `data`, and passing other arguments without an explicit keyword will result in an error or misinterpretation.
  warnings.warn(
c:\users\suley\documents\github\windfarmproject\venv\lib\site-packages\seaborn\_decorators.py:36: FutureWarning: Pass the following variables as keyword args: x, y. From version 0.12, the only valid positional argument will be `data`, and passing other arguments without an explicit keyword will result in an error or misinterpretation.
  warnings.warn(
c:\users\suley\documents\github\windfarmproject\venv\lib\site-packages\seaborn\_decorators.py:36: FutureWarning: Pass the following variables as keyword args: x, y. From version 0.12, the only valid positional argument will be `data`, and passing other arguments without an explicit keyword will result in an error or misinterpretation.
  warnings.warn(
c:\users\suley\documents\github\windfarmproject\venv\lib\site-packages\seaborn\_decorators.py:36: FutureWarning: Pass the following variables as keyword args: x, y. From version 0.12, the only valid positional argument will be `data`, and passing other arguments without an explicit keyword will result in an error or misinterpretation.
  warnings.warn(
c:\users\suley\documents\github\windfarmproject\venv\lib\site-packages\seaborn\_decorators.py:36: FutureWarning: Pass the following variables as keyword args: x, y. From version 0.12, the only valid positional argument will be `data`, and passing other arguments without an explicit keyword will result in an error or misinterpretation.
  warnings.warn(
c:\users\suley\documents\github\windfarmproject\venv\lib\site-packages\seaborn\_decorators.py:36: FutureWarning: Pass the following variables as keyword args: x, y. From version 0.12, the only valid positional argument will be `data`, and passing other arguments without an explicit keyword will result in an error or misinterpretation.
  warnings.warn(
c:\users\suley\documents\github\windfarmproject\venv\lib\site-packages\seaborn\_decorators.py:36: FutureWarning: Pass the following variables as keyword args: x, y. From version 0.12, the only valid positional argument will be `data`, and passing other arguments without an explicit keyword will result in an error or misinterpretation.
  warnings.warn(
c:\users\suley\documents\github\windfarmproject\venv\lib\site-packages\seaborn\_decorators.py:36: FutureWarning: Pass the following variables as keyword args: x, y. From version 0.12, the only valid positional argument will be `data`, and passing other arguments without an explicit keyword will result in an error or misinterpretation.
  warnings.warn(
c:\users\suley\documents\github\windfarmproject\venv\lib\site-packages\seaborn\_decorators.py:36: FutureWarning: Pass the following variables as keyword args: x, y. From version 0.12, the only valid positional argument will be `data`, and passing other arguments without an explicit keyword will result in an error or misinterpretation.
  warnings.warn(
c:\users\suley\documents\github\windfarmproject\venv\lib\site-packages\seaborn\_decorators.py:36: FutureWarning: Pass the following variables as keyword args: x, y. From version 0.12, the only valid positional argument will be `data`, and passing other arguments without an explicit keyword will result in an error or misinterpretation.
  warnings.warn(
c:\users\suley\documents\github\windfarmproject\venv\lib\site-packages\seaborn\_decorators.py:36: FutureWarning: Pass the following variables as keyword args: x, y. From version 0.12, the only valid positional argument will be `data`, and passing other arguments without an explicit keyword will result in an error or misinterpretation.
  warnings.warn(
c:\users\suley\documents\github\windfarmproject\venv\lib\site-packages\seaborn\_decorators.py:36: FutureWarning: Pass the following variables as keyword args: x, y. From version 0.12, the only valid positional argument will be `data`, and passing other arguments without an explicit keyword will result in an error or misinterpretation.
  warnings.warn(
c:\users\suley\documents\github\windfarmproject\venv\lib\site-packages\seaborn\_decorators.py:36: FutureWarning: Pass the following variables as keyword args: x, y. From version 0.12, the only valid positional argument will be `data`, and passing other arguments without an explicit keyword will result in an error or misinterpretation.
  warnings.warn(
c:\users\suley\documents\github\windfarmproject\venv\lib\site-packages\seaborn\_decorators.py:36: FutureWarning: Pass the following variables as keyword args: x, y. From version 0.12, the only valid positional argument will be `data`, and passing other arguments without an explicit keyword will result in an error or misinterpretation.
  warnings.warn(
c:\users\suley\documents\github\windfarmproject\venv\lib\site-packages\seaborn\_decorators.py:36: FutureWarning: Pass the following variables as keyword args: x, y. From version 0.12, the only valid positional argument will be `data`, and passing other arguments without an explicit keyword will result in an error or misinterpretation.
  warnings.warn(
c:\users\suley\documents\github\windfarmproject\venv\lib\site-packages\seaborn\_decorators.py:36: FutureWarning: Pass the following variables as keyword args: x, y. From version 0.12, the only valid positional argument will be `data`, and passing other arguments without an explicit keyword will result in an error or misinterpretation.
  warnings.warn(
c:\users\suley\documents\github\windfarmproject\venv\lib\site-packages\seaborn\_decorators.py:36: FutureWarning: Pass the following variables as keyword args: x, y. From version 0.12, the only valid positional argument will be `data`, and passing other arguments without an explicit keyword will result in an error or misinterpretation.
  warnings.warn(
c:\users\suley\documents\github\windfarmproject\venv\lib\site-packages\seaborn\_decorators.py:36: FutureWarning: Pass the following variables as keyword args: x, y. From version 0.12, the only valid positional argument will be `data`, and passing other arguments without an explicit keyword will result in an error or misinterpretation.
  warnings.warn(
c:\users\suley\documents\github\windfarmproject\venv\lib\site-packages\seaborn\_decorators.py:36: FutureWarning: Pass the following variables as keyword args: x, y. From version 0.12, the only valid positional argument will be `data`, and passing other arguments without an explicit keyword will result in an error or misinterpretation.
  warnings.warn(
c:\users\suley\documents\github\windfarmproject\venv\lib\site-packages\seaborn\_decorators.py:36: FutureWarning: Pass the following variables as keyword args: x, y. From version 0.12, the only valid positional argument will be `data`, and passing other arguments without an explicit keyword will result in an error or misinterpretation.
  warnings.warn(

We can see that Turbine 6 is producing less energy that the other turbines during several months. We can also note that the turbines are producing more energy during certain seasons and less during others._____no_output_____# 3) Failures_____no_output_____Exploring available failures data is very important. It will give us some insights and information about predicting failures before they happen in the future._____no_output_____

#load the failures dataset
failures = pd.read_csv(os.path.join(DATA_ROOT, DATA_FAILURE), delimiter=',')
failures.Timestamp = pd.to_datetime(failures.Timestamp)_____no_output_____failures.head()_____no_output_____

Failures data shows which component failed in which turbine and gives information about the time when the failure occurs and some remarks about the failure._____no_output_____## a) Failures/turbine_____no_output_____Let's see how many failures occured in each turbine._____no_output_____

failures.Turbine_ID.value_counts()_____no_output_____

It seems that Turbine 06 had 6 failures during the year while Turbine 01 had only one failure._____no_output_____

failures = failures.loc[failures.Turbine_ID.isin(turbines)]_____no_output_____

## b) Failures/components_____no_output_____Now, let's see how many failures occured on each component._____no_output_____

failures.Component.value_counts()_____no_output_____

The generator failed 6 times during the year while the generator bearing failed 4 times. This can indicate that we should "focus" on these components that tend to fail often.

If we assume that the more training data we have, the more accurate the models will be, we can note that the models for the gearbox, transformer and hydraulic group would perform equally well (2 failures each) while the model for the generator would perform better (6 failures)._____no_output_____

failures.Component.value_counts().plot(kind='bar', color='r')_____no_output_____

## c) Failures/Turbine and component_____no_output_____Let's check if there's any turbine that particularly suffered from a certain failure._____no_output_____

failures_counts = failures.groupby('Turbine_ID').apply(
    lambda x: x.Component.value_counts())
failures_counts = failures_counts.unstack(fill_value=0)_____no_output_____failures_counts_____no_output_____fig, ax = plt.subplots(figsize=(8,4))
failures_counts.plot(kind='bar', ax=ax)_____no_output_____

The generator of Turbine T06 has failed 5 times while the generator bearing of turbine T09 failed 3 times!_____no_output_____## c) Failures/time_____no_output_____Let's check when the failures happened over time._____no_output_____

failures_timeline = failures.set_index('Timestamp').resample('M').Turbine_ID.apply(
    lambda x: x.value_counts())
failures_timeline = failures_timeline.unstack(fill_value=0)_____no_output_____fig, ax = plt.subplots(figsize=(12,3))
failures_timeline.plot(kind='bar', ax=ax)
ax.set_xticklabels([i.get_text()[:10] for i in ax.get_xticklabels()])
plt.show()_____no_output_____

## d) Failures/time & turbine_____no_output_____Let's see when the faults happened for different turbines by plotting the produced power generated by each._____no_output_____

fig, axes = plt.subplots(figsize=(16,16), nrows=len(turbines),
                         sharex=True, sharey=True)

for i, t in enumerate(turbines):
    energy_turbine = df.loc[t].Grd_Prod_Pwr_Avg.resample('D').sum() / 6
    failures_turbine = failures.loc[failures.Turbine_ID == t]
    ax = axes.ravel()[i]
    ax.plot(energy_turbine)
    ax.set_title(t, fontdict=dict(size=14, weight='bold'))

    ax.vlines(
        failures_turbine.Timestamp.values, 0,
        energy_turbine.max(), color='red')
    for q, failure in failures_turbine.iterrows():
        ax.text(failure.Timestamp, energy_turbine.max(), failure.Component)

    ax.set_xlim(energy_turbine.index.min(), energy_turbine.index.max())
    ax.set_ylim(0)
    ax.set_ylabel("Power production kWh")
    ax.set_xlabel("Time")

plt.suptitle('Daily production of turbines and failures',
             fontsize=20, fontweight='bold', y=.92)
plt.show()_____no_output_____

# 4) Power curve_____no_output_____The power curve is a plot of the power produced by the generator (y axis) as a function of the wind speed (x axis). Often, when a failure occurs, the power production decreases or becomes null. Therefore, this plot can gives us insight about failures. However, it is not always sufficient to predict failures._____no_output_____

df = pd.read_csv(os.path.join(DATA_ROOT, DATA_SIGNAL))
df.head()
df.set_index('Timestamp', drop=True, inplace=True)
df.sort_index(inplace=True)
df.sort_values(by = ['Timestamp', 'Turbine_ID'], inplace=True)_____no_output_____# Here's the code to plot the power curve.

fig, ax = plt.subplots()

ax.scatter(
    df[df["Turbine_ID"] == "T01"]["Amb_WindSpeed_Avg"],
    df[df["Turbine_ID"] == "T01"]["Grd_Prod_Pwr_Avg"],
    s=1
)
ax.set_xlabel("Wind speed [m/s]")
ax.set_ylabel("Power production [kW]")
# _ = is a trick to stop jupyter from printing out text.
_ = ax.set_title("Power curve")_____no_output_____

Based on the power curve in the figure above, we can determine the cut-in and the cut-out speed:

- The cut-in speed is the minimum wind speed that permits power production (on which the turbine starts rotation and generates power). Here it is approximately 4.5 m/s.
- The cut-out speed is the minimum speed that permits maximum power production (the braking system is used to avoid rotor damage). Here it is approximately 12 m/s.

Therefore, we can note that many points have 0-power generation in the operating region of the turbine (between cut-in and cut-out speed). These points could correspond to when the turbine is stopped for maintenance._____no_output_____Reference: *A part of this code was inspired by Gianluca Mauro, AI Academy, who developed a similar case study for KIC InnoEnergy SE.*_____no_output_____


# Notebook from yuwei-star/scbean
Path: docs/source/tutorials/vimcca_rna_protein_tutorial.ipynb

## integrating scRNA-seq and ADT counts with VIMCCA
This tutorial shows loading, preprocessing, VIMCCA joint analyse of multimodal dataset.

## import packages
Here, we’ll import scbean along with other popular packages._____no_output_____

import matplotlib
matplotlib.use('TkAgg')
import scanpy as sc
from scbean.model import vimcca

# Command for Jupyter Notebooks only
%matplotlib inline
import warnings
warnings.filterwarnings('ignore')
from matplotlib.axes._axes import _log as matplotlib_axes_logger
matplotlib_axes_logger.setLevel('ERROR')Using TensorFlow backend.

## loading dataset
This use the CITE-seq dataset from [(Stuart*, Butler* et al, Cell 2019)](https://www.cell.com/cell/fulltext/S0092-8674(19)30559-8), which consists of 30,672 scRNA-seq profiles measured alongside a panel of 25 antibodies from bone marrow. The object contains two assays, RNA and antibody-derived tags (ADT)._____no_output_____

base_path = '/Users/wangyuwei/data/'
file_rna = base_path + 'bmcite_rna.h5ad'
file_atac = base_path + 'bmcite_adt.h5ad'

adata_rna = sc.read_h5ad(file_rna)
adata_adt = sc.read_h5ad(file_atac)_____no_output_____

## Processing with scanpy
Here, we filter and normalize scRNA-seq data, but only normalize adt data._____no_output_____

sc.pp.filter_genes(adata_rna, min_cells=10)
sc.pp.log1p(adata_rna)
sc.pp.log1p(adata_adt)
sc.pp.scale(adata_rna)
sc.pp.scale(adata_adt)_____no_output_____

## Joint analyse using VIMCCA
The returned z represents the complementary low-dimensional features of the two modalities learned by VIMCCA, which can be used for downstream analysis._____no_output_____

z = vimcca.fit_integration(
    adata_rna,
    adata_adt,
    sparse_x=False,
    sparse_y=False,
    hidden_layers=[128, 64, 16, 8],
    epochs=50,
    weight=5
)Model: "VIMCCA_mlp"
__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to
==================================================================================================
inputs_x (InputLayer)           [(None, 16313)]      0
__________________________________________________________________________________________________
encoder_z (Functional)          [(None, 8), (None, 8 2098384     inputs_x[0][0]
                                                                 inputs_x[0][0]
__________________________________________________________________________________________________
inputs_y (InputLayer)           [(None, 25)]         0
__________________________________________________________________________________________________
decoder_x (Functional)          (None, 16313)        2114553     encoder_z[0][2]
__________________________________________________________________________________________________
decoder_y (Functional)          (None, 25)           13401       encoder_z[1][2]
__________________________________________________________________________________________________
dense (Dense)                   (None, 128)          2088192     inputs_x[0][0]
__________________________________________________________________________________________________
batch_normalization (BatchNorma (None, 128)          384         dense[0][0]
__________________________________________________________________________________________________
activation (Activation)         (None, 128)          0           batch_normalization[0][0]
__________________________________________________________________________________________________
dropout (Dropout)               (None, 128)          0           activation[0][0]
__________________________________________________________________________________________________
dense_1 (Dense)                 (None, 64)           8256        dropout[0][0]
__________________________________________________________________________________________________
batch_normalization_1 (BatchNor (None, 64)           192         dense_1[0][0]
__________________________________________________________________________________________________
activation_1 (Activation)       (None, 64)           0           batch_normalization_1[0][0]
__________________________________________________________________________________________________
dropout_1 (Dropout)             (None, 64)           0           activation_1[0][0]
__________________________________________________________________________________________________
tf.math.subtract_2 (TFOpLambda) (None, 25)           0           inputs_y[0][0]
                                                                 decoder_y[0][0]
__________________________________________________________________________________________________
dense_2 (Dense)                 (None, 16)           1040        dropout_1[0][0]
__________________________________________________________________________________________________
tf.math.subtract (TFOpLambda)   (None, 16313)        0           inputs_x[0][0]
                                                                 decoder_x[0][0]
__________________________________________________________________________________________________
tf.math.reduce_mean_4 (TFOpLamb (1, 1)               0           tf.math.subtract_2[0][0]
__________________________________________________________________________________________________
batch_normalization_2 (BatchNor (None, 16)           48          dense_2[0][0]
__________________________________________________________________________________________________
tf.math.reduce_mean_2 (TFOpLamb (1, 1)               0           tf.math.subtract[0][0]
__________________________________________________________________________________________________
tf.math.subtract_3 (TFOpLambda) (None, 25)           0           tf.math.subtract_2[0][0]
                                                                 tf.math.reduce_mean_4[0][0]
__________________________________________________________________________________________________
activation_2 (Activation)       (None, 16)           0           batch_normalization_2[0][0]
__________________________________________________________________________________________________
tf.math.subtract_1 (TFOpLambda) (None, 16313)        0           tf.math.subtract[0][0]
                                                                 tf.math.reduce_mean_2[0][0]
__________________________________________________________________________________________________
tf.convert_to_tensor_1 (TFOpLam (None, 25)           0           decoder_y[0][0]
__________________________________________________________________________________________________
tf.cast_1 (TFOpLambda)          (None, 25)           0           inputs_y[0][0]
__________________________________________________________________________________________________
tf.math.square_1 (TFOpLambda)   (None, 25)           0           tf.math.subtract_3[0][0]
__________________________________________________________________________________________________
dropout_2 (Dropout)             (None, 16)           0           activation_2[0][0]
__________________________________________________________________________________________________
tf.convert_to_tensor (TFOpLambd (None, 16313)        0           decoder_x[0][0]
__________________________________________________________________________________________________
tf.cast (TFOpLambda)            (None, 16313)        0           inputs_x[0][0]
__________________________________________________________________________________________________
tf.math.square (TFOpLambda)     (None, 16313)        0           tf.math.subtract_1[0][0]
__________________________________________________________________________________________________
tf.math.squared_difference_1 (T (None, 25)           0           tf.convert_to_tensor_1[0][0]
                                                                 tf.cast_1[0][0]
__________________________________________________________________________________________________
tf.math.reduce_mean_5 (TFOpLamb ()                   0           tf.math.square_1[0][0]
__________________________________________________________________________________________________
z_log_var (Dense)               (None, 8)            136         dropout_2[0][0]
__________________________________________________________________________________________________
z_mean (Dense)                  (None, 8)            136         dropout_2[0][0]
__________________________________________________________________________________________________
tf.math.squared_difference (TFO (None, 16313)        0           tf.convert_to_tensor[0][0]
                                                                 tf.cast[0][0]
__________________________________________________________________________________________________
tf.math.reduce_mean_3 (TFOpLamb ()                   0           tf.math.square[0][0]
__________________________________________________________________________________________________
tf.math.reduce_mean_1 (TFOpLamb (None,)              0           tf.math.squared_difference_1[0][0
__________________________________________________________________________________________________
tf.math.truediv_2 (TFOpLambda)  ()                   0           tf.math.reduce_mean_5[0][0]
__________________________________________________________________________________________________
tf.math.truediv_3 (TFOpLambda)  ()                   0           tf.math.reduce_mean_5[0][0]
__________________________________________________________________________________________________
tf.math.log_1 (TFOpLambda)      ()                   0           tf.math.reduce_mean_5[0][0]
__________________________________________________________________________________________________
tf.__operators__.add_2 (TFOpLam (None, 8)            0           z_log_var[0][0]
__________________________________________________________________________________________________
tf.math.square_2 (TFOpLambda)   (None, 8)            0           z_mean[0][0]
__________________________________________________________________________________________________
tf.math.reduce_mean (TFOpLambda (None,)              0           tf.math.squared_difference[0][0]
__________________________________________________________________________________________________
tf.math.truediv (TFOpLambda)    ()                   0           tf.math.reduce_mean_3[0][0]
__________________________________________________________________________________________________
tf.math.truediv_1 (TFOpLambda)  ()                   0           tf.math.reduce_mean_3[0][0]
__________________________________________________________________________________________________
tf.math.log (TFOpLambda)        ()                   0           tf.math.reduce_mean_3[0][0]
__________________________________________________________________________________________________
tf.math.multiply_2 (TFOpLambda) (None,)              0           tf.math.reduce_mean_1[0][0]
                                                                 tf.math.truediv_2[0][0]
__________________________________________________________________________________________________
tf.math.multiply_3 (TFOpLambda) ()                   0           tf.math.truediv_3[0][0]
                                                                 tf.math.log_1[0][0]
__________________________________________________________________________________________________
tf.math.subtract_4 (TFOpLambda) (None, 8)            0           tf.__operators__.add_2[0][0]
                                                                 tf.math.square_2[0][0]
__________________________________________________________________________________________________
tf.math.exp (TFOpLambda)        (None, 8)            0           z_log_var[0][0]
__________________________________________________________________________________________________
tf.math.multiply (TFOpLambda)   (None,)              0           tf.math.reduce_mean[0][0]
                                                                 tf.math.truediv[0][0]
__________________________________________________________________________________________________
tf.math.multiply_1 (TFOpLambda) ()                   0           tf.math.truediv_1[0][0]
                                                                 tf.math.log[0][0]
__________________________________________________________________________________________________
tf.__operators__.add_1 (TFOpLam (None,)              0           tf.math.multiply_2[0][0]
                                                                 tf.math.multiply_3[0][0]
__________________________________________________________________________________________________
tf.math.subtract_5 (TFOpLambda) (None, 8)            0           tf.math.subtract_4[0][0]
                                                                 tf.math.exp[0][0]
__________________________________________________________________________________________________
tf.__operators__.add (TFOpLambd (None,)              0           tf.math.multiply[0][0]
                                                                 tf.math.multiply_1[0][0]
__________________________________________________________________________________________________
tf.math.multiply_5 (TFOpLambda) (None,)              0           tf.__operators__.add_1[0][0]
__________________________________________________________________________________________________
tf.math.reduce_sum (TFOpLambda) (None,)              0           tf.math.subtract_5[0][0]
__________________________________________________________________________________________________
tf.__operators__.add_3 (TFOpLam (None,)              0           tf.__operators__.add[0][0]
                                                                 tf.math.multiply_5[0][0]
__________________________________________________________________________________________________
tf.math.multiply_4 (TFOpLambda) (None,)              0           tf.math.reduce_sum[0][0]
__________________________________________________________________________________________________
tf.__operators__.add_4 (TFOpLam (None,)              0           tf.__operators__.add_3[0][0]
                                                                 tf.math.multiply_4[0][0]
__________________________________________________________________________________________________
tf.math.reduce_mean_6 (TFOpLamb ()                   0           tf.__operators__.add_4[0][0]
__________________________________________________________________________________________________
add_loss (AddLoss)              ()                   0           tf.math.reduce_mean_6[0][0]
==================================================================================================
Total params: 4,226,338
Trainable params: 4,225,090
Non-trainable params: 1,248
__________________________________________________________________________________________________
Epoch 1/50

## Visualization_____no_output_____

adata_rna.obsm['X_vimcca'] = z
sc.pp.neighbors(adata_rna, use_rep='X_vimcca')
sc.tl.leiden(adata_rna, resolution=3)
sc.tl.umap(adata_rna)_____no_output_____sc.pl.umap(adata_rna, color='celltype.l2')_____no_output_____


# Notebook from michaelmcabral/udacity-mlnd
Path: student_intervention/student_intervention.ipynb

# Machine Learning Engineer Nanodegree
## Supervised Learning
## Project: Building a Student Intervention System_____no_output_____Welcome to the second project of the Machine Learning Engineer Nanodegree! In this notebook, some template code has already been provided for you, and it will be your job to implement the additional functionality necessary to successfully complete this project. Sections that begin with **'Implementation'** in the header indicate that the following block of code will require additional functionality which you must provide. Instructions will be provided for each section and the specifics of the implementation are marked in the code block with a `'TODO'` statement. Please be sure to read the instructions carefully!

In addition to implementing code, there will be questions that you must answer which relate to the project and your implementation. Each section where you will answer a question is preceded by a **'Question X'** header. Carefully read each question and provide thorough answers in the following text boxes that begin with **'Answer:'**. Your project submission will be evaluated based on your answers to each of the questions and the implementation you provide.

>**Note:** Code and Markdown cells can be executed using the **Shift + Enter** keyboard shortcut. In addition, Markdown cells can be edited by typically double-clicking the cell to enter edit mode._____no_output_____### Question 1 - Classification vs. Regression
*Your goal for this project is to identify students who might need early intervention before they fail to graduate. Which type of supervised learning problem is this, classification or regression? Why?*_____no_output_____
**Answer:**

The proposed student intervention system is based on a **classification** problem, because the output space is discrete (categorical, finite). In this case, the output is binary: whether or not the student belongs to the group that might need early intervention.

P.S. In a regression problem, the output space is continuous.
_____no_output_____## Exploring the Data
Run the code cell below to load necessary Python libraries and load the student data. Note that the last column from this dataset, `'passed'`, will be our target label (whether the student graduated or didn't graduate). All other columns are features about each student._____no_output_____

# Import libraries
import numpy as np
import pandas as pd
from time import time
from sklearn.metrics import f1_score

# Read student data
student_data = pd.read_csv("student-data.csv")
print "Student data read successfully!"Student data read successfully!

### Implementation: Data Exploration
Let's begin by investigating the dataset to determine how many students we have information on, and learn about the graduation rate among these students. In the code cell below, you will need to compute the following:
- The total number of students, `n_students`.
- The total number of features for each student, `n_features`.
- The number of those students who passed, `n_passed`.
- The number of those students who failed, `n_failed`.
- The graduation rate of the class, `grad_rate`, in percent (%).
_____no_output_____

# Looking at the shape of the dataset
shape = student_data.shape
print 'The dataset contains {} rows and {} columns.'.format(shape[0], shape[1])The dataset contains 395 rows and 31 columns.
# Looking at the data types
student_data.dtypes.unique()_____no_output_____# Segregating numeric and non-numeric columns
numeric = list(student_data.dtypes[student_data.dtypes <> object].index)
non_numeric = list(student_data.dtypes[student_data.dtypes == object].index)
print "There are {} numeric columns and {} non-numeric ones.".format(len(numeric), len(non_numeric))There are 13 numeric columns and 18 non-numeric ones.
# Taking a look at the big picture
student_data.head()_____no_output_____# Summary statistics on numeric data
student_data[numeric].describe()_____no_output_____# Summary statistics on non-numeric data
student_data[non_numeric].describe()_____no_output_____# Plotting histograms of non-numeric data

# Importing the necessary tools from matplotlib
%matplotlib inline
import matplotlib.pyplot as plt

# Creating a 5x4 subplot matrix, to plot data from the 18 non-numeric columns
fig, axes = plt.subplots(5, 4, figsize=(15, 15), sharey=True)
fig.subplots_adjust(hspace=0.4)
fig.delaxes(axes[4, 2]), fig.delaxes(axes[4, 3])

# Creating a loop to plot data from the non-numeric features (17 columns)
n = 0
for feature in non_numeric[:-1]:
    axes[n/4, np.mod(n, 4)].hist(student_data[feature])
    axes[n/4, np.mod(n, 4)].set_title(feature, color='darkblue')
    axes[n/4, np.mod(n, 4)].set_ylim([0, 400])
    n += 1

# Plotting data from the non-numeric label (1 column: 'passed')
axes[4, 1].hist(student_data['passed'], color='darkred')
axes[4, 1].set_title('passed', color='darkred')
axes[4, 1].set_ylim([0, 400])

print 'Histograms of non-numeric data:'Histograms of non-numeric data:
# TODO: Calculate number of students
n_students = student_data.shape[0]

# TODO: Calculate number of features
n_features = student_data.shape[1] - 1

# TODO: Calculate passing students
n_passed = np.sum(student_data['passed'] == 'yes')

# TODO: Calculate failing students
n_failed = np.sum(student_data['passed'] == 'no')

# TODO: Calculate graduation rate
grad_rate = (float(n_passed) / float(n_students)) * 100

# Print the results
print "Total number of students: {}".format(n_students)
print "Number of features: {}".format(n_features)
print "Number of students who passed: {}".format(n_passed)
print "Number of students who failed: {}".format(n_failed)
print "Graduation rate of the class: {:.2f}%".format(grad_rate)Total number of students: 395
Number of features: 30
Number of students who passed: 265
Number of students who failed: 130
Graduation rate of the class: 67.09%

## Preparing the Data
In this section, we will prepare the data for modeling, training and testing.

### Identify feature and target columns
It is often the case that the data you obtain contains non-numeric features. This can be a problem, as most machine learning algorithms expect numeric data to perform computations with.

Run the code cell below to separate the student data into feature and target columns to see if any features are non-numeric._____no_output_____

# Extract feature columns
feature_cols = list(student_data.columns[:-1])

# Extract target column 'passed'
target_col = student_data.columns[-1]

# Show the list of columns
print "Feature columns:\n{}".format(feature_cols)
print "\nTarget column: {}".format(target_col)

# Separate the data into feature data and target data (X_all and y_all, respectively)
X_all = student_data[feature_cols]
y_all = student_data[target_col]

# Show the feature information by printing the first five rows
print "\nFeature values:"
print X_all.head()Feature columns:
['school', 'sex', 'age', 'address', 'famsize', 'Pstatus', 'Medu', 'Fedu', 'Mjob', 'Fjob', 'reason', 'guardian', 'traveltime', 'studytime', 'failures', 'schoolsup', 'famsup', 'paid', 'activities', 'nursery', 'higher', 'internet', 'romantic', 'famrel', 'freetime', 'goout', 'Dalc', 'Walc', 'health', 'absences']

Target column: passed

Feature values:
  school sex  age address famsize Pstatus  Medu  Fedu     Mjob      Fjob  \
0     GP   F   18       U     GT3       A     4     4  at_home   teacher
1     GP   F   17       U     GT3       T     1     1  at_home     other
2     GP   F   15       U     LE3       T     1     1  at_home     other
3     GP   F   15       U     GT3       T     4     2   health  services
4     GP   F   16       U     GT3       T     3     3    other     other

    ...    higher internet  romantic  famrel  freetime goout Dalc Walc health  \
0   ...       yes       no        no       4         3     4    1    1      3
1   ...       yes      yes        no       5         3     3    1    1      3
2   ...       yes      yes        no       4         3     2    2    3      3
3   ...       yes      yes       yes       3         2     2    1    1      5
4   ...       yes       no        no       4         3     2    1    2      5

  absences
0        6
1        4
2       10
3        2
4        4

[5 rows x 30 columns]

### Preprocess Feature Columns

As you can see, there are several non-numeric columns that need to be converted! Many of them are simply `yes`/`no`, e.g. `internet`. These can be reasonably converted into `1`/`0` (binary) values.

Other columns, like `Mjob` and `Fjob`, have more than two values, and are known as _categorical variables_. The recommended way to handle such a column is to create as many columns as possible values (e.g. `Fjob_teacher`, `Fjob_other`, `Fjob_services`, etc.), and assign a `1` to one of them and `0` to all others.

These generated columns are sometimes called _dummy variables_, and we will use the [`pandas.get_dummies()`](http://pandas.pydata.org/pandas-docs/stable/generated/pandas.get_dummies.html?highlight=get_dummies#pandas.get_dummies) function to perform this transformation. Run the code cell below to perform the preprocessing routine discussed in this section._____no_output_____

def preprocess_features(X):
    ''' Preprocesses the student data and converts non-numeric binary variables into
        binary (0/1) variables. Converts categorical variables into dummy variables. '''

    # Initialize new output DataFrame
    output = pd.DataFrame(index = X.index)

    # Investigate each feature column for the data
    for col, col_data in X.iteritems():

        # If data type is non-numeric, replace all yes/no values with 1/0
        if col_data.dtype == object:
            col_data = col_data.replace(['yes', 'no'], [1, 0])

        # If data type is categorical, convert to dummy variables
        if col_data.dtype == object:
            # Example: 'school' => 'school_GP' and 'school_MS'
            col_data = pd.get_dummies(col_data, prefix = col)

        # Collect the revised columns
        output = output.join(col_data)

    return output

X_all = preprocess_features(X_all)
print "Processed feature columns ({} total features):\n{}".format(len(X_all.columns), list(X_all.columns))Processed feature columns (48 total features):
['school_GP', 'school_MS', 'sex_F', 'sex_M', 'age', 'address_R', 'address_U', 'famsize_GT3', 'famsize_LE3', 'Pstatus_A', 'Pstatus_T', 'Medu', 'Fedu', 'Mjob_at_home', 'Mjob_health', 'Mjob_other', 'Mjob_services', 'Mjob_teacher', 'Fjob_at_home', 'Fjob_health', 'Fjob_other', 'Fjob_services', 'Fjob_teacher', 'reason_course', 'reason_home', 'reason_other', 'reason_reputation', 'guardian_father', 'guardian_mother', 'guardian_other', 'traveltime', 'studytime', 'failures', 'schoolsup', 'famsup', 'paid', 'activities', 'nursery', 'higher', 'internet', 'romantic', 'famrel', 'freetime', 'goout', 'Dalc', 'Walc', 'health', 'absences']
# Checking out the new features
X_all.head()_____no_output_____# Confirming they are all numerical - int8 or int64
X_all.dtypes.unique()_____no_output_____

### Implementation: Training and Testing Data Split
So far, we have converted all _categorical_ features into numeric values. For the next step, we split the data (both features and corresponding labels) into training and test sets. In the following code cell below, you will need to implement the following:
- Randomly shuffle and split the data (`X_all`, `y_all`) into training and testing subsets.
  - Use 300 training points (approximately 75%) and 95 testing points (approximately 25%).
  - Set a `random_state` for the function(s) you use, if provided.
  - Store the results in `X_train`, `X_test`, `y_train`, and `y_test`._____no_output_____

# TODO: Import any additional functionality you may need here
from sklearn.cross_validation import train_test_split

# TODO: Set the number of training points
num_train = 300

# Set the number of testing points
num_test = X_all.shape[0] - num_train

# TODO: Shuffle and split the dataset into the number of training and testing points above

X_train, X_test, y_train, y_test = train_test_split(X_all, y_all, test_size=num_test, random_state=1)

# Show the results of the split
print "Training set has {} samples.".format(X_train.shape[0])
print "Testing set has {} samples.".format(X_test.shape[0])Training set has 300 samples.
Testing set has 95 samples.

## Training and Evaluating Models
In this section, you will choose 3 supervised learning models that are appropriate for this problem and available in `scikit-learn`. You will first discuss the reasoning behind choosing these three models by considering what you know about the data and each model's strengths and weaknesses. You will then fit the model to varying sizes of training data (100 data points, 200 data points, and 300 data points) and measure the F1 score. You will need to produce three tables (one for each model) that shows the training set size, training time, prediction time, F1 score on the training set, and F1 score on the testing set.

**The following supervised learning models are currently available in** [`scikit-learn`](http://scikit-learn.org/stable/supervised_learning.html) **that you may choose from:**
- Gaussian Naive Bayes (GaussianNB)
- Decision Trees
- Ensemble Methods (Bagging, AdaBoost, Random Forest, Gradient Boosting)
- K-Nearest Neighbors (KNeighbors)
- Stochastic Gradient Descent (SGDC)
- Support Vector Machines (SVM)
- Logistic Regression_____no_output_____### Question 2 - Model Application
*List three supervised learning models that are appropriate for this problem. For each model chosen*
- Describe one real-world application in industry where the model can be applied. *(You may need to do a small bit of research for this — give references!)*
- What are the strengths of the model; when does it perform well?
- What are the weaknesses of the model; when does it perform poorly?
- What makes this model a good candidate for the problem, given what you know about the data?_____no_output_____
**Answer:**

**Decision trees:**

Real-world application: I can give a personal example of an application of decision trees in the aviation industry. Working as operations planning analyst at GRU Airport, I needed to develop an aircraft stand allocation simulation model. There were a lot of groups of rules to be considered, such as:

     - *Geometric rules* (max aircraft size per stand; adjacencies restrictions);

     - *Type of flight* (the available contact gates depends on whether the flight is domestic or international);

     - *Towing rules* (the simulator needs to be able to decide whether or not the aircraft turnaround will be broke by tows)

     - *Business rules* (to consider the preferable allocations of the airlines)

     - *Efficiency rules* (to emulate the expertise of the human allocators)

     - *Boundary rules* (as the simulator considers just one day, boundary rules are necessary)

Among all of them, the towing rules were the most difficult to model. The insights provided by the allocation personnel weren't enough. The simulator needed to be able to look to a turnaround (paired movement: arrival + departure, performed by the same aircraft) and tell if it:

     1- won't be towed

     2- will perform a single tow, from stand A to stand B

     3- will perform a multiple tow, from stand A to B, and from B to C.

I developed a decision tree model to perform this classification, extracted the rules from it and inserted them as inputs in the simulator. So this machine learning model was one of the pieces of a broader simulation model that was then presented at ANAC and is a benchmark in our sector.

Strengths: Trees are easy to understand and to visualize. Their outputs are rule-based, thus they can be translated into if-then-else rules and be incorporated in a broader range of softwares (as the airport simulator mentioned above). Other advantages, as listed in the scikit-learn documentation, are: little data preparation requirement; ability to handle both numerical and categorical data.

Weaknesses: They can create over-complex trees, thus causing overfitting and they also create biased trees if some classes dominate.

Decision trees are usually good candidates due to their simplicity and easiness to interpret. Even if their results aren't the best, good information can be extracted from the model, like an estimate of the relative importance of each feature.

**Support Vector Machines:**

Real-world application: As described by the website *Data Science Central*, SVMs can be applied to face detection, text and hypertext categorization, classification of images, bioinformatics (protein and cancer classification) and handwriting recognition.

Strengths: One of the main strengths of SVMs, as mentioned in the Udacity's classes, is their ability to use different kernel functions in order to run the algorithm in other domains, in which the data can be more easily separable. SVMs work with the concept of margin (they don't only look for lines that separate the data, but for the ones that do so with the biggest margins), so they are less prone to overfitting when comparing to other linear models. Another advantage, as listed by Kotu and Deshpande (2014), is the fact that, once an SVM model is built, small changes to the training data will not result in significant changes to the model coefficients as long as the support vectors do not change.

Weaknesses: As stated by Géron (2017), the training complexity is usually between O(n²) and O(n³). So it can get very slow when the number of training instances gets large. As mentioned by Kelleher and Namee (2015), they are not very interpretable, and, especially when kernel functions are used, it is very difficult to understand why a particular prediction has been made.

This model is a good candidate because, as described in the scikit-learn documentation, it is still effective in cases where the number of dimensions is large when compared with the number of samples, which is the case in this study.

**Ensemble Method - AdaBoost:**

Real-world application: The website *Medium* lists as real use cases of ensemble learning algorithms: reccomendation and marketing personalization engines as well as fraud detection systems.

Strengths: As described by Géron (2017), in general, ensemble modeling reduces the generalization error that arises due to overfitting the training set. This is possible, as explained in the scikit-learning documentation, because several weak learners (less prone to overfitting) are combined to produce a powerful ensemble.

Weaknesses: The weak learner needs to be chosen carefully. If the weak learner itself is over-complex, the ensemble method may not avoid overfitting.

This model is a good candidate because it tends to boost the results of base learners. In its default configuration, AdaBoost's base estimator is the DecisionTreeClassifier. So as we have chosen decision trees as one of our candidate models, it may be a good idea to take AdaBoost too and compare them.

**References:**

Books:

Kelleher, John; Namee, Brian. Fundamentals of Machine Learning for Predictive Data Analytics. MIT Press, 2015.

Kotu, Vijay; Deshpande, Bala. Predictive Analytics and Data Mining. Morgan Kaufmann, 2014.

Géron, Aurélien. Hands-On Machine Learning with Scikit-Learn and TensorFlow. O'Reilly, 2017.

Websites:

https://www.datasciencecentral.com/profiles/blogs/real-life-applications-of-support-vector-machines

https://medium.com/@ailabs/5-machine-learning-algorithms-and-their-proper-use-cases-a8cfd0cedb51

Other:

Scikit-learn documentation and Udacity's classes.

_____no_output_____### Setup
Run the code cell below to initialize three helper functions which you can use for training and testing the three supervised learning models you've chosen above. The functions are as follows:
- `train_classifier` - takes as input a classifier and training data and fits the classifier to the data.
- `predict_labels` - takes as input a fit classifier, features, and a target labeling and makes predictions using the F1 score.
- `train_predict` - takes as input a classifier, and the training and testing data, and performs `train_clasifier` and `predict_labels`.
 - This function will report the F1 score for both the training and testing data separately._____no_output_____

def train_classifier(clf, X_train, y_train):
    ''' Fits a classifier to the training data. '''

    # Start the clock, train the classifier, then stop the clock
    start = time()
    clf.fit(X_train, y_train)
    end = time()

    # Print the results
    print "Trained model in {:.4f} seconds".format(end - start)

def predict_labels(clf, features, target):
    ''' Makes predictions using a fit classifier based on F1 score. '''

    # Start the clock, make predictions, then stop the clock
    start = time()
    y_pred = clf.predict(features)
    end = time()

    # Print and return results
    print "Made predictions in {:.4f} seconds.".format(end - start)
    return f1_score(target.values, y_pred, pos_label='yes')

def train_predict(clf, X_train, y_train, X_test, y_test):
    ''' Train and predict using a classifer based on F1 score. '''

    # Indicate the classifier and the training set size
    print "Training a {} using a training set size of {}. . .".format(clf.__class__.__name__, len(X_train))

    # Train the classifier
    train_classifier(clf, X_train, y_train)

    # Print the results of prediction for both training and testing
    print "F1 score for training set: {:.4f}.".format(predict_labels(clf, X_train, y_train))
    print "F1 score for test set: {:.4f}.".format(predict_labels(clf, X_test, y_test))_____no_output_____

### Implementation: Model Performance Metrics
With the predefined functions above, you will now import the three supervised learning models of your choice and run the `train_predict` function for each one. Remember that you will need to train and predict on each classifier for three different training set sizes: 100, 200, and 300. Hence, you should expect to have 9 different outputs below — 3 for each model using the varying training set sizes. In the following code cell, you will need to implement the following:
- Import the three supervised learning models you've discussed in the previous section.
- Initialize the three models and store them in `clf_A`, `clf_B`, and `clf_C`.
 - Use a `random_state` for each model you use, if provided.
 - **Note:** Use the default settings for each model — you will tune one specific model in a later section.
- Create the different training set sizes to be used to train each model.
 - *Do not reshuffle and resplit the data! The new training points should be drawn from `X_train` and `y_train`.*
- Fit each model with each training set size and make predictions on the test set (9 in total).
**Note:** Three tables are provided after the following code cell which can be used to store your results._____no_output_____

# TODO: Import the three supervised learning models from sklearn
from sklearn.tree import DecisionTreeClassifier
from sklearn.svm import SVC
from sklearn.ensemble import AdaBoostClassifier

# TODO: Initialize the three models
clf_A = DecisionTreeClassifier(random_state=1)
clf_B = SVC(random_state=1)
clf_C = AdaBoostClassifier(random_state=1)

# TODO: Set up the training set sizes
X_train_100 = X_train.sample(n=100, random_state=1)
y_train_100 = y_train.sample(n=100, random_state=1)

X_train_200 = X_train.sample(n=200, random_state=1)
y_train_200 = y_train.sample(n=200, random_state=1)

X_train_300 = X_train
y_train_300 = y_train

# TODO: Execute the 'train_predict' function for each classifier and each training set size
# train_predict(clf, X_train, y_train, X_test, y_test)_____no_output_____# Classifier 1
train_predict(clf_A, X_train_100, y_train_100, X_test, y_test); print '\n'
train_predict(clf_A, X_train_200, y_train_200, X_test, y_test); print '\n'
train_predict(clf_A, X_train_300, y_train_300, X_test, y_test); print '\n'Training a DecisionTreeClassifier using a training set size of 100. . .
Trained model in 0.0000 seconds
Made predictions in 0.0000 seconds.
F1 score for training set: 1.0000.
Made predictions in 0.0020 seconds.
F1 score for test set: 0.7231.

Training a DecisionTreeClassifier using a training set size of 200. . .
Trained model in 0.0030 seconds
Made predictions in 0.0000 seconds.
F1 score for training set: 1.0000.
Made predictions in 0.0000 seconds.
F1 score for test set: 0.6719.

Training a DecisionTreeClassifier using a training set size of 300. . .
Trained model in 0.0040 seconds
Made predictions in 0.0000 seconds.
F1 score for training set: 1.0000.
Made predictions in 0.0000 seconds.
F1 score for test set: 0.6984.

# Classifier 2
train_predict(clf_B, X_train_100, y_train_100, X_test, y_test); print '\n'
train_predict(clf_B, X_train_200, y_train_200, X_test, y_test); print '\n'
train_predict(clf_B, X_train_300, y_train_300, X_test, y_test); print '\n'Training a SVC using a training set size of 100. . .
Trained model in 0.0270 seconds
Made predictions in 0.0010 seconds.
F1 score for training set: 0.8553.
Made predictions in 0.0010 seconds.
F1 score for test set: 0.8354.

Training a SVC using a training set size of 200. . .
Trained model in 0.0040 seconds
Made predictions in 0.0020 seconds.
F1 score for training set: 0.8660.
Made predictions in 0.0020 seconds.
F1 score for test set: 0.8408.

Training a SVC using a training set size of 300. . .
Trained model in 0.0450 seconds
Made predictions in 0.0060 seconds.
F1 score for training set: 0.8584.
Made predictions in 0.0020 seconds.
F1 score for test set: 0.8462.

# Classifier 3
train_predict(clf_C, X_train_100, y_train_100, X_test, y_test); print '\n'
train_predict(clf_C, X_train_200, y_train_200, X_test, y_test); print '\n'
train_predict(clf_C, X_train_300, y_train_300, X_test, y_test); print '\n'Training a AdaBoostClassifier using a training set size of 100. . .
Trained model in 0.0980 seconds
Made predictions in 0.0020 seconds.
F1 score for training set: 0.9855.
Made predictions in 0.0100 seconds.
F1 score for test set: 0.6612.

Training a AdaBoostClassifier using a training set size of 200. . .
Trained model in 0.0800 seconds
Made predictions in 0.0100 seconds.
F1 score for training set: 0.9135.
Made predictions in 0.0000 seconds.
F1 score for test set: 0.7188.

Training a AdaBoostClassifier using a training set size of 300. . .
Trained model in 0.1020 seconds
Made predictions in 0.0070 seconds.
F1 score for training set: 0.8578.
Made predictions in 0.0050 seconds.
F1 score for test set: 0.8116.

In order to get the estimates of processing times in a more robust way, it was applied the **%timeit magic command**:_____no_output_____

print 'Training times from Classifier 1 (training set sizes of 100, 200 and 300)'
%timeit clf_A.fit(X_train_100, y_train_100)
%timeit clf_A.fit(X_train_200, y_train_200)
%timeit clf_A.fit(X_train_300, y_train_300)

print '\nTraining times from Classifier 2 (training set sizes of 100, 200 and 300)'
%timeit clf_B.fit(X_train_100, y_train_100)
%timeit clf_B.fit(X_train_200, y_train_200)
%timeit clf_B.fit(X_train_300, y_train_300)

print '\nTraining times from Classifier 3 (training set sizes of 100, 200 and 300)'
%timeit clf_C.fit(X_train_100, y_train_100)
%timeit clf_C.fit(X_train_200, y_train_200)
%timeit clf_C.fit(X_train_300, y_train_300)Training times from Classifier 1 (training set sizes of 100, 200 and 300)
1000 loops, best of 3: 616 µs per loop
1000 loops, best of 3: 1.16 ms per loop
1000 loops, best of 3: 1.83 ms per loop

Training times from Classifier 2 (training set sizes of 100, 200 and 300)
1000 loops, best of 3: 1.05 ms per loop
100 loops, best of 3: 3.16 ms per loop
100 loops, best of 3: 6 ms per loop

Training times from Classifier 3 (training set sizes of 100, 200 and 300)
10 loops, best of 3: 76.5 ms per loop
10 loops, best of 3: 79.6 ms per loop
10 loops, best of 3: 84.4 ms per loop
print 'Prediction times (test) from Classifier 1 (training set sizes of 100, 200 and 300)'
clf_A.fit(X_train_100, y_train_100)
%timeit clf_A.predict(X_test)
clf_A.fit(X_train_200, y_train_200)
%timeit clf_A.predict(X_test)
clf_A.fit(X_train_300, y_train_300)
%timeit clf_A.predict(X_test)

print '\nPrediction times (test) from Classifier 2 (training set sizes of 100, 200 and 300)'
clf_B.fit(X_train_100, y_train_100)
%timeit clf_B.predict(X_test)
clf_B.fit(X_train_200, y_train_200)
%timeit clf_B.predict(X_test)
clf_B.fit(X_train_300, y_train_300)
%timeit clf_B.predict(X_test)

print '\nPrediction times (test) from Classifier 3 (training set sizes of 100, 200 and 300)'
clf_C.fit(X_train_100, y_train_100)
%timeit clf_C.predict(X_test)
clf_C.fit(X_train_200, y_train_200)
%timeit clf_C.predict(X_test)
clf_C.fit(X_train_300, y_train_300)
%timeit clf_C.predict(X_test)Prediction times (test) from Classifier 1 (training set sizes of 100, 200 and 300)
10000 loops, best of 3: 128 µs per loop
10000 loops, best of 3: 117 µs per loop
10000 loops, best of 3: 117 µs per loop

Prediction times (test) from Classifier 2 (training set sizes of 100, 200 and 300)
1000 loops, best of 3: 638 µs per loop
1000 loops, best of 3: 1.1 ms per loop
1000 loops, best of 3: 1.5 ms per loop

Prediction times (test) from Classifier 3 (training set sizes of 100, 200 and 300)
100 loops, best of 3: 4.17 ms per loop
100 loops, best of 3: 4.19 ms per loop
100 loops, best of 3: 4.16 ms per loop

### Tabular Results
Edit the cell below to see how a table can be designed in [Markdown](https://github.com/adam-p/markdown-here/wiki/Markdown-Cheatsheet#tables). You can record your results from above in the tables provided._____no_output_____** Classifer 1 - Decision Trees**

| Training Set Size | Training Time (s) | Prediction Time (test) (s) | F1 Score (train) | F1 Score (test) |
| :---------------: | :---------------------: | :--------------------: | :--------------: | :-------------: |
| 100               |         0.000616        |         0.000128       |      1.0000      |     0.7231      |
| 200               |         0.001160        |         0.000117       |      1.0000      |     0.6719      |
| 300               |         0.001830        |         0.000117       |      1.0000      |     0.6984      |

** Classifer 2 - Support Vector Machines**

| Training Set Size | Training Time (s) | Prediction Time (test) (s) | F1 Score (train) | F1 Score (test) |
| :---------------: | :---------------------: | :--------------------: | :--------------: | :-------------: |
| 100               |         0.001050        |         0.000638       |      0.8553      |     0.8354      |
| 200               |         0.003160        |         0.001100       |      0.8660      |     0.8408      |
| 300               |         0.006000        |         0.001500       |      0.8584      |     0.8462      |

** Classifer 3 - Ensemble Method: AdaBoost**

| Training Set Size | Training Time (s) | Prediction Time (test) (s) | F1 Score (train) | F1 Score (test) |
| :---------------: | :---------------------: | :--------------------: | :--------------: | :-------------: |
| 100               |         0.076500        |         0.004170       |      0.9855      |     0.6612      |
| 200               |         0.079600        |         0.004190       |      0.9135      |     0.7188      |
| 300               |         0.084400        |         0.004160       |      0.8578      |     0.8116      |_____no_output_____

# Analysing the Learning Curves
fig, axes = plt.subplots(1, 3, figsize=(15, 3))
f1_train_clf = pd.DataFrame([[1., 1., 1.], [0.8553, 0.8660, 0.8584], [0.9855, 0.9135, 0.8578]],
                            index=['clf_A', 'clf_B', 'clf_C'], columns=[100, 200, 300])

f1_test_clf = pd.DataFrame([[0.7231, 0.6719, 0.6984], [0.8354, 0.8408, 0.8462], [0.6612, 0.7188, 0.8116]],
                            index=['clf_A', 'clf_B', 'clf_C'], columns=[100, 200, 300])

for n in range(3):
    axes[n].plot(f1_train_clf.iloc[n], 'o-', label='F1 Score - Train')
    axes[n].plot(f1_test_clf.iloc[n], 'o-', label='F1 Score - Test')
    axes[n].set_ylim([0., 1.1])
    axes[n].set_xticks([100, 200, 300])
    axes[n].set_title('Classifier '+str(n+1))
    axes[n].set_xlabel('Training set size')
    axes[n].legend(loc='best')

print 'Learning Curves:'Learning Curves:

## Choosing the Best Model
In this final section, you will choose from the three supervised learning models the *best* model to use on the student data. You will then perform a grid search optimization for the model over the entire training set (`X_train` and `y_train`) by tuning at least one parameter to improve upon the untuned model's F1 score. _____no_output_____### Question 3 - Choosing the Best Model
*Based on the experiments you performed earlier, in one to two paragraphs, explain to the board of supervisors what single model you chose as the best model. Which model is generally the most appropriate based on the available data, limited resources, cost, and performance?*_____no_output_____
**Answer:**

The **SVM model was the chosen one**. It has achieved the best performance in terms of f1_score for all the training set sizes. This robustness relatively to the training set size is important if future available training sets are of the same magnitude. This behavior was expected because, as anticipated in question 2, SVM models are still effective in cases where the number of dimensions is large when compared with the number of samples. Even in the case of a training set of 100 samples, the 48-features SVM model has achieved a f1-score of 0.8354.

In terms of training times, the SVM models presented the higher complexity (a little below quadratic on the sample size). However, in absolute terms, it will only be an issue if future training sets are much larger than the one used in this study. In this context, with more than 10.000 samples, the process starts to be slow if you consider cross validation tuning with a large grid of parameters/values.

A weakness of the SVM model is that it is not very interpretable. Although you can identify the students that might need early intervention, you may not have valuable information about the causes of the imminent failures. The knowledge of the causes can be a good driver to establish better approaches o these early interventions (when and how to apply them). That's why we have also used decision tree's models. In the following cell we present the 3 most relevant features in order to segregate the ones that probably will pass and the ones that probably won't.
_____no_output_____

# The 3 most important features according to the decision tree's model
feature_ranking = pd.DataFrame({'importance': list(clf_A.feature_importances_), 'feature': list(X_train.columns)})
feature_ranking.set_index('feature', inplace=True)
feature_ranking.sort_values('importance', ascending=False)[:3]_____no_output_____

### Question 4 - Model in Layman's Terms
*In one to two paragraphs, explain to the board of directors in layman's terms how the final model chosen is supposed to work. Be sure that you are describing the major qualities of the model, such as how the model is trained and how the model makes a prediction. Avoid using advanced mathematical or technical jargon, such as describing equations or discussing the algorithm implementation.*_____no_output_____
**Answer:**

SVMs work on the principle of fitting boundaries to regions of points that are all alike (that is, belong to the same class). In our case, we have two classes: students that will pass and the ones that won't. The learning algorithm won't just look for lines that separate the data, but for the ones that do so with the biggest margins of safety. This characteristic is useful to increase the model's ability to keep good performances when it is tested with different datasets.

After the boundaries are set, predictions can be made just analysing in which region a new data point is located. If the features of a student constitutes a data point that falls in a region labeled by the model as one where students will pass, the model will predict that this student will do so.

Just to explain the name of this class of models, but not bothering will technical details, SVMs just use a core set of points do identify the boundaries. These data points are called support vectors because each one of them is a "vector" that "support" the boundary. This explains another strength of the model: small changes to the training data will not result in significant changes to the model coefficients as long as the support vectors do not change. This means the model tends to be more robust.
_____no_output_____### Implementation: Model Tuning
Fine tune the chosen model. Use grid search (`GridSearchCV`) with at least one important parameter tuned with at least 3 different values. You will need to use the entire training set for this. In the code cell below, you will need to implement the following:
- Import [`sklearn.grid_search.GridSearchCV`](http://scikit-learn.org/0.17/modules/generated/sklearn.grid_search.GridSearchCV.html) and [`sklearn.metrics.make_scorer`](http://scikit-learn.org/stable/modules/generated/sklearn.metrics.make_scorer.html).
- Create a dictionary of parameters you wish to tune for the chosen model.
 - Example: `parameters = {'parameter' : [list of values]}`.
- Initialize the classifier you've chosen and store it in `clf`.
- Create the F1 scoring function using `make_scorer` and store it in `f1_scorer`.
 - Set the `pos_label` parameter to the correct value!
- Perform grid search on the classifier `clf` using `f1_scorer` as the scoring method, and store it in `grid_obj`.
- Fit the grid search object to the training data (`X_train`, `y_train`), and store it in `grid_obj`._____no_output_____

# TODO: Import 'GridSearchCV' and 'make_scorer'
from sklearn.grid_search import GridSearchCV
from sklearn.metrics import make_scorer

# TODO: Create the parameters list you wish to tune
parameters = {'kernel': ['rbf', 'linear', 'poly', 'sigmoid'],
              'C': [0.6, 0.8, 1.0, 1.2, 1.4, 1.6],
              'gamma': ['auto', 0.05, 0.10, 0.15, 0.20]}

# TODO: Initialize the classifier
clf = SVC(random_state=1)

# TODO: Make an f1 scoring function using 'make_scorer'
f1_scorer = make_scorer(f1_score, pos_label='yes')

# TODO: Perform grid search on the classifier using the f1_scorer as the scoring method
grid_obj = GridSearchCV(estimator=clf, param_grid=parameters, scoring=f1_scorer)

# TODO: Fit the grid search object to the training data and find the optimal parameters
grid_obj = grid_obj.fit(X_train, y_train)

# Get the estimator
clf = grid_obj.best_estimator_

# Report the final F1 score for training and testing after parameter tuning
print "Tuned model has a training F1 score of {:.4f}.".format(predict_labels(clf, X_train, y_train))
print "Tuned model has a testing F1 score of {:.4f}.".format(predict_labels(clf, X_test, y_test))Made predictions in 0.0100 seconds.
Tuned model has a training F1 score of 0.9754.
Made predictions in 0.0000 seconds.
Tuned model has a testing F1 score of 0.8481.
# Taking a look at the final parameters after the tuning
clf_____no_output_____

### Question 5 - Final F1 Score
*What is the final model's F1 score for training and testing? How does that score compare to the untuned model?*_____no_output_____
**Answer:**

The final model's f1 score for training is **0.9754** and for testing is **0.8481**.

The final testing score is a little higher than the untuned model (that was 0.8462).

Three parameters were tested: **C**, **gamma** and the **kernel**.

The final model kept the default values for C and the kernel (*C=1* and *kernel=rbf*), but **changed the gamma value from 1/300 (default value) to 0.1**. The higher value for the gamma parameter means that each training point has a smaller area of influence when defining the boundary lines. This means that the points closer to the boundary have more relevance and can make it become more sinuous. **This can lead to overfitting and, judging by the large gap formed between training scores and test ones, that's what actually happened in our case**.

**I prefer to use the untuned model, with the default value of gamma**, that has almost the same performance with a total error more relative to the hypothesis set bias instead of its variance. This means that it has a higher probability to generalize well in future datasets, in other words, a higher probability to keep similiar good performances.
_____no_output_____> **Note**: Once you have completed all of the code implementations and successfully answered each question above, you may finalize your work by exporting the iPython Notebook as an HTML document. You can do this by using the menu above and navigating to
**File -> Download as -> HTML (.html)**. Include the finished document along with this notebook as your submission._____no_output_____


# Notebook from bmorris3/rms
Path: notebooks/hat7.ipynb

<code>
%matplotlib inline
%config InlineBackend.figure_format = 'retina'
import matplotlib.pyplot as plt
import numpy as np
from astropy.time import Time
import astropy.units as u
from astropy.utils.console import ProgressBar

from rms import STSP, Star, Spot, Planet_____no_output_____t0 = 2454954.357463
rotation_period = 25
inc_stellar = -20

transit_times = Time(t0, format='jd') + np.arange(-3/24, 3/24, 1/60/24) * u.day

planet = Planet.from_hat7()
planet.lam = 90

star = Star(planet=planet, rotation_period=rotation_period,
            inc_stellar=inc_stellar)_____no_output_____for attr in dir(star):
    if not attr.startswith('_'):
        print(attr, getattr(star, attr))

for attr in dir(planet):
    if not attr.startswith('_'):
        print(attr, getattr(planet, attr))inc_stellar -20
per_rot 25
planet <rms.planet.Planet object at 0x1119ac208>
spot_contrast 0.7
a 4.1502
b 0.5002125407344931
duration 0.12813004872191647
ecc 0.0
from_hat11 <bound method Planet.from_hat11 of <class 'rms.planet.Planet'>>
from_hat7 <bound method Planet.from_hat7 of <class 'rms.planet.Planet'>>
inc 83.111
lam 90
limb_dark quadratic
per 2.204737
rp 0.058330305324663184
t0 2454954.357463
u [0.3525, 0.168]
w 0.0
n_transits = 100
fluxes = np.zeros((len(transit_times), n_transits))
times = np.zeros((len(transit_times), n_transits))

no_transit_spot = Spot(radius=0.0001, latitude=-88*u.deg, longitude=0*u.deg)

i = 0
with ProgressBar(n_transits, ipython_widget=True) as bar:
    while i < n_transits:

        # Draw spots from the sunspot distribution
        n_spots = 50
        spots = [Spot.from_sunspot_distribution(mean_latitude=20) for i in range(n_spots)]

        # Draw observing times from uniform distribution
        random_times = transit_times + 1 * (np.random.rand() - 0.5) * u.min

        with STSP(random_times, star, spots, quiet=True) as stsp:
            lc = stsp.generate_lightcurve(n_ld_rings=100)

        with STSP(random_times, star, no_transit_spot, quiet=True) as stsp:
            no_spot_lc = stsp.generate_lightcurve(n_ld_rings=100)

        # If STSP ran successfully:
        f_test = lc.fluxes if not hasattr(lc.fluxes, 'value') else lc.fluxes.value

        if not np.any(np.isnan(lc.fluxes)):
            oot = lc.mask_out_of_transit(star, flip=True)
            oot_fluxes = oot['fluxes']
            median_oot_flux = np.median(oot_fluxes)

            min_length = min([len(lc.fluxes), len(no_spot_lc.fluxes)])
            #fluxes[:, i] = lc.fluxes / median_oot_flux - no_spot_lc.fluxes
            fluxes[:, i] = lc.fluxes / median_oot_flux - no_spot_lc.fluxes
            times[:, i] = random_times.jd

#             # Remove linear polynomial to OOT
#             oot_mask = ((random_times.jd < t0 - planet.duration/2) |
#                         (random_times.jd > t0 + planet.duration/2))
#             fit_params = np.polyfit(times[:, i][oot_mask] -
#                                     times[:, i][oot_mask].mean(),
#                                     fluxes[:, i][oot_mask], 1)
#             fluxes[:, i] -= np.polyval(fit_params, times[:, i] - times[:, i].mean())
    #         fluxes[:, i] -= np.median(fluxes[:, i][oot_mask])
            i += 1
            bar.update()
fig, ax = plt.subplots(1, 3, figsize=(10, 4))
ax[0].hist([s.latitude.to(u.deg).value for s in spots])
ax[0].set_title('lat')
ax[1].set_title('lon')
ax[1].hist([s.longitude.to(u.deg).value for s in spots])

ax[2].set_title("rad")
ax[2].hist([s.radius for s in spots])
plt.show()_____no_output_____mf = np.median(fluxes, axis=1)

for t, f in zip(times.T, fluxes.T):
    plt.plot_date(Time(t, format='jd').plot_date, f*1e6, fmt='.', alpha=1)#, color='k')
#plt.ylim([-0.003, 0.001])
ax = plt.gca()
ax.set(xlabel='Time (MM-DD HH)', ylabel='Transit Residuals [ppm]')
plt.setp(ax.get_xticklabels(), rotation=45, ha='right')
#plt.savefig('sun_with_h11_residuals.png', bbox_inches='tight', dpi=150)_____no_output_____from scipy.ndimage import gaussian_filter1d
from scipy.stats import binned_statistic

t, f = times.ravel(), fluxes.ravel()
f = f[np.argsort(t)]
t = t[np.argsort(t)]

bs = binned_statistic(t, f, statistic='mean', bins=50)
bincenters = 0.5*(bs.bin_edges[1:] + bs.bin_edges[:-1])
plt.plot(t, 1e6*f, '.')
plt.plot(bincenters, 1e6*bs.statistic, 'r', lw=4)
#plt.plot(t, gaussian_filter1d(f, 50), lw=4)_____no_output_____
</code>