# Notebook from ThomasRanvier/machine_learning
Path: ml_supervised_learning/tp_as_ranvier_desilets.ipynb

import numpy as np
import pandas as pd
import warnings
import matplotlib.pyplot as plt
warnings.filterwarnings('ignore')
from sklearn.model_selection import train_test_split
from sklearn import tree, metrics, neighbors, preprocessing,\
    decomposition, ensemble, model_selection, pipeline,\
    naive_bayes, neural_network, feature_extraction
from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer, TfidfVectorizer
import time
import copy
import pickle
from gensim import models
import re_____no_output_____

## I. Apprentissage supervisé : Feature engineering et Classification_____no_output_____### Chargement des données et préparation_____no_output_____

credit_scoring_data = pd.read_csv('credit_scoring.csv', sep=';')_____no_output_____credit_scoring_data.head()_____no_output_____# Data
X = credit_scoring_data.ix[:, :-1].values
# Target
y = credit_scoring_data.ix[:, -1].values_____no_output_____print('Dimensions des données :', X.shape)
print('Dimensions des targets :', y.shape)
plt.hist(y, bins=3)
plt.title('Histogramme des targets')
plt.show()Dimensions des données : (4375, 13)
Dimensions des targets : (4375,)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.5, random_state=1)
print('Dimensions de X_train :', X_train.shape)
print('Dimensions de X_test :', X_test.shape)
print('Dimensions de y_train :', y_train.shape)
print('Dimensions de y_test :', y_test.shape)Dimensions de X_train : (2187, 13)
Dimensions de X_test : (2188, 13)
Dimensions de y_train : (2187,)
Dimensions de y_test : (2188,)

### Apprentissage et évaluation de modèles_____no_output_____#### Normalisation des données_____no_output_____

# StandardScaler
SS = preprocessing.StandardScaler()
SS.fit(X_train)
X_train_ss = SS.transform(X_train)
SS.fit(X_test)
X_test_ss = SS.transform(X_test)

# MinMaxScaler
MM = preprocessing.MinMaxScaler()
MM.fit(X_train)
X_train_mm = MM.transform(X_train)
MM.fit(X_test)
X_test_mm = MM.transform(X_test)_____no_output_____

#### Evaluation des modèles_____no_output_____

def evaluate(model, X_train, X_test, y_train, y_test, save_to = ''):
    model.fit(X_train, y_train)
    y_predict = model.predict(X_test)
    acc = metrics.accuracy_score(y_test, y_predict)
    pre = metrics.precision_score(y_test, y_predict)
    rec = metrics.recall_score(y_test, y_predict)

    print('Accuracy du modèle :', acc)
    print('Precision du modèle :', pre)
    print('Recall du modèle :', rec)
    print()
    print('Matrice de confusion :')
    print()
    print(pd.DataFrame(
        metrics.confusion_matrix(y_test, y_predict),
        columns=['Prédit 0', 'Prédit 1'],
        index=['Vrai 0', 'Vrai 1']
    ))

    if len(save_to) > 0:
        with open(save_to, 'wb') as handle:
            pickle.dump(model, handle)_____no_output_____

#### CART_____no_output_____

cart = tree.DecisionTreeClassifier(random_state=1)
print('Données non normalisées :\n')
evaluate(cart, X_train, X_test, y_train, y_test)
print('\n\nDonnées normalisées avec StandardScaler :\n')
evaluate(cart, X_train_ss, X_test_ss, y_train, y_test)
print('\n\nDonnées normalisées avec MinMaxScaler :\n')
evaluate(cart, X_train_mm, X_test_mm, y_train, y_test)Données non normalisées :

Accuracy du modèle : 0.7271480804387569
Precision du modèle : 0.8194174757281554
Recall du modèle : 0.7992424242424242

Matrice de confusion :

        Prédit 0  Prédit 1
Vrai 0       325       279
Vrai 1       318      1266

Données normalisées avec StandardScaler :

Accuracy du modèle : 0.7180073126142597
Precision du modèle : 0.8183015141540487
Recall du modèle : 0.7847222222222222

Matrice de confusion :

        Prédit 0  Prédit 1
Vrai 0       328       276
Vrai 1       341      1243

Données normalisées avec MinMaxScaler :

Accuracy du modèle : 0.7221206581352834
Precision du modèle : 0.8116219667943806
Recall du modèle : 0.80239898989899

Matrice de confusion :

        Prédit 0  Prédit 1
Vrai 0       309       295
Vrai 1       313      1271

#### KNN_____no_output_____

knn = neighbors.KNeighborsClassifier(n_neighbors=5)
print('Données non normalisées :\n')
evaluate(knn, X_train, X_test, y_train, y_test)
print('\n\nDonnées normalisées avec StandardScaler :\n')
evaluate(knn, X_train_ss, X_test_ss, y_train, y_test)
print('\n\nDonnées normalisées avec MinMaxScaler :\n')
evaluate(knn, X_train_mm, X_test_mm, y_train, y_test)Données non normalisées :

Accuracy du modèle : 0.7248628884826326
Precision du modèle : 0.7709713024282561
Recall du modèle : 0.8819444444444444

Matrice de confusion :

        Prédit 0  Prédit 1
Vrai 0       189       415
Vrai 1       187      1397

Données normalisées avec StandardScaler :

Accuracy du modèle : 0.7495429616087751
Precision du modèle : 0.8087008343265792
Recall du modèle : 0.8566919191919192

Matrice de confusion :

        Prédit 0  Prédit 1
Vrai 0       283       321
Vrai 1       227      1357

Données normalisées avec MinMaxScaler :

Accuracy du modèle : 0.7513711151736746
Precision du modèle : 0.8002309468822171
Recall du modèle : 0.875

Matrice de confusion :

        Prédit 0  Prédit 1
Vrai 0       258       346
Vrai 1       198      1386

### Création de nouvelles variables_____no_output_____

pca = decomposition.PCA(n_components=.9)

pca.fit(X_train_ss)
X_train_pca_ss = pca.transform(X_train_ss)
X_train_pca_ss = np.append(X_train_ss[:], X_train_pca_ss[:, :3], axis=1)

pca.fit(X_train_mm)
X_train_pca_mm = pca.transform(X_train_mm)
X_train_pca_mm = np.append(X_train_mm[:], X_train_pca_mm[:, :3], axis=1)

pca.fit(X_test_ss)
X_test_pca_ss = pca.transform(X_test_ss)
X_test_pca_ss = np.append(X_test_ss[:], X_test_pca_ss[:, :3], axis=1)

pca.fit(X_test_mm)
X_test_pca_mm = pca.transform(X_test_mm)
X_test_pca_mm = np.append(X_test_mm[:], X_test_pca_mm[:, :3], axis=1)_____no_output_____

#### CART_____no_output_____

print('Données normalisées avec StandardScaler + PCA best variables :\n')
evaluate(cart, X_train_pca_ss, X_test_pca_ss, y_train, y_test)
print('\n\nDonnées normalisées avec MinMaxScaler + PCA best variables :\n')
evaluate(cart, X_train_pca_mm, X_test_pca_mm, y_train, y_test)Données normalisées avec StandardScaler + PCA best variables :

Accuracy du modèle : 0.7239488117001828
Precision du modèle : 0.8279785809906292
Recall du modèle : 0.7809343434343434

Matrice de confusion :

        Prédit 0  Prédit 1
Vrai 0       347       257
Vrai 1       347      1237

Données normalisées avec MinMaxScaler + PCA best variables :

Accuracy du modèle : 0.6992687385740403
Precision du modèle : 0.8050065876152833
Recall du modèle : 0.7714646464646465

Matrice de confusion :

        Prédit 0  Prédit 1
Vrai 0       308       296
Vrai 1       362      1222

#### KNN_____no_output_____

print('Données normalisées avec StandardScaler + PCA best variables :\n')
evaluate(knn, X_train_pca_ss, X_test_pca_ss, y_train, y_test)
print('\n\nDonnées normalisées avec MinMaxScaler + PCA best variables :\n')
evaluate(knn, X_train_pca_mm, X_test_pca_mm, y_train, y_test)Données normalisées avec StandardScaler + PCA best variables :

Accuracy du modèle : 0.7600548446069469
Precision du modèle : 0.8101933216168717
Recall du modèle : 0.8731060606060606

Matrice de confusion :

        Prédit 0  Prédit 1
Vrai 0       280       324
Vrai 1       201      1383

Données normalisées avec MinMaxScaler + PCA best variables :

Accuracy du modèle : 0.7458866544789763
Precision du modèle : 0.7900677200902935
Recall du modèle : 0.8838383838383839

Matrice de confusion :

        Prédit 0  Prédit 1
Vrai 0       232       372
Vrai 1       184      1400

### Sélection de variables_____no_output_____

clf = ensemble.RandomForestClassifier(n_estimators=100)
clf.fit(X_train_pca_ss, y_train)
importances = clf.feature_importances_
std = np.std([tree.feature_importances_ for tree in clf.estimators_],axis=0)
sorted_idx = np.argsort(importances)[::-1]
features = np.append(credit_scoring_data.columns[:-1].values, ['pca_1', 'pca_2', 'pca_3'])
print(features[sorted_idx])
padding = np.arange(X_train_pca_ss.size/len(X_train_pca_ss)) + 0.5
plt.barh(padding, importances[sorted_idx],xerr=std[sorted_idx], align='center')
plt.yticks(padding, features[sorted_idx])
plt.xlabel("Relative Importance")
plt.title("Variable Importance")
plt.show()['Income' 'Seniority' 'pca_2' 'pca_3' 'Price' 'pca_1' 'Amount' 'Age'
 'Records' 'Assets' 'Expenses' 'Job' 'Time' 'Debt' 'Home' 'Marital']
scores = np.zeros(X_train_pca_ss.shape[1] + 1)
for f in np.arange(0, X_train_pca_ss.shape[1] + 1):
    X1_f = X_train_pca_ss[:,sorted_idx[:f+1]]
    X2_f = X_test_pca_ss[:,sorted_idx[:f+1]]
    knn.fit(X1_f,y_train)
    yknn = knn.predict(X2_f)
    scores[f] = np.round(metrics.accuracy_score(y_test, yknn), 3)
plt.plot(scores)
plt.xlabel("Nombre de Variables")
plt.ylabel("Accuracy")
plt.title("Evolution de l'accuracy en fonction des variables")
plt.show()_____no_output_____clf = ensemble.RandomForestClassifier(n_estimators=100)
clf.fit(X_train_pca_mm, y_train)
importances = clf.feature_importances_
std = np.std([tree.feature_importances_ for tree in clf.estimators_],axis=0)
sorted_idx = np.argsort(importances)[::-1]
features = np.append(credit_scoring_data.columns[:-1].values, ['pca_1', 'pca_2', 'pca_3'])
print(features[sorted_idx])
padding = np.arange(X_train_pca_mm.size/len(X_train_pca_mm)) + 0.5
plt.barh(padding, importances[sorted_idx],xerr=std[sorted_idx], align='center')
plt.yticks(padding, features[sorted_idx])
plt.xlabel("Relative Importance")
plt.title("Variable Importance")
plt.show()['pca_3' 'Income' 'pca_2' 'Seniority' 'pca_1' 'Price' 'Amount' 'Age'
 'Assets' 'Expenses' 'Records' 'Time' 'Debt' 'Job' 'Marital' 'Home']
scores = np.zeros(X_train_pca_mm.shape[1] + 1)
for f in np.arange(0, X_train_pca_mm.shape[1] + 1):
    X1_f = X_train_pca_mm[:,sorted_idx[:f+1]]
    X2_f = X_test_pca_mm[:,sorted_idx[:f+1]]
    knn.fit(X1_f,y_train)
    yknn = knn.predict(X2_f)
    scores[f] = np.round(metrics.accuracy_score(y_test, yknn), 3)
plt.plot(scores)
plt.xlabel("Nombre de Variables")
plt.ylabel("Accuracy")
plt.title("Evolution de l'accuracy en fonction des variables")
plt.show()_____no_output_____

### Paramétrage des classifieurs_____no_output_____#### CART_____no_output_____

criterions = ['gini', 'entropy']
depths = list(range(1, 15))
tunable_params = {'criterion': criterions, 'max_depth': depths}
grid = model_selection.GridSearchCV(tree.DecisionTreeClassifier(), param_grid=tunable_params, scoring='accuracy')
grid.fit(X_train_pca_ss, y_train)

for i, criterion in enumerate(criterions):
    plt.plot(depths, grid.cv_results_['mean_test_score'][i*len(depths):(i+1)*len(depths)])
    plt.title('Scores du CART pour données normalisée avec StandardScaler, criterion : ' + criterion)
    plt.show()
    print('Meilleur score obtenu :', max(grid.cv_results_['mean_test_score'][i*len(depths):(i+1)*len(depths)]),
          ' pour max_depth =', np.argmax(grid.cv_results_['mean_test_score'][i*len(depths):(i+1)*len(depths)]) + 1)_____no_output_____criterions = ['gini', 'entropy']
depths = list(range(1, 15))
tunable_params = {'criterion': criterions, 'max_depth': depths}
grid = model_selection.GridSearchCV(tree.DecisionTreeClassifier(), param_grid=tunable_params, scoring='accuracy')
grid.fit(X_train_pca_mm, y_train)

for i, criterion in enumerate(criterions):
    plt.plot(depths, grid.cv_results_['mean_test_score'][i*len(depths):(i+1)*len(depths)])
    plt.title('Scores du CART pour données normalisée avec MinMaxScaler, criterion : ' + criterion)
    plt.show()
    print('Meilleur score obtenu :', max(grid.cv_results_['mean_test_score'][i*len(depths):(i+1)*len(depths)]),
          ' pour max_depth =', np.argmax(grid.cv_results_['mean_test_score'][i*len(depths):(i+1)*len(depths)]) + 1)_____no_output_____

#### KNN_____no_output_____

k_range = list(range(1, 40))
tunable_params = {'n_neighbors': k_range}
grid = model_selection.GridSearchCV(neighbors.KNeighborsClassifier(), param_grid=tunable_params, scoring='accuracy')
grid.fit(X_train_pca_ss, y_train)
plt.plot(k_range, grid.cv_results_['mean_test_score'])
plt.title('Scores du KNN pour données normalisée avec StandardScaler, selon k')
plt.show()
print('Meilleur score obtenu :', max(grid.cv_results_['mean_test_score']),
      ' pour k =', np.argmax(grid.cv_results_['mean_test_score']) + 1)_____no_output_____k_range = list(range(1, 40))
tunable_params = {'n_neighbors': k_range}
grid = model_selection.GridSearchCV(neighbors.KNeighborsClassifier(), param_grid=tunable_params, scoring='accuracy')
grid.fit(X_train_pca_mm, y_train)
plt.plot(k_range, grid.cv_results_['mean_test_score'])
plt.title('Scores du KNN pour données normalisée avec MinMaxScaler, selon k')
plt.show()
print('Meilleur score obtenu :', max(grid.cv_results_['mean_test_score']),
      ' pour k =', np.argmax(grid.cv_results_['mean_test_score']) + 1)_____no_output_____

### Création d’un pipeline

#### On ne conserve que le StandardScaler comme technique de normalisation puisqu'elle donne globalement des meilleurs résultats dans notre cas
#### On sauvegarde les deux modèles entrainés sur les données de training, on pourra ensuite les charger à nouveau._____no_output_____

pipe = pipeline.Pipeline(steps=[
    ('scaler', preprocessing.StandardScaler()),
    ('pca', decomposition.PCA(n_components=.9)),
    ('model', neighbors.KNeighborsClassifier(n_neighbors=19)),
])

# Saves the model trained on X_train and y_train
evaluate(pipe, X_train, X_test, y_train, y_test, save_to='knn_k19.pickle')

print('\n')

pipe = pipeline.Pipeline(steps=[
    ('scaler', preprocessing.StandardScaler()),
    ('pca', decomposition.PCA(n_components=.9)),
    ('model', tree.DecisionTreeClassifier(random_state=1)),
])

# Saves the model trained on X_train and y_train
evaluate(pipe, X_train, X_test, y_train, y_test, save_to='scart.pickle')Accuracy du modèle : 0.7728519195612431
Precision du modèle : 0.810394060536836
Recall du modèle : 0.8958333333333334

Matrice de confusion :

        Prédit 0  Prédit 1
Vrai 0       272       332
Vrai 1       165      1419

Accuracy du modèle : 0.6937842778793418
Precision du modèle : 0.7967532467532468
Recall du modèle : 0.7746212121212122

Matrice de confusion :

        Prédit 0  Prédit 1
Vrai 0       291       313
Vrai 1       357      1227

#### Chargement du modèle précédemment entrainé et prédictions sur les mêmes données qu'au dessus.
#### On obtient bien les mêmes résultats_____no_output_____

with open('knn_k19.pickle', 'rb') as handle:
    model = pickle.load(handle)
    y_predict = model.predict(X_test)
    acc = metrics.accuracy_score(y_test, y_predict)
    pre = metrics.precision_score(y_test, y_predict)
    rec = metrics.recall_score(y_test, y_predict)

    print('Accuracy du modèle :', acc)
    print('Precision du modèle :', pre)
    print('Recall du modèle :', rec)
    print()
    print('Matrice de confusion :')
    print()
    print(pd.DataFrame(
        metrics.confusion_matrix(y_test, y_predict),
        columns=['Prédit 0', 'Prédit 1'],
        index=['Vrai 0', 'Vrai 1']
    ))Accuracy du modèle : 0.7728519195612431
Precision du modèle : 0.810394060536836
Recall du modèle : 0.8958333333333334

Matrice de confusion :

        Prédit 0  Prédit 1
Vrai 0       272       332
Vrai 1       165      1419

### Comparaison de plusieurs algorithmes d’apprentissage_____no_output_____

def run_classifiers(clfs, X, y):
    for i in clfs:
        clf = clfs[i]
        print('-----------------')
        print(i)
        start = time.time()
        cv_acc = model_selection.cross_val_score(clf, copy.deepcopy(X), copy.deepcopy(y), cv=10)
        end = time.time()
        print("Accuracy: {1:.3f} +/- {2:.3f}".format(i, np.mean(cv_acc), np.std(cv_acc)))
        cv_acc = model_selection.cross_val_score(clf, copy.deepcopy(X), copy.deepcopy(y), cv=5, scoring='roc_auc')
        print("AUC: {1:.3f} +/- {2:.3f}".format(i, np.mean(cv_acc), np.std(cv_acc)))
        # Plus important que les oui soient vrais donc on choisit de conserver la precision et non le rappel
        cv_acc = model_selection.cross_val_score(clf, copy.deepcopy(X), copy.deepcopy(y), cv=5, scoring='precision')
        print("Precision: {1:.3f} +/- {2:.3f}".format(i, np.mean(cv_acc), np.std(cv_acc)))
        print("Exec time for a single run: {1:.3f}s".format(i, (end - start) / 10))_____no_output_____clfs = {
    'Bayes': naive_bayes.GaussianNB(),
    'RF': ensemble.RandomForestClassifier(n_estimators=50, random_state=1),
    'KNN': neighbors.KNeighborsClassifier(n_neighbors=19),
    'CART': tree.DecisionTreeClassifier(random_state=1),
    'AdaBoost': ensemble.AdaBoostClassifier(tree.DecisionTreeClassifier(max_depth=1), algorithm="SAMME", n_estimators=50),
    'MLP': neural_network.MLPClassifier(solver='lbfgs', alpha=1e-5, hidden_layer_sizes=(20, 10), random_state=1),
    'Bagging': ensemble.BaggingClassifier(n_estimators=50, random_state=1)
}

run_classifiers(clfs, X_train_pca_ss, y_train)-----------------
Bayes
Accuracy: 0.737 +/- 0.019
AUC: 0.777 +/- 0.016
Precision: 0.817 +/- 0.015
Exec time for a single run: 0.004s
-----------------
RF
Accuracy: 0.764 +/- 0.023
AUC: 0.815 +/- 0.020
Precision: 0.810 +/- 0.014
Exec time for a single run: 0.342s
-----------------
KNN
Accuracy: 0.764 +/- 0.023
AUC: 0.788 +/- 0.027
Precision: 0.799 +/- 0.019
Exec time for a single run: 0.053s
-----------------
CART
Accuracy: 0.696 +/- 0.030
AUC: 0.626 +/- 0.002
Precision: 0.791 +/- 0.002
Exec time for a single run: 0.032s
-----------------
AdaBoost
Accuracy: 0.780 +/- 0.023
AUC: 0.826 +/- 0.024
Precision: 0.817 +/- 0.016
Exec time for a single run: 0.254s
-----------------
MLP
Accuracy: 0.725 +/- 0.022
AUC: 0.765 +/- 0.023
Precision: 0.810 +/- 0.012
Exec time for a single run: 0.727s
-----------------
Bagging
Accuracy: 0.760 +/- 0.024
AUC: 0.801 +/- 0.019
Precision: 0.810 +/- 0.012
Exec time for a single run: 0.971s

#### Nous avons préféré conserver la precision plutôt que le rappel, il nous semble plus intéressant que la majorité des vrais prédits soient positifs.
#### On s'apperçoit que dans notre cas le classifier le plus performant est l'AdaBoost avec la meilleure accuracy, meilleure AUC et meilleure precision._____no_output_____## II. Apprentissage supervisé : Données hétérogènes_____no_output_____### Traitement en ne considérant qe les valeurs numériques_____no_output_____

credit_data = pd.read_table('credit.data', header=None)
# Replace ? with nan
credit_data = credit_data.replace('?', np.nan)
# Replace + and - with 1 and 0
credit_data = credit_data.replace('+', 1)
credit_data = credit_data.replace('-', 0)
# Drop nan
credit_data = credit_data.dropna()
credit_data = credit_data._get_numeric_data()_____no_output_____credit_data.head()_____no_output_____# Data
X = credit_data.ix[:, :14].values
# Target
y = credit_data.ix[:, 15].values_____no_output_____print('Dimensions des données :', X.shape)
print('Dimensions des targets :', y.shape)
plt.hist(y, bins=3)
plt.title('Histogramme des targets')
plt.show()Dimensions des données : (653, 4)
Dimensions des targets : (653,)
print('Dimensions de X :', X.shape)
print('Dimensions de y :', y.shape)Dimensions de X : (653, 4)
Dimensions de y : (653,)
# StandardScaler
SS = preprocessing.StandardScaler()
SS.fit(X)
X_ss = SS.transform(X)

# PCA
pca = decomposition.PCA(n_components=.9)

pca.fit(X_ss)
X_pca_ss = pca.transform(X_ss)
X_pca_ss = np.append(X_ss[:], X_pca_ss[:, :3], axis=1)_____no_output_____run_classifiers(clfs, X_pca_ss, y)-----------------
Bayes
Accuracy: 0.720 +/- 0.063
AUC: 0.805 +/- 0.054
Precision: 0.845 +/- 0.101
Exec time for a single run: 0.003s
-----------------
RF
Accuracy: 0.731 +/- 0.069
AUC: 0.792 +/- 0.058
Precision: 0.716 +/- 0.067
Exec time for a single run: 0.131s
-----------------
KNN
Accuracy: 0.753 +/- 0.076
AUC: 0.808 +/- 0.066
Precision: 0.859 +/- 0.086
Exec time for a single run: 0.009s
-----------------
CART
Accuracy: 0.698 +/- 0.053
AUC: 0.696 +/- 0.069
Precision: 0.667 +/- 0.078
Exec time for a single run: 0.007s
-----------------
AdaBoost
Accuracy: 0.778 +/- 0.067
AUC: 0.830 +/- 0.065
Precision: 0.796 +/- 0.056
Exec time for a single run: 0.136s
-----------------
MLP
Accuracy: 0.737 +/- 0.054
AUC: 0.779 +/- 0.061
Precision: 0.718 +/- 0.054
Exec time for a single run: 0.296s
-----------------
Bagging
Accuracy: 0.756 +/- 0.075
AUC: 0.791 +/- 0.062
Precision: 0.741 +/- 0.063
Exec time for a single run: 0.219s

### Traitement en considérant toutes les données_____no_output_____

credit_data = pd.read_table('credit.data', header=None)
# Replace + and - with 1 and 0
credit_data = credit_data.replace('+', 1)
credit_data = credit_data.replace('-', 0)

# Data
X = credit_data.ix[:, :14].values
# Target
y = credit_data.ix[:, 15].values

# Define columns
col_num = [1, 2, 7, 10, 13, 14]
col_cat = [0, 3, 4, 5, 6, 8, 9, 11, 12]_____no_output_____credit_data.head()_____no_output_____# Handle categorial columns
X_cat = np.copy(X)
X_cat = X_cat.take(col_cat, axis=1)
for col_id in range(len(col_cat)):
    unique_val, val_idx = np.unique(X_cat[:, col_id], return_inverse=True)
    X_cat[:, col_id] = val_idx
imp_cat = preprocessing.Imputer(missing_values=0, strategy='most_frequent')
X_cat[:, range(5)] = imp_cat.fit_transform(X_cat[:, range(5)])

# Handle numeric columns
X_num = np.copy(X[:, col_num])
X_num[X_num == '?'] = np.nan
X_num = X_num.astype(float)
imp_num = preprocessing.Imputer(missing_values=np.nan, strategy='mean')
X_num = imp_num.fit_transform(X_num)

# StandardScaler
SS = preprocessing.StandardScaler()
SS.fit(X_num)
X_num_ss = SS.transform(X_num)

# PCA
pca = decomposition.PCA(n_components=.9)

pca.fit(X_num_ss)
X_num_pca_ss = pca.transform(X_num_ss)
X_num_pca_ss = np.append(X_num_ss[:], X_num_pca_ss[:, :3], axis=1)C:\Users\Thomas Ranvier\Anaconda3\lib\site-packages\sklearn\utils\deprecation.py:66: DeprecationWarning: Class Imputer is deprecated; Imputer was deprecated in version 0.20 and will be removed in 0.22. Import impute.SimpleImputer from sklearn instead.
  warnings.warn(msg, category=DeprecationWarning)
C:\Users\Thomas Ranvier\Anaconda3\lib\site-packages\sklearn\utils\deprecation.py:66: DeprecationWarning: Class Imputer is deprecated; Imputer was deprecated in version 0.20 and will be removed in 0.22. Import impute.SimpleImputer from sklearn instead.
  warnings.warn(msg, category=DeprecationWarning)
# Convert categorial values to binary
X_cat_bin = preprocessing.OneHotEncoder().fit_transform(X_cat).toarray()_____no_output_____# Construct back the whole dataset set and run all classifiers
X_total = np.concatenate((X_cat, X_num_pca_ss), axis=1)

run_classifiers(clfs, X_total, y)-----------------
Bayes
Accuracy: 0.768 +/- 0.071
AUC: 0.877 +/- 0.065
Precision: 0.830 +/- 0.112
Exec time for a single run: 0.007s
-----------------
RF
Accuracy: 0.846 +/- 0.128
AUC: 0.914 +/- 0.073
Precision: 0.843 +/- 0.155
Exec time for a single run: 0.135s
-----------------
KNN
Accuracy: 0.773 +/- 0.077
AUC: 0.844 +/- 0.066
Precision: 0.808 +/- 0.121
Exec time for a single run: 0.013s
-----------------
CART
Accuracy: 0.782 +/- 0.102
AUC: 0.792 +/- 0.084
Precision: 0.772 +/- 0.138
Exec time for a single run: 0.011s
-----------------
AdaBoost
Accuracy: 0.838 +/- 0.147
AUC: 0.915 +/- 0.078
Precision: 0.836 +/- 0.175
Exec time for a single run: 0.193s
-----------------
MLP
Accuracy: 0.827 +/- 0.111
AUC: 0.899 +/- 0.066
Precision: 0.856 +/- 0.149
Exec time for a single run: 0.310s
-----------------
Bagging
Accuracy: 0.839 +/- 0.132
AUC: 0.903 +/- 0.084
Precision: 0.817 +/- 0.148
Exec time for a single run: 0.307s
# Test with binarized cat columns
X_total_bin = np.concatenate((X_cat_bin, X_num_pca_ss), axis=1)

run_classifiers(clfs, X_total_bin, y)-----------------
Bayes
Accuracy: 0.724 +/- 0.047
AUC: 0.863 +/- 0.052
Precision: 0.854 +/- 0.099
Exec time for a single run: 0.003s
-----------------
RF
Accuracy: 0.849 +/- 0.137
AUC: 0.908 +/- 0.079
Precision: 0.847 +/- 0.163
Exec time for a single run: 0.144s
-----------------
KNN
Accuracy: 0.839 +/- 0.102
AUC: 0.894 +/- 0.076
Precision: 0.863 +/- 0.136
Exec time for a single run: 0.017s
-----------------
CART
Accuracy: 0.802 +/- 0.091
AUC: 0.788 +/- 0.090
Precision: 0.776 +/- 0.152
Exec time for a single run: 0.011s
-----------------
AdaBoost
Accuracy: 0.836 +/- 0.156
AUC: 0.914 +/- 0.078
Precision: 0.838 +/- 0.180
Exec time for a single run: 0.170s
-----------------
MLP
Accuracy: 0.791 +/- 0.124
AUC: 0.844 +/- 0.093
Precision: 0.801 +/- 0.141
Exec time for a single run: 0.272s
-----------------
Bagging
Accuracy: 0.841 +/- 0.141
AUC: 0.906 +/- 0.086
Precision: 0.831 +/- 0.153
Exec time for a single run: 0.280s

#### On observe des résultats bien meilleurs lorsque l'on considère le dataset dans son ensemble.
#### On peut également constater que les résultats sont globalement légèrement suppérieurs avec les données catégorielles binarisées.
#### Le classifier semblant le plus performant avec ce dataset est la random forest_____no_output_____## III. Apprentissage supervisé sur des données textuelles : Feature engineering et Classification_____no_output_____

sms_data = pd.read_table('SMSSpamCollection.data', header=None)_____no_output_____sms_data.head()_____no_output_____

### Vectorizer_____no_output_____

# Create the vectorizer
vectorizer = CountVectorizer(stop_words='english')_____no_output_____# Fit it to our texts
X_cv = vectorizer.fit_transform(sms_data.ix[:, 1])
print('Matrice de co-occurences :')
X_cvMatrice de co-occurences :

#### Il n'est apparemment pas toujours recommandé de supprimer les stop words mais nous les supprimons ici avec l'option stop_words='english'.
#### Seuls les mots de plus de 2 caractères sont conservés._____no_output_____

# Extract targets in y
y = sms_data.ix[:, 0].values
y = np.where(y=='ham', 1, y)
y = np.where(y=='spam', 0, y)
y = y.astype(float)_____no_output_____clfs_light = {
    'Bayes': naive_bayes.GaussianNB(),
    'MLP': neural_network.MLPClassifier(solver='lbfgs', alpha=1e-5, hidden_layer_sizes=(20, 10), random_state=1),
}

run_classifiers(clfs_light, X_cv.toarray().astype(float), y)-----------------
Bayes
Accuracy: 0.884 +/- 0.009
AUC: 0.898 +/- 0.011
Precision: 0.985 +/- 0.003
Exec time for a single run: 2.488s
-----------------
MLP
Accuracy: 0.983 +/- 0.004
AUC: 0.988 +/- 0.004
Precision: 0.985 +/- 0.001
Exec time for a single run: 1.788s

#### On remarque que les résultats sont excellents, notamment pour le MLP, mais le temps d'exécution est extrémement long, nous avons dû créer une liste de classifiers très allégée pour éviter de devoir attendre trop longtemps.

#### Il est important de noter que l'exécution est longue à cause _____no_output_____### Tf-idf_____no_output_____

# Create the td-idf
vectorizer = TfidfVectorizer(stop_words='english')_____no_output_____# Fit to the data
X_tf = vectorizer.fit_transform(sms_data.ix[:, 1])
X_tf_____no_output_____run_classifiers(clfs_light, X_tf.toarray().astype(float), y)-----------------
Bayes
Accuracy: 0.881 +/- 0.010
AUC: 0.883 +/- 0.008
Precision: 0.980 +/- 0.002
Exec time for a single run: 1.095s
-----------------
MLP
Accuracy: 0.925 +/- 0.059
AUC: 0.984 +/- 0.006
Precision: 0.915 +/- 0.060
Exec time for a single run: 1.732s

#### Nous utilisons ici le tfidf vectorizer qui combine l'étape précédente avec une transformation par tfidf.

#### Les résultats obtenus sont moins bon que précédemment et le temps d'exécution est le même car la matrice présente les mêmes dimensions que précédemment._____no_output_____

transformer = decomposition.TruncatedSVD(n_components=100)_____no_output_____# Fit to the data
X_t = transformer.fit_transform(X_tf)
X_t.shape_____no_output_____run_classifiers(clfs, X_t, y)-----------------
Bayes
Accuracy: 0.697 +/- 0.015
AUC: 0.912 +/- 0.017
Precision: 0.978 +/- 0.005
Exec time for a single run: 0.013s
-----------------
RF
Accuracy: 0.975 +/- 0.005
AUC: 0.976 +/- 0.010
Precision: 0.973 +/- 0.004
Exec time for a single run: 1.092s
-----------------
KNN
Accuracy: 0.959 +/- 0.006
AUC: 0.968 +/- 0.006
Precision: 0.958 +/- 0.005
Exec time for a single run: 0.420s
-----------------
CART
Accuracy: 0.952 +/- 0.007
AUC: 0.895 +/- 0.022
Precision: 0.973 +/- 0.007
Exec time for a single run: 0.413s
-----------------
AdaBoost
Accuracy: 0.965 +/- 0.006
AUC: 0.978 +/- 0.010
Precision: 0.974 +/- 0.006
Exec time for a single run: 2.252s
-----------------
MLP
Accuracy: 0.950 +/- 0.043
AUC: 0.984 +/- 0.005
Precision: 0.984 +/- 0.005
Exec time for a single run: 0.286s
-----------------
Bagging
Accuracy: 0.973 +/- 0.007
AUC: 0.972 +/- 0.010
Precision: 0.974 +/- 0.005
Exec time for a single run: 11.829s

#### Le SVD permet d'obtenir une matrice de dimensions plus basses, on peut donc ainsi faire tourner la liste complète des classifiers.
#### On peut constater que les résultats sont globalement bons et meilleurs qu'avant le SVD_____no_output_____### Création d'un pipeline_____no_output_____

pipe = pipeline.Pipeline(steps=[
    ('vectorizer+tfidf', TfidfVectorizer(stop_words='english')),
    ('svd', decomposition.TruncatedSVD(n_components=100)),
    ('model', ensemble.RandomForestClassifier(n_estimators=50, random_state=1))
])_____no_output_____

### Application à un autre jeu de données_____no_output_____

yelp_data = pd.read_csv('yelp-text-by-stars.csv', delimiter=';', encoding = 'ISO-8859-1')_____no_output_____print(yelp_data.shape)
yelp_data.head()(47371, 2)
# Change 0 to 3 to 0
yelp_data = yelp_data.replace([0, 1, 2, 3], 0)
# Change 4 and 5 to 1
yelp_data = yelp_data.replace([4, 5], 1)_____no_output_____X = yelp_data.ix[:, 1].values

# Extract targets in y
y = yelp_data.ix[:, 0].values
print(y)[0 0 0 ... 1 1 1]
clfs = {'pipe': pipe}

run_classifiers(clfs, X, y)-----------------
pipe
Accuracy: 0.796 +/- 0.048
AUC: 0.861 +/- 0.037
Precision: 0.801 +/- 0.068
Exec time for a single run: 23.876s

#### On peut constater que pour l'ensemble du dataset yelp, qui est assez imposant nous obtenons une accuracy d'environ 80%, de même pour la précision.
#### Le SVD du pipe est de 100 composant et nous supprimons les mots clés stop, comme précédemment.
#### Le temps d'exécution indiqué est l'estimation du temps d'une seule exécution de l'algorithme._____no_output_____## IV. Application d'une approche Word2Vec

2 types, Skip-gram et CBOW : Continuous Bag Of Words.

### Skip-gram
* Input: Target word
* Outputs: Surrounding words

### CBOW
Similar to Skip-gram but input and outputs are swapped

It is claimed that Skip-gram tends to do better in rare words.
Nevertheless, the performance of Skip-gram and CBOW are generally similar._____no_output_____#### Première étape : Adapter les données pour le Word2Vec model_____no_output_____

cleaned_X = []
for text in X:
    # Application d'une regex qui ne grade que les lettres et chiffres, on passe tous les mots en minuscules
    cleaned_X.append(re.sub(r'[^a-z0-9]+', ' ', text.lower()).split())

print(cleaned_X[0][:10])['boarded', 'my', 'english', 'mastiff', 'here', 'over', 'new', 'year', 's', 'they']
model_w2v = models.Word2Vec(sentences=cleaned_X, size=100, window=5, min_count=5, workers=4, sg=0)_____no_output_____

#### La commande suivante permet ensuite de visualiser les mots les plus similaire au mot choisit_____no_output_____

model_w2v.wv.most_similar('are')_____no_output_____

### FastText

Permet de ne pas seulement analyser les mots entiers, mais par groupes de n lettres.

Le principal intérêt est de permettre de trouver des mots similaires à des mots ne faisant pas partie du dataset de base !

Exemple :
* n = 4
* word = 'tomate'
* ngrams = ['toma','omat','mate']_____no_output_____

model_ft = models.FastText(cleaned_X, size=100, window=5, min_count=5, workers=4,sg=1)_____no_output_____# Find a word not in dataset

word_not_in_dataset = 'anomalies'

for sentence in cleaned_X:
    if word_not_in_dataset in sentence:
        print('Word does exist in dataset!')
        break
else:
    print('Word does NOT exist in dataset!')Word does NOT exist in dataset!
model_ft.wv.most_similar(word_not_in_dataset)_____no_output_____


# Notebook from mukherjeetejas/Machine-learning
Path: data science notebooks/The Android App Market on Google Play_notebook (1).ipynb

## 1. Google Play Store apps and reviews

Mobile apps are everywhere. They are easy to create and can be lucrative. Because of these two factors, more and more apps are being developed. In this notebook, we will do a comprehensive analysis of the Android app market by comparing over ten thousand apps in Google Play across different categories. We'll look for insights in the data to devise strategies to drive growth and retention.

Let's take a look at the data, which consists of two files:

apps.csv: contains all the details of the applications on Google Play. There are 13 features that describe a given app.

user_reviews.csv: contains 100 reviews for each app, most helpful first. The text in each review has been pre-processed and attributed with three new features: Sentiment (Positive, Negative or Neutral), Sentiment Polarity and Sentiment Subjectivity.

_____no_output_____

# Read in dataset
import pandas as pd
apps_with_duplicates = pd.read_csv('datasets/apps.csv')

# Drop duplicates
apps = apps_with_duplicates.drop_duplicates()

# Print the total number of apps
print('Total number of apps in the dataset = ', len(apps))

# Have a look at a random sample of 6 rows
n = 6
apps.sample(n)Total number of apps in the dataset =  9659

## 2. Data cleaning

The four features that we will be working with most frequently henceforth are Installs, Size, Rating and Price. The info() function (from the previous task)  told us that Installs and Price columns are of type object and not int64 or float64 as we would expect. This is because the column contains some characters more than just [0,9] digits. Ideally, we would want these columns to be numeric as their name suggests.

Hence, we now proceed to data cleaning and prepare our data to be consumed in our analyis later. Specifically, the presence of special characters (, $ +) in the Installs and Price columns make their conversion to a numerical data type difficult.
_____no_output_____

# List of characters to remove
chars_to_remove = ['+', ',', '$']
# List of column names to clean
cols_to_clean = ['Installs', 'Price']

# Loop for each column
for col in cols_to_clean:
    # Replace each character with an empty string
    for char in chars_to_remove:
        apps[col] = apps[col].astype(str).str.replace(char, '')
    # Convert col to numeric
    apps[col] = pd.to_numeric(apps[col])_____no_output_____

## 3. Exploring app categories

With more than 1 billion active users in 190 countries around the world, Google Play continues to be an important distribution platform to build a global audience. For businesses to get their apps in front of users, it's important to make them more quickly and easily discoverable on Google Play. To improve the overall search experience, Google has introduced the concept of grouping apps into categories.

This brings us to the following questions:

Which category has the highest share of (active) apps in the market?

Is any specific category dominating the market?

Which categories have the fewest number of apps?

We will see that there are 33 unique app categories present in our dataset. Family and Game apps have the highest market prevalence. Interestingly, Tools, Business and Medical apps are also at the top.
_____no_output_____

import plotly
plotly.offline.init_notebook_mode(connected=True)
import plotly.graph_objs as go

# Print the total number of unique categories
num_categories = len(apps['Category'].unique())
print('Number of categories = ', num_categories)

# Count the number of apps in each 'Category' and sort them in descending order
num_apps_in_category = apps['Category'].value_counts().sort_values(ascending = False)

data = [go.Bar(
        x = num_apps_in_category.index, # index = category name
        y = num_apps_in_category.values, # value = count
)]

plotly.offline.iplot(data)_____no_output_____

## 4. Distribution of app ratings

After having witnessed the market share for each category of apps, let's see how all these apps perform on an average. App ratings (on a scale of 1 to 5) impact the discoverability, conversion of apps as well as the company's overall brand image. Ratings are a key performance indicator of an app.

From our research, we found that the average volume of ratings across all app categories is 4.17. The histogram plot is skewed to the right indicating that the majority of the apps are highly rated with only a few exceptions in the low-rated apps.
_____no_output_____

# Average rating of apps
avg_app_rating = apps['Rating'].mean()
print('Average app rating = ', avg_app_rating)

# Distribution of apps according to their ratings
data = [go.Histogram(
        x = apps['Rating']
)]

# Vertical dashed line to indicate the average app rating
layout = {'shapes': [{
              'type' :'line',
              'x0': avg_app_rating,
              'y0': 0,
              'x1': avg_app_rating,
              'y1': 1000,
              'line': { 'dash': 'dashdot'}
          }]
          }

plotly.offline.iplot({'data': data, 'layout': layout})Average app rating =  4.173243045387994

## 5. Size and price of an app

Let's now examine app size and app price. For size, if the mobile app is too large, it may be difficult and/or expensive for users to download. Lengthy download times could turn users off before they even experience your mobile app. Plus, each user's device has a finite amount of disk space. For price, some users expect their apps to be free or inexpensive. These problems compound if the developing world is part of your target market; especially due to internet speeds, earning power and exchange rates.

How can we effectively come up with strategies to size and price our app?

Does the size of an app affect its rating?

Do users really care about system-heavy apps or do they prefer light-weighted apps?

Does the price of an app affect its rating?

Do users always prefer free apps over paid apps?

We find that the majority of top rated apps (rating over 4) range from 2 MB to 20 MB. We also find that the vast majority of apps price themselves under \$10.
_____no_output_____

%matplotlib inline
import seaborn as sns
sns.set_style("darkgrid")
import warnings
warnings.filterwarnings("ignore")

# Filter rows where both Rating and Size values are not null
apps_with_size_and_rating_present = apps[(~apps['Rating'].isnull()) & (~apps['Size'].isnull())]

# Subset for categories with at least 250 apps
large_categories = apps_with_size_and_rating_present.groupby(['Category']).filter(lambda x: len(x) >= 250).reset_index()

# Plot size vs. rating
plt1 = sns.jointplot(x = large_categories['Size'], y = large_categories['Rating'], kind = 'hex')

# Subset apps whose 'Type' is 'Paid'
paid_apps = apps_with_size_and_rating_present[apps_with_size_and_rating_present['Type'] == 'Paid']

# Plot price vs. rating
plt2 = sns.jointplot(x = paid_apps['Price'], y = paid_apps['Rating'])_____no_output_____

## 6. Relation between app category and app price

So now comes the hard part. How are companies and developers supposed to make ends meet? What monetization strategies can companies use to maximize profit? The costs of apps are largely based on features, complexity, and platform.

There are many factors to consider when selecting the right pricing strategy for your mobile app. It is important to consider the willingness of your customer to pay for your app. A wrong price could break the deal before the download even happens. Potential customers could be turned off by what they perceive to be a shocking cost, or they might delete an app they’ve downloaded after receiving too many ads or simply not getting their money's worth.

Different categories demand different price ranges. Some apps that are simple and used daily, like the calculator app, should probably be kept free. However, it would make sense to charge for a highly-specialized medical app that diagnoses diabetic patients. Below, we see that Medical and Family apps are the most expensive. Some medical apps extend even up to \$80! All game apps are reasonably priced below \$20.
_____no_output_____

import matplotlib.pyplot as plt
fig, ax = plt.subplots()
fig.set_size_inches(15, 8)

# Select a few popular app categories
popular_app_cats = apps[apps.Category.isin(['GAME', 'FAMILY', 'PHOTOGRAPHY',
                                            'MEDICAL', 'TOOLS', 'FINANCE',
                                            'LIFESTYLE','BUSINESS'])]

# Examine the price trend by plotting Price vs Category
ax = sns.stripplot(x = popular_app_cats['Price'], y = popular_app_cats['Category'], jitter=True, linewidth=1)
ax.set_title('App pricing trend across categories')

# Apps whose Price is greater than 200
apps_above_200 = popular_app_cats[['Category', 'App', 'Price']][popular_app_cats['Price'] > 200]
apps_above_200
_____no_output_____

## 7. Filter out "junk" apps

It looks like a bunch of the really expensive apps are "junk" apps. That is, apps that don't really have a purpose. Some app developer may create an app called I Am Rich Premium or most expensive app (H) just for a joke or to test their app development skills. Some developers even do this with malicious intent and try to make money by hoping people accidentally click purchase on their app in the store.

Let's filter out these junk apps and re-do our visualization. The distribution of apps under \$20 becomes clearer.
_____no_output_____

# Select apps priced below $100
apps_under_100 = popular_app_cats[popular_app_cats['Price'] < 100]

fig, ax = plt.subplots()
fig.set_size_inches(15, 8)

# Examine price vs category with the authentic apps
ax = sns.stripplot(x='Price', y='Category', data=apps_under_100,
                   jitter=True, linewidth=1)
ax.set_title('App pricing trend across categories after filtering for junk apps')_____no_output_____

## 8. Popularity of paid apps vs free apps

For apps in the Play Store today, there are five types of pricing strategies: free, freemium, paid, paymium, and subscription. Let's focus on free and paid apps only. Some characteristics of free apps are:

Free to download.

Main source of income often comes from advertisements.

Often created by companies that have other products and the app serves as an extension of those products.

Can serve as a tool for customer retention, communication, and customer service.

Some characteristics of paid apps are:

Users are asked to pay once for the app to download and use it.

The user can't really get a feel for the app before buying it.

Are paid apps installed as much as free apps? It turns out that paid apps have a relatively lower number of installs than free apps, though the difference is not as stark as I would have expected!
_____no_output_____

trace0 = go.Box(
    # Data for paid apps
    y=apps[apps['Type'] == 'Paid']['Installs'],
    name = 'Paid'
)

trace1 = go.Box(
    # Data for free apps
    y=apps[apps['Type'] == 'Free']['Installs'],
    name = 'Free'
)

layout = go.Layout(
    title = "Number of downloads of paid apps vs. free apps",
    yaxis = dict(
        type = 'log',
        autorange = True
    )
)

# Add trace0 and trace1 to a list for plotting
data = [trace0, trace1]
plotly.offline.iplot({'data': data, 'layout': layout})_____no_output_____

## 9. Sentiment analysis of user reviews

Mining user review data to determine how people feel about your product, brand, or service can be done using a technique called sentiment analysis. User reviews for apps can be analyzed to identify if the mood is positive, negative or neutral about that app. For example, positive words in an app review might include words such as 'amazing', 'friendly', 'good', 'great', and 'love'. Negative words might be words like 'malware', 'hate', 'problem', 'refund', and 'incompetent'.

By plotting sentiment polarity scores of user reviews for paid and free apps, we observe that free apps receive a lot of harsh comments, as indicated by the outliers on the negative y-axis. Reviews for paid apps appear never to be extremely negative. This may indicate something about app quality, i.e., paid apps being of higher quality than free apps on average. The median polarity score for paid apps is a little higher than free apps, thereby syncing with our previous observation.

In this notebook, we analyzed over ten thousand apps from the Google Play Store. We can use our findings to inform our decisions should we ever wish to create an app ourselves.
_____no_output_____

# Read in the user reviews
reviews_df = pd.read_csv('datasets/user_reviews.csv')

# Join and merge the two dataframe
merged_df = pd.merge(apps, reviews_df, on = 'App', how = "inner")

# Drop NA values from Sentiment and Translated_Review columns
merged_df = merged_df.dropna(subset=['Sentiment', 'Translated_Review'])

sns.set_style('ticks')
fig, ax = plt.subplots()
fig.set_size_inches(11, 8)

# User review sentiment polarity for paid vs. free apps
ax = sns.boxplot(x = 'Type', y = 'Sentiment_Polarity', data = merged_df)
ax.set_title('Sentiment Polarity Distribution')_____no_output_____


# Notebook from ClaudiuCreanga/DataScienceSchool
Path: AlexisBoukouvalasTutorial/Lab3_DEtime.ipynb

# Notebook 3: Differential expression in time

### This exercise is based on the publication [Yang et al. (2016) "Inferring the perturbation time from biological time course data" Bioinformatics, 32(19), 2956-2964.](https://doi.org/10.1093/bioinformatics/btw329)

### We used a branching Gaussian process to identify when each gene's expression profile begins to change when comparing plants with a wild-type pathogen infection (control) to plants with a modified pathogen infection (perturbation).

### Look at the results for some different genes - Genes 1 and 8 and from Fig. 4 in the handbook chapter and genes 6 and 19 are from Fig. 5.

Data are from Lewis et al. (2015). "Transcriptional dynamics driving MAMP-triggered immunity and pathogen effector-mediated immunosuppression in Arabidopsis leaves following infection with Pseudomonas syringae pv tomato DC3000" The Plant Cell, 27(11), 3038-3064.

Notebook created by Sumon Ahmed, Jing Yang, Magnus Rattray and Alexis Boukouvalas_____no_output_____

%matplotlib inline
import matplotlib
import pandas as pd
import numpy as np
from matplotlib import pyplot as plt
import gpflow
import tensorflow as tf
from sklearn.preprocessing import scale
from branchingKernel import BranchKernel
from branchingKernel import plotCovarianceMatrix
tf.logging.set_verbosity(tf.logging.ERROR) # Suppresses output logs from tensorflow_____no_output_____

## Before looking at the data, let's visualize the covariance of data from the branching GP model

Illustration of the covariance matrix, for two functions $f$ and $g$ evaluated at points evenly distributed in [0,100] and crossing at $x_p=40$ (left) and the resulting data covariance matrix, for time course data $y^c$ and $y^p$ from a wild-type and perturbed system respectively (right) _____no_output_____

tc = np.linspace(0., 100., 100)[:, None]
tc = np.c_[tc, np.ones(len(tc))[:,None]]
tp = np.linspace(0., 100., 100)[:, None]
tp = np.c_[tp, (np.ones(len(tp)) * 2)[:,None]]
t = np.concatenate((tc, tp))
xp = 40.

sess = tf.InteractiveSession()
kern = BranchKernel(gpflow.kernels.RBF(1, lengthscales=20.), xp)

fig, ax = plt.subplots(1, 2, figsize=(12, 6))
fig.subplots_adjust(left=0.02, bottom=0.06, right=0.95, top=0.94, wspace=0.3)

covMatrix = kern.K(t)
plotCovarianceMatrix(covMatrix.eval(), ax[0], xLabel=('$f$','$g$'), yLabel=('$f$','$g$'))

t[np.where(t[:,0] <= xp),1]=1
covMatrix_DE = kern.K(t)
plotCovarianceMatrix(covMatrix_DE.eval(), ax[1], xLabel=('$y^c$','$y^p$'), yLabel=('$y^c$','$y^p$'))
sess.close()Tensor("mul_21:0", shape=(200, 200), dtype=float64)
Tensor("add_24:0", shape=(1, 1), dtype=float64)
Tensor("invKbb_2:0", shape=(1, 1), dtype=float64)
Tensor("mul_31:0", shape=(200, 200), dtype=float64)
Tensor("add_34:0", shape=(1, 1), dtype=float64)
Tensor("invKbb_3:0", shape=(1, 1), dtype=float64)
ERROR:tensorflow:==================================
Object was never used (type <class 'tensorflow.python.framework.ops.Tensor'>):
<tf.Tensor 'SquaredExponential-84acf556-0/lengthscales/IsVariableInitialized:0' shape=() dtype=bool>
If you want to mark it as used call its "mark_used()" method.
It was originally created here:
  File "/home/alexis/.pyenv/versions/3.6.8/envs/handbook_tutorial/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 3346, in run_code
    return outflag  File "<ipython-input-2-22bcab834f8e>", line 9, in <module>
    kern = BranchKernel(gpflow.kernels.RBF(1, lengthscales=20.), xp)  File "</home/alexis/.pyenv/versions/3.6.8/envs/handbook_tutorial/lib/python3.6/site-packages/decorator.py:decorator-gen-135>", line 2, in __init__  File "/home/alexis/.pyenv/versions/3.6.8/envs/handbook_tutorial/lib/python3.6/site-packages/gpflow/core/compilable.py", line 157, in init_wrapper
    self.initialize(force=True)  File "/home/alexis/.pyenv/versions/3.6.8/envs/handbook_tutorial/lib/python3.6/site-packages/gpflow/core/node.py", line 156, in build
    self._build()  File "/home/alexis/.pyenv/versions/3.6.8/envs/handbook_tutorial/lib/python3.6/site-packages/gpflow/params/parameterized.py", line 302, in _build
    self._prior_tensor = self._build_prior(priors)  File "/home/alexis/.pyenv/versions/3.6.8/envs/handbook_tutorial/lib/python3.6/site-packages/gpflow/core/node.py", line 156, in build
    self._build()  File "/home/alexis/.pyenv/versions/3.6.8/envs/handbook_tutorial/lib/python3.6/site-packages/gpflow/params/parameter.py", line 373, in _build
    self._prior_tensor = prior  File "/home/alexis/.pyenv/versions/3.6.8/envs/handbook_tutorial/lib/python3.6/site-packages/tensorflow/python/util/tf_should_use.py", line 193, in wrapped
    return _add_should_use_warning(fn(*args, **kwargs))
==================================

## Load some real data from the wild-type and perturbed condition

I've only included 3 replicates to make this run a bit faster in the lab, so the results will differ a bit from those in the handbook chapter figures. We've taken the first 20 genes in the dataset. Genes 1 and 8 and from Fig. 4 in the handbook chapter and genes 6 and 19 are from Fig. 5. _____no_output_____

controlData = pd.read_csv('data/controlData3reps.csv', index_col=[0])
perturbedData = pd.read_csv('data/perturbedData3reps.csv', index_col=[0])_____no_output_____controlTimes = controlData['controlTimes'].values[:,None]
controlTimes = np.c_[controlTimes, np.ones(len(controlTimes))[:,None]]
perturbedTimes = perturbedData['perturbedTimes'].values[:,None]
perturbedTimes = np.c_[perturbedTimes, ( np.ones(len(controlTimes)) * 2 )[:,None]]
times = np.concatenate((controlTimes, perturbedTimes))_____no_output_____

## Let's take a look at an example - when does this gene begin to differ between the perturbed and wild-type datasets? _____no_output_____

import matplotlib.lines as mlines
gene_id = '1'
xc = controlTimes[:,0][:,None]
yc = controlData[gene_id].values[:,None]
xp = perturbedTimes[:,0][:,None]
yp = perturbedData[gene_id].values[:,None]

fig, ax = plt.subplots(1, 1, figsize=(6, 6))

ax.plot(xc, yc, 'ko', mew=2, color='b')
ax.set_title('Wild-type (blue) and Perturbation (red)')
ax.plot(xp, yp, 'ko', mew=2, color='r')
# ax[1].set_title('Perturbation')_____no_output_____

## Before considering branching, let's check whether the gene is changing over time

It's useful to pre-screen the genes and save a bit of computation time, as we are mainly interested in genes with measurable dynamics in the wild-type. Applying this filter is faster than applying the branching model to every gene because the branching model is applied across a grid of possible branching times. _____no_output_____

xc = controlTimes[:,0][:,None]
yc = controlData[gene_id].values[:,None]
mean_y = np.mean(yc)
sd_y = np.std(yc)
yc = scale(yc)
Xtest = np.linspace(min(xc) - 0.5, max(xc) + 0.5, 100)_____no_output_____

### First we fit a Gaussian process model with a squared exponential covariance_____no_output_____

with gpflow.defer_build():
    mGP = gpflow.models.GPR(xc, yc, kern=gpflow.kernels.RBF(1))
mGP.compile()
opt = gpflow.train.ScipyOptimizer()
opt.minimize(mGP)                    # Optimise hyper-parameters
muGP, varGP = mGP.predict_y(Xtest)   # Predictive distribution of data
muGP = sd_y * muGP + mean_y          # Transform back to original scale
varGP = np.square(sd_y) * varGP
mGP_log_likelihood = mGP.compute_log_likelihood()       # Compute the log-likelihood_____no_output_____

### Then we fit a constant model with only noise - this is a white noise kernel_____no_output_____

with gpflow.defer_build():
    mConst = gpflow.models.GPR(xc, yc, kern=gpflow.kernels.White(1))
mConst.compile()
opt = gpflow.train.ScipyOptimizer()
opt.minimize(mConst)
muConst, varConst = mConst.predict_y(Xtest)
muConst = sd_y * muConst + mean_y
varConst = np.square(sd_y) * varConst
mConst_log_likelihood = mConst.compute_log_likelihood()       # Compute the log-likelihood_____no_output_____

### Plot the fit to each model_____no_output_____

import matplotlib.lines as mlines

fig, ax = plt.subplots(1, 2, figsize=(16, 6))

ax[0].plot(Xtest, muGP, '-', lw=2, color='b')
ax[0].fill_between(Xtest.reshape(-1), (muGP - 2 * np.sqrt(varGP)).reshape(-1), \
                   (muGP + 2 * np.sqrt(varGP)).reshape(-1), color='lightblue', alpha=0.6)
ax[0].plot(xc, sd_y * yc + mean_y, 'ko', mew=2, color='b')
ax[0].set_title('GP regression')

ax[1].plot(Xtest, muConst, '-', lw=2, color='b')
ax[1].fill_between(Xtest.reshape(-1), (muConst - 2 * np.sqrt(varConst)).reshape(-1), \
                   (muConst + 2 * np.sqrt(varConst)).reshape(-1), color='lightblue', alpha=0.6)
ax[1].plot(xc, sd_y * yc + mean_y, 'ko', mew=2, color='b')
ax[1].set_title('Constant model')_____no_output_____

### The likelihood ratio quantifies whether the GP model is a better fit than the constant one

### The larger the number, the more evidence that the gene is dynamic. In practice it's useful to eyeball some genes close to zero to identify a reasonable cut-off_____no_output_____

LL_ratio = mGP.compute_log_likelihood() - mConst.compute_log_likelihood()
print(LL_ratio)21.96448954719847

## Now use the branching GP to identify changes between two time course experiments  _____no_output_____

yc = controlData[gene_id].values[:,None]
yp = perturbedData[gene_id].values[:,None]
y = np.concatenate((yc, yp))_____no_output_____N = 20 # We'll approximate the perturbation time posterior by a hitogram with 20 bins
testTimes = np.linspace(min(times[:,0]), max(times[:,0]), N) _____no_output_____

### Get the optimized hyperparameters

To learn the hyper-parameters we fit an independent GP model to the control and perturbation time series with shared hyper-parameters (lengthscale and variance). We can do that be putting the perturbation time (where the functions branch) very far to the left of the data. _____no_output_____

xp = -1000. # Put branching time much earlier than zero time
x = times.copy()
x[np.where(x[:,0] <= xp),1]=1
# with gpflow.settings.temp_settings(custom_config), gpflow.session_manager.get_session().as_default():
with gpflow.defer_build():
    base_kern = gpflow.kernels.RBF(1)
    k = BranchKernel(base_kern, xp)
    m = gpflow.models.GPR(x, y, kern=k)
m.compile()
opt = gpflow.train.ScipyOptimizer()
opt.minimize(m, disp=False)

ls = m.kern.kern.lengthscales.read_value()
variance = m.kern.kern.variance.read_value()
likelihood = m.likelihood.variance.read_value()Tensor("GPR-27924832-35/likelihood_1/mul_1:0", dtype=float64)
Tensor("GPR-27924832-35/likelihood_1/add_4:0", shape=(1, 1), dtype=float64)
Tensor("GPR-27924832-35/likelihood_1/invKbb:0", shape=(1, 1), dtype=float64)

### Fit the branching model for many different branch times so that we can use the model likelihood to calculate the posterior probability of branching at each time, using a histogram approach_____no_output_____

ll = np.zeros(len(testTimes))
models = list()

for i in range(0, N):
    x = times.copy()
    x[np.where(x[:,0] <= testTimes[i]),1]=1
    with gpflow.defer_build():
        base_kernel_new = gpflow.kernels.RBF(1)
        base_kernel_new.lengthscales = ls
        base_kernel_new.variance = variance

        base_kernel_new.lengthscales.set_trainable(False)
        base_kernel_new.variance.set_trainable(False)

        k_new = BranchKernel(base_kernel_new, testTimes[i])
        m_new = gpflow.models.GPR(x, y, kern=k_new)
    m_new.compile()
    opt = gpflow.train.ScipyOptimizer()
    opt.minimize(m_new, disp=True)

    ll[i] = np.exp(m_new.compute_log_likelihood())
    models.append(m_new)
    del x
    del k_new
    del m_new
normalized_ll = ll / ll.sum(0)Tensor("GPR-a6a16282-46/likelihood_1/mul_1:0", dtype=float64)
Tensor("GPR-a6a16282-46/likelihood_1/add_4:0", shape=(1, 1), dtype=float64)
Tensor("GPR-a6a16282-46/likelihood_1/invKbb:0", shape=(1, 1), dtype=float64)
Tensor("GPR-84796a1a-57/likelihood_1/mul_1:0", dtype=float64)
Tensor("GPR-84796a1a-57/likelihood_1/add_4:0", shape=(1, 1), dtype=float64)
Tensor("GPR-84796a1a-57/likelihood_1/invKbb:0", shape=(1, 1), dtype=float64)
Tensor("GPR-523ec483-68/likelihood_1/mul_1:0", dtype=float64)
Tensor("GPR-523ec483-68/likelihood_1/add_4:0", shape=(1, 1), dtype=float64)
Tensor("GPR-523ec483-68/likelihood_1/invKbb:0", shape=(1, 1), dtype=float64)
Tensor("GPR-4698f2a5-79/likelihood_1/mul_1:0", dtype=float64)
Tensor("GPR-4698f2a5-79/likelihood_1/add_4:0", shape=(1, 1), dtype=float64)
Tensor("GPR-4698f2a5-79/likelihood_1/invKbb:0", shape=(1, 1), dtype=float64)
Tensor("GPR-3b5f1b51-90/likelihood_1/mul_1:0", dtype=float64)
Tensor("GPR-3b5f1b51-90/likelihood_1/add_4:0", shape=(1, 1), dtype=float64)
Tensor("GPR-3b5f1b51-90/likelihood_1/invKbb:0", shape=(1, 1), dtype=float64)
Tensor("GPR-51d88072-101/likelihood_1/mul_1:0", dtype=float64)
Tensor("GPR-51d88072-101/likelihood_1/add_4:0", shape=(1, 1), dtype=float64)
Tensor("GPR-51d88072-101/likelihood_1/invKbb:0", shape=(1, 1), dtype=float64)
Tensor("GPR-8cbb36b7-112/likelihood_1/mul_1:0", dtype=float64)
Tensor("GPR-8cbb36b7-112/likelihood_1/add_4:0", shape=(1, 1), dtype=float64)
Tensor("GPR-8cbb36b7-112/likelihood_1/invKbb:0", shape=(1, 1), dtype=float64)
Tensor("GPR-76925b84-123/likelihood_1/mul_1:0", dtype=float64)
Tensor("GPR-76925b84-123/likelihood_1/add_4:0", shape=(1, 1), dtype=float64)
Tensor("GPR-76925b84-123/likelihood_1/invKbb:0", shape=(1, 1), dtype=float64)
Tensor("GPR-f8472335-134/likelihood_1/mul_1:0", dtype=float64)
Tensor("GPR-f8472335-134/likelihood_1/add_4:0", shape=(1, 1), dtype=float64)
Tensor("GPR-f8472335-134/likelihood_1/invKbb:0", shape=(1, 1), dtype=float64)
Tensor("GPR-74545c50-145/likelihood_1/mul_1:0", dtype=float64)
Tensor("GPR-74545c50-145/likelihood_1/add_4:0", shape=(1, 1), dtype=float64)
Tensor("GPR-74545c50-145/likelihood_1/invKbb:0", shape=(1, 1), dtype=float64)
Tensor("GPR-123b295e-156/likelihood_1/mul_1:0", dtype=float64)
Tensor("GPR-123b295e-156/likelihood_1/add_4:0", shape=(1, 1), dtype=float64)
Tensor("GPR-123b295e-156/likelihood_1/invKbb:0", shape=(1, 1), dtype=float64)
Tensor("GPR-888cc2f2-167/likelihood_1/mul_1:0", dtype=float64)
Tensor("GPR-888cc2f2-167/likelihood_1/add_4:0", shape=(1, 1), dtype=float64)
Tensor("GPR-888cc2f2-167/likelihood_1/invKbb:0", shape=(1, 1), dtype=float64)
Tensor("GPR-2a2d0497-178/likelihood_1/mul_1:0", dtype=float64)
Tensor("GPR-2a2d0497-178/likelihood_1/add_4:0", shape=(1, 1), dtype=float64)
Tensor("GPR-2a2d0497-178/likelihood_1/invKbb:0", shape=(1, 1), dtype=float64)
Tensor("GPR-f8935c9f-189/likelihood_1/mul_1:0", dtype=float64)
Tensor("GPR-f8935c9f-189/likelihood_1/add_4:0", shape=(1, 1), dtype=float64)
Tensor("GPR-f8935c9f-189/likelihood_1/invKbb:0", shape=(1, 1), dtype=float64)
Tensor("GPR-91643ca2-200/likelihood_1/mul_1:0", dtype=float64)
Tensor("GPR-91643ca2-200/likelihood_1/add_4:0", shape=(1, 1), dtype=float64)
Tensor("GPR-91643ca2-200/likelihood_1/invKbb:0", shape=(1, 1), dtype=float64)
Tensor("GPR-0d30424c-211/likelihood_1/mul_1:0", dtype=float64)
Tensor("GPR-0d30424c-211/likelihood_1/add_4:0", shape=(1, 1), dtype=float64)
Tensor("GPR-0d30424c-211/likelihood_1/invKbb:0", shape=(1, 1), dtype=float64)
Tensor("GPR-89e84009-222/likelihood_1/mul_1:0", dtype=float64)
Tensor("GPR-89e84009-222/likelihood_1/add_4:0", shape=(1, 1), dtype=float64)
Tensor("GPR-89e84009-222/likelihood_1/invKbb:0", shape=(1, 1), dtype=float64)
Tensor("GPR-e9897bab-233/likelihood_1/mul_1:0", dtype=float64)
Tensor("GPR-e9897bab-233/likelihood_1/add_4:0", shape=(1, 1), dtype=float64)
Tensor("GPR-e9897bab-233/likelihood_1/invKbb:0", shape=(1, 1), dtype=float64)
Tensor("GPR-4b314e3f-244/likelihood_1/mul_1:0", dtype=float64)
Tensor("GPR-4b314e3f-244/likelihood_1/add_4:0", shape=(1, 1), dtype=float64)
Tensor("GPR-4b314e3f-244/likelihood_1/invKbb:0", shape=(1, 1), dtype=float64)
Tensor("GPR-dd751f66-255/likelihood_1/mul_1:0", dtype=float64)
Tensor("GPR-dd751f66-255/likelihood_1/add_4:0", shape=(1, 1), dtype=float64)
Tensor("GPR-dd751f66-255/likelihood_1/invKbb:0", shape=(1, 1), dtype=float64)
iMAP = np.argmax(ll)
MAP_model = models[iMAP]_____no_output_____

### We will show the GP regression fit for the model with the maximum posterior (MAP) perturbation time _____no_output_____

Xnew = np.linspace(min(times[:,0]) - 0.5, max(times[:,0]) + 0.5, 100)[:, None]
x1 = np.c_[Xnew, np.ones(len(Xnew))[:, None]]
x2 = np.c_[Xnew, ( np.ones(len(Xnew)) * 2 )[:, None]]
Xtest = np.concatenate((x1, x2))
Xtest[np.where(Xtest[:,0] <= testTimes[iMAP]),1]=1_____no_output_____mu, var = MAP_model.predict_y(Xtest)
upper = mu+2*np.sqrt(var)
lower = mu-2*np.sqrt(var)Tensor("autoflow/GPR/predict_y_2/predict/mul_1:0", dtype=float64)
Tensor("autoflow/GPR/predict_y_2/predict/add_4:0", shape=(1, 1), dtype=float64)
Tensor("autoflow/GPR/predict_y_2/predict/invKbb:0", shape=(1, 1), dtype=float64)
Tensor("autoflow/GPR/predict_y_2/predict/mul_10:0", dtype=float64)
Tensor("autoflow/GPR/predict_y_2/predict/add_13:0", shape=(1, 1), dtype=float64)
Tensor("autoflow/GPR/predict_y_2/predict/invKbb_1:0", shape=(1, 1), dtype=float64)
import matplotlib.lines as mlines

fig, ax = plt.subplots(2, 1, figsize=(12, 12))

ax[0].plot(Xnew, mu[0:100,:], '-', lw=2, color='b')
ax[0].fill_between(Xnew.reshape(-1), lower[0:100, :].reshape(-1), upper[0:100, :].reshape(-1), color='lightblue', alpha=0.2)

ax[0].plot(Xnew, mu[100:200,:], '-', lw=2, color='r')
ax[0].fill_between(Xnew.reshape(-1), lower[100:200, :].reshape(-1), upper[100:200, :].reshape(-1), color='yellow', alpha=0.2)

ax[0].plot(controlTimes[:,0], yc, 'ko', mew=2, color='b')
ax[0].plot(perturbedTimes[:,0], yp, 'ko', mew=2, color='r')
ax[0].set_ylabel('Gene Expression', fontsize=16)

blue_line = mlines.Line2D([], [], color='blue', linewidth=3., label='Predicted Mean (Control data)')
red_line = mlines.Line2D([], [], color='red', linestyle='-', linewidth=3., label='Predicted Mean (Perturbed data)')
blue_dot = mlines.Line2D([], [], color='blue', marker='o', markersize='8', linestyle='', label='Control Data')
red_dot = mlines.Line2D([], [], color='red', marker='o', markersize='8', linestyle='', label='Control Data')
ax[0].legend(handles=[blue_line, red_line, blue_dot, red_dot], bbox_to_anchor=(1.28, 0.5), loc=10, fontsize=16, frameon=False)

width = testTimes[1]-testTimes[0]
ax[1].bar(testTimes, normalized_ll, color='y', align='center', edgecolor = "none", width=width)
ax[1].set_xlabel('Times', fontsize=16)
ax[1].set_ylabel('Probability of perturbation', fontsize=16)
plt.subplots_adjust(wspace=0, hspace=0)_____no_output_____

gene expression
differential expression
bioinformatics