# Notebook from Owinoh-victor/CNN-Image-classification-Using-Fashion-MNIST-dataset
Path: CNN Image classifocationn python  Notebook.ipynb

# Importing libraries_____no_output_____

import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import seaborn as sns
import keras
import tensorflow as tfUsing TensorFlow backend.
train = pd.read_csv('C:/Users/pc/Desktop/Fashion MNIST/fashion-mnist_train.csv')
test = pd.read_csv('C:/Users/pc/Desktop/Fashion MNIST/fashion-mnist_test.csv')
df = train.copy()
test_df = test.copy()
print("Dataset imported succesfully")Dataset imported succesfully
#view starting values of data set
print("Dataset Head")
print(train.head())
print("=" * 30)Dataset Head
   label  pixel1  pixel2  pixel3  pixel4  pixel5  pixel6  pixel7  pixel8  \
0      2       0       0       0       0       0       0       0       0
1      9       0       0       0       0       0       0       0       0
2      6       0       0       0       0       0       0       0       5
3      0       0       0       0       1       2       0       0       0
4      3       0       0       0       0       0       0       0       0

   pixel9  ...  pixel775  pixel776  pixel777  pixel778  pixel779  pixel780  \
0       0  ...         0         0         0         0         0         0
1       0  ...         0         0         0         0         0         0
2       0  ...         0         0         0        30        43         0
3       0  ...         3         0         0         0         0         1
4       0  ...         0         0         0         0         0         0

   pixel781  pixel782  pixel783  pixel784
0         0         0         0         0
1         0         0         0         0
2         0         0         0         0
3         0         0         0         0
4         0         0         0         0

[5 rows x 785 columns]
==============================
# View How many samples and how many missing values for each feature
print("Dataset Features Details")
print(train.info())
print("=" * 30)Dataset Features Details
<class 'pandas.core.frame.DataFrame'>
RangeIndex: 60000 entries, 0 to 59999
Columns: 785 entries, label to pixel784
dtypes: int64(785)
memory usage: 359.3 MB
None
==============================

# Exploring the dataset
it is iportant to note that Label is the Target variable._____no_output_____# Dataset Label Description
0 T-shirt/top

1 Trouser

2 Pullover

3 Dress

4 Coat

5 Sandal

6 Shirt

7 Sneaker

8 Bag

9 Ankle boot_____no_output_____# Examine Dimensions_____no_output_____

print('Train: ', df.shape)
print('Test: ', test_df.shape)Train:  (60000, 785)
Test:  (10000, 785)

So, there are 60,000 Training Samples and 10,000 Test Samples.
Each example is a 28x28 grayscale image, associated with a label from 10 classes._____no_output_____

df.label.unique()_____no_output_____

# Checking for Null Values_____no_output_____

 #Train DATASET
df.isnull().any().sum()_____no_output_____# Test DATASET
test_df.isnull().any().sum()_____no_output_____

Wonderful, So there are no Null Values in both Train and Test Set._____no_output_____# Visualizing the Dataset_____no_output_____

# Mapping Classes lebels
clothes = {0 : 'T-shirt/top',
            1 : 'Trouser',
            2 : 'Pullover',
            3 : 'Dress',
            4 : 'Coat',
            5 : 'Sandal',
            6 : 'Shirt',
            7 : 'Sneaker',
            8 : 'Bag',
            9 : 'Ankle boot'}_____no_output_____

Plotting Random Images_____no_output_____

fig, axes = plt.subplots(4, 4, figsize = (15,15))
for row in axes:
    for axe in row:
        index = np.random.randint(60000)
        img = df.drop('label', axis=1).values[index].reshape(28,28)
        cloths = df['label'][index]
        axe.imshow(img, cmap='gray')
        axe.set_title(clothes[cloths])
        axe.set_axis_off()_____no_output_____

Let's look at the Distribution of labels to anaylze if there are any skewed classes_____no_output_____

df['label'].value_counts()_____no_output_____sns.factorplot(x='label', data=df, kind='count', size=3, aspect= 1.5)C:\Users\pc\Anaconda3\lib\site-packages\seaborn\categorical.py:3666: UserWarning: The `factorplot` function has been renamed to `catplot`. The original name will be removed in a future release. Please update your code. Note that the default `kind` in `factorplot` (`'point'`) has changed `'strip'` in `catplot`.
  warnings.warn(msg)
C:\Users\pc\Anaconda3\lib\site-packages\seaborn\categorical.py:3672: UserWarning: The `size` paramter has been renamed to `height`; please update your code.
  warnings.warn(msg, UserWarning)

It beautiful that all classes are equally Distributed.There is no anomalies in the dataset_____no_output_____

# Setting Random Seeds for Reproducibilty.
seed = 66
np.random.seed(seed)_____no_output_____

# Split the data into train/validation/test datasets
Now we are gonna split the training data into Train and Validation Set.

Train set is used for Training the model and Validation set is used for Evaluating our Model's Performance on the Dataset._____no_output_____

from sklearn.model_selection import train_test_split
X = train.iloc[:,1:]
Y = train.iloc[:,0]
x_train, x_validate, y_train, y_validate = train_test_split(x_train, y_train, test_size = 0.2, random_state = 12345)_____no_output_____x_train.shape
_____no_output_____x_test.shape_____no_output_____# * unpack the tuple
x_train = x_train.reshape(x_train.shape[0], *(28, 28, 1))
x_test = x_test.reshape(x_test.shape[0], *(28, 28, 1))
x_validate = x_validate.reshape(x_validate.shape[0], *(28, 28, 1))_____no_output_____

The first parameter in reshape indicates the number of examples.
 We pass it as -1, which means that it is an unknown dimension and we want numpy to figure it out._____no_output_____

print(x_train.shape)
print(x_train.shape)
print(x_validate.shape)(43200, 28, 28, 1)
(43200, 28, 28, 1)
(10800, 28, 28, 1)

# Building the Model _____no_output_____

from keras.models import Sequential
from keras.layers import Conv2D, MaxPooling2D, Dense, Flatten, Dropout
from keras.optimizers import Adam
from keras.callbacks import TensorBoard_____no_output_____model = tf.keras.Sequential()
# Must define the input shape in the first layer of the neural network
model.add(tf.keras.layers.Conv2D(filters=64, kernel_size=2, padding='same', activation='relu', input_shape=(28,28,1)))
model.add(tf.keras.layers.MaxPooling2D(pool_size=2))
model.add(tf.keras.layers.Dropout(0.3))
model.add(tf.keras.layers.Conv2D(filters=32, kernel_size=2, padding='same', activation='relu'))
model.add(tf.keras.layers.MaxPooling2D(pool_size=2))
model.add(tf.keras.layers.Dropout(0.3))
model.add(tf.keras.layers.Flatten())
model.add(tf.keras.layers.Dense(256, activation='relu'))
model.add(tf.keras.layers.Dropout(0.5))
model.add(tf.keras.layers.Dense(10, activation='softmax'))
# Take a look at the model summary
model.summary()WARNING:tensorflow:From C:\Users\pc\Anaconda3\lib\site-packages\tensorflow\python\ops\init_ops.py:1251: calling VarianceScaling.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.
Instructions for updating:
Call initializer instance with the dtype argument instead of passing it to the constructor
Model: "sequential"
_________________________________________________________________
Layer (type)                 Output Shape              Param #
=================================================================
conv2d (Conv2D)              (None, 28, 28, 64)        320
_________________________________________________________________
max_pooling2d (MaxPooling2D) (None, 14, 14, 64)        0
_________________________________________________________________
dropout (Dropout)            (None, 14, 14, 64)        0
_________________________________________________________________
conv2d_1 (Conv2D)            (None, 14, 14, 32)        8224
_________________________________________________________________
max_pooling2d_1 (MaxPooling2 (None, 7, 7, 32)          0
_________________________________________________________________
dropout_1 (Dropout)          (None, 7, 7, 32)          0
_________________________________________________________________
flatten (Flatten)            (None, 1568)              0
_________________________________________________________________
dense (Dense)                (None, 256)               401664
_________________________________________________________________
dropout_2 (Dropout)          (None, 256)               0
_________________________________________________________________
dense_1 (Dense)              (None, 10)                2570
=================================================================
Total params: 412,778
Trainable params: 412,778
Non-trainable params: 0
_________________________________________________________________
model.compile(loss ='sparse_categorical_crossentropy', optimizer='adam' ,metrics =['accuracy'])
_____no_output_____epochs = 150

history = cnn_model.fit(x_train, y_train, batch_size = 512, epochs = epochs,
                        verbose = 1, validation_data = (x_validate, y_validate))Train on 43200 samples, validate on 10800 samples
Epoch 1/150
43200/43200 [==============================] - 76s 2ms/sample - loss: 4.3887 - acc: 0.4623 - val_loss: 0.8466 - val_acc: 0.6875
Epoch 2/150
43200/43200 [==============================] - 73s 2ms/sample - loss: 0.9502 - acc: 0.6544 - val_loss: 0.6472 - val_acc: 0.7543
Epoch 3/150
43200/43200 [==============================] - 73s 2ms/sample - loss: 0.7803 - acc: 0.7049 - val_loss: 0.5486 - val_acc: 0.7912
Epoch 4/150
43200/43200 [==============================] - 71s 2ms/sample - loss: 0.6893 - acc: 0.7388 - val_loss: 0.5318 - val_acc: 0.7856
Epoch 5/150
43200/43200 [==============================] - 71s 2ms/sample - loss: 0.6426 - acc: 0.7534 - val_loss: 0.4876 - val_acc: 0.8058
Epoch 6/150
43200/43200 [==============================] - 72s 2ms/sample - loss: 0.6032 - acc: 0.7686 - val_loss: 0.4773 - val_acc: 0.8037
Epoch 7/150
43200/43200 [==============================] - 72s 2ms/sample - loss: 0.5719 - acc: 0.7796 - val_loss: 0.4438 - val_acc: 0.8319
Epoch 8/150
43200/43200 [==============================] - 72s 2ms/sample - loss: 0.5530 - acc: 0.7882 - val_loss: 0.4262 - val_acc: 0.8419
Epoch 9/150
43200/43200 [==============================] - 72s 2ms/sample - loss: 0.5355 - acc: 0.7971 - val_loss: 0.4175 - val_acc: 0.8485
Epoch 10/150
43200/43200 [==============================] - 72s 2ms/sample - loss: 0.5166 - acc: 0.8053 - val_loss: 0.4020 - val_acc: 0.8494
Epoch 11/150
43200/43200 [==============================] - 71s 2ms/sample - loss: 0.5024 - acc: 0.8104 - val_loss: 0.3837 - val_acc: 0.8569
Epoch 12/150
43200/43200 [==============================] - 70s 2ms/sample - loss: 0.4818 - acc: 0.8186 - val_loss: 0.3787 - val_acc: 0.8648
Epoch 13/150
43200/43200 [==============================] - 71s 2ms/sample - loss: 0.4718 - acc: 0.8245 - val_loss: 0.3624 - val_acc: 0.8648
Epoch 14/150
43200/43200 [==============================] - 72s 2ms/sample - loss: 0.4584 - acc: 0.8309 - val_loss: 0.3587 - val_acc: 0.8645
Epoch 15/150
43200/43200 [==============================] - 70s 2ms/sample - loss: 0.4484 - acc: 0.8326 - val_loss: 0.3505 - val_acc: 0.8716
Epoch 16/150
43200/43200 [==============================] - 71s 2ms/sample - loss: 0.4395 - acc: 0.8363 - val_loss: 0.3400 - val_acc: 0.8751
Epoch 17/150
43200/43200 [==============================] - 71s 2ms/sample - loss: 0.4243 - acc: 0.8438 - val_loss: 0.3330 - val_acc: 0.8764
Epoch 18/150
43200/43200 [==============================] - 71s 2ms/sample - loss: 0.4207 - acc: 0.8444 - val_loss: 0.3300 - val_acc: 0.8788
Epoch 19/150
43200/43200 [==============================] - 74s 2ms/sample - loss: 0.4071 - acc: 0.8506 - val_loss: 0.3266 - val_acc: 0.8781
Epoch 20/150
43200/43200 [==============================] - 72s 2ms/sample - loss: 0.4028 - acc: 0.8515 - val_loss: 0.3222 - val_acc: 0.8799
Epoch 21/150
43200/43200 [==============================] - 76s 2ms/sample - loss: 0.3949 - acc: 0.8556 - val_loss: 0.3164 - val_acc: 0.8843
Epoch 22/150
43200/43200 [==============================] - 74s 2ms/sample - loss: 0.3861 - acc: 0.8562 - val_loss: 0.3123 - val_acc: 0.8816
Epoch 23/150
43200/43200 [==============================] - 75s 2ms/sample - loss: 0.3811 - acc: 0.8602 - val_loss: 0.3126 - val_acc: 0.8831
Epoch 24/150
43200/43200 [==============================] - 75s 2ms/sample - loss: 0.3785 - acc: 0.8612 - val_loss: 0.3035 - val_acc: 0.8895
Epoch 25/150
43200/43200 [==============================] - 74s 2ms/sample - loss: 0.3728 - acc: 0.8604 - val_loss: 0.3070 - val_acc: 0.8833
Epoch 26/150
43200/43200 [==============================] - 75s 2ms/sample - loss: 0.3639 - acc: 0.8648 - val_loss: 0.2972 - val_acc: 0.8915
Epoch 27/150
43200/43200 [==============================] - 73s 2ms/sample - loss: 0.3582 - acc: 0.8682 - val_loss: 0.2934 - val_acc: 0.8919
Epoch 28/150
43200/43200 [==============================] - 71s 2ms/sample - loss: 0.3564 - acc: 0.8686 - val_loss: 0.2949 - val_acc: 0.8894
Epoch 29/150
43200/43200 [==============================] - 72s 2ms/sample - loss: 0.3521 - acc: 0.8720 - val_loss: 0.2916 - val_acc: 0.8946
Epoch 30/150
43200/43200 [==============================] - 71s 2ms/sample - loss: 0.3504 - acc: 0.8738 - val_loss: 0.2851 - val_acc: 0.8932
Epoch 31/150
43200/43200 [==============================] - 71s 2ms/sample - loss: 0.3415 - acc: 0.8733 - val_loss: 0.2797 - val_acc: 0.8974
Epoch 32/150
43200/43200 [==============================] - 71s 2ms/sample - loss: 0.3392 - acc: 0.8761 - val_loss: 0.2863 - val_acc: 0.8931
Epoch 33/150
43200/43200 [==============================] - 72s 2ms/sample - loss: 0.3310 - acc: 0.8769 - val_loss: 0.2834 - val_acc: 0.8935
Epoch 34/150
43200/43200 [==============================] - 71s 2ms/sample - loss: 0.3307 - acc: 0.8785 - val_loss: 0.2802 - val_acc: 0.8941
Epoch 35/150
43200/43200 [==============================] - 71s 2ms/sample - loss: 0.3264 - acc: 0.8797 - val_loss: 0.2753 - val_acc: 0.8994
Epoch 36/150
43200/43200 [==============================] - 71s 2ms/sample - loss: 0.3233 - acc: 0.8799 - val_loss: 0.2784 - val_acc: 0.8963
Epoch 37/150
43200/43200 [==============================] - 71s 2ms/sample - loss: 0.3224 - acc: 0.8821 - val_loss: 0.2714 - val_acc: 0.9002
Epoch 38/150
43200/43200 [==============================] - 71s 2ms/sample - loss: 0.3216 - acc: 0.8827 - val_loss: 0.2753 - val_acc: 0.8992
Epoch 39/150
43200/43200 [==============================] - 74s 2ms/sample - loss: 0.3158 - acc: 0.8828 - val_loss: 0.2680 - val_acc: 0.9019
Epoch 40/150
43200/43200 [==============================] - 70s 2ms/sample - loss: 0.3134 - acc: 0.8833 - val_loss: 0.2727 - val_acc: 0.9006
Epoch 41/150
43200/43200 [==============================] - 71s 2ms/sample - loss: 0.3097 - acc: 0.8855 - val_loss: 0.2676 - val_acc: 0.9025
Epoch 42/150
43200/43200 [==============================] - 71s 2ms/sample - loss: 0.3079 - acc: 0.8861 - val_loss: 0.2826 - val_acc: 0.8946
Epoch 43/150
43200/43200 [==============================] - 70s 2ms/sample - loss: 0.2972 - acc: 0.8890 - val_loss: 0.2708 - val_acc: 0.9012
Epoch 44/150
43200/43200 [==============================] - 71s 2ms/sample - loss: 0.3034 - acc: 0.8864 - val_loss: 0.2678 - val_acc: 0.9000
Epoch 45/150
43200/43200 [==============================] - 71s 2ms/sample - loss: 0.2953 - acc: 0.8921 - val_loss: 0.2659 - val_acc: 0.9010
Epoch 46/150
43200/43200 [==============================] - 71s 2ms/sample - loss: 0.2957 - acc: 0.8918 - val_loss: 0.2601 - val_acc: 0.9039
Epoch 47/150
43200/43200 [==============================] - 71s 2ms/sample - loss: 0.2949 - acc: 0.8915 - val_loss: 0.2656 - val_acc: 0.9026
Epoch 48/150
43200/43200 [==============================] - 71s 2ms/sample - loss: 0.2866 - acc: 0.8933 - val_loss: 0.2624 - val_acc: 0.9041
Epoch 49/150
43200/43200 [==============================] - 71s 2ms/sample - loss: 0.2876 - acc: 0.8936 - val_loss: 0.2599 - val_acc: 0.9055
Epoch 50/150
43200/43200 [==============================] - 72s 2ms/sample - loss: 0.2870 - acc: 0.8949 - val_loss: 0.2611 - val_acc: 0.9044
Epoch 51/150
43200/43200 [==============================] - 74s 2ms/sample - loss: 0.2834 - acc: 0.8961 - val_loss: 0.2577 - val_acc: 0.9072
Epoch 52/150
43200/43200 [==============================] - 78s 2ms/sample - loss: 0.2808 - acc: 0.8983 - val_loss: 0.2586 - val_acc: 0.9075
Epoch 53/150
43200/43200 [==============================] - 71s 2ms/sample - loss: 0.2788 - acc: 0.8973 - val_loss: 0.2649 - val_acc: 0.9025
Epoch 54/150
43200/43200 [==============================] - 73s 2ms/sample - loss: 0.2760 - acc: 0.8990 - val_loss: 0.2566 - val_acc: 0.9081
Epoch 55/150
43200/43200 [==============================] - 77s 2ms/sample - loss: 0.2766 - acc: 0.8974 - val_loss: 0.2571 - val_acc: 0.9069
Epoch 56/150
43200/43200 [==============================] - 76s 2ms/sample - loss: 0.2708 - acc: 0.8992 - val_loss: 0.2572 - val_acc: 0.9101
Epoch 57/150
43200/43200 [==============================] - 75s 2ms/sample - loss: 0.2696 - acc: 0.9001 - val_loss: 0.2508 - val_acc: 0.9128
Epoch 58/150
43200/43200 [==============================] - 85s 2ms/sample - loss: 0.2650 - acc: 0.9021 - val_loss: 0.2561 - val_acc: 0.9081
Epoch 59/150
43200/43200 [==============================] - 82s 2ms/sample - loss: 0.2669 - acc: 0.8999 - val_loss: 0.2553 - val_acc: 0.9078
Epoch 60/150
43200/43200 [==============================] - 78s 2ms/sample - loss: 0.2653 - acc: 0.9026 - val_loss: 0.2564 - val_acc: 0.9070
Epoch 61/150
43200/43200 [==============================] - 79s 2ms/sample - loss: 0.2616 - acc: 0.9036 - val_loss: 0.2512 - val_acc: 0.9107
Epoch 62/150
43200/43200 [==============================] - 79s 2ms/sample - loss: 0.2618 - acc: 0.9027 - val_loss: 0.2499 - val_acc: 0.9121
Epoch 63/150
43200/43200 [==============================] - 78s 2ms/sample - loss: 0.2606 - acc: 0.9022 - val_loss: 0.2535 - val_acc: 0.9092
Epoch 64/150
43200/43200 [==============================] - 76s 2ms/sample - loss: 0.2568 - acc: 0.9033 - val_loss: 0.2504 - val_acc: 0.9073
Epoch 65/150
43200/43200 [==============================] - 74s 2ms/sample - loss: 0.2505 - acc: 0.9053 - val_loss: 0.2481 - val_acc: 0.9119
Epoch 66/150
43200/43200 [==============================] - 76s 2ms/sample - loss: 0.2579 - acc: 0.9052 - val_loss: 0.2516 - val_acc: 0.9131
Epoch 67/150
43200/43200 [==============================] - 76s 2ms/sample - loss: 0.2536 - acc: 0.9057 - val_loss: 0.2504 - val_acc: 0.9142
Epoch 68/150
43200/43200 [==============================] - 76s 2ms/sample - loss: 0.2520 - acc: 0.9078 - val_loss: 0.2526 - val_acc: 0.9108
Epoch 69/150
43200/43200 [==============================] - 78s 2ms/sample - loss: 0.2501 - acc: 0.9059 - val_loss: 0.2472 - val_acc: 0.9111
Epoch 70/150
43200/43200 [==============================] - 76s 2ms/sample - loss: 0.2481 - acc: 0.9080 - val_loss: 0.2469 - val_acc: 0.9132
Epoch 71/150
43200/43200 [==============================] - 80s 2ms/sample - loss: 0.2480 - acc: 0.9082 - val_loss: 0.2453 - val_acc: 0.9151
Epoch 72/150
43200/43200 [==============================] - 76s 2ms/sample - loss: 0.2467 - acc: 0.9079 - val_loss: 0.2451 - val_acc: 0.9122
Epoch 73/150
43200/43200 [==============================] - 81s 2ms/sample - loss: 0.2447 - acc: 0.9082 - val_loss: 0.2441 - val_acc: 0.9158
Epoch 74/150
43200/43200 [==============================] - 79s 2ms/sample - loss: 0.2400 - acc: 0.9114 - val_loss: 0.2469 - val_acc: 0.9138
Epoch 75/150
43200/43200 [==============================] - 79s 2ms/sample - loss: 0.2401 - acc: 0.9101 - val_loss: 0.2510 - val_acc: 0.9164
Epoch 76/150
43200/43200 [==============================] - 72s 2ms/sample - loss: 0.2322 - acc: 0.9122 - val_loss: 0.2395 - val_acc: 0.9148
Epoch 77/150
43200/43200 [==============================] - 74s 2ms/sample - loss: 0.2348 - acc: 0.9123 - val_loss: 0.2458 - val_acc: 0.9131
Epoch 78/150
43200/43200 [==============================] - 73s 2ms/sample - loss: 0.2435 - acc: 0.9099 - val_loss: 0.2447 - val_acc: 0.9135
Epoch 79/150
43200/43200 [==============================] - 73s 2ms/sample - loss: 0.2320 - acc: 0.9148 - val_loss: 0.2466 - val_acc: 0.9134
Epoch 80/150
43200/43200 [==============================] - 73s 2ms/sample - loss: 0.2281 - acc: 0.9134 - val_loss: 0.2444 - val_acc: 0.9172
Epoch 81/150
43200/43200 [==============================] - 76s 2ms/sample - loss: 0.2327 - acc: 0.9120 - val_loss: 0.2406 - val_acc: 0.9180
Epoch 82/150
43200/43200 [==============================] - 76s 2ms/sample - loss: 0.2282 - acc: 0.9139 - val_loss: 0.2472 - val_acc: 0.9184
Epoch 83/150
43200/43200 [==============================] - 76s 2ms/sample - loss: 0.2329 - acc: 0.9127 - val_loss: 0.2430 - val_acc: 0.9153
Epoch 84/150
43200/43200 [==============================] - 73s 2ms/sample - loss: 0.2275 - acc: 0.9139 - val_loss: 0.2527 - val_acc: 0.9144
Epoch 85/150
43200/43200 [==============================] - 75s 2ms/sample - loss: 0.2249 - acc: 0.9162 - val_loss: 0.2443 - val_acc: 0.9156
Epoch 86/150
43200/43200 [==============================] - 77s 2ms/sample - loss: 0.2266 - acc: 0.9158 - val_loss: 0.2436 - val_acc: 0.9175
Epoch 87/150
43200/43200 [==============================] - 82s 2ms/sample - loss: 0.2223 - acc: 0.9172 - val_loss: 0.2388 - val_acc: 0.9170
Epoch 88/150
43200/43200 [==============================] - 73s 2ms/sample - loss: 0.2163 - acc: 0.9188 - val_loss: 0.2455 - val_acc: 0.9169
Epoch 89/150
43200/43200 [==============================] - 71s 2ms/sample - loss: 0.2225 - acc: 0.9156 - val_loss: 0.2456 - val_acc: 0.9192
Epoch 90/150
43200/43200 [==============================] - 71s 2ms/sample - loss: 0.2147 - acc: 0.9180 - val_loss: 0.2446 - val_acc: 0.9175
Epoch 91/150
43200/43200 [==============================] - 72s 2ms/sample - loss: 0.2186 - acc: 0.9184 - val_loss: 0.2479 - val_acc: 0.9176
Epoch 92/150
43200/43200 [==============================] - 70s 2ms/sample - loss: 0.2182 - acc: 0.9164 - val_loss: 0.2446 - val_acc: 0.9197
Epoch 93/150
43200/43200 [==============================] - 70s 2ms/sample - loss: 0.2158 - acc: 0.9194 - val_loss: 0.2399 - val_acc: 0.9210
Epoch 94/150
43200/43200 [==============================] - 70s 2ms/sample - loss: 0.2161 - acc: 0.9208 - val_loss: 0.2481 - val_acc: 0.9178
Epoch 95/150
43200/43200 [==============================] - 71s 2ms/sample - loss: 0.2146 - acc: 0.9180 - val_loss: 0.2380 - val_acc: 0.9186
Epoch 96/150
43200/43200 [==============================] - 70s 2ms/sample - loss: 0.2160 - acc: 0.9184 - val_loss: 0.2400 - val_acc: 0.9200
Epoch 97/150
43200/43200 [==============================] - 70s 2ms/sample - loss: 0.2109 - acc: 0.9213 - val_loss: 0.2425 - val_acc: 0.9161
Epoch 98/150
43200/43200 [==============================] - 70s 2ms/sample - loss: 0.2119 - acc: 0.9207 - val_loss: 0.2384 - val_acc: 0.9184
Epoch 99/150
43200/43200 [==============================] - 75s 2ms/sample - loss: 0.2080 - acc: 0.9224 - val_loss: 0.2416 - val_acc: 0.9167
Epoch 100/150
43200/43200 [==============================] - 75s 2ms/sample - loss: 0.2090 - acc: 0.9231 - val_loss: 0.2424 - val_acc: 0.9165
Epoch 101/150
43200/43200 [==============================] - 72s 2ms/sample - loss: 0.2065 - acc: 0.9226 - val_loss: 0.2426 - val_acc: 0.9188
Epoch 102/150
43200/43200 [==============================] - 76s 2ms/sample - loss: 0.2090 - acc: 0.9211 - val_loss: 0.2365 - val_acc: 0.9196
Epoch 103/150
43200/43200 [==============================] - 74s 2ms/sample - loss: 0.2039 - acc: 0.9238 - val_loss: 0.2432 - val_acc: 0.9171
Epoch 104/150
43200/43200 [==============================] - 76s 2ms/sample - loss: 0.2062 - acc: 0.9232 - val_loss: 0.2422 - val_acc: 0.9194
Epoch 105/150
43200/43200 [==============================] - 80s 2ms/sample - loss: 0.2006 - acc: 0.9244 - val_loss: 0.2491 - val_acc: 0.9188
Epoch 106/150
43200/43200 [==============================] - 96s 2ms/sample - loss: 0.2019 - acc: 0.9251 - val_loss: 0.2374 - val_acc: 0.9211
Epoch 107/150
43200/43200 [==============================] - 89s 2ms/sample - loss: 0.2024 - acc: 0.9239 - val_loss: 0.2368 - val_acc: 0.9184
Epoch 108/150
43200/43200 [==============================] - 88s 2ms/sample - loss: 0.2014 - acc: 0.9234 - val_loss: 0.2389 - val_acc: 0.9206
Epoch 109/150
43200/43200 [==============================] - 90s 2ms/sample - loss: 0.1996 - acc: 0.9249 - val_loss: 0.2461 - val_acc: 0.9170
Epoch 110/150
43200/43200 [==============================] - 85s 2ms/sample - loss: 0.1999 - acc: 0.9246 - val_loss: 0.2430 - val_acc: 0.9192
Epoch 111/150
43200/43200 [==============================] - 87s 2ms/sample - loss: 0.1960 - acc: 0.9259 - val_loss: 0.2524 - val_acc: 0.9166
Epoch 112/150
43200/43200 [==============================] - 87s 2ms/sample - loss: 0.1954 - acc: 0.9267 - val_loss: 0.2382 - val_acc: 0.9209
Epoch 113/150
43200/43200 [==============================] - 89s 2ms/sample - loss: 0.1964 - acc: 0.9270 - val_loss: 0.2353 - val_acc: 0.9195
Epoch 114/150
43200/43200 [==============================] - 89s 2ms/sample - loss: 0.1921 - acc: 0.9274 - val_loss: 0.2415 - val_acc: 0.9195
Epoch 115/150
43200/43200 [==============================] - 81s 2ms/sample - loss: 0.1982 - acc: 0.9257 - val_loss: 0.2374 - val_acc: 0.9203
Epoch 116/150
43200/43200 [==============================] - 85s 2ms/sample - loss: 0.1923 - acc: 0.9262 - val_loss: 0.2551 - val_acc: 0.9148
Epoch 117/150
43200/43200 [==============================] - 85s 2ms/sample - loss: 0.1951 - acc: 0.9281 - val_loss: 0.2483 - val_acc: 0.9171
Epoch 118/150
43200/43200 [==============================] - 87s 2ms/sample - loss: 0.1912 - acc: 0.9285 - val_loss: 0.2417 - val_acc: 0.9184
Epoch 119/150
43200/43200 [==============================] - 89s 2ms/sample - loss: 0.1918 - acc: 0.9268 - val_loss: 0.2442 - val_acc: 0.9198
Epoch 120/150
43200/43200 [==============================] - 88s 2ms/sample - loss: 0.1921 - acc: 0.9261 - val_loss: 0.2489 - val_acc: 0.9185
Epoch 121/150
43200/43200 [==============================] - 84s 2ms/sample - loss: 0.1949 - acc: 0.9268 - val_loss: 0.2383 - val_acc: 0.9215
Epoch 122/150
43200/43200 [==============================] - 81s 2ms/sample - loss: 0.1902 - acc: 0.9294 - val_loss: 0.2370 - val_acc: 0.9194
Epoch 123/150
43200/43200 [==============================] - 81s 2ms/sample - loss: 0.1914 - acc: 0.9278 - val_loss: 0.2538 - val_acc: 0.9147
Epoch 124/150
43200/43200 [==============================] - 84s 2ms/sample - loss: 0.1875 - acc: 0.9293 - val_loss: 0.2443 - val_acc: 0.9186
Epoch 125/150
43200/43200 [==============================] - 84s 2ms/sample - loss: 0.1923 - acc: 0.9262 - val_loss: 0.2494 - val_acc: 0.9130
Epoch 126/150
43200/43200 [==============================] - 85s 2ms/sample - loss: 0.1878 - acc: 0.9296 - val_loss: 0.2411 - val_acc: 0.9197
Epoch 127/150
43200/43200 [==============================] - 86s 2ms/sample - loss: 0.1851 - acc: 0.9294 - val_loss: 0.2441 - val_acc: 0.9182
Epoch 128/150
43200/43200 [==============================] - 82s 2ms/sample - loss: 0.1920 - acc: 0.9273 - val_loss: 0.2419 - val_acc: 0.9193
Epoch 129/150
43200/43200 [==============================] - 81s 2ms/sample - loss: 0.1881 - acc: 0.9293 - val_loss: 0.2430 - val_acc: 0.9193
Epoch 130/150
43200/43200 [==============================] - 81s 2ms/sample - loss: 0.1849 - acc: 0.9315 - val_loss: 0.2463 - val_acc: 0.9196
Epoch 131/150
43200/43200 [==============================] - 81s 2ms/sample - loss: 0.1817 - acc: 0.9322 - val_loss: 0.2459 - val_acc: 0.9203
Epoch 132/150
43200/43200 [==============================] - 84s 2ms/sample - loss: 0.1918 - acc: 0.9290 - val_loss: 0.2431 - val_acc: 0.9196
Epoch 133/150
43200/43200 [==============================] - 82s 2ms/sample - loss: 0.1854 - acc: 0.9300 - val_loss: 0.2430 - val_acc: 0.9190
Epoch 134/150
43200/43200 [==============================] - 82s 2ms/sample - loss: 0.1837 - acc: 0.9297 - val_loss: 0.2480 - val_acc: 0.9189
Epoch 135/150
43200/43200 [==============================] - 82s 2ms/sample - loss: 0.1828 - acc: 0.9319 - val_loss: 0.2496 - val_acc: 0.9174
Epoch 136/150
43200/43200 [==============================] - 83s 2ms/sample - loss: 0.1843 - acc: 0.9310 - val_loss: 0.2379 - val_acc: 0.9195
Epoch 137/150
43200/43200 [==============================] - 82s 2ms/sample - loss: 0.1781 - acc: 0.9325 - val_loss: 0.2404 - val_acc: 0.9198
Epoch 138/150
43200/43200 [==============================] - 81s 2ms/sample - loss: 0.1779 - acc: 0.9334 - val_loss: 0.2458 - val_acc: 0.9204
Epoch 139/150
43200/43200 [==============================] - 82s 2ms/sample - loss: 0.1778 - acc: 0.9325 - val_loss: 0.2497 - val_acc: 0.9168
Epoch 140/150
43200/43200 [==============================] - 81s 2ms/sample - loss: 0.1809 - acc: 0.9332 - val_loss: 0.2509 - val_acc: 0.9153
Epoch 141/150
43200/43200 [==============================] - 80s 2ms/sample - loss: 0.1818 - acc: 0.9307 - val_loss: 0.2417 - val_acc: 0.9196
Epoch 142/150
43200/43200 [==============================] - 81s 2ms/sample - loss: 0.1795 - acc: 0.9313 - val_loss: 0.2516 - val_acc: 0.9173
Epoch 143/150
43200/43200 [==============================] - 85s 2ms/sample - loss: 0.1842 - acc: 0.9306 - val_loss: 0.2524 - val_acc: 0.9173
Epoch 144/150
43200/43200 [==============================] - 82s 2ms/sample - loss: 0.1774 - acc: 0.9332 - val_loss: 0.2414 - val_acc: 0.9198
Epoch 145/150
43200/43200 [==============================] - 82s 2ms/sample - loss: 0.1781 - acc: 0.9331 - val_loss: 0.2417 - val_acc: 0.9219
Epoch 146/150
43200/43200 [==============================] - 82s 2ms/sample - loss: 0.1716 - acc: 0.9345 - val_loss: 0.2409 - val_acc: 0.9210
Epoch 147/150
43200/43200 [==============================] - 85s 2ms/sample - loss: 0.1703 - acc: 0.9355 - val_loss: 0.2544 - val_acc: 0.9165
Epoch 148/150
43200/43200 [==============================] - 82s 2ms/sample - loss: 0.1765 - acc: 0.9334 - val_loss: 0.2529 - val_acc: 0.9174
Epoch 149/150
43200/43200 [==============================] - 86s 2ms/sample - loss: 0.1759 - acc: 0.9341 - val_loss: 0.2432 - val_acc: 0.9204
Epoch 150/150
43200/43200 [==============================] - 83s 2ms/sample - loss: 0.1734 - acc: 0.9350 - val_loss: 0.2497 - val_acc: 0.9186

# Vizualizing the Loss and Accuracy evolution _____no_output_____

plt.figure(figsize=(12, 8))

plt.subplot(2, 2, 1)
plt.plot(history.history['loss'], label='Loss')
plt.plot(history.history['val_loss'], label='val_Loss')
plt.legend()
plt.title('Loss evolution')
_____no_output_____plt.plot(history.history['acc'])
plt.plot(history.history['val_acc'])
plt.title(" Accuracy Evolution")
plt.xlabel('Epochs')
plt.ylabel('Accuracy')
plt.legend(['Accuracy', 'val_accuracy'])
plt.show()_____no_output_____

# Evaluating the model _____no_output_____

evaluation = model.evaluate(x_test, y_test)
print(f'Test Accuracy : {evaluation[1]:.3f}')6000/6000 [==============================] - 4s 655us/sample - loss: 0.2320 - acc: 0.9265
Test Accuracy : 0.927

# Predictions _____no_output_____

 #get the predictions for the test data
predicted_classes = model.predict_classes(x_test)_____no_output_____test_img = x_test[0]
prediction = model.predict(x_test)
prediction[0]_____no_output_____np.argmax(prediction[0])_____no_output_____L = 5
W = 5
fig, axes = plt.subplots(L, W, figsize = (12,12))
axes = axes.ravel()

for i in np.arange(0, L * W):
    axes[i].imshow(x_test[i].reshape(28,28))
    axes[i].set_title(f"Prediction Class = {predicted_classes[i]:0.1f}\n True Class = {y_test[i]:0.1f}")
    axes[i].axis('off')

plt.subplots_adjust(wspace=0.5)_____no_output_____

# Confusion matrix_____no_output_____

from sklearn.metrics import confusion_matrix
cmatrix = confusion_matrix(y_test, predicted_classes)
plt.figure(figsize = (14,10))
sns.heatmap(cmatrix, annot=True)
# Sum the diagonal element to get the total true correct values_____no_output_____

# Precision and Recall _____no_output_____

from sklearn.metrics import classification_report

num_classes = 10
target_names = [f"Class {i}" for i in range(num_classes)]

print(classification_report(y_test, predicted_classes, target_names = target_names))              precision    recall  f1-score   support

     Class 0       0.88      0.89      0.89       629
     Class 1       1.00      0.99      0.99       580
     Class 2       0.89      0.87      0.88       550
     Class 3       0.91      0.93      0.92       612
     Class 4       0.86      0.88      0.87       598
     Class 5       0.99      0.98      0.99       613
     Class 6       0.80      0.78      0.79       592
     Class 7       0.97      0.99      0.98       594
     Class 8       0.98      0.98      0.98       623
     Class 9       0.98      0.97      0.98       609

    accuracy                           0.93      6000
   macro avg       0.93      0.93      0.93      6000
weighted avg       0.93      0.93      0.93      6000


# Notebook from AlexRMU/art
Path: ArtList.ipynb

# ART_____no_output_____### Version : 4.9_____no_output_____

from art import *_____no_output_____

## Art Counter_____no_output_____

ART_COUNTER_____no_output_____

## Art List_____no_output_____

art_list()3
ᕙ༼ ,,ԾܫԾ,, ༽ᕗ
******************************
5
ᕙ༼ ,,இܫஇ,, ༽ᕗ
******************************
9/11 truth
✈__✈ █ █ ▄
******************************
acid
⊂(◉‿◉)つ
******************************
afraid
(　ﾟ Дﾟ)
******************************
airplane1
 ‛¯¯٭٭¯¯(▫▫)¯¯٭٭¯¯’
******************************
airplane2
✈
******************************
ak-47
︻┳デ═—
******************************
alien
::)
******************************
almost cared
╰╏ ◉ 〜 ◉ ╏╯
******************************
american money1
[($)]
******************************
american money2
[̲̅$̲̅(̲̅1̲̅)̲̅$̲̅]
******************************
american money3
[̲̅$̲̅(̲̅5̲̅)̲̅$̲̅]
******************************
american money4
[̲̅$̲̅(̲̅ιοο̲̅)̲̅$̲̅]
******************************
american money5
[̲̅$̲̅(̲̅2οο̲̅)̲̅$̲̅]
******************************
angel1
^i^
******************************
angel2
O:-)
******************************
angry
 ლ(ಠ益ಠ)ლ
******************************
angry birds
( ఠൠఠ )ﾉ
******************************
angry face
(⋟﹏⋞)
******************************
angry face2
(╬ ಠ益ಠ)
******************************
angry troll
ヽ༼ ಠ益ಠ ༽ﾉ
******************************
angry2
( ͠° ͟ʖ ͡°)
******************************
ankush
︻デ┳═ー*----*
******************************
arrow1
»»---------------------►
******************************
arrow2
XXX-------->
******************************
arrowhead
⤜(ⱺ ʖ̯ⱺ)⤏
******************************
at what cost
ლ(ಠ益ಠლ)
******************************
atish
(| - _ - |)
******************************
awesome
<:3 )~~~
******************************
awkward
•͡˘㇁•͡˘
******************************
bad hair1
=:-)
******************************
bad hair2
=:-(
******************************
bagel
nln >_< nln
******************************
band aid
(̲̅:̲̅:̲̅:̲̅[̲̅ ̲̅]̲̅:̲̅:̲̅:̲̅ )
******************************
barbell
▐━━━━━▌
******************************
barcode1
█║▌│ █│║▌ ║││█║▌ │║║█║ │║║█║
******************************
barcode2
║█║▌║█║▌│║▌║▌█║
******************************
barf
(´ж｀ς)
******************************
baseball fan
q:o)
******************************
basking in glory
ヽ(´ー｀)ノ
******************************
bat1
^O^
******************************
bat2
 ^v^
******************************
bautista
(╯°_°）╯︵ ━━━
******************************
bear
ʕ•ᴥ•ʔ
******************************
bear GTFO
ʕ •`ᴥ•´ʔ
******************************
bear squiting
ʕᵔᴥᵔʔ
******************************
because
∵
******************************
bee
¸.·´¯`·¸¸.·´¯`·.¸.-<\^}0=:
******************************
being draged
╰(◣﹏◢)╯
******************************
bender
 ¦̵̱ ̵̱ ̵̱ ̵̱ ̵̱(̢ ̡͇̅└͇̅┘͇̅ (▤8כ−◦
******************************
big eyes
⺌∅‿∅⺌
******************************
big kiss
:-X
******************************
big nose
˚∆˚
******************************
big smile
:-D
******************************
bird
 (⌒▽⌒)
******************************
birds
~(‾▿‾)~
******************************
blackeye
0__#
******************************
bomb
!!(　’ ‘)ﾉﾉ⌒●~*
******************************
boobies
(. )( .)
******************************
boobs
(.)(.)
******************************
boombox1
♫♪.ılılıll|̲̅̅●̲̅̅|̲̅̅=̲̅̅|̲̅̅●̲̅̅|llılılı.♫♪
******************************
boombox2
♫♪ |̲̅̅●̲̅̅|̲̅̅=̲̅̅|̲̅̅●̲̅̅| ♫♪
******************************
boxing
ლ(•́•́ლ)
******************************
breakdown
ಥ﹏ಥ
******************************
british money
[£::]
******************************
buck teeth
:-B
******************************
bugs bunny
E=B
******************************
bullshit
|3ᵕᶦᶦᶳᶣᶨᶵ
******************************
bunny
(\_/)
******************************
butt
(‿|‿)
******************************
butterfly
Ƹ̵̡Ӝ̵̨̄Ʒ
******************************
camera
[◉"]
******************************
canoe
.,.,\______/,..,.,
******************************
car
`o##o>
******************************
car race
∙،°.  ˘Ô≈ôﺣ   » » »
******************************
care crowd
(-(-_(-_-)_-)-)
******************************
careless
◔_◔
******************************
carpet roll
@__
******************************
cassette1
|[●▪▪●]|
******************************
cassette2
[¯ↂ■■ↂ¯]
******************************
cat face
⦿⽘⦿
******************************
cat smile
≧◔◡◔≦
******************************
cat1
=^..^=
******************************
cat2
龴ↀ◡ↀ龴
******************************
cat3
^.--.^
******************************
caterpillar
,/\,/\,/\,/\,/\,/\,o
******************************
catlenny
( ͡° ᴥ ͡°)
******************************
chair
╦╣
******************************
charly
+:)
******************************
chasing
''⌐(ಠ۾ಠ)¬''
******************************
cheer
  ^(¤o¤)^
******************************
cheers
（ ^_^）o自自o（^_^ ）
******************************
chess
♞▀▄▀▄♝▀▄
******************************
chess pieces
♚ ♛ ♜ ♝ ♞ ♟ ♔ ♕ ♖ ♗ ♘ ♙
******************************
chicken
ʚ(•｀
******************************
chu
(´ε｀ )
******************************
cigarette1
(̅_̅_̅_̅(̅_̅_̅_̅_̅_̅_̅_̅_̅̅_̅()ڪے
******************************
cigarette2
(____((____________()~~~
******************************
cigarette3
()___)____________)
******************************
clowning
*:o)
******************************
club bold
♣
******************************
club regular
♧
******************************
coffee now
{zzz}°°°( -_-)>c[_]
******************************
coffee1
c[_]
******************************
coffee2
l_D
******************************
coffee3
l_P
******************************
coffee4
l_B
******************************
computer mouse
[E}
******************************
concerned
(@_@)
******************************
confused scratch
(⊙.☉)7
******************************
confused1
:-/
******************************
confused2
:-\
******************************
crab
(\|) ._. (|/)
******************************
crayons
((̲̅ ̲̅(̲̅C̲̅r̲̅a̲̅y̲̅o̲̅l̲̲̅̅a̲̅( ̲̅((>
******************************
crazy
ミ●﹏☉ミ
******************************
creeper
ƪ(ړײ)ƪ
******************************
crotch shot
\*/
******************************
cry
 (╯︵╰,)
******************************
cry face
｡ﾟ( ﾟஇ‸இﾟ)ﾟ｡
******************************
cry troll
༼ ༎ຶ ෴ ༎ຶ༽
******************************
crying1
Ỏ̷͖͈̞̩͎̻̫̫̜͉̠̫͕̭̭̫̫̹̗̹͈̼̠̖͍͚̥͈̮̼͕̠̤̯̻̥̬̗̼̳̤̳̬̪̹͚̞̼̠͕̼̠̦͚̫͔̯̹͉͉̘͎͕̼̣̝͙̱̟̹̩̟̳̦̭͉̮̖̭̣̣̞̙̗̜̺̭̻̥͚͙̝̦̲̱͉͖͉̰̦͎̫̣̼͎͍̠̮͓̹̹͉̤̰̗̙͕͇͔̱͕̭͈̳̗̭͔̘̖̺̮̜̠͖̘͓̳͕̟̠̱̫̤͓͔̘̰̲͙͍͇̙͎̣̼̗̖͙̯͉̠̟͈͍͕̪͓̝̩̦̖̹̼̠̘̮͚̟͉̺̜͍͓̯̳̱̻͕̣̳͉̻̭̭̱͍̪̩̭̺͕̺̼̥̪͖̦̟͎̻̰_Ỏ̷͖͈̞̩͎̻̫̫̜͉̠̫͕̭̭̫̫̹̗̹͈̼̠̖͍͚̥͈̮̼͕̠̤̯̻̥̬̗̼̳̤̳̬̪̹͚̞̼̠͕̼̠̦͚̫͔̯̹͉͉̘͎͕̼̣̝͙̱̟̹̩̟̳̦̭͉̮̖̭̣̣̞̙̗̜̺̭̻̥͚͙̝̦̲̱͉͖͉̰̦͎̫̣̼͎͍̠̮͓̹̹͉̤̰̗̙͕͇͔̱͕̭͈̳̗̭͔̘̖̺̮̜̠͖̘͓̳͕̟̠̱̫̤͓͔̘̰̲͙͍͇̙͎̣̼̗̖͙̯͉̠̟͈͍͕̪͓̝̩̦̖̹̼̠̘̮͚̟͉̺̜͍͓̯̳̱̻͕̣̳͉̻̭̭̱͍̪̩̭̺͕̺̼̥̪͖̦̟͎̻̰
******************************
crying2
:~(
******************************
cthulhu
^(;,;)^
******************************
cup1
(▓
******************************
cup2
\̅_̅/̷̚ʾ
******************************
cussing
:-#
******************************
cute cat
^⨀ᴥ⨀^
******************************
cute face
(｡◕‿◕｡)
******************************
dab
ヽ( •_)ᕗ
******************************
dagger
cxxx|;:;:;:;:;:;:;:;>
******************************
dalek
 ̵̄/͇̐\
******************************
damnyou
(ᕗ ͠° ਊ ͠° )ᕗ
******************************
dance
(>'-')> <('_'<) ^('_')\- \m/(-_-)\m/ <( '-')> \_( .")> <(._.)-`
******************************
dance2
♪♪ ヽ(ˇ∀ˇ )ゞ
******************************
dancee
♪┏(°.°)┛┗(°.°)┓┗(°.°)┛┏(°.°)┓ ♪
******************************
dancing
┌(ㆆ㉨ㆆ)ʃ
******************************
dancing people
(/.__.)/   \(.__.\)
******************************
dead child
'-=,o
******************************
dead eyes
¿ⓧ_ⓧﮌ
******************************
dead girl
'==>x\9
******************************
dead guy
'==xx\0
******************************
dear god why
щ（ﾟДﾟщ）
******************************
death star defense team
|-o-| (-o-) |-o-|
******************************
decorate
▂▃▅▇█▓▒░۩۞۩        ۩۞۩░▒▓█▇▅▃▂
******************************
depressed
(︶︹︶)
******************************
derp
ヘ（。□°）ヘ
******************************
devil
]:->
******************************
devilish grin
>:-D
******************************
devilish smile
>:)
******************************
devious smile
ಠ‿ಠ
******************************
dgaf
┌∩┐(◣ _ ◢)┌∩┐
******************************
diamond bold
♦
******************************
diamond regular
♢
******************************
dice
[: :]
******************************
dick
8====D
******************************
disagree
٩◔̯◔۶
******************************
discombobulated
⊙﹏⊙
******************************
dislike1
(Ծ‸ Ծ)
******************************
dislike2
( ಠ ʖ̯ ಠ)
******************************
dna sample
~
******************************
do you even lift bro?
ᕦ(ò_óˇ)ᕤ
******************************
domino
[: :|:::]
******************************
don king
==8-)
******************************
double flip
┻━┻ ︵ヽ(`Д´)ﾉ︵ ┻━┻
******************************
drowning
人人人ヾ( ;×o×)〃 人人人
******************************
druling1
:-...
******************************
druling2
:-P~~~
******************************
drunkenness
ヽ（´ー｀）┌
******************************
dude glasses1
@[O],[O]
******************************
dude glasses2
@(o),(o)
******************************
dummy
<-|-'_'-|->
******************************
dunno
¯\(°_o)/¯
******************************
eastbound fish
><((((>
******************************
eaten apple
[===]-'
******************************
eds dick
8=D
******************************
eeriemob
(-(-_-(-_(-_(-_-)_-)-_-)_-)_-)-)
******************************
electrocardiogram1
√v^√v^√v^√v^√♥
******************************
electrocardiogram2
v^v^v^v^√\/♥
******************************
electrocardiogram3
/\/\/\/\/\/\/\/\/\/\/\v^♥
******************************
electrocardiogram4
√v^√v^♥√v^√v^√
******************************
elephant
°j°m
******************************
emo
(///_ ;)
******************************
emo dance
ヾ(-_- )ゞ
******************************
energy
 つ ◕_◕ ༽つ  つ ◕_◕ ༽つ
******************************
envelope
✉
******************************
epic gun
︻┳デ═—
******************************
equalizer
▇ ▅ █ ▅ ▇ ▂ ▃ ▁ ▁ ▅ ▃ ▅ ▅ ▄ ▅ ▇
******************************
eric
>--) ) ) )*>
******************************
error
(╯°□°)╯︵ ɹoɹɹƎ
******************************
exchange
(╯°□°）╯︵ ǝƃuɐɥɔxǝ
******************************
excited
☜(⌒▽⌒)☞
******************************
exorcism
ح(•̀ж•́)ง †
******************************
eye closed
 (╯_╰)
******************************
eye roll
⥀.⥀
******************************
eyes
℃ↂ_ↂↃ
******************************
face
•|龴◡龴|•
******************************
facepalm
(>ლ)
******************************
fail
o(╥﹏╥)o
******************************
fart
(ˆ⺫ˆ๑)<3
******************************
fat ass
(__!__)
******************************
faydre
(U) [^_^] (U)
******************************
feel perky
(`･ω･´)
******************************
fido
V•ᴥ•V
******************************
fight
(ง'̀-'́)ง
******************************
finger1
╭∩╮(Ο_Ο)╭∩╮
******************************
finger2
┌∩┐(◣_◢)┌∩┐
******************************
finn
| (• ◡•)|
******************************
fish invasion
›(̠̄:̠̄c  ›(̠̄:̠̄c  (¦Ҝ  (¦Ҝ  ҉  -  -  -   ¦̺͆¦  ▪▌
******************************
fish skeleton1
>-}-}-}->
******************************
fish skeleton2
>++('>
******************************
fish swim
¸.·´¯`·.´¯`·.¸¸.·´¯`·.¸><(((º>
******************************
fish1
><(((('>
******************************
fish2
><>
******************************
fish3
`·.¸¸ ><((((º>.·´¯`·><((((º>
******************************
fish4
><>     ><>
******************************
fish5
<><
******************************
fish6
<`)))><
******************************
fisticuffs
ლ(｀ー´ლ)
******************************
flex
ᕙ(⇀‸↼‶)ᕗ
******************************
flip friend
(ノಠ ∩ಠ)ノ彡( \o°o)\
******************************
fly away
⁽⁽ଘ( ˊᵕˋ )ଓ⁾⁾
******************************
flying
ح˚௰˚づ
******************************
fork
---=
******************************
formula1 car
\ō͡≡o˞̶
******************************
fox
-^^,--,~
******************************
french kiss
:-XÞ
******************************
frown
(ღ˘⌣˘ღ)
******************************
fu
(ಠ_ಠ)┌∩┐
******************************
fuck off
t(-.-t)
******************************
fuck you
nlm (-_-) mln
******************************
fuck you2
(° ͜ʖ͡°)╭∩╮
******************************
fuckall
 	╭∩╮（︶︿︶）╭∩╮
******************************
full mouth
:-I
******************************
fungry
Σ_(꒪ཀ꒪」∠)_
******************************
ghost
‹’’›(Ͼ˳Ͽ)‹’’›
******************************
gimme
༼ つ ◕_◕ ༽つ
******************************
glasses
-@-@-
******************************
glasses2
ᒡ◯ᵔ◯ᒢ
******************************
glitter
(*・‿・)ノ⌒*:･ﾟ✧
******************************
go away bear
╭∩╮ʕ•ᴥ•ʔ╭∩╮
******************************
gotit
(☞ﾟ∀ﾟ)☞
******************************
gtalk fit
(•̪̀●́)=ε/̵͇̿̿/'̿̿ ̿ ̿̿  N --------{---(@
******************************
guitar
c====(=#O| ) ~~ ♬·¯·♩¸¸♪·¯·♫¸
******************************
gun1
︻╦╤─
******************************
gun2
︻デ═一
******************************
gun3
╦̵̵̿╤─ ҉ ~ •
******************************
hacksaw
[|^^^^^^^
******************************
hairstyle
⨌⨀_⨀⨌
******************************
hal
@_'-'
******************************
hammer
#==
******************************
happy
 ۜ\(סּںסּَ` )/ۜ
******************************
happy birthday 1
ዞᏜ℘℘Ꮍ ℬℹℛʈዞᗬᏜᎽ
******************************
happy face
ヽ(´▽`)/
******************************
happy hug
\(ᵔᵕᵔ)/
******************************
happy square
【ツ】
******************************
happy2
⎦˚◡˚⎣
******************************
happy3
㋡
******************************
happy4
^_^
******************************
happy5
[^_^]
******************************
head shot
->~∑≥_≤)
******************************
headphone1
d[-_-]b
******************************
headphone2
d(-_-)b
******************************
headphone3
(W)
******************************
heart bold
♥
******************************
heart regular
♡
******************************
heart1
»-(¯`·.·´¯)->
******************************
heart2
♡♡
******************************
heart3
<3
******************************
hell yeah
(òÓ,)_\,,/
******************************
hello
(ʘ‿ʘ)╯
******************************
help
٩(͡๏̯͡๏)۶
******************************
high five
( ⌒o⌒)人(⌒-⌒ )v
******************************
hitchhicking
(งツ)ว
******************************
homer
(_8(|)
******************************
homer simpson
=(:o)
******************************
honeycute
❤◦.¸¸.  ◦✿
******************************
house
__̴ı̴̴̡̡̡ ̡͌l̡̡̡ ̡͌l̡*̡̡ ̴̡ı̴̴̡ ̡̡͡|̲̲̲͡͡͡ ̲▫̲͡ ̲̲̲͡͡π̲̲͡͡ ̲̲͡▫̲̲͡͡ ̲|̡̡̡ ̡ ̴̡ı̴̡̡ ̡͌l̡̡̡̡.___
******************************
hoxom
h(o x o )m
******************************
hug me
(っ◕‿◕)っ
******************************
hugger
(づ￣ ³￣)づ
******************************
huhu
█▬█ █▄█ █▬█ █▄█
******************************
hybrix
ʕʘ̅͜ʘ̅ʔ
******************************
i dont care
╭∩╮（︶︿︶）╭∩╮
******************************
i kill you
 ̿ ̿̿'̿̿\̵͇̿̿\=(•̪●)=/̵͇̿̿/'̿̿ ̿ ̿
******************************
im a hugger
(⊃｡•́‿•̀｡)⊃
******************************
infinity
(X)
******************************
injured
(҂◡_◡)
******************************
inlove
(✿ ♥‿♥)
******************************
innocent face
ʘ‿ʘ
******************************
japanese lion face
°‿‿°
******************************
jaymz
 (•̪●)==ε/̵͇̿̿/’̿’̿ ̿ ̿̿    `(•.°)~
******************************
jazz musician
ヽ(⌐■_■)ノ♪♬
******************************
john lennon
((ºjº))
******************************
jokeranonimous
╭∩╮ (òÓ,) ╭∩╮
******************************
jokeranonimous2
╭∩╮(ô¿ô)╭∩╮
******************************
joy
n_n
******************************
judgemental
\{ಠʖಠ\}
******************************
judging
( ఠ ͟ʖ ఠ)
******************************
kablewee
 	̿' ̿'\̵͇̿̿\з=( ͡ °_̯͡° )=ε/̵͇̿̿/'̿'̿ ̿
******************************
killer
(⌐■_■)--︻╦╤─ - - - (╥﹏╥)
******************************
kilroy was here
" Ü "
******************************
king
-_-
******************************
kirby
(つ -‘ _ ‘- )つ
******************************
kirby dance
<(''<)  <( ' ' )>  (> '')>
******************************
kiss
(o'3'o)
******************************
kiss my ass
(_x_)
******************************
kissing
( ˘ ³˘)♥
******************************
kitty
=^. .^=
******************************
kitty emote
ᵒᴥᵒ#
******************************
knife1
)xxxxx[;;;;;;;;;>
******************************
knife2
)xxx[::::::::::>
******************************
koala
@( * O * )@
******************************
kokain
 ̿ ̿' ̿'\̵͇̿̿\з=(•̪●)=ε/̵͇̿̿/'̿''̿ ̿
******************************
kyubey
／人 ⌒ ‿‿ ⌒ 人＼
******************************
kyubey2
／人 ◕‿‿◕ 人＼
******************************
laughing
(＾▽＾)
******************************
lenny
( ͡° ͜ʖ ͡°)
******************************
licking lips
:-9
******************************
line brack
●▬▬▬▬๑۩۩๑▬▬▬▬▬●
******************************
linqan
:Q___
******************************
listening to headphones
◖ᵔᴥᵔ◗ ♪ ♫
******************************
loading1
█▒▒▒▒▒▒▒▒▒
******************************
loading2
███▒▒▒▒▒▒▒
******************************
loading3
█████▒▒▒▒▒
******************************


# Notebook from murakrishn/uq-course
Path: lectures/lecture_09.ipynb

# Lecture 9 - Uncertainty Propagation: Advanced Monte Carlo Sampling

## Objectives

+ To use importance sampling to accelerate MC
+ To use latin hyper-cube designs to accelerate MC in moderate dimensions
+ To propagate uncertainty through an initial value problem

## Readings

+ These notes.
+ Monte Carlo Strategies in Scientific Computing (Jun S. Liu, 2001): Chapters 1 and 2.
+ Monte Carlo Statistical Methods (Christian P. Robert and George Casella, 1999): Chapter 3.
+ McKay, M.D.; Beckman, R.J.; Conover, W.J. (May 1979). A Comparison of Three Methods for Selecting Values of Input Variables in the Analysis of Output from a Computer Code. Technometrics (JSTOR Abstract) (American Statistical Association) 21 (2): 239–245. doi:10.2307/1268522. ISSN 0040-1706. JSTOR 1268522. OSTI 5236110_____no_output_____

import numpy as np
import scipy.stats as st
import matplotlib as mpl
import matplotlib.pyplot as plt
%matplotlib inline
import seaborn as sns
sns.set_style('white')
sns.set_context('talk')_____no_output_____

## Importance Sampling
Let $X$ be a random variable with probability density $p(x)$ and $f(x)$ be a function.
As in the lecture 13, we wish to estimate the expectation:
$$
I = \mathbb{E}_p[f(X)]=\int f(x)p(x)dx.
$$
Note that we have explicitly marked the expectation operator with $p$.
This is because in a while we will change $p$ to something else.

The idea in *importance sampling* is to re-write $I$ as an expectation with respect to another probability density, say $q(x)$.
The only requirement is that the support of $q(x)$ must be broader than the support of $p(x)$ ($q(x)$ must be positive wherever $p(x)$ is positive).
Here we go:
$$
I = \mathbb{E}_p[f(X)] = \int f(x)p(x)dx = \int \frac{f(x)p(x)}{q(x)}q(x)dx = \mathbb{E}_q\left[\frac{f(x)p(x)}{q(x)}\right].
$$
This suggests that you can estimate $I$ by sampling from $q(x)$ instead of $p(x)$.
A trivial estimator is:
$$
I_{n} = \frac{1}{n}\sum_{j=1}^n\frac{f(X_j)p(X_j)}{q(X_j)},
$$
where $X_1,\dots,X_n\sim q$ iid.

Why on earth is this useful?

+ First, it may be much easier to sample from $q(x)$ than it is to sample from $p(x)$.
+ You can reuse samples from $q(x)$ with different $f(x)$ and different $p(x)$.
+ With the right choice of $q(x)$ you can actually reduce the variance of the MC estimator.

Let's see how it works in practice._____no_output_____### Example: Small Tail Probabilities (Casella 3.11)
Let $X\sim \mathcal{N}(0,1)$ and assume that we want to compute:
$$
I = \mathbb{P}[X > 4.5].
$$
The standard MC estimator is:
$$
I_n = \frac{1}{n}\sum_{j=1}^n1_{[4.5,+\infty]}(X_j),
$$
for $X_j\sim \mathcal{N}(0,1)$ iid.
Let's see how it does._____no_output_____

num_mc = 10000
s = 0.
s2 = 0.
for i in range(num_mc):
    x = np.random.randn()
    if x > 4.5:
        s += 1.
        s2 += 1.
I_n = s / num_mc
V_n = (s2 / num_mc - I_n ** 2) / num_mc
print('I_n = %1.2e +- %1.2e' % (I_n, 2 * np.sqrt(V_n)))I_n = 0.00e+00 +- 0.00e+00

It doesn't look right for $n=10,000$. We did not count a single event. You would have to go to $n=1,000,000$ to hit one or two events.
To get an accurate answer you probably need hundrends of millions of samples.

Let's see if we can remedy this situtation with importance sampling.
Ideally, we only want to sample $X$'s that are greater than $4.5$.
Let us introduce a random variable $Y\sim \mathcal{TE}(4.5, 1)$ following an exponential distribution with rate parameter $1$ which is truncated at $4.5$.
The density of $Y$ is:
$$
q(y) = \frac{e^{-y}}{c}1_{[4.5,+\infty)}(y),
$$
where $c$ is a normalization constant.
To find the normalization constant we impose:
$$
\int q(y) dy = 1 \Rightarrow c = \int_0^{+\infty}e^{-y}1_{[4.5,+\infty)}(y)dy = \int_{4.5}^{+\infty}e^{-y}dy = \left[-e^{-y}\right]_{4.5}^{+\infty}=e^{-4.5}.
$$
So, the density becomes:
$$
q(y) = e^{-(y-4.5)}1_{[4.5,+\infty)}(y).
$$
Let's visualize it comparing it to the standard normal._____no_output_____

fig, ax = plt.subplots()
x = np.linspace(-4., 10., 200)
p = st.norm.pdf(x)
q = np.exp(-(x-4.5))
q[x < 4.5] = 0.
ax.plot(x, p, label='$p(x)$')
ax.plot(x, q, label='$q(x)$')
ax.set_xlabel('$x$')
ax.set_ylabel('Probability density')
plt.legend(loc='best');_____no_output_____

The importance sampling estimator is:
$$
I_{2,n} = \frac{1}{n}\sum_{j=1}^n\frac{f(X_j)p(X_j)}{q(X_j)}=\frac{1}{n\sqrt{2\pi}}\sum_{j=1}^ne^{-\frac{1}{2}X_j^2+X_j-4.5},
$$
for $X_1,\dots,X_n\sim q$ iid.
Let's see what we are going to get:_____no_output_____

num_mc = 10000
s = 0.
s2 = 0.
for i in range(num_mc):
    x = np.random.exponential() + 4.5 # Just add 4.5 to a standard exponential
    f = np.exp(-0.5 * x ** 2 + x - 4.5) / np.sqrt(2. * np.pi)
    s += f
    s2 += f ** 2
I_n = s / num_mc
V_n = (s2 / num_mc - I_n ** 2) / num_mc
print('I_n = %1.4e +- %1.4e' % (I_n, 2*np.sqrt(V_n)))I_n = 3.3932e-06 +- 8.7954e-08

### Finite variance concerns
The variance of an importance sampling estimator is:
$$
V_n = \mathbb{V}[I_n] = \frac{1}{n}\mathbb{V}_q\left[\frac{f(X_j)p(X_j)}{q(X_j)}\right]=\frac{1}{n}\left(\mathbb{E}_q\left[\frac{f^2(X_j)p^2(X_j)}{q^2(X_j)}\right]-\left(\mathbb{E}_q\left[\frac{f(X_j)p(X_j)}{q(X_j)}\right]\right)^2\right).
$$
So, in order to have a finite variance estimator, we must have that:
$$
\mathbb{E}_q\left[\frac{f^2(X_j)p^2(X_j)}{q^2(X_j)}\right] = \int f^2(x)\frac{p^2(x)}{q(x)}dx < \infty.
$$
Unless, this condition is satisfied, you will get garbage._____no_output_____### Example: Infinite Variance Estimator.
It is absolutely crucial that the variance of your estimator is finite.
Otherwise, you cannot trust it even if it theoretically converges.
Let us demonstrate this by constructing a really stupid importance sampling desnity.
Let $X\sim\mathcal{E}(1)$ (exponential with unit rate) so that the density is:
$$
p(x) = e^{-x}_{[0,\infty)}(x).
$$
Pick $f(x)=x$ so that we want to compute the mean $\mathbb{E}[f(X)] = \mathbb{E}_p[X]=1$ (analytically known).
As importance sampling density let's use the density of a $\mathcal{N}(0,1)$:
$$
q(x) = \frac{1}{\sqrt{2\pi}}e^{-\frac{1}{2}x^2}.
$$
Let's look at that integral that dominates the variance:
$$
\int f^2(x)\frac{p^2(x)}{q(x)}dx = \int_0^\infty x^2 \frac{e^{-2x}}{\frac{1}{\sqrt{2\pi}}e^{-\frac{1}{2}x^2}}dx = \infty.
$$
Let's implement an MC estimate using this importance sampling density and see what we get._____no_output_____

# How many times to try
num_tries = 10
# How many MC steps to take per try
num_mc = 1000
# For plotting purposes
fig, ax = plt.subplots()
for i in range(num_tries):
    I = np.ndarray((num_mc, ))
    s = 0.
    for j in range(num_mc):
        x = np.random.randn()
        if x >= 0.:
            y = x * np.exp(-x) / st.norm.pdf(x)
            s += y
        I[j] = s / (j + 1)
    plt.plot(np.arange(1, num_mc + 1), I, color=sns.color_palette()[0], lw=1)
ax.set_xlabel('$n$')
ax.set_ylabel('$I_n$')
plt.plot(np.arange(1, num_mc + 1), np.ones((num_mc, )), color='r');_____no_output_____

### Questions

+ Do a single trial for a large number of samples. Does it converge? Remember, that in reality you never know the truth (red line). If you think that you have found a number of samples that guarantees convergence, run it once more. Does it converge now?

+ Increase the number of tries to 10, 100, 1000 (decrease the number of samples per trial if it becomes to slow). What do you observe?_____no_output_____### The Best Importance Sampling Distribution
So, the choice of $q(x)$ is completely up to us.
What if we picked it so that we minimize the variance of the estimator?
That is, we would like to find the $q^*(x)$ that solves minimizes:
$$
\min_{q}\int f^2(x)\frac{p^2(x)}{q(x)}dx.
$$
We will do it by taking the derivative of the functional:
$$
J[q] = \int f^2(x)\frac{p^2(x)}{q(x)}dx,
$$
with respect to an arbitrary direction and setting it equal to zero.
To this end, let $\eta(x)$ be any non-negative function such that
$$
\int \eta(x)dx = 0.
$$
Then, for any $\epsilon>0$, we have that the pertubation $q(x) + \epsilon\eta(x)$ is a valid probability density since:
$$
\int (q(x) + \epsilon\eta(x)) dx = \int q(x) dx + \epsilon\int\eta(x)dx = 1.
$$
The first variation of $J[q]$ in the direction of $\eta(x)$ is:
$$
\begin{split}
\frac{\delta J[q]}{\delta \eta} &= \lim_{\epsilon\rightarrow 0}\frac{J[q + \epsilon\eta]-J[q]}{\epsilon}\\
&= \lim_{\epsilon\rightarrow 0}\frac{\int \left[f^2(x)\frac{p^2(x)}{q(x)+\epsilon\eta(x)}-f^2(x)\frac{p^2(x)}{q(x)}\right]dx}{\epsilon}\\
&= \lim_{\epsilon\rightarrow 0}\frac{\int f^2(x)p^2(x)\left[\frac{1}{q(x)+\epsilon\eta(x)}-\frac{1}{q(x)}\right]dx}{\epsilon}\\
&= \lim_{\epsilon\rightarrow 0}\frac{\int f^2(x)p^2(x)\left[\frac{q(x)-q(x)-\epsilon\eta(x)}{\left(q(x)+\epsilon\eta(x)\right)q(x)}\right]dx}{\epsilon}\\
&=-\lim_{\epsilon\rightarrow 0}\int f^2(x)p^2(x)\left[\frac{\eta(x)}{\left(q(x)+\epsilon\eta(x)\right)q(x)}\right]dx\\
&=-\int f^2(x)p^2(x)\left[\frac{\eta(x)}{q^2(x)}\right]dx.
\end{split}
$$
For an extremum, the first variation must be equal to zero (necessary condition) for all possible $\eta$'s.
We see that this is trivially satisfied for:
$$
q^*(x) \propto |f(x)|p(x).
$$
To show that the above defined $q^*$ is a minimum, we need to take the second variation of $J[q]$ and show that it is positive for all $\eta$'s.
We leave this as an excersise.

### Questions

+ Is it really possible to use the minimum variance importance sampling density? _____no_output_____## Latin Hyper-Cube Designs

Latin hyper-cube designs (LHS) are quasi-random sequences that resemble uniform random nummbers, but have better convergence properties than truly random numbers.

+ A **latin square** is a square grid containing samples only one sample in each row and each column.
+ A **latin hypercube** is the generalisation of this concept to many dimensions.

Here is how they look like in 2D._____no_output_____

import pyDOE # Library the implements latin hyper-cube designs you must install it: pip install py-design
fig, ax = plt.subplots()
ax.grid(which='major')
for i in range(1):
    X = pyDOE.lhs(2, 5) # 2 is the dimension, 5 is the number of samples,
    ax.scatter(X[:,0], X[:,1], 50., color=sns.color_palette()[i])
ax.set_xlabel('$x_1$')
ax.set_ylabel('$x_2$');_____no_output_____

Let's compare to uniform samples in 2D:_____no_output_____

num_samples = 5
fig, ax = plt.subplots()
X_lhs = pyDOE.lhs(2, num_samples, 'c')
ax.scatter(X_lhs[:,0], X_lhs[:,1], 50., color=sns.color_palette()[0])
X_lhs = pyDOE.lhs(2, num_samples, 'c')
ax.grid(which='major')
ax.scatter(X_lhs[:,0], X_lhs[:,1], 50., color=sns.color_palette()[2])
ax.set_title('Latin hypercube samples')
ax.set_xlim(0, 1)
ax.set_ylim(0, 1)
ax.set_xlabel('$x_1$')
ax.set_ylabel('$x_2$');
fig2, ax2 = plt.subplots()
X_unif = np.random.rand(num_samples, 2)
ax2.grid(which='major')
ax2.scatter(X_unif[:,0], X_unif[:,1], 50., color=sns.color_palette()[1])
X_unif = np.random.rand(num_samples, 2)
ax2.scatter(X_unif[:,0], X_unif[:,1], 50., color=sns.color_palette()[3])
ax2.set_title('Uniform samples')
ax2.set_xlim(0, 1)
ax2.set_ylim(0, 1)
ax2.set_xlabel('$x_1$')
ax2.set_ylabel('$x_2$');_____no_output_____

## Uniform Sampling vs Latin Hyper-cubes
Consider the ODE:
  \begin{align*}
    \dot{y} = \frac{d y(t)}{dt} & =-ay(t),\qquad y(0) = y_0.
  \end{align*}
The solution is: $y(t) = Ie^{-at}$.
Let's make $a$ and $I$ random and use MC and LHS to find the mean, $E[y(t)]$, and the variance $V[y(t)]$.
We'll take:
$$
a \sim \mathcal{U}(0, 0.1),
$$
and
$$
y_0 \sim \mathcal{U}(8, 10).
$$
For convenience, let's map these random variables to standardized uniform random variables:
$$
x_i \sim \mathcal{U}(0, 1), i=1,2.
$$
This can be done by defining:
$$
a = 0.1 x_1.
$$
Using the new random variables, the ODE can be written as:
$$
\dot{y} = -0.1 x_1 y, y(0) = 8 + 2x_2.
$$_____no_output_____### The "Solver" object
Let's develop a solver for this problem. We'll make the solver work as a nice function that accepts a vector $\mathbf{x} = (x_1, x_2)$ as an input, and returns the solution of the ODE on a finite set of timesteps $0=t_1<t_2<\dots<t_{n_t}=T$. The solver, will make use of the functionality of [scipy.integrate.odeint](http://docs.scipy.org/doc/scipy-0.14.0/reference/generated/scipy.integrate.odeint.html). This is a trivial problem, but studying its structure will teach you a few useful Python tricks._____no_output_____

import scipy.integrate

class Ex1Solver(object):
    """
    An object that can solver the afforementioned ODE problem.
    It will work just like a multivariate function.
    """

    def __init__(self, nt=100, T=5):
        """
        This is the initializer of the class.

        Arguments:
            nt - The number of timesteps.
            T  - The final time.
        """
        self.nt = nt
        self.T = T
        self.t = np.linspace(0, T, nt) # The timesteps on which we will get the solution
        # The following are not essential, but they are convenient
        self.num_input = 2             # The number of inputs the class accepts
        self.num_output = nt           # The number of outputs the class returns

    def __call__(self, x):
        """
        This special class method emulates a function call.

        Arguments:
            x - A 1D numpy array with 2 elements. This represents the stochastic input x = (x1, x2).
        """
        def rhs(y, t, x1):
            """
            This is the right hand side of the ODE.
            """
            return -.1 * x1 * y
        # The initial condition
        y0 = [8 + 2 * x[1]]
        # We are ready to solve the ODE
        y = scipy.integrate.odeint(rhs, y0, self.t, args=(x[0],)).flatten()
        # The only strange thing here is the use of ``args`` to pass they x1 argument to the rhs().
        # That's it
        return y_____no_output_____

Since this is the first time we see a Python class, let's play with it._____no_output_____

# Here is how you can initialize it:
solver = Ex1Solver()
# You can access its arguments:
print("Number of timesteps:", solver.nt)
print("Final time:", solver.T)
print("Timesteps:", solver.t)
print("Num inputs:", solver.num_input)
print("Num outputs:", solver.num_output)Number of timesteps: 100
Final time: 5
Timesteps: [0.         0.05050505 0.1010101  0.15151515 0.2020202  0.25252525
 0.3030303  0.35353535 0.4040404  0.45454545 0.50505051 0.55555556
 0.60606061 0.65656566 0.70707071 0.75757576 0.80808081 0.85858586
 0.90909091 0.95959596 1.01010101 1.06060606 1.11111111 1.16161616
 1.21212121 1.26262626 1.31313131 1.36363636 1.41414141 1.46464646
 1.51515152 1.56565657 1.61616162 1.66666667 1.71717172 1.76767677
 1.81818182 1.86868687 1.91919192 1.96969697 2.02020202 2.07070707
 2.12121212 2.17171717 2.22222222 2.27272727 2.32323232 2.37373737
 2.42424242 2.47474747 2.52525253 2.57575758 2.62626263 2.67676768
 2.72727273 2.77777778 2.82828283 2.87878788 2.92929293 2.97979798
 3.03030303 3.08080808 3.13131313 3.18181818 3.23232323 3.28282828
 3.33333333 3.38383838 3.43434343 3.48484848 3.53535354 3.58585859
 3.63636364 3.68686869 3.73737374 3.78787879 3.83838384 3.88888889
 3.93939394 3.98989899 4.04040404 4.09090909 4.14141414 4.19191919
 4.24242424 4.29292929 4.34343434 4.39393939 4.44444444 4.49494949
 4.54545455 4.5959596  4.64646465 4.6969697  4.74747475 4.7979798
 4.84848485 4.8989899  4.94949495 5.        ]
Num inputs: 2
Num outputs: 100
# If you wish, you may intiialize it with a different initial arguments:
solver = Ex1Solver(nt=200, T=50)
print("Number of timesteps:", solver.nt)
print("Final time:", solver.T)
print("Timesteps:", solver.t)
print("Num inputs:", solver.num_input)
print("Num outputs:", solver.num_output)Number of timesteps: 200
Final time: 50
Timesteps: [ 0.          0.25125628  0.50251256  0.75376884  1.00502513  1.25628141
  1.50753769  1.75879397  2.01005025  2.26130653  2.51256281  2.7638191
  3.01507538  3.26633166  3.51758794  3.76884422  4.0201005   4.27135678
  4.52261307  4.77386935  5.02512563  5.27638191  5.52763819  5.77889447
  6.03015075  6.28140704  6.53266332  6.7839196   7.03517588  7.28643216
  7.53768844  7.78894472  8.04020101  8.29145729  8.54271357  8.79396985
  9.04522613  9.29648241  9.54773869  9.79899497 10.05025126 10.30150754
 10.55276382 10.8040201  11.05527638 11.30653266 11.55778894 11.80904523
 12.06030151 12.31155779 12.56281407 12.81407035 13.06532663 13.31658291
 13.5678392  13.81909548 14.07035176 14.32160804 14.57286432 14.8241206
 15.07537688 15.32663317 15.57788945 15.82914573 16.08040201 16.33165829
 16.58291457 16.83417085 17.08542714 17.33668342 17.5879397  17.83919598
 18.09045226 18.34170854 18.59296482 18.84422111 19.09547739 19.34673367
 19.59798995 19.84924623 20.10050251 20.35175879 20.60301508 20.85427136
 21.10552764 21.35678392 21.6080402  21.85929648 22.11055276 22.36180905
 22.61306533 22.86432161 23.11557789 23.36683417 23.61809045 23.86934673
 24.12060302 24.3718593  24.62311558 24.87437186 25.12562814 25.37688442
 25.6281407  25.87939698 26.13065327 26.38190955 26.63316583 26.88442211
 27.13567839 27.38693467 27.63819095 27.88944724 28.14070352 28.3919598
 28.64321608 28.89447236 29.14572864 29.39698492 29.64824121 29.89949749
 30.15075377 30.40201005 30.65326633 30.90452261 31.15577889 31.40703518
 31.65829146 31.90954774 32.16080402 32.4120603  32.66331658 32.91457286
 33.16582915 33.41708543 33.66834171 33.91959799 34.17085427 34.42211055
 34.67336683 34.92462312 35.1758794  35.42713568 35.67839196 35.92964824
 36.18090452 36.4321608  36.68341709 36.93467337 37.18592965 37.43718593
 37.68844221 37.93969849 38.19095477 38.44221106 38.69346734 38.94472362
 39.1959799  39.44723618 39.69849246 39.94974874 40.20100503 40.45226131
 40.70351759 40.95477387 41.20603015 41.45728643 41.70854271 41.95979899
 42.21105528 42.46231156 42.71356784 42.96482412 43.2160804  43.46733668
 43.71859296 43.96984925 44.22110553 44.47236181 44.72361809 44.97487437
 45.22613065 45.47738693 45.72864322 45.9798995  46.23115578 46.48241206
 46.73366834 46.98492462 47.2361809  47.48743719 47.73869347 47.98994975
 48.24120603 48.49246231 48.74371859 48.99497487 49.24623116 49.49748744
 49.74874372 50.        ]
Num inputs: 2
Num outputs: 200
# Now let's evaluate the solver at a specific input.
# You can just use it as a function
x = [0.5, 0.5]
y = solver(x)
print(y)[9.         8.88764176 8.77668666 8.66711647 8.55891419 8.45206273
 8.34654522 8.24234502 8.13944568 8.03783094 7.9374848  7.8383914
 7.74053511 7.64390048 7.54847226 7.45423538 7.36117497 7.26927633
 7.17852497 7.08890658 7.00040701 6.91301231 6.82670868 6.7414825
 6.6573203  6.57420881 6.49213489 6.41108558 6.3310481  6.25200982
 6.17395828 6.09688115 6.02076628 5.94560164 5.87137538 5.79807577
 5.72569126 5.65421041 5.58362194 5.51391471 5.44507772 5.37710012
 5.30997115 5.24368025 5.17821693 5.11357087 5.04973187 4.98668985
 4.92443486 4.86295708 4.8022468  4.74229444 4.68309054 4.62462576
 4.56689086 4.50987674 4.4535744  4.39797495 4.34306962 4.28884974
 4.23530676 4.18243223 4.13021781 4.07865524 4.02773638 3.97745321
 3.92779777 3.87876223 3.83033886 3.78252001 3.73529813 3.68866578
 3.6426156  3.59714032 3.55223278 3.50788589 3.46409264 3.42084611
 3.37813949 3.33596603 3.29431907 3.25319203 3.21257843 3.17247185
 3.13286597 3.09375454 3.05513139 3.01699042 2.97932561 2.94213102
 2.90540078 2.86912908 2.83331021 2.79793852 2.7630084  2.72851437
 2.69445096 2.66081281 2.62759461 2.59479111 2.56239714 2.53040758
 2.49881739 2.46762157 2.43681521 2.40639345 2.37635147 2.34668455
 2.317388   2.28845719 2.25988756 2.2316746  2.20381386 2.17630094
 2.1491315  2.12230124 2.09580594 2.06964141 2.04380353 2.01828822
 1.99309144 1.96820922 1.94363764 1.91937282 1.89541092 1.87174817
 1.84838083 1.82530521 1.80251768 1.78001463 1.75779251 1.73584782
 1.7141771  1.69277691 1.67164389 1.6507747  1.63016604 1.60981467
 1.58971737 1.56987096 1.55027233 1.53091837 1.51180603 1.4929323
 1.47429418 1.45588875 1.4377131  1.41976435 1.40203968 1.38453629
 1.36725142 1.35018233 1.33332635 1.31668079 1.30024304 1.28401051
 1.26798062 1.25215086 1.23651871 1.22108172 1.20583745 1.1907835
 1.17591748 1.16123706 1.1467399  1.13242374 1.1182863  1.10432535
 1.09053869 1.07692415 1.06347958 1.05020286 1.03709188 1.02414458
 1.01135892 0.99873289 0.98626447 0.97395172 0.96179268 0.94978543
 0.93792809 0.92621878 0.91465565 0.90323687 0.89196065 0.88082521
 0.86982879 0.85896964 0.84824606 0.83765636 0.82719887 0.81687192
 0.8066739  0.7966032  0.78665822 0.77683739 0.76713918 0.75756203
 0.74810445 0.73876494]
# Let's plot it:
fig, ax = plt.subplots()
ax.plot(solver.t, y)
ax.set_xlabel('$t$')
ax.set_ylabel('$y(t)$')_____no_output_____# Now, let's just plot a few random samples.
fig, ax = plt.subplots()
for i in range(10):
    x = np.random.rand(2)
    y = solver(x)
    plt.plot(solver.t, y)
ax.set_xlabel('$t$')
ax.set_ylabel('$y(t)$');_____no_output_____

### Propagating Uncertainties with MC

Let's propagate the uncertainties through the ODE using MC._____no_output_____

# Let's now do Monte Carlo to compute the mean and the variance
# This is to accumulate the sum of all outputs
y_mc = np.zeros(solver.num_output)
# This is to accumlate the square of all outputs
y2_mc = np.zeros(solver.num_output)
# Pick the number of samples you wish to do:
num_samples = 10000
# Let's do it
data_mc = []
for i in range(num_samples):
    if i % 1000 == 0:
        print('sample', i + 1, 'from', num_samples)
    x = np.random.rand(2)
    y = solver(x)
    y_mc += y
    y2_mc += y ** 2
    data_mc.append(y)
data_mc = np.array(data_mc)
# Now we are ready for the mean estimate:
y_m_mc = y_mc / num_samples
# And the variance estimate
y_v_mc = y2_mc / num_samples - y_m_mc ** 2sample 1 from 10000
sample 1001 from 10000
sample 2001 from 10000
sample 3001 from 10000
sample 4001 from 10000
sample 5001 from 10000
sample 6001 from 10000
sample 7001 from 10000
sample 8001 from 10000
sample 9001 from 10000
# Let's plot the mean
fig, ax = plt.subplots()
ax.plot(solver.t, y_m_mc)
ax.set_xlabel('$t$')
ax.set_ylabel('$\mathbb{E}[y(t)]$')_____no_output_____# Let's plot the variance
fig, ax = plt.subplots()
ax.plot(solver.t, y_v_mc)
ax.set_xlabel('$t$')
ax.set_ylabel('$\mathbb{V}[y(t)]$')_____no_output_____# Now, let's draw the predictive envelop
# We need the standard deviation:
y_s_mc = np.sqrt(y_v_mc)
# A lower bound for the prediction
y_l_mc = np.percentile(data_mc, 2.75, axis=0)
# An upper bound for the prediction
y_u_mc = np.percentile(data_mc, 97.5, axis=0)
# And let's plot it:
fig, ax = plt.subplots()
ax.plot(solver.t, y_m_mc)
ax.fill_between(solver.t, y_l_mc, y_u_mc, alpha=0.25)_____no_output_____

### Propagating Uncertainties with LHS

Let's propagate the uncertainties through the ODE using LHS.
All we have to do is to copy and paste the above code and make some modifications._____no_output_____

# Let's now do LHS to compute the mean and the variance
# This is to accumulate the sum of all outputs
y_lhs = np.zeros(solver.num_output)
# This is to accumlate the square of all outputs
y2_lhs = np.zeros(solver.num_output)
# Pick the number of samples you wish to do:
num_samples = 10000
# You have to create the LHS design prior to looping:
X = pyDOE.lhs(2, num_samples)
# Let's do it
data_lhs = []
for i in range(num_samples):
    if i % 1000 == 0:
        print('sample', i + 1, 'from', num_samples)
    x = X[i, :]
    y = solver(x)
    y_lhs += y
    y2_lhs += y ** 2
    data_lhs.append(y)
data_lhs = np.array(data_lhs)
# Now we are ready for the mean estimate:
y_m_lhs = y_lhs / num_samples
# And the variance estimate
y_v_lhs = y2_lhs / num_samples - y_m_lhs ** 2sample 1 from 10000
sample 1001 from 10000
sample 2001 from 10000
sample 3001 from 10000
sample 4001 from 10000
sample 5001 from 10000
sample 6001 from 10000
sample 7001 from 10000
sample 8001 from 10000
sample 9001 from 10000
# Let's plot the mean
plt.plot(solver.t, y_m_lhs)
plt.xlabel('$t$')
plt.ylabel('$\mathbb{E}[y(t)]$')_____no_output_____# Let's plot the variance
plt.plot(solver.t, y_v_lhs)
plt.xlabel('$t$')
plt.ylabel('$\mathbb{V}[y(t)]$')_____no_output_____# Now, let's draw the predictive envelop
# We need the standard deviation:
y_s_lhs = np.sqrt(y_v_lhs)
y_l_lhs = np.percentile(data_lhs, 2.75, axis=0)
# An upper bound for the prediction
y_u_lhs = np.percentile(data_lhs, 97.5, axis=0)
# And let's plot it:
plt.plot(solver.t, y_m_lhs)
plt.fill_between(solver.t, y_l_lhs, y_u_lhs, alpha=0.25)_____no_output_____

## Comparing MC to LHS

So, which one is better? MC or LHS? To test this, we need to establish a ground truth.
Fortunately, we can get an analytic solution to:
$$
\dot{y} = -0.1 x_1 y, y(0) = 8 + 2x_2.
$$
It is:
$$
y(t;x_1,x_2) = (8 + 2x_2)e^{-0.1 x_1 t}.
$$
We can integrate $x_1$ and $x_2$ analytically from this expression.

Doing so, we get the following mean solution and variance:
$$
\mu = \mathbb{E}[y] = \frac{90}{t} (1 - \exp^{-0.1t}), \\
\mathrm{S} = \mathbb{V}[y] =  \frac{2440}{6t}(1 - \exp^{-0.2t}) - \mu^2
$$_____no_output_____

def E_y(t):
    if t == 0:
        return 9.
    return (90*(1-np.exp(-0.1*t))) / t
E_y =  np.vectorize(E_y)

def V_y(t):
    if t == 0:
        return (2440*0.2)/6. - E_y(t) **2

    return (2440.*(1-np.exp(-0.2*t)))/(6.*t) - E_y(t)**2
V_y = np.vectorize(V_y)_____no_output_____# Let's start by doing some plots that compare the variance obtained by MC and LHS to the true one
plt.plot(solver.t, V_y(solver.t), label='True variance')
plt.plot(solver.t, y_v_mc, '--', label='MC (10,000)')
plt.plot(solver.t, y_v_lhs, '-.', label='LHS (10,000))')
plt.legend(loc='best')_____no_output_____

They seem close. But this is for 10,000 samples.
Let's test the convergence of each method. To do this, we will compute the evolution of the root square error in the variance:
$$
\text{RSE}_\alpha(n) = \left(\sum_{i=1}^{n_t}\left(V[y(t_i)] - V_{\alpha,n}[y(t_i)]\right)^2\right)^{\frac{1}{2}},
$$
where $\alpha = \{\text{MC}, \text{LHS}\}$ stands for the method, $n$ for the number of samples used, and $V_{\alpha,n}[y(t_i)]$ for the estimate of the variance of $y(t_i)$.
It will probably take a while to develop the code._____no_output_____

def get_MC_rse(max_num_samples=100):
    """
    Get the maximum error of MC.
    """
    y_v_true = V_y(solver.t)
    y = np.zeros(solver.num_output)
    y2 = np.zeros(solver.num_output)
    n = []
    rse = []
    for i in range(max_num_samples):
        x = np.random.rand(2)
        y_sample = solver(x)
        y += y_sample
        y2 += y_sample ** 2
        if i % 1 == 0:    # Produce estimate every 100 steps
            n.append(i + 1)
            y_m = y / (i + 1)
            y_v = y2 / (i + 1) - y_m ** 2
            rse.append(np.linalg.norm(y_v_true - y_v))
    return n, rse

def _get_LHS_rse(max_num_samples=100):
    y_v_true = V_y(solver.t)
    y = np.zeros(solver.num_output)
    y2 = np.zeros(solver.num_output)
    n = []
    rse = []
    X = pyDOE.lhs(2, num_samples)
    for i in range(max_num_samples):
        x = X[i, :]
        y_sample = solver(x)
        y += y_sample
        y2 += y_sample ** 2
        if i % 1 == 0:    # Produce estimate every 100 steps
            n.append(i + 1)
            y_m = y / (i + 1)
            y_v = y2 / (i + 1) - y_m ** 2
            rse.append(np.linalg.norm(y_v_true - y_v))
    return n, rse

def get_LHS_rse(max_num_samples=100):
    n = []
    rse = []
    for i in range(max_num_samples):
        if i % 1 == 0:
            _n, _rse = _get_LHS_rse(i + 1)
            n.append(_n[-1])
            rse.append(_rse[-1])
    return n, rse_____no_output_____n_mc, rse_mc = get_MC_rse(50)
n_lhs, rse_lhs = get_LHS_rse(50)
plt.plot(n_mc, rse_mc, label='MC RSE')
plt.plot(n_lhs, rse_lhs, label='LHS RSE')
plt.xlabel('$n$')
plt.ylabel('RSE')
plt.legend(loc='best')_____no_output_____

It is hard to tell which one is better...
There is a considerable ammount of epistemic uncertainty.
How would you go about quantifying it?
Notice, that everytime you run the above code, it produces a different estimate. This suggests, that if you repeat the procedure, say 100 times, you will be able to get predictive error bars for the error!_____no_output_____

num_exper = 1
num_samples = 20
rse_mc_samples = []
rse_lhs_samples = []
for i in range(num_exper):
    print(i)
    n_mc, rse_mc = get_MC_rse(num_samples)
    n_lhs, rse_lhs = get_LHS_rse(num_samples)
    rse_mc_samples.append(rse_mc)
    rse_lhs_samples.append(rse_lhs)
rse_mc_samples = np.array(rse_mc_samples)
rse_lhs_samples = np.array(rse_lhs_samples)
rse_mc_m = np.mean(rse_mc_samples, axis=0)
rse_mc_l = np.percentile(rse_mc_samples, 2.75, axis=0)
rse_mc_u = np.percentile(rse_mc_samples, 97.5, axis=0)
rse_lhs_m = np.mean(rse_lhs_samples, axis=0)
rse_lhs_l = np.percentile(rse_lhs_samples, 2.75, axis=0)
rse_lhs_u = np.percentile(rse_lhs_samples, 97.5, axis=0)
plt.plot(n_mc, rse_mc_m, label='MC')
plt.plot(n_lhs, rse_lhs_m, '--', label='LHS')
plt.fill_between(n_mc, rse_mc_l, rse_mc_u, alpha=0.25)
plt.fill_between(n_lhs, rse_lhs_l, rse_lhs_u, color='g', alpha=0.25)
plt.legend(loc='best')
plt.xlabel('$n$')
plt.ylabel('RSE')0

### Questions

+ To get a more accurate picture of the convergence rate of the two methods increaes the number of trials ``num_expr``. You will have to wait...

+ LHS seems to perform better. Does it have a different convergence rate than MC?_____no_output_____


# Notebook from theislab/pertpy
Path: notebooks/mixscape_data_preparation/00_2_load_normalized.ipynb

%load_ext autoreload
%autoreload 2_____no_output_____cd ../mnt/znas/icb_zstore01/groups/ml01/workspace/ignacio.ibarra/theislab/pertpy
import pertpy as pt
import scanpy as sc_____no_output_____cd notebooks/mnt/znas/icb_zstore01/groups/ml01/workspace/ignacio.ibarra/theislab/pertpy/notebooks

### To load datasets the path has to be communicated_____no_output_____

DATA_DIR = ''_____no_output_____

### The CRISPR dataset_____no_output_____

adata = pt.datasets.mixscape_vignette_crispr(backup_dir=DATA_DIR)_____no_output_____sc.set_figure_params(facecolor='white')
sc.pl.umap(adata, color=['replicate', 'Phase'])
sc.pl.umap(adata, color=['crispr'], groups=['NT'])
sc.pl.umap(adata, color=['crispr'], groups=['Perturbed'])_____no_output_____

### The Crohn's disease dataset_____no_output_____

adata_crohn = pt.datasets.mixscape_vignette_crohn(datasetdir=DATA_DIR)Variable names are not unique. To make them unique, call `.var_names_make_unique`.
adata_crohn_____no_output_____