# Notebook from AlgoAIBoss/Data-Mining-Cup
Path: Preprocessing/Variable_Engineering_and_Merging_Datasets.ipynb

<code>
## Merging the two data sets with the language variable

import pandas as pd
import numpy as np

items_1 = pd.read_csv('items_1.csv')
items_2 = pd.read_csv('items_2.csv')

df = pd.concat([items_1, items_2])_____no_output_____##########################
## Variable Engineering ##
##########################_____no_output_____## General topic 1 (first character of main topic) and general topic 2 (first two characters of main topic)

df['general_topic'] = df['main topic'].astype(str).str[0]
df['general_topic_2'] = df['main topic'].astype(str).str[0:2]
df['general_topic_3'] = df['main topic'].astype(str).str[0:3]_____no_output_____## Looking at first character of main topic

## Categorical variable for each topic on the "Thema Subject Categories 1.4" chart_____no_output_____df['arts'] = np.where(df['general_topic'] == 'A', 1, 0)
df['language_linguistics'] = np.where(df['general_topic'] == 'C', 1, 0)

df['literature'] = np.where(df['general_topic'] == 'D', 1, 0)
df['poetry'] = np.where(df['general_topic_2'] == 'DC', 1, 0)

df['fiction'] = np.where(df['general_topic'] == 'F', 1, 0)
df['science_fiction'] = np.where(df['general_topic_2'] == 'FL', 1, 0)
df['fiction_fantasy'] = np.where(df['general_topic_2'] == 'FM', 1, 0)

df['reference'] = np.where(df['general_topic'] == 'G', 1, 0)

df['social_sciences'] = np.where(df['general_topic'] == 'J', 1, 0)
df['education'] = np.where(df['general_topic_2'] == 'JN', 1, 0)
df['politics_government'] = np.where(df['general_topic_2'] == 'JP', 1, 0)
df['warfare'] = np.where(df['general_topic_2'] == 'JW', 1, 0)

df['business'] = np.where(df['general_topic'] == 'K', 1, 0)
df['law'] = np.where(df['general_topic'] == 'L', 1, 0)
df['medicine'] = np.where(df['general_topic'] == 'M', 1, 0)
df['history'] = np.where(df['general_topic'] == 'N', 1, 0)

df['math_sciences'] = np.where(df['general_topic'] == 'P', 1, 0)
df['math'] = np.where(df['general_topic_2'] == 'PB', 1, 0)
df['astronomy'] = np.where(df['general_topic_2'] == 'PG', 1, 0)
df['physics'] = np.where(df['general_topic_2'] == 'PH', 1, 0)
df['chemistry'] = np.where(df['general_topic_2'] == 'PN', 1, 0)
df['biology'] = np.where(df['general_topic_2'] == 'PS', 1, 0)

df['philosophy_religion'] = np.where(df['general_topic'] == 'Q', 1, 0)
df['philosophy'] = np.where(df['general_topic_2'] == 'QD', 1, 0)
df['religion'] = np.where(df['general_topic_2'] == 'QR', 1, 0)

df['earth_sciences'] = np.where(df['general_topic'] == 'R', 1, 0)
df['geography'] = np.where(df['general_topic_2'] == 'RG', 1, 0)

df['sports'] = np.where(df['general_topic'] == 'S', 1, 0)
df['general_sports'] = np.where(df['general_topic_2'] == 'SC', 1, 0)
df['ball_sports'] = np.where(df['general_topic_2'] == 'SF', 1, 0)
df['athletics_gynastics'] = np.where(df['general_topic_2'] == 'SH', 1, 0)
df['equestrian'] = np.where(df['general_topic_2'] == 'SK', 1, 0)
df['vehicle_sports'] = np.where(df['general_topic_2'] == 'SM', 1, 0)
df['water_sports'] = np.where(df['general_topic_2'] == 'SP', 1, 0)
df['combat_sports'] = np.where(df['general_topic_2'] == 'SR', 1, 0)
df['field_sports'] = np.where(df['general_topic_2'] == 'SV', 1, 0)
df['outdoors'] = np.where(df['general_topic_2'] == 'SZ', 1, 0)

df['technology'] = np.where(df['general_topic'] == 'T', 1, 0)
df['agriculture'] = np.where(df['general_topic_2'] == 'TV', 1, 0)

df['computing'] = np.where(df['general_topic'] == 'U', 1, 0)

df['health'] = np.where(df['general_topic'] == 'V', 1, 0)
df['family_health'] = np.where(df['general_topic_2'] == 'VF', 1, 0)
df['self_help'] = np.where(df['general_topic_2'] == 'VS', 1, 0)
df['mind_body_spirit'] = np.where(df['general_topic_2'] == 'VX', 1, 0)

df['lifestyle_hobbies'] = np.where(df['general_topic'] == 'W', 1, 0)
df['cooking'] = np.where(df['general_topic_2'] == 'WB', 1, 0)
df['antiques'] = np.where(df['general_topic_2'] == 'WC', 1, 0)
df['quizzes_games'] = np.where(df['general_topic_2'] == 'WD', 1, 0)
df['crafts'] = np.where(df['general_topic_2'] == 'WF', 1, 0)
df['humour'] = np.where(df['general_topic_2'] == 'WH', 1, 0)
df['personal_style'] = np.where(df['general_topic_2'] == 'WJ', 1, 0)
df['home'] = np.where(df['general_topic_2'] == 'WK', 1, 0)
df['gardening'] = np.where(df['general_topic_2'] == 'WM', 1, 0)
df['nature'] = np.where(df['general_topic_2'] == 'WN', 1, 0)
df['travel_holiday'] = np.where(df['general_topic_2'] == 'WT', 1, 0)
df['stationary'] = np.where(df['general_topic_2'] == 'WZ', 1, 0)

df['graphic_novels'] = np.where(df['general_topic'] == 'X', 1, 0)
df['cartoons'] = np.where(df['general_topic_2'] == 'XY', 1, 0)

df['childrens'] = np.where(df['general_topic'] == 'Y', 1, 0)
df['childrens_young'] = np.where(df['general_topic_2'] == 'YB', 1, 0)
df['childrens_fiction'] = np.where(df['general_topic_2'] == 'YF', 1, 0)
df['childrens_educational'] = np.where(df['general_topic_2'] == 'YP', 1, 0)_____no_output_____## Categorical variables to go along with language
## for the top six languages in the data set

df['english'] = np.where(df['language'] == 'en', 1, 0)
df['german'] = np.where(df['language'] == 'de', 1, 0)
df['spanish'] = np.where(df['language'] == 'es', 1, 0)
df['italian'] = np.where(df['language'] == 'it', 1, 0)
df['french'] = np.where(df['language'] == 'fr', 1, 0)
df['portuguese'] = np.where(df['language'] == 'pt', 1, 0)

df['other_language'] = np.where(~np.isin(df['language'], ['en', 'de', 'es', 'it', 'fr', 'pt']), 1, 0)_____no_output_____## Categorical variables to go along with author
## for the top 50 authors in the data set

df['GarciaSantiago']=np.where(df['author']=='Garcia Santiago',1,0)
df['ShelleyAdmont,KidkiddosBooks']=np.where(df['author']=='Shelley Admont, Kidkiddos Books',1,0)
df['JamesManning']=np.where(df['author']=='James Manning',1,0)
df['JulesVerne']=np.where(df['author']=='Jules Verne',1,0)
df['IdriesShah']=np.where(df['author']=='Idries Shah',1,0)
df['ErinHunter']=np.where(df['author']=='Erin Hunter',1,0)
df['EnidBlyton']=np.where(df['author']=='Enid Blyton',1,0)
df['R.L.Stine']=np.where(df['author']=='R. L. Stine',1,0)
df['TerryPratchett']=np.where(df['author']=='Terry Pratchett',1,0)
df['SpeedyPublishingLlc']=np.where(df['author']=='Speedy Publishing Llc',1,0)
df['H.G.Wells']=np.where(df['author']=='H. G. Wells',1,0)
df['RogerHargreaves']=np.where(df['author']=='Roger Hargreaves',1,0)
df['SteveHerman']=np.where(df['author']=='Steve Herman',1,0)
df['TuulaPere']=np.where(df['author']=='Tuula Pere',1,0)
df['DaisyMeadows']=np.where(df['author']=='Daisy Meadows',1,0)
df['ElBlokehead']=np.where(df['author']=='El Blokehead',1,0)
df['RoaldDahl']=np.where(df['author']=='Roald Dahl',1,0)
df['LewisCarroll']=np.where(df['author']=='Lewis Carroll',1,0)
df['AlfredBekker']=np.where(df['author']=='Alfred Bekker',1,0)
df['PenelopeDyan']=np.where(df['author']=='Penelope Dyan',1,0)
df['FionaWatt']=np.where(df['author']=='Fiona Watt',1,0)
df['MichelleM.Pillow']=np.where(df['author']=='Michelle M. Pillow',1,0)
df['L.FrankBaum']=np.where(df['author']=='L. Frank Baum',1,0)
df['MatildeCorreia']=np.where(df['author']=='Matilde Correia',1,0)
df['GinoBianchi']=np.where(df['author']=='Gino Bianchi',1,0)
df['NorbertPautner']=np.where(df['author']=='Norbert Pautner',1,0)
df['BrandonSanderson']=np.where(df['author']=='Brandon Sanderson',1,0)
df['CaroleMarsh']=np.where(df['author']=='Carole Marsh',1,0)
df['CarolynKeene']=np.where(df['author']=='Carolyn Keene',1,0)
df['MichaelMorpurgo']=np.where(df['author']=='Michael Morpurgo',1,0)
df['JenniferL.Armentrout']=np.where(df['author']=='Jennifer L. Armentrout',1,0)
df['DanGutman']=np.where(df['author']=='Dan Gutman',1,0)
df['RickRiordan']=np.where(df['author']=='Rick Riordan',1,0)
df['L.M.Montgomery']=np.where(df['author']=='L. M. Montgomery',1,0)
df['Thithiajobs']=np.where(df['author']=='Thithiajobs',1,0)
df['PeterHertzberg']=np.where(df['author']=='Peter Hertzberg',1,0)
df['JacquelineWilson']=np.where(df['author']=='Jacqueline Wilson',1,0)
df['GraceGoodwin']=np.where(df['author']=='Grace Goodwin',1,0)
df['CassandraClare']=np.where(df['author']=='Cassandra Clare',1,0)
df['LouisaMayAlcott']=np.where(df['author']=='Louisa May Alcott',1,0)
df['JeffKinney']=np.where(df['author']=='Jeff Kinney',1,0)
df['J.R.R.Tolkien']=np.where(df['author']=='J. R. R. Tolkien',1,0)
df['MaryPopeOsborne']=np.where(df['author']=='Mary Pope Osborne',1,0)
df['Panini']=np.where(df['author']=='Panini',1,0)
df['J.R.Ward']=np.where(df['author']=='J. R. Ward',1,0)
df['GeorgeR.R.Martin']=np.where(df['author']=='George R. R. Martin',1,0)
df['SaraShepard']=np.where(df['author']=='Sara Shepard',1,0)
df['JakeMaddox']=np.where(df['author']=='Jake Maddox',1,0)
df['MarkusHeitz']=np.where(df['author']=='Markus Heitz',1,0)
df['LincolnPeirce']=np.where(df['author']=='Lincoln Peirce',1,0)_____no_output_____df.to_csv('items_final_newer.csv', index = False, header = True)_____no_output_____
</code>


# Notebook from MichaelAllen1966/2004_reinforcement_learning
Path: es/es_simple_equation.ipynb

<code>
import numpy as np
import matplotlib.pyplot as plt

def evolution_strategy(
    f,
    population_size,
    sigma,
    lr,
    initial_params,
    num_iters):

    # assume initial params is a 1-D array
    num_params = len(initial_params)
    reward_per_iteration = np.zeros(num_iters)
    # Initalise parameters
    params = initial_params
    # Loop through num_iters
    for t in range(num_iters):
      # N = noise
        N = np.random.randn(population_size, num_params)
        # R strores the reward
        R = np.zeros(population_size)
        # loop through each "offspring" and get reward
        for j in range(population_size):
        # Add noise to 'parent' to get offspring
            params_try = params + sigma*N[j]
            # Get reward for a single offspring
            # 'f' is function to optimise paramaters
            R[j] = f(params_try)
        # m = mean reward for all offspring
        m = R.mean()
        # A = standardised reward for each offspring
        A = (R - m) / R.std()
        # Store reward progress
        reward_per_iteration[t] = m
        # Update parameters
        update_rate = lr/sigma
        for i in range(len(params)):
            adjustment = N[:, i] * A
            mean_adjustment = np.mean(adjustment)
            params[i] = params[i] + update_rate * mean_adjustment

    return params, reward_per_iteration

def reward_function(params):
    # Aim to optimise parameters in function (max reward = 0)

    x0 = params[0]
    x1 = params[1]
    x2 = params[2]

    # Oprimal parameters: x0 = 0, x1 = 4,  x2 = -2
    reward = -(x0**2 + 0.1*(x1 - 4)**2 + 0.5*(x2 + 2)**2)

    return reward

# Call evolution strategy
best_params, rewards = evolution_strategy(
  f=reward_function,
  population_size=50,
  sigma=0.1,
  lr=3e-3,
  initial_params=np.random.randn(3),
  num_iters=500)

# plot the rewards per iteration
plt.plot(rewards)
plt.show()

# final params
best_params = [np.round(param, 3) for param in best_params]
print("Final params:", best_params)_____no_output_____
</code>


# Notebook from kzymgch/pyprobml
Path: notebooks/figures/chapter20_figures.ipynb

# Copyright 2021 Google LLC
# Use of this source code is governed by an MIT-style
# license that can be found in the LICENSE file or at
# https://opensource.org/licenses/MIT.

# Author(s): Kevin P. Murphy (murphyk@gmail.com) and Mahmoud Soliman (mjs@aucegypt.edu)_____no_output_____

_____no_output__________no_output_____# Cloning the pyprobml repo_____no_output_____

!git clone https://github.com/probml/pyprobml
%cd pyprobml/scripts_____no_output_____

# Installing required software (This may take few minutes)_____no_output_____

!apt-get install octave  -qq > /dev/null
!apt-get install liboctave-dev -qq > /dev/null_____no_output_____%%capture
%load_ext autoreload
%autoreload 2
DISCLAIMER = 'WARNING : Editing in VM - changes lost after reboot!!'
from google.colab import files

def interactive_script(script, i=True):
  if i:
    s = open(script).read()
    if not s.split('\n', 1)[0]=="## "+DISCLAIMER:
      open(script, 'w').write(
          f'## {DISCLAIMER}\n' + '#' * (len(DISCLAIMER) + 3) + '\n\n' + s)
    files.view(script)
    %run $script
  else:
      %run $script

def show_image(img_path):
  from google.colab.patches import cv2_imshow
  import cv2
  img = cv2.imread(img_path, cv2.IMREAD_UNCHANGED)
  img=cv2.resize(img,(600,600))
  cv2_imshow(img)_____no_output_____

## Figure 20.1:  _____no_output_____
  An illustration of PCA where we project from 2d to 1d. Circles are the original data points, crosses are the reconstructions. The red star is the data mean.
Figure(s) generated by [pcaDemo2d.py](https://github.com/probml/pyprobml/blob/master/scripts/pcaDemo2d.py) _____no_output_____

interactive_script("pcaDemo2d.py")_____no_output_____

## Figure 20.2:  _____no_output_____
  An illustration of PCA applied to MNIST digits from class 9. Grid points are at the 5, 25, 50, 75, 95 \% quantiles of the data distribution along each dimension. The circled points are the closest projected images to the vertices of the grid. Adapted from Figure 14.23 of [HTF09] .
Figure(s) generated by [pca_digits.py](https://github.com/probml/pyprobml/blob/master/scripts/pca_digits.py) _____no_output_____

interactive_script("pca_digits.py")_____no_output_____

## Figure 20.3:  _____no_output_____
  a) Some randomly chosen $64 \times 64$ pixel images from the Olivetti face database. (b) The mean and the first three PCA components represented as images.
Figure(s) generated by [pcaImageDemo.m](https://github.com/probml/pmtk3/blob/master/demos/pcaImageDemo.m) _____no_output_____

!octave -W pcaImageDemo.m >> ______no_output_____

## Figure 20.4:  _____no_output_____
  Illustration of the variance of the points projected onto different 1d vectors. $v_1$ is the first principal component, which maximizes the variance of the projection. $v_2$ is the second principal component which is direction orthogonal to $v_1$. Finally $v'$ is some other vector in between $v_1$ and $v_2$. Adapted from Figure 8.7 of [Aur19] .
Figure(s) generated by [pca_projected_variance.py](https://github.com/probml/pyprobml/blob/master/scripts/pca_projected_variance.py) _____no_output_____

interactive_script("pca_projected_variance.py")_____no_output_____

## Figure 20.5:  _____no_output_____
  Effect of standardization on PCA applied to the height/weight dataset. (Red=female, blue=male.) Left: PCA of raw data. Right: PCA of standardized data.
Figure(s) generated by [pcaStandardization.py](https://github.com/probml/pyprobml/blob/master/scripts/pcaStandardization.py) _____no_output_____

interactive_script("pcaStandardization.py")_____no_output_____

## Figure 20.6:  _____no_output_____
  Reconstruction error on MNIST vs number of latent dimensions used by PCA. (a) Training set. (b) Test set.
Figure(s) generated by [pcaOverfitDemo.m](https://github.com/probml/pmtk3/blob/master/demos/pcaOverfitDemo.m) _____no_output_____

!octave -W pcaOverfitDemo.m >> ______no_output_____

## Figure 20.7:  _____no_output_____
  (a) Scree plot for training set, corresponding to \cref  fig:pcaErr (a). (b) Fraction of variance explained.
Figure(s) generated by [pcaOverfitDemo.m](https://github.com/probml/pmtk3/blob/master/demos/pcaOverfitDemo.m) _____no_output_____

!octave -W pcaOverfitDemo.m >> ______no_output_____

## Figure 20.8:  _____no_output_____
  Profile likelihood corresponding to PCA model in \cref  fig:pcaErr (a).
Figure(s) generated by [pcaOverfitDemo.m](https://github.com/probml/pmtk3/blob/master/demos/pcaOverfitDemo.m) _____no_output_____

!octave -W pcaOverfitDemo.m >> ______no_output_____

## Figure 20.9:  _____no_output_____
  Illustration of the FA generative process, where we have $L=1$ latent dimension generating $D=2$ observed dimensions; we assume $\boldsymbol  \Psi  =\sigma ^2 \mathbf  I $. The latent factor has value $z \in \mathbb  R $, sampled from $p(z)$; this gets mapped to a 2d offset $\boldsymbol  \delta  = z \mathbf  w $, where $\mathbf  w \in \mathbb  R ^2$, which gets added to $\boldsymbol  \mu  $ to define a Gaussian $p(\mathbf  x |z) = \mathcal  N (\mathbf  x |\boldsymbol  \mu  + \boldsymbol  \delta  ,\sigma ^2 \mathbf  I )$. By integrating over $z$, we ``slide'' this circular Gaussian ``spray can'' along the principal component axis $\mathbf  w $, which induces elliptical Gaussian contours in $\mathbf  x $ space centered on $\boldsymbol  \mu  $. Adapted from Figure 12.9 of [Bis06] . _____no_output_____

show_image("/content/pyprobml/notebooks/figures/images/{PPCAsprayCan}.png")_____no_output_____

## Figure 20.10:  _____no_output_____
  Illustration of EM for PCA when $D=2$ and $L=1$. Green stars are the original data points, black circles are their reconstructions. The weight vector $\mathbf  w $ is represented by blue line. (a) We start with a random initial guess of $\mathbf  w $. The E step is represented by the orthogonal projections. (b) We update the rod $\mathbf  w $ in the M step, keeping the projections onto the rod (black circles) fixed. (c) Another E step. The black circles can 'slide' along the rod, but the rod stays fixed. (d) Another M step. Adapted from Figure 12.12 of [Bis06] .
Figure(s) generated by [pcaEmStepByStep.m](https://github.com/probml/pmtk3/blob/master/demos/pcaEmStepByStep.m) _____no_output_____

!octave -W pcaEmStepByStep.m >> ______no_output_____

## Figure 20.11:  _____no_output_____
  Mixture of factor analyzers as a PGM. _____no_output_____

show_image("/content/pyprobml/notebooks/figures/images/{mixFAdgmC}.png")_____no_output_____

## Figure 20.12:  _____no_output_____
  Mixture of PPCA models fit to a 2d dataset, using $L=1$ latent dimensions and $K=1$ and $K=10$ mixture components.
Figure(s) generated by [mixPpcaDemoNetlab.m](https://github.com/probml/pmtk3/blob/master/demos/mixPpcaDemoNetlab.m) _____no_output_____

!octave -W mixPpcaDemoNetlab.m >> ______no_output_____

## Figure 20.13:  _____no_output_____
  Random samples from the MixFA model fit to CelebA. From Figure 4 of [EY18] . Used with kind permission of Yair Weiss _____no_output_____

show_image("/content/pyprobml/notebooks/figures/images/MFAGAN-samples.png")_____no_output_____

## Figure 20.14:  _____no_output_____
  (a) 150 synthetic 16 dimensional bit vectors. (b) The 2d embedding learned by binary PCA, fit using variational EM. We have color coded points by the identity of the true ``prototype'' that generated them. (c) Predicted probability of being on. (d) Thresholded predictions.
Figure(s) generated by [binaryFaDemoTipping.m](https://github.com/probml/pmtk3/blob/master/demos/binaryFaDemoTipping.m) _____no_output_____

!octave -W binaryFaDemoTipping.m >> ______no_output_____

## Figure 20.15:  _____no_output_____
  Gaussian latent factor models for paired data. (a) Supervised PCA. (b) Partial least squares. _____no_output_____

show_image("/content/pyprobml/notebooks/figures/images/eSPCAxy.png")_____no_output_____show_image("/content/pyprobml/notebooks/figures/images/ePLSxy.png")_____no_output_____

## Figure 20.16:  _____no_output_____
  Canonical correlation analysis as a PGM. _____no_output_____

show_image("/content/pyprobml/notebooks/figures/images/eCCAxy.png")_____no_output_____

## Figure 20.17:  _____no_output_____
  An autoencoder with one hidden layer. _____no_output_____

show_image("/content/pyprobml/notebooks/figures/images/{autoencoder}.png")_____no_output_____

## Figure 20.18:  _____no_output_____
  Results of applying an autoencoder to the Fashion MNIST data. Top row are first 5 images from validation set. Bottom row are reconstructions. (a) MLP model (trained for 20 epochs). The encoder is an MLP with architecture 784-100-30. The decoder is the mirror image of this. (b) CNN model (trained for 5 epochs). The encoder is a CNN model with architecture Conv2D(16, 3x3, same, selu), MaxPool2D(2x2), Conv2D(32, 3x3, same, selu), MaxPool2D(2x2), Conv2D(64, 3x3, same, selu), MaxPool2D(2x2). The decoder is the mirror image of this, using transposed convolution and without the max pooling layers. Adapted from Figure 17.4 of [Aur19] . _____no_output_____

show_image("/content/pyprobml/notebooks/figures/images/ae_fashion_mlp_recon.png")_____no_output_____show_image("/content/pyprobml/notebooks/figures/images/ae_fashion_cnn_recon.png")_____no_output_____

## Figure 20.19:  _____no_output_____
  tSNE plot of the first 2 latent dimensions of the Fashion MNIST validation set computed using an MLP-based autoencoder. Adapted from Figure 17.5 of [Aur19] . _____no_output_____

show_image("/content/pyprobml/notebooks/figures/images/ae-mlp-fashion-tsne.png")_____no_output_____

## Figure 20.20:  _____no_output_____
  Denoising autoencoder (MLP architecture) applied to some noisy Fashion MNIST images from the validation set. (a) Gaussian noise. (b) Bernoulli dropout noise. Top row: input. Bottom row: output Adapted from Figure 17.9 of [Aur19] . _____no_output_____

show_image("/content/pyprobml/notebooks/figures/images/ae-denoising-gaussian.png")_____no_output_____show_image("/content/pyprobml/notebooks/figures/images/ae-denoising-dropout.png")_____no_output_____

## Figure 20.21:  _____no_output_____
  The residual error from a DAE, $\mathbf  e (\mathbf  x )=r( \cc@accent  "707E  \mathbf  x   )-\mathbf  x $, can learn a vector field corresponding to the score function. Arrows point towards higher probability regions. The length of the arrow is proportional to $||\mathbf  e (\mathbf  x )||$, so points near the 1d data manifold (represented by the curved line) have smaller arrows. From Figure 5 of [GY14] . Used with kind permission of Guillaume Alain. _____no_output_____

show_image("/content/pyprobml/notebooks/figures/images/DAE.png")_____no_output_____

## Figure 20.22:  _____no_output_____
  Neuron activity (in the bottleneck layer) for an autoencoder applied to Fashion MNIST. We show results for three models, with different kinds of sparsity penalty: no penalty (left column), $\ell _1$ penalty (middle column), KL penalty (right column). Top row: Heatmap of 300 neuron activations (columns) across 100 examples (rows). Middle row: Histogram of activation levels derived from this heatmap. Bottom row: Histogram of the mean activation per neuron, averaged over all examples in the validation set. Adapted from Figure 17.11 of [Aur19] . _____no_output_____

show_image("/content/pyprobml/notebooks/figures/images/ae-sparse-noreg-heatmap.png")_____no_output_____show_image("/content/pyprobml/notebooks/figures/images/ae-sparse-L1reg-heatmap.png")_____no_output_____show_image("/content/pyprobml/notebooks/figures/images/ae-sparse-KLreg-heatmap.png")_____no_output_____show_image("/content/pyprobml/notebooks/figures/images/ae-sparse-noreg-act.png")_____no_output_____show_image("/content/pyprobml/notebooks/figures/images/ae-sparse-L1reg-act.png")_____no_output_____show_image("/content/pyprobml/notebooks/figures/images/ae-sparse-KLreg-act.png")_____no_output_____show_image("/content/pyprobml/notebooks/figures/images/ae-sparse-noreg-neurons.png")_____no_output_____show_image("/content/pyprobml/notebooks/figures/images/ae-sparse-L1reg-neurons.png")_____no_output_____show_image("/content/pyprobml/notebooks/figures/images/ae-sparse-KLreg-neurons.png")_____no_output_____

## Figure 20.23:  _____no_output_____
  Schematic illustration of a VAE. From a figure from   http://krasserm.github.io/2018/07/27/dfc-vae/ . Used with kind permission of Martin Krasser. _____no_output_____

show_image("/content/pyprobml/notebooks/figures/images/vae-krasser.png")_____no_output_____

## Figure 20.24:  _____no_output_____
  Comparison of reconstruction abilities of an autoencoder and VAE. Top row: Original images. Middle row: Reconstructions from a VAE. Bottom row: Reconstructions from an AE. We see that the VAE reconstructions (middle) are blurrier. Both models have the same shallow convolutional architecture (3 hidden layers, 200 latents), and are trained on identical data (20k images of size $64 \times 64$ extracted from CelebA) for the same number of epochs (20). _____no_output_____

show_image("/content/pyprobml/notebooks/figures/images/ae-celeba-orig.png")_____no_output_____show_image("/content/pyprobml/notebooks/figures/images/vae-celeba-recon.png")_____no_output_____show_image("/content/pyprobml/notebooks/figures/images/ae-celeba-recon.png")_____no_output_____

## Figure 20.25:  _____no_output_____
  Unconditional samples from a VAE (top row) or AE (bottom row) trained on CelebA. Both models have the same structure and both are trained for 20 epochs. _____no_output_____

show_image("/content/pyprobml/notebooks/figures/images/vae-celeba-samples.png")_____no_output_____show_image("/content/pyprobml/notebooks/figures/images/ae-celeba-samples.png")_____no_output_____

## Figure 20.26:  _____no_output_____
  Interpolation between two real images (first and last columns) in the latent space of a VAE. Adapted from Figure 3.22 of [Dav19] . _____no_output_____

show_image("/content/pyprobml/notebooks/figures/images/vae-celeba-interp-gender.png")_____no_output_____

## Figure 20.27:  _____no_output_____
  Adding or removing the ``sunglasses'' vector to an image using a VAE. The first column is an input image, with embedding $\mathbf  z $. Subsequent columns show the decoding of $\mathbf  z + s \boldsymbol  \Delta  $, where $s \in \ -4,-3,-2,-1,0,1,2,3,4\ $ and $\boldsymbol  \Delta  = \overline  \mathbf  z  ^+ - \overline  \mathbf  z  ^-$ is the difference in the average embeddings of images of people with or without sunglasses. Adapted from Figure 3.21 of [Dav19] . _____no_output_____

show_image("/content/pyprobml/notebooks/figures/images/vae-celeba-glasses-scale.png")_____no_output_____

## Figure 20.28:  _____no_output_____
  Illustration of the tangent space and tangent vectors at two different points on a 2d curved manifold. From Figure 1 of [MM+17] . Used with kind permission of Michael Bronstein _____no_output_____

show_image("/content/pyprobml/notebooks/figures/images/tangentSpace.png")_____no_output_____

## Figure 20.29:  _____no_output_____
  Illustration of the image manifold. (a) An image of the digit 6 from the USPS dataset, of size $64 \times 57 = 3,648$. (b) A random sample from the space $\ 0,1\ ^ 3648 $ reshaped as an image. (c) A dataset created by rotating the original image by one degree 360 times. We project this data onto its first two principal components, to reveal the underlying 2d circular manifold. From Figure 1 of [Nei12] . Used with kind permission of Neil Lawrence _____no_output_____

show_image("/content/pyprobml/notebooks/figures/images/manifold-6-original.png")_____no_output_____show_image("/content/pyprobml/notebooks/figures/images/manifold-6-rnd.png")_____no_output_____show_image("/content/pyprobml/notebooks/figures/images/manifold-6-rotated.png")_____no_output_____

## Figure 20.30:  _____no_output_____
  Illustration of some data generated from low-dimensional manifolds. (a) The 2d Swiss-roll manifold embedded into 3d.
Figure(s) generated by [manifold_swiss_sklearn.py](https://github.com/probml/pyprobml/blob/master/scripts/manifold_swiss_sklearn.py) [manifold_digits_sklearn.py](https://github.com/probml/pyprobml/blob/master/scripts/manifold_digits_sklearn.py) _____no_output_____

interactive_script("manifold_swiss_sklearn.py")_____no_output_____interactive_script("manifold_digits_sklearn.py")_____no_output_____

## Figure 20.31:  _____no_output_____
  Metric MDS applied to (a) Swiss roll.
Figure(s) generated by [manifold_swiss_sklearn.py](https://github.com/probml/pyprobml/blob/master/scripts/manifold_swiss_sklearn.py) [manifold_digits_sklearn.py](https://github.com/probml/pyprobml/blob/master/scripts/manifold_digits_sklearn.py) _____no_output_____

interactive_script("manifold_swiss_sklearn.py")_____no_output_____interactive_script("manifold_digits_sklearn.py")_____no_output_____

## Figure 20.32:  _____no_output_____
  (a) If we measure distances along the manifold, we find $d(1,6) > d(1,4)$, whereas if we measure in ambient space, we find $d(1,6) < d(1,4)$. The plot at the bottom shows the underlying 1d manifold. (b) The $K$-nearest neighbors graph for some datapoints; the red path is the shortest distance between A and B on this graph. From [Hin13] . Used with kind permission of Geoff Hinton. _____no_output_____

show_image("/content/pyprobml/notebooks/figures/images/hinton-isomap1.png")_____no_output_____show_image("/content/pyprobml/notebooks/figures/images/hinton-isomap2.png")_____no_output_____

## Figure 20.33:  _____no_output_____
  Isomap applied to (a) Swiss roll.
Figure(s) generated by [manifold_swiss_sklearn.py](https://github.com/probml/pyprobml/blob/master/scripts/manifold_swiss_sklearn.py) [manifold_digits_sklearn.py](https://github.com/probml/pyprobml/blob/master/scripts/manifold_digits_sklearn.py) _____no_output_____

interactive_script("manifold_swiss_sklearn.py")_____no_output_____interactive_script("manifold_digits_sklearn.py")_____no_output_____

## Figure 20.34:  _____no_output_____
  (a) Noisy version of Swiss roll data. We perturb each point by adding $\mathcal  N (0, 0.5^2)$ noise. (b) Results of Isomap applied to this data.
Figure(s) generated by [manifold_swiss_sklearn.py](https://github.com/probml/pyprobml/blob/master/scripts/manifold_swiss_sklearn.py) _____no_output_____

interactive_script("manifold_swiss_sklearn.py")_____no_output_____

## Figure 20.35:  _____no_output_____
  Visualization of the first 8 kernel principal component basis functions derived from some 2d data. We use an RBF kernel with $\sigma ^2=0.1$.
Figure(s) generated by [kpcaScholkopf.m](https://github.com/probml/pmtk3/blob/master/demos/kpcaScholkopf.m) _____no_output_____

!octave -W kpcaScholkopf.m >> ______no_output_____

## Figure 20.36:  _____no_output_____
  Kernel PCA applied to (a) Swiss roll.
Figure(s) generated by [manifold_swiss_sklearn.py](https://github.com/probml/pyprobml/blob/master/scripts/manifold_swiss_sklearn.py) [manifold_digits_sklearn.py](https://github.com/probml/pyprobml/blob/master/scripts/manifold_digits_sklearn.py) _____no_output_____

interactive_script("manifold_swiss_sklearn.py")_____no_output_____interactive_script("manifold_digits_sklearn.py")_____no_output_____

## Figure 20.37:  _____no_output_____
  LLE applied to (a) Swiss roll.
Figure(s) generated by [manifold_swiss_sklearn.py](https://github.com/probml/pyprobml/blob/master/scripts/manifold_swiss_sklearn.py) [manifold_digits_sklearn.py](https://github.com/probml/pyprobml/blob/master/scripts/manifold_digits_sklearn.py) _____no_output_____

interactive_script("manifold_swiss_sklearn.py")_____no_output_____interactive_script("manifold_digits_sklearn.py")_____no_output_____

## Figure 20.38:  _____no_output_____
  Laplacian eigenmaps applied to (a) Swiss roll.
Figure(s) generated by [manifold_swiss_sklearn.py](https://github.com/probml/pyprobml/blob/master/scripts/manifold_swiss_sklearn.py) [manifold_digits_sklearn.py](https://github.com/probml/pyprobml/blob/master/scripts/manifold_digits_sklearn.py) _____no_output_____

interactive_script("manifold_swiss_sklearn.py")_____no_output_____interactive_script("manifold_digits_sklearn.py")_____no_output_____

## Figure 20.39:  _____no_output_____
  Illustration of the Laplacian matrix derived from an undirected graph. From   https://en.wikipedia.org/wiki/Laplacian_matrix . Used with kind permission of Wikipedia author AzaToth. _____no_output_____

show_image("/content/pyprobml/notebooks/figures/images/graphLaplacian.png")_____no_output_____

## Figure 20.40:  _____no_output_____
  Illustration of a (positive) function defined on a graph. From Figure 1 of [DI+13] . Used with kind permission of Pascal Frossard. _____no_output_____

show_image("/content/pyprobml/notebooks/figures/images/graphFun.png")_____no_output_____

## Figure 20.41:  _____no_output_____
  tSNE applied to (a) Swiss roll.
Figure(s) generated by [manifold_swiss_sklearn.py](https://github.com/probml/pyprobml/blob/master/scripts/manifold_swiss_sklearn.py) [manifold_digits_sklearn.py](https://github.com/probml/pyprobml/blob/master/scripts/manifold_digits_sklearn.py) _____no_output_____

interactive_script("manifold_swiss_sklearn.py")_____no_output_____interactive_script("manifold_digits_sklearn.py")_____no_output_____

## Figure 20.42:  _____no_output_____
  Illustration of the effect of changing the perplexity parameter when t-SNE is applied to some 2d data. From [MFI16] . See   http://distill.pub/2016/misread-tsne  for an animated version of these figures. Used with kind permission of Martin Wattenberg. _____no_output_____

show_image("/content/pyprobml/notebooks/figures/images/tSNE-wattenberg0.png.png")_____no_output_____

## References:
 [Aur19] G. Aur'elien "Hands-On Machine Learning with Scikit-Learn and TensorFlow: Concepts, Tools, and Techniques for BuildingIntelligent Systems (2nd edition)". (2019).

[Bis06] C. Bishop "Pattern recognition and machine learning". (2006).

[DI+13] S. DI, N. SK, F. P, O. A and V. P. "The emerging field of signal processing on graphs: Extendinghigh-dimensional data analysis to networks and other irregulardomains". In: IEEE Signal Process. Mag. (2013).

[Dav19] F. David "Generative Deep Learning: Teaching Machines to Paint, WriteCompose, and Play". (2019).

[EY18] R. Eitan and W. Yair. "On GANs and GMMs". (2018).

[GY14] A. Guillaume and B. Yoshua. "What Regularized Auto-Encoders Learn from the Data-GeneratingDistribution". In: jmlr (2014).

[HTF09] T. Hastie, R. Tibshirani and J. Friedman. "The Elements of Statistical Learning". (2009).

[Hin13] G. Hinton "CSC 2535 Lecture 11: Non-linear dimensionality reduction". (2013).

[MFI16] W. Martin, V. Fernanda and J. Ian. "How to Use t-SNE Effectively". In: Distill (2016).

[MM+17] B. MM, B. J, L. Y, S. A and V. P. "Geometric Deep Learning: Going beyond Euclidean data". In: IEEE Signal Process. Mag. (2017).

[Nei12] L. NeilD "A Unifying Probabilistic Perspective for Spectral DimensionalityReduction: Insights and New Models". In: jmlr (2012).

_____no_output_____


# Notebook from Anuraag287/AI-Classification
Path: Projects/4/COVID_FastAI_ResNet34.ipynb

# Peter Moss Acute Myeloid & Lymphoblastic Leukemia AI Research Project

## Covid FastAI Resnet34 Classifier
**Using The SARS-CoV-2 CT-scan dataset for Image Processing and Image Classification Using Convolutional Neural Network**

The SARS-CoV-2 FastAI  Classifier was created by [Salvatore Raieli](https://github.com/salvatorera) based on his [Resnet50](https://github.com/AMLResearchProject/ALL-FastAI-2019/tree/master/Colab-notebooks/ALL-FastAI-Resnet-50.ipynb) project. The classifier provides a Google Colab notebook that uses FastAI and SARS-CoV-2 from  [A large dataset of CT scans for SARS-CoV-2 (COVID-19) identification](https://www.kaggle.com/plameneduardo/sarscov2-ctscan-dataset).

## FastAI Classifier Project Contributors

- [Salvatore Raieli](https://github.com/salvatorera "Salvatore Raieli") - PhD Immunolgy / Bioinformatics, Bologna, Italy

## Disclaimer

This project should be used for research purposes only. The purpose of the project is to show the potential of Artificial Intelligence for medical support systems such as diagnosis systems.

Although the program is fairly accurate and shows good results both on paper and in real world testing, it is not meant to be an alternative to professional medical diagnosis.

Salvatore Raieli is a bioinformatician researcher and PhD in Immunology, but does not work in medical diagnosis.

Please use these systems responsibly.

## Dataset

SARS-CoV-2 CT scan dataset is a public dataset, containing 1252 CT scans (computed tomography scan) that are from positive patients for SARS-CoV-2 infection (COVID-19) and 1230 CT scans for SARS-CoV-2 non-infected patients. This dataset was created by our partners at Lancaster University, PlamenLancaster: Professor Angelov Plamen, and Eduardo Soares.

The dataset has been collected from real patients in Sao Paulo, Brazil. The dataset is available in [Kaggle](www.kaggle.com/plameneduardo/sarscov2-ctscan-dataset). As baseline result for this dataset Lancaster university LIRA group, [LIRA](https://www.lancaster.ac.uk/lira/) used an eXplainable Deep Learning approach (xDNN), you can find the available code on [github](https://www.medrxiv.org/content/10.1101/2020.04.24.20078584v2).

In using the dataset please cite the original paper by our partners:
Angelov, Plamen, and Eduardo Almeida Soares. "EXPLAINABLE-BY-DESIGN APPROACH FOR COVID-19 CLASSIFICATION VIA CT-SCAN." medRxiv (2020).
Soares, Eduardo, Angelov, Plamen, Biaso, Sarah, Higa Froes, Michele, and Kanda Abe, Daniel. "SARS-CoV-2 CT-scan dataset: A large dataset of real patients CT scans for SARS-CoV-2 identification." medRxiv (2020). doi: https://doi.org/10.1101/2020.04.24.20078584.

# Installation

Please follow the [Installation Guide](Documentation/Installation.md) to install COVID-19 FastAI Classifiers.

## Import data to Google Drive
You need to download and import **SARS-Cov-2 dataset** from the kaggle page to your Google Drive,

In this tutorial we assume you have uploaded your copy of the dataset to a folder located on your Google drive with the location: *fastai-v3/Covid-fastAI*. Once you have uploaded the dataset you can continue with this tutorial.

## Google Colab

*You should now be running this tutorial on Google Colab, if not please read this tutorial from the beginning.*

First we need import the Google Colab Drive library, mount our dataset drive from Google Drive, and set the path to the SARS-Cov-2 folder on your drive. Run the following code block to do this. You will be asked to click a link that will authorize the application with the permissions it needs to mount your drive etc. Follow the steps and then past the authorization key into this application._____no_output_____

%reload_ext autoreload
%autoreload 2
%matplotlib inline

from google.colab import drive
drive.mount('/content/gdrive', force_remount=True)
dataset_dir = "/content/gdrive/My Drive/fastai-v3/Covid-fastAI/SARS-Cov-2/"Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly

Enter your authorization code:
··········
Mounted at /content/gdrive

# Import required libraries and the dataset
## Import required
We need to import the relevant FastAI libraries, running the first code block with do this and get the paths to the dataset files.

## Import datset
We are importing the list of files present in the folders in a list named COVID. We are then extracting the label from the images (using from sklearn.preprocessing the function LabelEncoder). We are then dividing the dataset in the train and test set (we using from sklearn.model_selection  train_test_split). We are using the random_state argument in order to replicate the result. The test_size argument represents the proportion of the dataset to include in the test (in our case the 20%). Since we set the shufflue argument to True, the data will be shufflued before the split.

Now we need to import the dataset into this notebook. run the following code blocks to import the COVID dataset as a FastAI [DataBunch](https://docs.fast.ai/basic_data.html#DataBunch). In the ImageDataBunch.from_name_list function we can see that we pass the *dataset_dir* we created earlier in the tutorial, fileNames that we created earlier,  pattern for the files, some augmentation, the size of the images we need to replicate original model input sizes.

For more information about getting datasets ready with FastAI you can check out [this article](https://docs.fast.ai/vision.data.html#Quickly-get-your-data-ready-for-training).

## data.show_batch()
Now we use the [data.show_batch()](https://docs.fast.ai/basic_data.html#DataBunch.show_batch) function to show a batch of our data. Run the  code block to do this and view the results.

## View classes info
Now we can run the  code block which will print out the classes list and lengths.

_____no_output_____

import os
from fastai.vision import *
from fastai.metrics import error_rate
import numpy as np
import glob_____no_output_____CTRL_files = glob.glob("/content/gdrive/My Drive/fastai-v3/Covid_fastAI/SARS-Cov-2/non-COVID/*")
COVID_files = glob.glob("/content/gdrive/My Drive/fastai-v3/Covid_fastAI/SARS-Cov-2/COVID/*")
COVID = np.concatenate([CTRL_files, COVID_files])
len(CTRL_files), len(COVID_files), COVID[0]_____no_output_____from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import train_test_split

COVID_labels = [fn.split("/")[-1].split(" (")[0] for fn in COVID]
# encode text category labels

le = LabelEncoder()
le.fit(COVID_labels)
COVID_LE = le.transform(COVID_labels)
COVID_LE[0], list(le.classes_)
X_train, X_test, y_train, y_test = train_test_split(COVID, COVID_LE, stratify = COVID_LE, shuffle = True, test_size=0.2, random_state=42)
len(X_train), len(X_test)_____no_output_____#Import datset
data = ImageDataBunch.from_lists(dataset_dir, X_train, y_train, ds_tfms=get_transforms(do_flip = True, flip_vert = True),
                                   size=224, bs=128).normalize(imagenet_stats)
data_____no_output_____#data.show_batch()
data.show_batch(rows=3, figsize=(7,6))_____no_output_____# View classes info
print(data.classes)
len(data.classes),data.c[0, 1]

# The Resnet34 model

## What and why to use transfer learning?

Transfer learning is meaning use a pre-trained model to build our classifier. A pre-trained model is a model that has been previously trained on a dataset. The model comprehends the updated weights and bias.  Using pre-trained model you are saving time and computational resources. Another vantage is that pre-trained models often perform better that architecture designed from scratch. To better understand this point, suppose you want to build a classifier able to sort different sailboat types. A model pre-trained on ships would have already capture in its first layers some boat features, learning faster and with better accuracy among the different sailboat types.

## The Resnet34 architecture

Resnet34 generally is considered a good choice as first architecture to test, it shows good performance without an excessive size allowing to use a higher batch size and thus less computation time. For this reason, before to test more complex architectures (such as Resner5o and so on) Resnet34 is a good compromise.

Residual net have been ideated to solve the problem of the **vanishing gradient**. Highly intricate networks with a large number of hidden layer are working effectively in solving complicated tasks. Their structures allow them to catch pattern in complicated data. When we train the network the early layer tend to be trained slower (the gradient are smaller during backpropagation). The initial layers are important because they learn the basic feature of an object (edge, corner and so on). Failing to proper train these layers lead to a decrease in the overall accuracy of the model.

**Residual neural network** have been ideated to solve this issue. The Resnet model presents the possibility to skip the training of some layer during the initial training. The skipped layer is reusing the learned weights from the previous layer. [Original research article](https://arxiv.org/pdf/1512.03385.pdf)

##Test the Resnet34 architecture with our dataset
Now we are going to test how the FastaAI implementation of this architechture works with the COVID dataset.

Create the convolutional neural network
First we will create the convolutional neural network based on this architechture, to do this we can use the following code block which uses FastAI ( cnn_learner previously create_cnn) function. We pass the loaded data, specify the model, pass error_rate & accuracy as a list for the metrics parameter specifying we want to see both error_rate and accuracy, and finally specify a weight decay of 1e-1 (1.0).

## learn.lr_find() & learn.recorder.plot()
Now we will use the [learn.lr_find()](https://docs.fast.ai/basic_train.html#lr_find) function to run LR Finder. LR Finder help to find the best learning rate to use with our network. For more information the [original paper](https://arxiv.org/pdf/1506.01186.pdf).
As shown from the output of above,  [learn.recorder.plot()](https://docs.fast.ai/basic_train.html#Recorder.plot) function plot the loss over learning rate. Run the following code block to view the graph. The best learning rate should be chosen as the learning rate value where the curve is the steepest. You may try different learning rate values in order to pick up the best.

## learn.fit_one_cycle() & learn.recorder.plot_losses()
The [learn.fit_one_cycle()](https://docs.fast.ai/basic_train.html#fit_one_cycle) function can be used to fit the model. Fit one cycle reach a comparable accuracy faster than th *fit* function in training of complex models. Fit one cycle instead of maintain fix the learning rate during all the iterations is linearly increasing the learning rate and then it is decreasing again (this process is what is called one cycle). Moreover, this learning rate variation is helping in preventing overfitting. We use 5 for the parameter *cyc_len* to specify the number of cycles to run (on cycle can be considered equivalent to an epoch), and *max_lr* to specify the maximum learning rate to use which we set as *0.001*. Fit one cycle varies the learning rate from 10 fold less the maximum learning rate selected. For more information about fit one cycle: [article](https://arxiv.org/pdf/1803.09820.pdf). We then use [learn.recorder.plot_losses()](https://docs.fast.ai/basic_train.html#Recorder.plot_losses) to plot the losses from *fit_one_cycle* as a graph._____no_output_____

#load the model
learn = cnn_learner(data, models.resnet34, metrics=[error_rate,accuracy], wd=1e-1)Downloading: "https://download.pytorch.org/models/resnet34-333f7ec4.pth" to /root/.cache/torch/checkpoints/resnet34-333f7ec4.pth
learn.lr_find()
learn.recorder.plot()_____no_output_____lr = 0.001
learn.fit_one_cycle(cyc_len=5, max_lr=lr)
learn.recorder.plot_losses()_____no_output_____

## Save the model
We can save the model once it has been trained.

## ClassificationInterpretation()
We use  [ClassificationInterpretation()](https://docs.fast.ai/vision.learner.html#ClassificationInterpretation) to visualize interpretations of our model.

## interp.plot_top_losses()
We can use [interp.plot_top_losses()](https://docs.fast.ai/vision.learner.html#plot_top_losses) to view our top losses and their details.

## interp.plot_confusion_matrix()
We will use  [interp.plot_confusion_matrix()](https://docs.fast.ai/vision.learner.html#ClassificationInterpretation.plot_confusion_matrix) to display a [confusion matrix](https://en.wikipedia.org/wiki/Confusion_matrix). Below, 2 in the top left square represents  true negatives, while  in the top right square represent false positives, in the bottom left square represents false negatives, and in the bottom right represents true positives._____no_output_____

#learn.save('resnet34_covid_model')
preds, y, losses = learn.get_preds(with_loss=True)
interp = ClassificationInterpretation(learn, preds, y, losses)
interp.plot_top_losses(9, figsize=(7,7))_____no_output_____interp.plot_confusion_matrix()_____no_output_____

#Unfreeze the model

## learn.unfreeze()
Next we use [learn.unfreeze()](https://docs.fast.ai/basic_train.html#Learner.unfreeze) to unfreeze the model. The original model was trained on ImageNet to classify images among 1000 categories. None of these categories is a leukemia cell, for these reason when fast.ai *cnn_learner* function is behind line substituting the last layer with 2 other layers. The last layer is a matrix that has the same size of our data class (*data.c*). Before, we just trained these two layers while the other model's layers were still keeping the downloaded weight. Unfreezing our model allow us to train also these other layers and updates their weights.

## Train the entire (unfrozen) model
Now that we have unfrozen our model, we will use the following code blocks to train the whole model.

##Slice parameter

Initial layers are activated by simple patterns (like edge, lines, circles etc...)  while the following layers are acquiring the ability to recognize more sophisticated patterns. Update too much the weight of these layers would probably decrease our accuracy. The scope of transfer learning is to exploit this ability of a pre-trained model in recognizing particular patterns and to adapt to our dataset. The parameter *slice* allows to apply ** discriminative learning rate**. In other words, we apply a smaller learning rate (in this case, 1e-5) to the earlier layer and a higher learning rate to the last layer.

## Save the model

We save our model after the un-freezing_____no_output_____

learn.unfreeze()
learn.lr_find()
learn.recorder.plot()_____no_output_____nlr = slice(1e-5, 1e-4)
learn.fit_one_cycle(5, nlr)_____no_output_____preds,y,losses = learn.get_preds(with_loss=True)
interp = ClassificationInterpretation(learn, preds, y, losses)
interp.plot_top_losses(9, figsize=(7,7))_____no_output_____interp.plot_confusion_matrix()_____no_output_____#learn.save('resnet34_covid_unfreeze')_____no_output_____

## prediction_____no_output_____

#prediction
im= open_image(X_test[0])
_, pred_class, preds = learn.predict(im)
np.argmax(preds)_____no_output_____

# Contributing

The Peter Moss Acute Myeloid & Lymphoblastic Leukemia AI Research project encourages and welcomes code contributions, bug fixes and enhancements from the Github.

Please read the [CONTRIBUTING](https://github.com/AMLResearchProject/ALL-FastAI-2019/blob/master/CONTRIBUTING.md "CONTRIBUTING") document for a full guide to forking our repositories and submitting your pull requests. You will also find information about our code of conduct on this page.

## Contributors

- [Salvatore Raieli](https://github.com/salvatorera "Salvatore Raieli") - [Peter Moss Leukemia AI Research](https://www.leukemiaresearchassociation.ai "Peter Moss Leukemia AI Research") & Biogenera, Bologna, Italy

# Versioning

We use SemVer for versioning. For the versions available, see [Releases](https://github.com/AMLResearchProject/ALL-FastAI-2019/releases "Releases").

# License

This project is licensed under the **MIT License** - see the [LICENSE](https://github.com/AMLResearchProject/ALL-FastAI-2019/blob/master/LICENSE "LICENSE") file for details.

# Bugs/Issues

We use the [repo issues](https://github.com/AMLResearchProject/ALL-FastAI-2019/issues "repo issues") to track bugs and general requests related to using this project. See [CONTRIBUTING](https://github.com/AMLResearchProject/ALL-FastAI-2019/blob/master/CONTRIBUTING.md "CONTRIBUTING") for more info on how to submit bugs, feature requests and proposals._____no_output_____


# Notebook from xxxhycl2010/pytorch-lightning
Path: notebooks/02-datamodules.ipynb

_____no_output_____# PyTorch Lightning DataModules ⚡

With the release of `pytorch-lightning` version 0.9.0, we have included a new class called `LightningDataModule` to help you decouple data related hooks from your `LightningModule`.

This notebook will walk you through how to start using Datamodules.

The most up to date documentation on datamodules can be found [here](https://pytorch-lightning.readthedocs.io/en/latest/extensions/datamodules.html).

---

  - Give us a ⭐ [on Github](https://www.github.com/PytorchLightning/pytorch-lightning/)
  - Check out [the documentation](https://pytorch-lightning.readthedocs.io/en/latest/)
  - Join us [on Slack](https://join.slack.com/t/pytorch-lightning/shared_invite/zt-pw5v393p-qRaDgEk24~EjiZNBpSQFgQ)_____no_output_____### Setup
Lightning is easy to install. Simply ```pip install pytorch-lightning```_____no_output_____

! pip install pytorch-lightning --quiet_____no_output_____

# Introduction

First, we'll go over a regular `LightningModule` implementation without the use of a `LightningDataModule`_____no_output_____

import pytorch_lightning as pl
from pytorch_lightning.metrics.functional import accuracy
import torch
from torch import nn
import torch.nn.functional as F
from torch.utils.data import random_split, DataLoader

# Note - you must have torchvision installed for this example
from torchvision.datasets import MNIST, CIFAR10
from torchvision import transforms_____no_output_____

## Defining the LitMNISTModel

Below, we reuse a `LightningModule` from our hello world tutorial that classifies MNIST Handwritten Digits.

Unfortunately, we have hardcoded dataset-specific items within the model, forever limiting it to working with MNIST Data. 😢

This is fine if you don't plan on training/evaluating your model on different datasets. However, in many cases, this can become bothersome when you want to try out your architecture with different datasets._____no_output_____

class LitMNIST(pl.LightningModule):

    def __init__(self, data_dir='./', hidden_size=64, learning_rate=2e-4):

        super().__init__()

        # We hardcode dataset specific stuff here.
        self.data_dir = data_dir
        self.num_classes = 10
        self.dims = (1, 28, 28)
        channels, width, height = self.dims
        self.transform = transforms.Compose([
            transforms.ToTensor(),
            transforms.Normalize((0.1307,), (0.3081,))
        ])

        self.hidden_size = hidden_size
        self.learning_rate = learning_rate

        # Build model
        self.model = nn.Sequential(
            nn.Flatten(),
            nn.Linear(channels * width * height, hidden_size),
            nn.ReLU(),
            nn.Dropout(0.1),
            nn.Linear(hidden_size, hidden_size),
            nn.ReLU(),
            nn.Dropout(0.1),
            nn.Linear(hidden_size, self.num_classes)
        )

    def forward(self, x):
        x = self.model(x)
        return F.log_softmax(x, dim=1)

    def training_step(self, batch, batch_idx):
        x, y = batch
        logits = self(x)
        loss = F.nll_loss(logits, y)
        return loss

    def validation_step(self, batch, batch_idx):
        x, y = batch
        logits = self(x)
        loss = F.nll_loss(logits, y)
        preds = torch.argmax(logits, dim=1)
        acc = accuracy(preds, y)
        self.log('val_loss', loss, prog_bar=True)
        self.log('val_acc', acc, prog_bar=True)
        return loss

    def configure_optimizers(self):
        optimizer = torch.optim.Adam(self.parameters(), lr=self.learning_rate)
        return optimizer

    ####################
    # DATA RELATED HOOKS
    ####################

    def prepare_data(self):
        # download
        MNIST(self.data_dir, train=True, download=True)
        MNIST(self.data_dir, train=False, download=True)

    def setup(self, stage=None):

        # Assign train/val datasets for use in dataloaders
        if stage == 'fit' or stage is None:
            mnist_full = MNIST(self.data_dir, train=True, transform=self.transform)
            self.mnist_train, self.mnist_val = random_split(mnist_full, [55000, 5000])

        # Assign test dataset for use in dataloader(s)
        if stage == 'test' or stage is None:
            self.mnist_test = MNIST(self.data_dir, train=False, transform=self.transform)

    def train_dataloader(self):
        return DataLoader(self.mnist_train, batch_size=32)

    def val_dataloader(self):
        return DataLoader(self.mnist_val, batch_size=32)

    def test_dataloader(self):
        return DataLoader(self.mnist_test, batch_size=32)_____no_output_____

## Training the ListMNIST Model_____no_output_____

model = LitMNIST()
trainer = pl.Trainer(max_epochs=2, gpus=1, progress_bar_refresh_rate=20)
trainer.fit(model)_____no_output_____

# Using DataModules

DataModules are a way of decoupling data-related hooks from the `LightningModule` so you can develop dataset agnostic models._____no_output_____## Defining The MNISTDataModule

Let's go over each function in the class below and talk about what they're doing:

1. ```__init__```
    - Takes in a `data_dir` arg that points to where you have downloaded/wish to download the MNIST dataset.
    - Defines a transform that will be applied across train, val, and test dataset splits.
    - Defines default `self.dims`, which is a tuple returned from `datamodule.size()` that can help you initialize models.

2. ```prepare_data```
    - This is where we can download the dataset. We point to our desired dataset and ask torchvision's `MNIST` dataset class to download if the dataset isn't found there.
    - **Note we do not make any state assignments in this function** (i.e. `self.something = ...`)

3. ```setup```
    - Loads in data from file and prepares PyTorch tensor datasets for each split (train, val, test).
    - Setup expects a 'stage' arg which is used to separate logic for 'fit' and 'test'.
    - If you don't mind loading all your datasets at once, you can set up a condition to allow for both 'fit' related setup and 'test' related setup to run whenever `None` is passed to `stage`.
    - **Note this runs across all GPUs and it *is* safe to make state assignments here**

4. ```x_dataloader```
    - `train_dataloader()`, `val_dataloader()`, and `test_dataloader()` all return PyTorch `DataLoader` instances that are created by wrapping their respective datasets that we prepared in `setup()`_____no_output_____

class MNISTDataModule(pl.LightningDataModule):

    def __init__(self, data_dir: str = './'):
        super().__init__()
        self.data_dir = data_dir
        self.transform = transforms.Compose([
            transforms.ToTensor(),
            transforms.Normalize((0.1307,), (0.3081,))
        ])

        # self.dims is returned when you call dm.size()
        # Setting default dims here because we know them.
        # Could optionally be assigned dynamically in dm.setup()
        self.dims = (1, 28, 28)
        self.num_classes = 10

    def prepare_data(self):
        # download
        MNIST(self.data_dir, train=True, download=True)
        MNIST(self.data_dir, train=False, download=True)

    def setup(self, stage=None):

        # Assign train/val datasets for use in dataloaders
        if stage == 'fit' or stage is None:
            mnist_full = MNIST(self.data_dir, train=True, transform=self.transform)
            self.mnist_train, self.mnist_val = random_split(mnist_full, [55000, 5000])

        # Assign test dataset for use in dataloader(s)
        if stage == 'test' or stage is None:
            self.mnist_test = MNIST(self.data_dir, train=False, transform=self.transform)

    def train_dataloader(self):
        return DataLoader(self.mnist_train, batch_size=32)

    def val_dataloader(self):
        return DataLoader(self.mnist_val, batch_size=32)

    def test_dataloader(self):
        return DataLoader(self.mnist_test, batch_size=32)_____no_output_____

## Defining the dataset agnostic `LitModel`

Below, we define the same model as the `LitMNIST` model we made earlier.

However, this time our model has the freedom to use any input data that we'd like 🔥._____no_output_____

class LitModel(pl.LightningModule):

    def __init__(self, channels, width, height, num_classes, hidden_size=64, learning_rate=2e-4):

        super().__init__()

        # We take in input dimensions as parameters and use those to dynamically build model.
        self.channels = channels
        self.width = width
        self.height = height
        self.num_classes = num_classes
        self.hidden_size = hidden_size
        self.learning_rate = learning_rate

        self.model = nn.Sequential(
            nn.Flatten(),
            nn.Linear(channels * width * height, hidden_size),
            nn.ReLU(),
            nn.Dropout(0.1),
            nn.Linear(hidden_size, hidden_size),
            nn.ReLU(),
            nn.Dropout(0.1),
            nn.Linear(hidden_size, num_classes)
        )

    def forward(self, x):
        x = self.model(x)
        return F.log_softmax(x, dim=1)

    def training_step(self, batch, batch_idx):
        x, y = batch
        logits = self(x)
        loss = F.nll_loss(logits, y)
        return loss

    def validation_step(self, batch, batch_idx):

        x, y = batch
        logits = self(x)
        loss = F.nll_loss(logits, y)
        preds = torch.argmax(logits, dim=1)
        acc = accuracy(preds, y)
        self.log('val_loss', loss, prog_bar=True)
        self.log('val_acc', acc, prog_bar=True)
        return loss

    def configure_optimizers(self):
        optimizer = torch.optim.Adam(self.parameters(), lr=self.learning_rate)
        return optimizer_____no_output_____

## Training the `LitModel` using the `MNISTDataModule`

Now, we initialize and train the `LitModel` using the `MNISTDataModule`'s configuration settings and dataloaders._____no_output_____

# Init DataModule
dm = MNISTDataModule()
# Init model from datamodule's attributes
model = LitModel(*dm.size(), dm.num_classes)
# Init trainer
trainer = pl.Trainer(max_epochs=3, progress_bar_refresh_rate=20, gpus=1)
# Pass the datamodule as arg to trainer.fit to override model hooks :)
trainer.fit(model, dm)_____no_output_____

## Defining the CIFAR10 DataModule

Lets prove the `LitModel` we made earlier is dataset agnostic by defining a new datamodule for the CIFAR10 dataset._____no_output_____

class CIFAR10DataModule(pl.LightningDataModule):

    def __init__(self, data_dir: str = './'):
        super().__init__()
        self.data_dir = data_dir
        self.transform = transforms.Compose([
            transforms.ToTensor(),
            transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
        ])

        self.dims = (3, 32, 32)
        self.num_classes = 10

    def prepare_data(self):
        # download
        CIFAR10(self.data_dir, train=True, download=True)
        CIFAR10(self.data_dir, train=False, download=True)

    def setup(self, stage=None):

        # Assign train/val datasets for use in dataloaders
        if stage == 'fit' or stage is None:
            cifar_full = CIFAR10(self.data_dir, train=True, transform=self.transform)
            self.cifar_train, self.cifar_val = random_split(cifar_full, [45000, 5000])

        # Assign test dataset for use in dataloader(s)
        if stage == 'test' or stage is None:
            self.cifar_test = CIFAR10(self.data_dir, train=False, transform=self.transform)

    def train_dataloader(self):
        return DataLoader(self.cifar_train, batch_size=32)

    def val_dataloader(self):
        return DataLoader(self.cifar_val, batch_size=32)

    def test_dataloader(self):
        return DataLoader(self.cifar_test, batch_size=32)_____no_output_____

## Training the `LitModel` using the `CIFAR10DataModule`

Our model isn't very good, so it will perform pretty badly on the CIFAR10 dataset.

The point here is that we can see that our `LitModel` has no problem using a different datamodule as its input data._____no_output_____

dm = CIFAR10DataModule()
model = LitModel(*dm.size(), dm.num_classes, hidden_size=256)
trainer = pl.Trainer(max_epochs=5, progress_bar_refresh_rate=20, gpus=1)
trainer.fit(model, dm)_____no_output_____

  Congratulations - Time to Join the Community!

Congratulations on completing this notebook tutorial! If you enjoyed this and would like to join the Lightning movement, you can do so in the following ways!

### Star [Lightning](https://github.com/PyTorchLightning/pytorch-lightning) on GitHub
The easiest way to help our community is just by starring the GitHub repos! This helps raise awareness of the cool tools we're building.

* Please, star [Lightning](https://github.com/PyTorchLightning/pytorch-lightning)

### Join our [Slack](https://join.slack.com/t/pytorch-lightning/shared_invite/zt-pw5v393p-qRaDgEk24~EjiZNBpSQFgQ)!
The best way to keep up to date on the latest advancements is to join our community! Make sure to introduce yourself and share your interests in `#general` channel

### Interested by SOTA AI models ! Check out [Bolt](https://github.com/PyTorchLightning/lightning-bolts)
Bolts has a collection of state-of-the-art models, all implemented in [Lightning](https://github.com/PyTorchLightning/pytorch-lightning) and can be easily integrated within your own projects.

* Please, star [Bolt](https://github.com/PyTorchLightning/lightning-bolts)

### Contributions !
The best way to contribute to our community is to become a code contributor! At any time you can go to [Lightning](https://github.com/PyTorchLightning/pytorch-lightning) or [Bolt](https://github.com/PyTorchLightning/lightning-bolts) GitHub Issues page and filter for "good first issue".

* [Lightning good first issue](https://github.com/PyTorchLightning/pytorch-lightning/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22)
* [Bolt good first issue](https://github.com/PyTorchLightning/lightning-bolts/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22)
* You can also contribute your own notebooks with useful examples !

### Great thanks from the entire Pytorch Lightning Team for your interest !

_____no_output_____