# Notebook from mkleesiek/CarND-P03-Traffic-Sign-Classifier
Path: P3.ipynb

# Self-Driving Car Engineer Nanodegree

## Deep Learning

## Project 3#: Build a Traffic Sign Recognition Classifier_____no_output_____---
### Imports, notebook configuration and utility functions_____no_output_____

%%javascript
IPython.OutputArea.auto_scroll_threshold = -1;_____no_output_____%%capture
# Imports
import warnings
import pickle
import os
import math
import random
import urllib.request

from zipfile import ZipFile
from io import BytesIO

import numpy as np
import pandas as pd

import skimage as ski
import skimage.color
import skimage.transform
import skimage.io
import skimage.util

import matplotlib.pyplot as plt
from matplotlib.patches import Rectangle

import tensorflow as tf

import sklearn as skl
import sklearn.utils

# Visualizations will be shown in the notebook.
%matplotlib inline

# Adjust warnings and log verbosity
warnings.filterwarnings('ignore')
warnings.simplefilter('ignore')
tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR)

def update_progress(progress, opt_text=""):
    """
    Print a rewritable single line ASCI progress bar.
    :param progress: Progress from 0.0 to 1.0.
    """
    bar_length = 20
    block = int(math.ceil(bar_length * progress))
    print("Progress: [{}] {:.1f}%   {:s}".format( "#" * block + "-" * (bar_length - block),
                                                 progress * 100, opt_text), end='\r')

def color_depth(dataset):
    """
    Determine the image color depth of a dataset.
    """
    return dataset.shape[-1] if (isinstance(dataset, np.ndarray) and dataset.ndim == 4) else 0
_____no_output_____

---
## Step 0: Load The Data
_____no_output_____

def region_of_interest(img, original_size, original_roi):
    """
    Given the original image size and region of interest (ROI), calculate the ROI according to the current shape.
    """
    return [
        original_roi[0] / original_size[0] * img.shape[0],
        original_roi[1] / original_size[1] * img.shape[1],
        original_roi[2] / original_size[0] * img.shape[0],
        original_roi[3] / original_size[1] * img.shape[1],
    ]

def load_from_pickle(file):
    """
    Load the pickled traffic sign image dataset from Udacity
    :param file: File handler to the pickle file.
    :return: A tuple of features, labels and rescaled ROIs.
    """
    data = pickle.load(file)
    X, y = data['features'], data['labels']

    assert(len(X) == len(y))

    if 'coords' in data:
        roi = np.zeros_like(data['coords'], dtype=np.float)
        for i in range(len(roi)):
            roi[i] = region_of_interest(data['features'][i], data['sizes'][i], data['coords'][i])
    else:
        roi = []

    return X, y, roi
_____no_output_____# This notebook's temporary data directory
DATA_DIR = 'data'
# URL pointing to Udacity's pickled traffic signs dataset
ARCHIVE_URL = 'https://s3-us-west-1.amazonaws.com/udacity-selfdrivingcar/traffic-signs-data.zip'

# Repository locations of the downloaded and extracted data
training_file = DATA_DIR + '/train.p'
validation_file = DATA_DIR + '/valid.p'
testing_file = DATA_DIR + '/test.p'

# If the pickle files do not exist yet, download and extract the archive
if any(not os.path.exists(file) for file in (training_file, validation_file, testing_file)):
    os.makedirs(DATA_DIR)

    url = urllib.request.urlopen(ARCHIVE_URL)

    with ZipFile(BytesIO(url.read())) as my_zip_file:
        my_zip_file.extractall(DATA_DIR)

# Load training, validation and testing data
with open(training_file, mode='rb') as f:
    X_train, y_train, roi_train = load_from_pickle(f)
with open(validation_file, mode='rb') as f:
    X_valid, y_valid, roi_valid = load_from_pickle(f)
with open(testing_file, mode='rb') as f:
    X_test, y_test, roi_test = load_from_pickle(f)

print("Data successfully loaded.")
Data successfully loaded.

---
## Step 1: Dataset Summary & Exploration_____no_output_____### Summary and Visualization of the Dataset_____no_output_____

# Location of the class index -> class name mapping file
SIGNNAMES_FILE = 'signnames.csv'

def summarize_dataset(train=([],[],[]), valid=([],[],[]), test=([],[],[]), plot_distribution=True):
    """
    Given the loaded features, lables and ROIs for training, validation and testing data,
    print some basic metrics and visualize the dataset.
    :return: A list of class indices
    """
    (X_train, y_train, roi_train) = train
    (X_valid, y_valid, roi_valid) = valid
    (X_test, y_test, roi_test) = test

    # Assess unique classes and distribution of the training dataset using NumPy
    classes, classes_count = np.unique(y_test, return_counts=True)

    # Number of training examples
    n_train = len(y_train)
    # Number of validation examples
    n_validation = len(y_valid)
    # Number of testing examples.
    n_test = len(y_test)
    # What's the shape of an traffic sign image?
    image_shape = X_test[0].shape if len(X_test) else X_train[0].shape
    # How many unique classes/labels there are in the dataset.
    n_classes = len(classes)

    # Print basic summary
    print("Number of training examples =", n_train)
    print("Number of validation examples =", n_validation)
    print("Number of testing examples =", n_test)
    print("Image data shape =", image_shape)
    print("Number of classes =", n_classes)

    # Plot a histogram, showing the distribution of samples across classes using Pandas
    classes_df = pd.read_csv(SIGNNAMES_FILE, index_col=0)
    classes_df['Training'] = pd.Series(y_train).value_counts().sort_index()
    classes_df['Validation'] = pd.Series(y_valid).value_counts().sort_index()
    classes_df['Testing']  = pd.Series(y_test) .value_counts().sort_index()

    if plot_distribution:
        classes_df.plot(kind='bar', stacked=True, figsize=(14,5))
        # print(classes_df)

    # prepare a grid of subplots
    ncols = 5
    nrows = (n_classes + ncols - 1) // ncols
    fig, ax = plt.subplots(nrows=nrows, ncols=ncols, figsize=(14,nrows*2.2), squeeze=True)

    # iterate over all subplot axes and image classes
    for i, axi in enumerate(ax.flat):
        # i runs from 0 to (nrows*ncols-1)
        if i >= n_classes:
            fig.delaxes(axi)
            continue

        cls_id = classes[i]
        cls_title = classes_df.iloc[:,0][cls_id]
        cls_count = np.nansum([classes_df.iloc[:,k][cls_id] for k in (1,2,3)], dtype=np.uint32)

        # select an image for plotting
        indices = np.where(y_test==cls_id)[0]
        sel_index = np.random.choice(indices)
        img = X_test[sel_index]

        # plot the image along with the class index, title and number of samples per class
        axi.imshow(img)
        axi.set_title("{:02d}: {:20.20s} [{}]".format(cls_id, cls_title, cls_count))

        # draw the rectangular region of interest onto the image
        if len(roi_test) > 0:
            roi = roi_test[sel_index]
            axi.add_patch(Rectangle(
                (roi[0], roi[1]), roi[2]-roi[0], roi[3]-roi[1],
                linewidth=1,
                edgecolor='r',
                facecolor='none'
            ))

    plt.tight_layout(pad=1.1)
    plt.show()

    return classes

# print summary and plots
classes = summarize_dataset(
    (X_train, y_train, roi_train),
    (X_valid, y_valid, roi_valid),
    (X_test, y_test, roi_test)
)

# number of available classes of traffic signs
n_classes = len(classes)Number of training examples = 34799
Number of validation examples = 4410
Number of testing examples = 12630
Image data shape = (32, 32, 3)
Number of classes = 43

----

## Step 2: Design and Test a Model Architecture

### Pre-process the Dataset (normalization, grayscale, rescaling)_____no_output_____

### Preprocess the data here. It is required to normalize the data. Other preprocessing steps could include
### converting to grayscale, etc.
### Feel free to use as many code cells as needed.

def pad(img, shape):
    """
    Pad the given image to fill a specific shape.
    The last pixel of the image is used for padding.
    """
    if img.shape >= shape:
        return img

    out = np.full(shape, np.mean(img.flat[-5:]))
    out[tuple(slice(0, d) for d in np.shape(img))] = img
    return out

def preprocess_images(dataset, dataset_roi=None, size=(32,32), debug=-1):
    """
    Preprocess a given image dataset.

    The following stages are executed:
    * Conversion to grayscale, if applicable.
    * If the region of interest ``dataset_roi`` is given, each image is cropped to that region,
      then padded and resized to match ``size``.
    * Normalization: The available color range for each image is rescaled to the range [-1.0, 1.0]
    * The dataset is reshaped to be suitable for tensorflow input.

    :return: The processed image dataset.
    """

    print("Processing image dataset ...")
    processed_data = np.zeros((len(dataset), size[0], size[1], 1), np.float)

    for i in range(len(dataset)):

        img = dataset[i]

        if i == debug:
            plt.figure()
            plt.imshow(img)

        # Convert to grayscale
        if img.shape[-1] == 3:
            img = ski.color.rgb2gray(img)

        img = img.astype(np.float)

        # Crop and resize to region of interest
        if not dataset_roi is None and i < len(dataset_roi):

            roi = dataset_roi[i]

            if i == debug:
                plt.figure()
                plt.imshow(img, cmap="gray")
                plt.gca().add_patch(Rectangle((roi[0],roi[1]),roi[2]-roi[0],roi[3]-roi[1],linewidth=1,edgecolor='r',facecolor='none'))

            # Calculate the area to crop, leaving some padding outside the given ROI on each side
            # of the reactangle

            w = roi[2]-roi[0]
            h = roi[3]-roi[1]

            padding = 0.09

            # croop coordinates
            x1 = math.floor(roi[0] - w*padding)
            x2 = math.ceil( roi[2] + w*padding) + 1
            x1 = max(x1, 0)
            x2 = min(x2, img.shape[1])
            p = x2-x1

            y1 = math.floor(roi[1] - h*padding)
            y2 = math.ceil( roi[3] + h*padding) + 1
            y1 = max(y1, 0)
            y2 = min(y2, img.shape[0])
            q = y2-y1

#             print(roi, (x1,y1,x2,y2), (img.shape[1],img.shape[0]))

            # If the calculated crop area does differ from the target image size go ahead.
            if size != (x2-x1, y2-y1):

                # crop the image
                img_cropped = img[y1:y2,x1:x2]

                if img_cropped.size > 0:

                    if i == debug:
                        plt.figure()
                        plt.imshow(img_cropped, cmap="gray")

                    # pad the image if necessary
#                     img = pad(img_cropped, (max(img_cropped.shape),max(img_cropped.shape)))

                    # resize the cropped image to match the target size again
                    img = ski.transform.resize(img_cropped, (size[0],size[1]), anti_aliasing=True)

                    if i == debug:
                        plt.figure()
                        plt.imshow(img, cmap="gray")

        # Normalize image data to a range [-1.0, +1.0]
        col_range = np.max(img) - np.min(img)
        img = (img - np.min(img)) / col_range * 2 - 1

        if i == debug:
            plt.figure()
            plt.imshow(img, cmap="gray")

        # Reshape for tensor input
        img = img.reshape((32, 32, 1))

        processed_data[i] = img

        # print progress bar
        if i % 1000 == 0 or i == len(dataset)-1:
            update_progress((i+1) / len(dataset))

    print()
    return processed_data
_____no_output_____# Preprocess images
X_train = preprocess_images(X_train, roi_train)
X_valid = preprocess_images(X_valid, roi_valid)
X_test = preprocess_images(X_test, roi_test)

# Determine input color depth
input_depth = color_depth(X_train)
Processing image dataset ...
Progress: [####################] 100.0%
Processing image dataset ...
Progress: [####################] 100.0%
Processing image dataset ...
Progress: [####################] 100.0%

### Augment and Balance the Training Data Set_____no_output_____

def transform_image(img):
    """
    Transform a single image by applying a random amount of rotation between [-15°,15°] and a random
    amount of Gaussian noise.
    """

    result = np.zeros_like(img)

    # apply a random amount of rotation between -15 and 15 degrees
    random_degree = random.uniform(-15, 15)
    result = ski.transform.rotate(img, random_degree)

    # apply a random amount of gaussian noise (no clipping)
    random_var = random.uniform(0.0, 0.01)
    result = ski.util.random_noise(result, var=random_var, clip=False)

    return result

def augment_data(X, y, roi, factor=2, debug=-1):
    """
    Augment the preprocessed image dataset.

    Source images are picked randomly within each image class to first balance the dataset.
    In a second pass, images are picked evenly across classes to increase the size of the balanced
    dataset by a given `factor`.
    Each copied image is applied a random amount of Gaussian noise and rotation using the function
    `transform_image()`.
    :return: The augmented dataset as tuple (features, labels and roi).
    """

    print("Augmenting and balancing dataset ...")

    # Assess unique classes and distribution of the training dataset
    classes, classes_count = np.unique(y, return_counts=True)
    # print(classes_count)

    # calculate the augmented dataset size
    n_per_class_max = np.nanmax(classes_count)
    final_dataset_size = n_per_class_max * len(classes) * factor
    augmented_size = final_dataset_size - len(X)

    # prepare result arrays
    X_augmented = np.zeros((augmented_size,) + X.shape[1:])
    y_augmented = np.zeros((augmented_size,) + y.shape[1:])
    roi_augmented = np.zeros((augmented_size,) + roi.shape[1:])

    # counter for images added to `X_augmented`
    j = 0
    # iterate over all classes and augment within tha class (choose images according to dataset imbalance)
    for (cls, count) in zip(classes, classes_count):
        for i in range(count, n_per_class_max*factor):
            # pick a random image from the source dataset, but with label == cls:
            p = np.random.choice(np.where(y == cls)[0])

            if j == debug:
                plt.figure()
                plt.imshow(X[p], cmap="gray")

            # transform the image
            X_augmented[j] = transform_image(X[p])

            if j == debug:
                plt.figure()
                plt.imshow(X_augmented[j] , cmap="gray")

            # keep the label and roi for this new addition to the dataset
            y_augmented[j] = y[p]
            roi_augmented[j] = roi[p]
            j += 1

            # print progress
            if j % 1000 == 0 or j == augmented_size-1:
                update_progress((j+1) / augmented_size)

    print()
    # Concetenate source and augmented dataset and return
    return np.concatenate((X, X_augmented)), np.concatenate((y, y_augmented)), np.concatenate((roi, roi_augmented))

print()

# Print imbalance ratios for augmented training dataset
classes, classes_count = np.unique(y_train, return_counts=True)
print("Minimum/maximum training data class count: {} / {}".format(min(classes_count), max(classes_count)))
print()

# Augment the dataset
X_train, y_train, roi_train = augment_data(X_train, y_train, roi_train)
print("Training dataset augmented. Number of examples = {}".format(len(y_train)))
print()

# Print imbalance ratios for augmented training dataset
classes, classes_count = np.unique(y_train, return_counts=True)
print("Minimum/maximum training data class count: {} / {}".format(min(classes_count), max(classes_count)))

Minimum/maximum training data class count: 180 / 2010

Augmenting and balancing dataset ...
Progress: [####################] 100.0%
Training dataset augmented. Number of examples = 172860

Minimum/maximum training data class count: 4020 / 4020

### Model Architecture_____no_output_____

# Helper function around tf.nn.conv2d so simplify 2D convolutions
def conv2d(x, W, b, strides=1, padding='VALID', name=None):
    conv = tf.nn.conv2d(x, W, strides=[1, strides, strides, 1], padding=padding, name=name)
    return tf.nn.bias_add(conv, b)

# Helper function around tf.nn.max_pool to simplify max pooling
def pool2d(x, k=2):
    return tf.nn.max_pool(x, ksize=[1, k, k, 1], strides=[1, k, k, 1], padding='VALID')

def LeNet(x):
    """
    Tensor flow model for a convoluted neural network, following the LeNet-5 architecture.
    :param x: Tensor holding an array of input images with shape (32,32,1).
    :return: Tensor holding the result logits for the given input.
    """

    # Arguments used for weight initialization via `tf.truncated_normal`
    MU = 0
    SIGMA = 0.1

    # Make the convolutional layer accessible outside for visualization
    global conv1, conv2, fc2_w, fc3_w

    # Layer 1: Convolutional. Input = 32x32xinput_depth. Output = 28x28x6.
    conv1_w = tf.Variable(tf.truncated_normal([5, 5, input_depth, 6], MU, SIGMA))
    conv1_b = tf.Variable(tf.zeros(6))
    conv1 = conv2d(x, conv1_w, conv1_b)
    # Activation.
    conv1 = tf.nn.tanh(conv1)
    # Pooling. Input = 28x28x6. Output = 14x14x6.
    conv1_p = pool2d(conv1, 2)

    # Layer 2: Convolutional. Output = 10x10x16.
    conv2_w = tf.Variable(tf.truncated_normal([5, 5, 6, 16], MU, SIGMA))
    conv2_b = tf.Variable(tf.zeros(16))
    conv2 = conv2d(conv1_p, conv2_w, conv2_b)
    # Activation.
    conv2 = tf.nn.tanh(conv2)
    # Pooling. Input = 10x10x16. Output = 5x5x16.
    conv2_p = pool2d(conv2, 2)

    # Flatten. Input = 5x5x16. Output = 400.
    fc0 = tf.contrib.layers.flatten(conv2_p)
    # 1st Dropout.
    fc0 = tf.nn.dropout(fc0, rate=dropout_rate)

    # Layer 3: Fully Connected. Input = 400. Output = 120.
    fc1_w = tf.Variable(tf.truncated_normal([400, 120], MU, SIGMA))
    fc1_b = tf.Variable(tf.zeros(120))
    fc1 = tf.add(tf.matmul(fc0, fc1_w), fc1_b)
    # Activation.
    fc1 = tf.nn.tanh(fc1)
    # 2nd Dropout.
    fc1 = tf.nn.dropout(fc1, rate=dropout_rate)

    # Layer 4: Fully Connected. Input = 120. Output = 84.
    fc2_w = tf.Variable(tf.truncated_normal([120, 84], MU, SIGMA))
    fc2_b = tf.Variable(tf.zeros(84))
    fc2 = tf.add(tf.matmul(fc1, fc2_w), fc2_b)
    # Activation.
    fc2 = tf.nn.tanh(fc2)
    # 3rd Dropout.
    fc2 = tf.nn.dropout(fc2, rate=dropout_rate)

    # Layer 5: Fully Connected. Input = 84. Output = n_classes.
    fc3_w = tf.Variable(tf.truncated_normal([84, n_classes], MU, SIGMA))
    fc3_b = tf.Variable(tf.zeros(n_classes))
    logits = tf.add(tf.matmul(fc2, fc3_w), fc3_b)

    return logits_____no_output_____

### Train, Validate and Test the Model

#### Hyperparameters and Operations_____no_output_____

# Learning rate
LEARNING_RATE = 0.0005
# Number of epochs
EPOCHS = 200
# Batch size
BATCH_SIZE = 128

# Probability of dropping input nodes in dropout layers (only used during training, otherwise = 0.0)
DROPOUT_RATE = 0.3
# Multiplier for L2 regularization (deactivated if 0.0)
BETA = 0.0

# Probability of dropping input elements to dropout nodes
dropout_rate = tf.placeholder_with_default(0.0, ())
# Placeholder for input features
x = tf.placeholder(tf.float32, (None, 32, 32, input_depth))
# Placeholder for input labels (scalar)
y = tf.placeholder(tf.int32, (None))
# One hot tensor representation of labels
one_hot_y = tf.one_hot(y, n_classes)

# Logits as defined by our LeNet model
logits = LeNet(x)
# Cross Entropy (incl. Soft-Max)
cross_entropy = tf.nn.softmax_cross_entropy_with_logits(labels=one_hot_y, logits=logits)
# Summation and mean
loss_operation = tf.reduce_mean(cross_entropy)

# L2 regularization (if BETA > 0)
if BETA > 0.0:
    regularizer = (tf.nn.l2_loss(fc2_w) + tf.nn.l2_loss(fc3_w))
    loss_operation = tf.reduce_mean(loss_operation + BETA * regularizer)

# Optimizer function
optimizer = tf.train.AdamOptimizer(learning_rate = LEARNING_RATE)
# Training op. for the TF runs
training_operation = optimizer.minimize(loss_operation)

# Operation returning the label with the highest logits score (the prediction)
prediction = tf.argmax(logits, 1)
# True if the prediction matches the input label
correct_prediction = tf.equal(prediction, tf.argmax(one_hot_y, 1))
# Accuracy averaged over a batch of inputs
accuracy_operation = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

def predict(X_data):
    """
    Calculate the predictions for an image dataset, using tf.Session.run() in batches.
    """
    sess = tf.get_default_session()
    result = np.zeros(len(X_data), dtype=np.uint8)

    for offset in range(0, len(X_data), BATCH_SIZE):
        batch_x = X_data[offset:offset+BATCH_SIZE]
        result[offset:offset+BATCH_SIZE] = sess.run(prediction, feed_dict={x: batch_x, dropout_rate: 0.0})

    return result

def evaluate(X_data, y_data):
    """
    Calculate the mean accuracy and loss for classifying an image dataset, using tf.Session.run() in batches.
    """
    sess = tf.get_default_session()
    total_accuracy = 0
    total_loss = 0

    for offset in range(0, len(X_data), BATCH_SIZE):
        batch_x, batch_y = X_data[offset:offset+BATCH_SIZE], y_data[offset:offset+BATCH_SIZE]

        accuracy = sess.run(accuracy_operation, feed_dict={x: batch_x, y: batch_y, dropout_rate: 0.0})
        loss = sess.run(loss_operation, feed_dict={x: batch_x, y: batch_y, dropout_rate: 0.0})

        total_accuracy += (accuracy * len(batch_x))
        total_loss += (loss * len(batch_x))

    return total_accuracy / len(X_data), total_loss / len(X_data)
WARNING: Entity <bound method Flatten.call of <tensorflow.python.layers.core.Flatten object at 0x7f4e5115d290>> could not be transformed and will be executed as-is. Please report this to the AutgoGraph team. When filing the bug, set the verbosity to 10 (on Linux, `export AUTOGRAPH_VERBOSITY=10`) and attach the full output. Cause: converting <bound method Flatten.call of <tensorflow.python.layers.core.Flatten object at 0x7f4e5115d290>>: AttributeError: module 'gast' has no attribute 'Index'

### Training and Saving the Model_____no_output_____

saver = tf.train.Saver()

with tf.Session() as sess:

    sess.run(tf.global_variables_initializer())

    # Store training and validation accuracy / loss per epoch
    training_accs = []
    validation_accs = []
    training_losses = []
    validation_losses = []

    print("Training model ...")
    print()
    for i in range(EPOCHS):
        X_train, y_train = skl.utils.shuffle(X_train, y_train)
        for offset in range(0, len(X_train), BATCH_SIZE):
            end = offset + BATCH_SIZE
            batch_x, batch_y = X_train[offset:end], y_train[offset:end]
            sess.run(training_operation, feed_dict={x: batch_x, y: batch_y, dropout_rate: DROPOUT_RATE})

        training_accuracy, training_loss = evaluate(X_train, y_train)
        validation_accuracy, validation_loss = evaluate(X_valid, y_valid)

        # store training/validation accuracies and losses for each epoch for later visualization
        training_accs.append(training_accuracy)
        validation_accs.append(validation_accuracy)
        training_losses.append(training_loss)
        validation_losses.append(validation_loss)

        # print progress
        update_progress((i+1) / EPOCHS, "({}: {:.3f}/{:.3f})".format((i+1), training_accuracy, validation_accuracy))

    # print final accuracies
    print()
    print()
    print("EPOCH {}:".format(EPOCHS))
    print("Training Accuracy = {:.3f}".format(training_accuracy))
    print("Validation Accuracy = {:.3f}".format(validation_accuracy))
    print()

    # save the model
    saver.save(sess, DATA_DIR+'/lenet')
    print("Model saved.")
    Training model ...

Progress: [####################] 100.0%   (200: 0.999/0.979)

EPOCH 200:
Training Accuracy = 0.999
Validation Accuracy = 0.979

Model saved.
# Plot training/validation accuracies and losses vs. epoch evolution to check for overfitting
if validation_accs:

    fig, ax = plt.subplots(nrows=2, ncols=1, figsize=(12,7), sharex=True, squeeze=True)

    ax[0].set_ylabel('Accuracy')
    ax[0].plot(range(1, len(training_accs)+1), training_accs, '.-b', label='Training')
    ax[0].plot(range(1, len(validation_accs)+1), validation_accs, '.-r', label='Validation')
    ax[0].set_ylim([0.95, 1.0])
    ax[0].legend(loc='lower right')
    ax[0].grid()
#     ax[0].set_xlabel('Epoch')

    ax[1].set_ylabel('Loss')
    ax[1].plot(range(1, len(training_losses)+1), training_losses, '.-b', label='Training')
    ax[1].plot(range(1, len(validation_losses)+1), validation_losses, '.-r', label='Validation')
    ax[1].set_ylim([0.0, 0.2])
    ax[1].legend(loc='upper right')
    ax[1].grid()
    ax[1].set_xlabel('Epoch')_____no_output_____

### Calculate Test Accuracy_____no_output_____

# Restore session
with tf.Session() as sess:
    saver.restore(sess, tf.train.latest_checkpoint(DATA_DIR))

    # Calculate accuracy for the test data
    test_accuracy, _ = evaluate(X_test, y_test)
    print("Test Accuracy = {:.3f}".format(test_accuracy))

    # Get the predictions for the test data
    p_test = predict(X_test)

    # Construct a confusion matrix
    cfm = tf.math.confusion_matrix(y_test, p_test).eval(session=sess)
    np.set_printoptions(threshold=np.inf)

    # Print parts of the confusion matrix
    print()
    print("Confusion matrix [0:15,0:15]")
    print(cfm[:15,:15])
    print()
    print("Confusion matrix [15:30,15:30]")
    print(cfm[15:30,15:30])
    print()
    print("Confusion matrix [30:43,30:43]")
    print(cfm[30:,30:])
    Test Accuracy = 0.970

Confusion matrix [0:15,0:15]
[[ 60   0   0   0   0   0   0   0   0   0   0   0   0   0   0]
 [  1 710   3   0   2   2   0   0   0   0   0   0   1   0   0]
 [  0   2 747   1   0   0   0   0   0   0   0   0   0   0   0]
 [  0   0   0 431   0  15   0   0   1   0   2   0   0   0   0]
 [  0   1   0   0 647   1   1   4   1   0   2   0   1   0   0]
 [  0   1   8   2   0 617   0   0   0   0   2   0   0   0   0]
 [  0   0   0   2   0  16 130   0   0   0   0   0   0   0   0]
 [  0   0   0   1   0   4   0 415  29   0   0   0   0   0   0]
 [  0   1   0   4   0   3   0   0 440   0   0   0   0   0   0]
 [  0   0   0   0   0   0   0   0   0 479   1   0   0   0   0]
 [  0   1   0   0   0   3   0   0   1   1 653   0   0   0   0]
 [  0   0   0   0   0   0   0   0   0   0   0 403   1   0   0]
 [  0   0   0   0   0   0   0   0   0   0   0   0 680   0   0]
 [  0   0   0   0   0   0   0   0   0   0   0   0   0 717   0]
 [  0   0   0   0   0   0   0   0   0   0   0   0   0   0 269]]

Confusion matrix [15:30,15:30]
[[210   0   0   0   0   0   0   0   0   0   0   0   0   0   0]
 [  0 149   0   0   0   1   0   0   0   0   0   0   0   0   0]
 [  0   0 355   0   0   0   0   0   0   0   0   0   0   0   0]
 [  0   0   0 340   1   0   4   0   0   0   0  17  25   1   1]
 [  0   0   0   0  60   0   0   0   0   0   0   0   0   0   0]
 [  0   0   0   1   0  89   0   0   0   0   0   0   0   0   0]
 [  0   0   0   0   0   0  78   0   0   0   0   0   0   0   0]
 [  0   0   0   0   0   0   0 117   0   0   0   1   0   0   1]
 [  0   0   0   0   0   0   0   0 150   0   0   0   0   0   0]
 [  0   0   0   0   0   0   0   0   0  88   0   0   0   0   2]
 [  0   0   0   0   0   0   0   0   0   0 455   0   8   0   0]
 [  0   0   0   2   0   0   0   3   0   0  11 162   0   0   0]
 [  0   0   0   2   0   1   0   0   0   0  15   6  30   0   0]
 [  0   0   0   0   0   0   0   0   0   0   0   0   0 149   0]
 [  0   0   0   0   0   0   0   0   0   0   0   0   0   0  90]]

Confusion matrix [30:43,30:43]
[[132   1   0   0   0   0   0   0   0   0   0   0   0]
 [  0 269   0   0   0   0   0   0   0   0   0   0   0]
 [  0   0  60   0   0   0   0   0   0   0   0   0   0]
 [  0   0   0 209   0   0   0   0   0   0   0   0   0]
 [  0   0   0   0 119   0   0   0   0   0   0   0   0]
 [  0   0   0   0   0 376   0   0   0   0   0   0   0]
 [  0   0   0   0   1   0 114   0   3   0   0   0   0]
 [  0   0   0   0   0   1   0  59   0   0   0   0   0]
 [  0   0   0   0   0   0   0   0 681   0   0   0   0]
 [  0   0   0   0   0   0   0   1   0  88   0   0   0]
 [  0   0   0   0   0   0   0   1   0   0  88   0   0]
 [  0   0   0   0   0   0   0   0   0   0   0  54   0]
 [  0   0   0   0   0   0   0   0   0   0   0   2  88]]

---
## Step 3: Test a Model on New Images

### Load and Output the Images_____no_output_____

# Directory for selected German traffic sign web images
IMAGE_DIR = 'images'

# Filenames of those images
filenames = [
#     'sign_20.jpg',
#     'sign_30.jpg',
    'sign_100.jpg',
    'sign_stop.jpg',
    'sign_noentry.jpg',
    'sign_children.jpg',
#     'sign_straight_turn_left.jpg',
    'sign_roundabout.jpg',
]

# Labels
y_new = np.array([
#      0,
#      1,
     7,
    14,
    17,
    28,
#     37,
    40
])

# Region of interest for the above images
roi_new = np.array([
#     (120, 125, 470, 477),
#     ( 41,  28, 154, 137),
    ( 23,   8, 332, 304),
    (266,   7, 882, 613),
    (116,  48, 507, 467),
    ( 49,   9, 389, 302),
#     ( 33,  22, 168, 149),
    ( 68,  71, 236, 239)
])

# Load images and construct feature array
X_new = []
for fn in filenames:
    img = ski.io.imread(IMAGE_DIR+'/'+fn, as_gray=False)
    img = ski.util.img_as_ubyte(img)
    X_new.append(img)

# Summarize the new dataset
summarize_dataset(test=(X_new, y_new, roi_new), plot_distribution=False);
Number of training examples = 0
Number of validation examples = 0
Number of testing examples = 5
Image data shape = (335, 384, 3)
Number of classes = 5

### Pre-Process the New Images_____no_output_____

# Preprocess the new images
X_new = preprocess_images(X_new, roi_new)

# Determine input depth
input_depth = color_depth(X_new)
Processing image dataset ...
Progress: [####----------------] 20.0%
Progress: [####################] 100.0%

### Predict the Sign Type for Each Image_____no_output_____

### Classify the new images
with tf.Session() as sess:
    saver.restore(sess, tf.train.latest_checkpoint(DATA_DIR))

    p_new = predict(X_new)
    print("New Data Predictions:", p_new)New Data Predictions: [ 8 14 37 28 40]

### Analyze Performance_____no_output_____

### Calculate the accuracy for these 5 new images.
with tf.Session() as sess:
    saver.restore(sess, tf.train.latest_checkpoint(DATA_DIR))

    new_accuracy, _ = evaluate(X_new, y_new)
    print("New Data Accuracy = {:.3f}".format(new_accuracy))
New Data Accuracy = 0.600

### Output Top 5 Softmax Probabilities For Each Image Found on the Web_____no_output_____

# Operation to retrieve the top 5 softmax probabilities
topk_operation = tf.nn.top_k(tf.nn.softmax(logits), k=5)

with tf.Session() as sess:
    saver.restore(sess, tf.train.latest_checkpoint(DATA_DIR))
    top5_softmax = sess.run(topk_operation, feed_dict={x: X_new, y: y_new, dropout_rate: 0.0})

# Prepare subplot grid.
ncols = 5
nrows = (len(X_new) + ncols - 1) // ncols
fig, ax = plt.subplots(nrows=nrows, ncols=ncols, figsize=(15,7), squeeze=True)

# Plot each image on the grid.
for i, axi in enumerate(ax.flat):
    # i runs from 0 to (nrows*ncols-1)
    if i >= len(X_new):
        fig.delaxes(axi)
        continue

    axi.imshow(X_new[i], cmap="gray")
    axi.set_title("Image {:02d}".format(i+1))

plt.show()

# Print out the top five softmax probabilities for the predictions on the German traffic sign images found on the web.
for i in range(len(X_new)):
    top5 = dict(zip(top5_softmax.indices[i], top5_softmax.values[i]))
    print("Image #{:02d} with label {:02d} was classified as {:02d}.".format(i+1, y_new[i], top5_softmax.indices[i][0]))
    print("Top 5 Probabilities:", ", ".join("{}: {:.2%}".format(k, v) for k, v in top5.items()))
    print()
_____no_output_____

---
## Step 4: Visualize the Neural Network's State with Test Images
_____no_output_____

def outputFeatureMap(image_input, tf_activation, activation_min=-1, activation_max=-1, plt_num=1):
    """
    Plot a feature map or our CNN to visualize the weights and features the network has identified after training.

    :param image_input: the test image being fed into the network to produce the feature maps
    :param tf_activation: should be a tf variable name used during your training procedure that represents the calculated state of a specific weight layer
    :param activation_min: can be used to view the activation contrast in more detail, by default matplot sets min and max to the actual min and max values of the output
    :param activation_max: see ``activation_min``
    :param plt_num: used to plot out multiple different weight feature map sets on the same block, just extend the plt number for each new feature map entry
    :return:
    """

    # If you get an error tf_activation is not defined it may be having trouble accessing the variable from inside a function
    activation = tf_activation.eval(session=sess,feed_dict={x : image_input, dropout_rate: 0.0})
    featuremaps = activation.shape[3]
    plt.figure(plt_num, figsize=(15,15))
    for featuremap in range(featuremaps):
        plt.subplot(6,8, featuremap+1) # sets the number of feature maps to show on each row and column
        plt.title('FeatureMap ' + str(featuremap)) # displays the feature map number
        if activation_min != -1 & activation_max != -1:
            plt.imshow(activation[0,:,:, featuremap], interpolation="nearest", vmin=activation_min, vmax=activation_max, cmap="gray")
        elif activation_max != -1:
            plt.imshow(activation[0,:,:, featuremap], interpolation="nearest", vmax=activation_max, cmap="gray")
        elif activation_min !=-1:
            plt.imshow(activation[0,:,:, featuremap], interpolation="nearest", vmin=activation_min, cmap="gray")
        else:
            plt.imshow(activation[0,:,:, featuremap], interpolation="nearest", cmap="gray")

# Print feature maps for convolution layer 1 and 2 for each of the 5 downloaded images.
with tf.Session() as sess:
    saver.restore(sess, tf.train.latest_checkpoint(DATA_DIR))

    for i, img in enumerate(X_new):

        outputFeatureMap([img], conv1, plt_num=i*2+1)
        outputFeatureMap([img], conv2, plt_num=i*2+2)
_____no_output_____


# Notebook from kabdilleh1/Community-Notebooks
Path: Notebooks/How_to_work_with_cloud_storage.ipynb

# ISB-CGC Community Notebooks

```
Title:   How to work with cloud storage
Author:  David L Gibbs
Created: 2019-07-17
Purpose: Demonstrate how to move files --in and out of-- GCS.
```
***
_____no_output_____## **How to work with cloud storage.**_____no_output_____### **In this notebook, we demonstrate how to work with files stored in GCS buckets.**
_____no_output_____Documentation at:  https://cloud.google.com/storage/docs/listing-objects_____no_output_____### Let's authenticate ourselves_____no_output_____

# with gcloud, we can authenticate ourselves

!gcloud auth loginYour browser has been opened to visit:

    https://accounts.google.com/o/oauth2/auth?redirect_uri=http%3A%2F%2Flocalhost%3A8085%2F&prompt=select_account&response_type=code&client_id=32555940559.apps.googleusercontent.com&scope=https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fuserinfo.email+https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fcloud-platform+https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fappengine.admin+https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fcompute+https%3A%2F%2Fwww.googleapis.com%2Fauth%2Faccounts.reauth&access_type=offline

[13678:13697:0501/104705.579283:ERROR:browser_process_sub_thread.cc(217)] Waited 3 ms for network service
Opening in existing browser session.
WARNING: `gcloud auth login` no longer writes application default credentials.
If you need to use ADC, see:
  gcloud auth application-default --help

You are now logged in as [dgibbs@systemsbiology.org].
Your current project is [isb-cgc-02-0001].  You can change this setting by running:
  $ gcloud config set project PROJECT_ID
# and we can select our project

!gcloud config set project PROJECT_ID
ERROR: (gcloud.config.set) The project property must be set to a valid project ID, not the project name [PROJECT_ID]
To set your project, run:

  $ gcloud config set project PROJECT_ID

or to unset it, run:

  $ gcloud config unset project

### Using 'bangs'._____no_output_____Using a 'bang', or the exclamaition point (!), we can run command line commands.
This includes file operations like 'ls', 'mv', and 'cp'.
We can also use Google's tools including 'gsutil'._____no_output_____

# here we get a list of files stored *locally*
!ls -lha total 2.1M
drwxr-xr-x 3 davidgibbs davidgibbs 4.0K Apr 30 12:13  .
drwxr-xr-x 8 davidgibbs davidgibbs 4.0K Apr 30 10:07  ..
-rw-r--r-- 1 davidgibbs davidgibbs 7.9K Apr 30 10:08 'BCGSC microRNA expression.ipynb'
-rw-r--r-- 1 davidgibbs davidgibbs  19K Apr 30 11:06 'BRAF-V600 study using CCLE data.ipynb'
-rw-r--r-- 1 davidgibbs davidgibbs 174K Apr 30 10:08 'Copy Number segments.ipynb'
-rw-r--r-- 1 davidgibbs davidgibbs 111K Apr 30 10:45 'Creating TCGA cohorts -- part 1.ipynb'
-rw-r--r-- 1 davidgibbs davidgibbs  26K Apr 30 10:08 'Creating TCGA cohorts -- part 2.ipynb'
-rw-r--r-- 1 davidgibbs davidgibbs 130K Apr 30 10:08 'Creating TCGA cohorts -- part 3.ipynb'
-rw-r--r-- 1 davidgibbs davidgibbs  65K Apr 30 10:08 'DNA Methylation.ipynb'
-rw-r--r-- 1 davidgibbs davidgibbs  15K Apr 30 12:13  how_to_move_files.ipynb
drwxr-xr-x 2 davidgibbs davidgibbs 4.0K Apr 30 11:08  .ipynb_checkpoints
-rw-r--r-- 1 davidgibbs davidgibbs 362K Apr 30 10:08  isb_cgc_bam_slicing_with_pysam.ipynb
-rw-r--r-- 1 davidgibbs davidgibbs 362K Apr 30 10:08  ISB_cgc_bam_slicing_with_pysam.ipynb
-rw-r--r-- 1 davidgibbs davidgibbs 116K Apr 30 10:08  ISB_CGC_Query_of_the_Month_November_2018.ipynb
-rw-r--r-- 1 davidgibbs davidgibbs  22K Apr 30 10:08 'Protein expression.ipynb'
-rw-r--r-- 1 davidgibbs davidgibbs 2.5K Apr 30 10:08  README.md
-rw-r--r-- 1 davidgibbs davidgibbs 382K Apr 30 10:08  RegulomeExplorer_1_Gexpr_CNV.ipynb
-rw-r--r-- 1 davidgibbs davidgibbs  28K Apr 30 10:46  renamed_test.bam
-rw-r--r-- 1 davidgibbs davidgibbs 106K Apr 30 10:08 'Somatic Mutations.ipynb'
-rw-r--r-- 1 davidgibbs davidgibbs  40K Apr 30 10:08 'TCGA Annotations.ipynb'
-rw-r--r-- 1 davidgibbs davidgibbs  15K Apr 30 10:08 'The ISB-CGC open-access TCGA tables in BigQuery.ipynb'
-rw-r--r-- 1 davidgibbs davidgibbs  51K Apr 30 10:08 'UNC HiSeq mRNAseq gene expression.ipynb'
# Now we use gsutil to list files in our bucket.
!gsutil ls gs://bam_bucket_1/gs://bam_bucket_1/renamed_test.bam
gs://bam_bucket_1/test.bam
gs://bam_bucket_1/test_2.bam
gs://bam_bucket_1/test_3.bam
# then we can copy to our local env.
!gsutil cp gs://bam_bucket_1/test.bam test_dl.bamCopying gs://bam_bucket_1/test.bam...
/ [1 files][ 27.5 KiB/ 27.5 KiB]
Operation completed over 1 objects/27.5 KiB.
# and it made it?
!ls -lha | grep test-rw-r--r-- 1 davidgibbs davidgibbs  28K Apr 30 10:46 renamed_test.bam
-rw-r--r-- 1 davidgibbs davidgibbs  28K May  1 10:47 test_dl.bam
# then we can copy it back to our bucket
!mv test_dl.bam renamed_test.bam
!gsutil cp renamed_test.bam gs://bam_bucket_1/renamed_test.bamCopying file://renamed_test.bam [Content-Type=application/octet-stream]...
/ [1 files][ 27.5 KiB/ 27.5 KiB]
Operation completed over 1 objects/27.5 KiB.

### Using 'pythons'._____no_output_____

# to install and import the library
#!pip3 install --upgrade --user google-cloud-storage_____no_output_____!gcloud auth application-default login
#Your browser has been opened to visit:
#
#    https://accounts.google.com/o/oauth2/auth?redirect_uri=.....
#
#Credentials saved to file: [/home/davidgibbs/.config/gcloud/application_default_credentials.json]
Your browser has been opened to visit:

    https://accounts.google.com/o/oauth2/auth?redirect_uri=http%3A%2F%2Flocalhost%3A8085%2F&prompt=select_account&response_type=code&client_id=764086051850-6qr4p6gpi6hn506pt8ejuq83di341hur.apps.googleusercontent.com&scope=https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fuserinfo.email+https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fcloud-platform&access_type=offline

[14422:14441:0501/104735.895643:ERROR:browser_process_sub_thread.cc(217)] Waited 8 ms for network service
Opening in existing browser session.

Credentials saved to file: [/home/davidgibbs/.config/gcloud/application_default_credentials.json]

These credentials will be used by any library that requests
Application Default Credentials.

To generate an access token for other uses, run:
  gcloud auth application-default print-access-token

_____no_output_____!export GOOGLE_APPLICATION_CREDENTIALS="/home/davidgibbs/.config/gcloud/application_default_credentials.json"
import google.auth
import google.cloud.storage as storage
credentials, project = google.auth.default()/home/davidgibbs/.local/lib/python3.6/site-packages/google/auth/_default.py:66: UserWarning: Your application has authenticated using end user credentials from Google Cloud SDK. We recommend that most server applications use service accounts instead. If your application continues to use end user credentials from Cloud SDK, you might receive a "quota exceeded" or "API not enabled" error. For more information about service accounts, see https://cloud.google.com/docs/authentication/
  warnings.warn(_CLOUD_SDK_CREDENTIALS_WARNING)
# connection to our project
storage_client = storage.Client()/home/davidgibbs/.local/lib/python3.6/site-packages/google/auth/_default.py:66: UserWarning: Your application has authenticated using end user credentials from Google Cloud SDK. We recommend that most server applications use service accounts instead. If your application continues to use end user credentials from Cloud SDK, you might receive a "quota exceeded" or "API not enabled" error. For more information about service accounts, see https://cloud.google.com/docs/authentication/
  warnings.warn(_CLOUD_SDK_CREDENTIALS_WARNING)
/home/davidgibbs/.local/lib/python3.6/site-packages/google/auth/_default.py:66: UserWarning: Your application has authenticated using end user credentials from Google Cloud SDK. We recommend that most server applications use service accounts instead. If your application continues to use end user credentials from Cloud SDK, you might receive a "quota exceeded" or "API not enabled" error. For more information about service accounts, see https://cloud.google.com/docs/authentication/
  warnings.warn(_CLOUD_SDK_CREDENTIALS_WARNING)

for b in storage_client.list_buckets():
    print(b)<Bucket: artifacts.isb-cgc-02-0001.appspot.com>
<Bucket: bam_bucket_1>
<Bucket: dataproc-6b064c10-086c-44db-b3b5-f14e410e0c13-us>
<Bucket: dave_scratch_cosmic_v86>
<Bucket: daves-cromwell-bucket>
<Bucket: gibbs_bucket_nov162016>
<Bucket: isb-cgc-02-0001>
<Bucket: isb-cgc-02-0001-datalab>
<Bucket: isb-cgc-02-0001-scratch>
<Bucket: isb-cgc-02-0001-workflows>
<Bucket: isb_dataproc_oct28>
<Bucket: may_2018_qotm>
<Bucket: pancan_staging>
<Bucket: public_bucket_for_data_file_lists>
<Bucket: qotm_nov>
<Bucket: qotm_oct_2018>
<Bucket: qotm_oct_20182018-10-29-23-47-53>
<Bucket: qotm_oct_20182018-10-31-23-34-51>
<Bucket: smr-workspace-mlengine>
<Bucket: test_bucket_888>
<Bucket: us.artifacts.isb-cgc-02-0001.appspot.com>
<Bucket: vm-config.isb-cgc-02-0001.appspot.com>
<Bucket: vm-containers.isb-cgc-02-0001.appspot.com>
<Bucket: wild_new_bucket>
# here we'll create a bucket
bucket = storage_client.create_bucket('wild_new_bucket_2000')
print('Bucket {} created'.format(bucket.name))Bucket wild_new_bucket_2000 created
# and then move a file to it
bucket = storage_client.get_bucket('wild_new_bucket_2000')
blob = bucket.blob('test_dl_upload.bam')
blob.upload_from_filename('renamed_test.bam')_____no_output_____# and check that it made it
blobs = bucket.list_blobs()
for blob in blobs:
    print(blob.name)test_dl_upload.bam
# end of notebook_____no_output_____