# Notebook from lgcharpe/Masters
Path: notebooks/Thesis Notebooks/Chapter 5/Case_study.ipynb

from hfunc import preprocessing
import tensorflow as tf
import pathlib
import os
import copy
import numpy as np
os.environ['TF_FORCE_GPU_ALLOW_GROWTH'] = 'true'_____no_output_____import tensorflow.keras as keras
from keras.backend import clear_session
import gc

# Reset Keras Session
def reset_keras():
    clear_session()
    print(gc.collect())
    print(gc.collect())Using TensorFlow backend.
import glob

def remove_logs(log_dir):
    files = glob.glob(log_dir)
    for f in files:
        os.remove(f)_____no_output_____DIR_PATH = pathlib.Path(
    'C:\\Users\\lucas\\Documents\\Masters\\data\\kvasir-dataset-v2'
)

ds = preprocessing.load_dataset_images(DIR_PATH, 224, 224)

for image, label in ds.take(8):
    print("Image shape:", image.numpy().shape)
    print('Label:', label.numpy())

del image, label

train_ds, val_ds, test_ds = preprocessing.train_val_test_split(
    ds,
    0.7,
    0.15
)

cachefile = "C:\\Users\\lucas\\Documents\\Masters\\cache\\kvasir"

train_ds = preprocessing.prepare_for_model_use(
    train_ds,
    cache=cachefile+'_train',
    prefetch=True
)
val_ds = preprocessing.prepare_for_model_use(
    val_ds,
    cache=cachefile+'_val',
    shuffle=False,
    repeat=False
)
test_ds = preprocessing.prepare_for_model_use(
    test_ds,
    cache=cachefile+'_test',
    shuffle=False,
    repeat=False
)Image shape: (224, 224, 3)
Label: [ True False False False False False False False]
Image shape: (224, 224, 3)
Label: [False  True False False False False False False]
Image shape: (224, 224, 3)
Label: [False False  True False False False False False]
Image shape: (224, 224, 3)
Label: [False False False  True False False False False]
Image shape: (224, 224, 3)
Label: [False False False False  True False False False]
Image shape: (224, 224, 3)
Label: [False False False False False  True False False]
Image shape: (224, 224, 3)
Label: [False False False False False False  True False]
Image shape: (224, 224, 3)
Label: [False False False False False False False  True]
ind = 14_____no_output_____

ind += 1
np.random.seed(2020)
vgg16 = tf.keras.applications.VGG16(weights=None, classes=8)_____no_output_____vgg16.summary()Model: "vgg16"
_________________________________________________________________
Layer (type)                 Output Shape              Param #
=================================================================
input_1 (InputLayer)         [(None, 224, 224, 3)]     0
_________________________________________________________________
block1_conv1 (Conv2D)        (None, 224, 224, 64)      1792
_________________________________________________________________
block1_conv2 (Conv2D)        (None, 224, 224, 64)      36928
_________________________________________________________________
block1_pool (MaxPooling2D)   (None, 112, 112, 64)      0
_________________________________________________________________
block2_conv1 (Conv2D)        (None, 112, 112, 128)     73856
_________________________________________________________________
block2_conv2 (Conv2D)        (None, 112, 112, 128)     147584
_________________________________________________________________
block2_pool (MaxPooling2D)   (None, 56, 56, 128)       0
_________________________________________________________________
block3_conv1 (Conv2D)        (None, 56, 56, 256)       295168
_________________________________________________________________
block3_conv2 (Conv2D)        (None, 56, 56, 256)       590080
_________________________________________________________________
block3_conv3 (Conv2D)        (None, 56, 56, 256)       590080
_________________________________________________________________
block3_pool (MaxPooling2D)   (None, 28, 28, 256)       0
_________________________________________________________________
block4_conv1 (Conv2D)        (None, 28, 28, 512)       1180160
_________________________________________________________________
block4_conv2 (Conv2D)        (None, 28, 28, 512)       2359808
_________________________________________________________________
block4_conv3 (Conv2D)        (None, 28, 28, 512)       2359808
_________________________________________________________________
block4_pool (MaxPooling2D)   (None, 14, 14, 512)       0
_________________________________________________________________
block5_conv1 (Conv2D)        (None, 14, 14, 512)       2359808
_________________________________________________________________
block5_conv2 (Conv2D)        (None, 14, 14, 512)       2359808
_________________________________________________________________
block5_conv3 (Conv2D)        (None, 14, 14, 512)       2359808
_________________________________________________________________
block5_pool (MaxPooling2D)   (None, 7, 7, 512)         0
_________________________________________________________________
flatten (Flatten)            (None, 25088)             0
_________________________________________________________________
fc1 (Dense)                  (None, 4096)              102764544
_________________________________________________________________
fc2 (Dense)                  (None, 4096)              16781312
_________________________________________________________________
predictions (Dense)          (None, 8)                 32776
=================================================================
Total params: 134,293,320
Trainable params: 134,293,320
Non-trainable params: 0
_________________________________________________________________
adam = tf.keras.optimizers.Adam(learning_rate=0.01, beta_1=0.8, beta_2=0.9)
sgd = tf.keras.optimizers.SGD(learning_rate=0.001, momentum=0.5, nesterov=True)
vgg16.compile(optimizer=sgd, loss='categorical_crossentropy', metrics=['accuracy'])_____no_output_____earl_stop = tf.keras.callbacks.EarlyStopping(patience=7, restore_best_weights=True)
tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=f"./logs/model_{ind}")_____no_output_____reset_keras()
if not os.path.isfile('./weights/my_weights.index'):
    hist = vgg16.fit(train_ds, epochs=200, validation_data=val_ds, steps_per_epoch=175, validation_steps=35, callbacks=[tensorboard_callback, earl_stop], use_multiprocessing=True)6462
0
import copy
def estimate_node_importance(model, tester_model, layer_sizes, tol_low, tol_high, data, folder_path=None):

    l, a = model.evaluate(data, verbose=0, steps=38)
    or_weights = model.get_weights()
    weight_len = len(or_weights) - 3
    amounts = []
    places = []

    if folder_path:
        # Retriving Zero Nodes
        try:
            r = open(folder_path+'/zero.txt', 'r')
            temp_z = [list(map(int,line.split())) for line in r.readlines()]
            r.close()
        except:
            print('No Zero file')
            temp_z = []
        # Retriving Important Nodes
        try:
            r = open(folder_path+'/important.txt', 'r')
            temp_imp = [list(map(int,line.split())) for line in r.readlines()]
        except:
            print("No Important File")
            temp_imp = []
        # Retriving Worse Nodes
        try:
            r = open(folder_path+'/worse.txt', 'r+')
            temp_wr = [list(map(int,line.split())) for line in r.readlines()]
            r.close()
        except:
            print("No Worse File")
            temp_wr = []

    for layer, size in enumerate(layer_sizes):

        if folder_path and layer < len(temp_z):
            z = temp_z[layer]
            wr = temp_wr[layer]
            imp = temp_imp[layer]
            num_zeros = len(z)
            num_worse = len(wr)
            num_important = len(imp)
            done = (num_zeros + num_worse + num_important)
        else:
            num_zeros, num_worse, num_important = (0, 0, 0)
            z = []
            wr = []
            imp = []
            done = 0

        print(f'Layer {len(layer_sizes)-layer}')
        for i in range(size):
            if i == 0:
                if done == size:
                    break
                else:
                    i += done
            w = copy.deepcopy(or_weights)
            w[weight_len - (2*layer+1)][...,i] = 0
            w[weight_len - 2*layer][i] = 0
            tester_model.set_weights(w)
            nl, na = tester_model.evaluate(data, verbose=0, steps=38)
            change = l - nl
            print(f'Node {i}: {change}')
            if change <= tol_high and change >= tol_low:
                num_zeros += 1
                z += [i]
                if folder_path:
                    f = open(folder_path+'/zero.txt', 'a+')
                    f.write(str(i)+" ")
                    f.close()
            elif change > 0:
                num_worse += 1
                wr += [i]
                if folder_path:
                    f = open(folder_path+'/worse.txt', 'a+')
                    f.write(str(i)+" ")
                    f.close()
            else:
                num_important += 1
                imp += [i]
                if folder_path:
                    f = open(folder_path+'/important.txt', 'a+')
                    f.write(str(i)+" ")
                    f.close()
        if folder_path:
            f = open(folder_path+'/zero.txt', 'a+')
            f.write("\n")
            f.close()
            f = open(folder_path+'/worse.txt', 'a+')
            f.write("\n")
            f.close()
            f = open(folder_path+'/important.txt', 'a+')
            f.write("\n")
            f.close()
        amounts.append((num_zeros, num_worse, num_important))
        places.append((z, wr, imp))

    return amounts, places_____no_output_____if not os.path.isfile('./weights/my_weights.index'):
    vgg16.save_weights('./weights/my_weights')
else:
    vgg16.load_weights('./weights/my_weights')_____no_output_____vgg16.evaluate(test_ds, steps=38)38/38 [==============================] - 4s 112ms/step - loss: 0.4099 - accuracy: 0.87250s - loss: 0.4246 - ac
layer_sizes = [4096, 4096, 512, 512, 512, 512, 512, 512, 256, 256, 256, 128, 128, 64, 64]
vgg16_test = tf.keras.applications.VGG16(weights=None, classes=8)
vgg16_test.compile(optimizer=sgd, loss='categorical_crossentropy', metrics=['accuracy'])
th = 1e-5
tl = -1e-5_____no_output_____th = 1e-5
tl = -1e-5
reset_keras()
amounts, places = estimate_node_importance(vgg16, vgg16_test, layer_sizes, tl, th, val_ds, 'Estimate')41
0
Layer 15
Layer 14
Layer 13
Layer 12
Layer 11
Layer 10
Layer 9
Layer 8
Layer 7
Layer 6
Layer 5
Layer 4
Layer 3
Layer 2
Layer 1
for i, (nz, nw, ni) in enumerate(reversed(amounts)):
    print(f'######### LAYER {i+1} #########')
    print("Zero Nodes:", nz)
    print("Worse Nodes:", nw)
    print("Important Nodes:", ni)######### LAYER 1 #########
Zero Nodes: 14
Worse Nodes: 16
Important Nodes: 34
######### LAYER 2 #########
Zero Nodes: 17
Worse Nodes: 22
Important Nodes: 25
######### LAYER 3 #########
Zero Nodes: 30
Worse Nodes: 42
Important Nodes: 56
######### LAYER 4 #########
Zero Nodes: 34
Worse Nodes: 47
Important Nodes: 47
######### LAYER 5 #########
Zero Nodes: 88
Worse Nodes: 93
Important Nodes: 75
######### LAYER 6 #########
Zero Nodes: 73
Worse Nodes: 98
Important Nodes: 85
######### LAYER 7 #########
Zero Nodes: 79
Worse Nodes: 92
Important Nodes: 85
######### LAYER 8 #########
Zero Nodes: 139
Worse Nodes: 175
Important Nodes: 198
######### LAYER 9 #########
Zero Nodes: 137
Worse Nodes: 193
Important Nodes: 182
######### LAYER 10 #########
Zero Nodes: 84
Worse Nodes: 206
Important Nodes: 222
######### LAYER 11 #########
Zero Nodes: 52
Worse Nodes: 189
Important Nodes: 271
######### LAYER 12 #########
Zero Nodes: 28
Worse Nodes: 202
Important Nodes: 282
######### LAYER 13 #########
Zero Nodes: 11
Worse Nodes: 202
Important Nodes: 299
######### LAYER 14 #########
Zero Nodes: 287
Worse Nodes: 1833
Important Nodes: 1976
######### LAYER 15 #########
Zero Nodes: 207
Worse Nodes: 1887
Important Nodes: 2002
def node_pruning(model, tester_model, data, layer_sizes, tol, ignore_cutoff=0, method='exhaustive', folder_path=None):

    if isinstance(data, tuple):
        (x, y) = data
    try:
        loss, acc = model.evaluate(data, verbose=0, steps=38)
    except:
        loss, acc = model.evaluate(x, y,verbose=0, steps=38)
    original = model.get_weights()
    weight_len = len(original) - 3
    bas = [acc]
    bls = [loss]
    best_weights = model.get_weights()
    best_acc = 0
    best_loss = 1e20
    ol = loss
    oa = acc
    amounts = []
    places = []

    if folder_path:
        # Retriving Nodes removed
        try:
            r = open(folder_path+'/removed.txt', 'r')
            temp_remove = [list(map(int,line.split())) for line in r.readlines()]
            r.close()
        except:
            print('No node removal file')
            temp_remove = []
        # Retriving loss
        try:
            r = open(folder_path+'/loss.txt', 'r')
            bls = list(map(np.double,r.readline().split()))
            ol = bls[-1]
            r.close()
        except:
            print("No Loss File")
            f_loss = open(folder_path+'/loss.txt', "a+")
            bls = [loss]
            f_loss.write(str(loss)+' ')
            f_loss.close()
        # Retriving accuracy
        try:
            r = open(folder_path+'/accuracy.txt', 'r+')
            bas = r.readline().split()
            bas = list(map(np.double,bas))
            oa = bas[-1]
            r.close()
        except:
            print("No Accuracy File")
            f_acc = open(folder_path+'/accuracy.txt', "a+")
            bas = [acc]
            f_acc.write(str(acc)+' ')
            f_acc.close()
        # Retriving best weights
        try:
            tester_model.load_weights(folder_path+'/weights')
            best_weights = tester_model.get_weights()
        except:
            print('No weights found')

    for layer, size in enumerate(layer_sizes):
        if folder_path and layer < len(temp_remove):
            nodes_removed = temp_remove[layer]
            num_removed = len(nodes_removed)
            if layer + 1 < len(temp_remove):
                end_not_reached = False
            else:
                end_not_reached = True
        else:
            num_removed = 0
            nodes_removed = []
            end_not_reached = True
        print(f'Starting removal in Layer {len(layer_sizes) - layer}')
        if method == 'exhaustive':
            current_pos = 0
            best_change = tol
            best_pos = -1
            improved = False
            while end_not_reached or improved:
                if not(end_not_reached):
                    end_not_reached = True
                    improved = False
                    current_pos = 0
                    size -= 1
                    nodes_removed += [best_pos]
                    best_weights[weight_len - (2*layer+1)][...,best_pos] = 0
                    best_weights[weight_len - 2*layer][best_pos] = 0
                    best_pos = -1
                    ol = best_loss
                    oa = best_acc
                    bas += [best_acc]
                    bls += [best_loss]
                    best_change = tol
                    num_removed += 1
                if current_pos in nodes_removed:
                    current_pos += 1
                    if current_pos - num_removed >= size:
                        end_not_reached = False
                    continue
                w = copy.deepcopy(best_weights)
                w[weight_len - (2*layer+1)][...,current_pos] = 0
                w[weight_len - 2*layer][current_pos] = 0
                tester_model.set_weights(w)
                try:
                    nl, na = tester_model.evaluate(data, verbose=0, steps=38)
                except:
                    nl, na = tester_model.evaluate(x, y, verbose=0, steps=38)
                if ol - nl >= best_change:
                    best_change = ol - nl
                    best_pos = current_pos
                    improved = True
                    best_acc = na
                    best_loss = nl
                current_pos += 1
                if current_pos - num_removed >= size:
                    end_not_reached = False
        elif method == 'greedy':
            nodes_to_estimate = list(np.arange(size))
            if folder_path and layer < len(temp_remove):
                for i in nodes_removed:
                    nodes_to_estimate.remove(i)
                size -= num_removed
            current_pos = nodes_to_estimate[0]
            idx = 0
            while end_not_reached:
                w = copy.deepcopy(best_weights)
                w[weight_len - (2*layer+1)][...,current_pos] = 0
                w[weight_len - 2*layer][current_pos] = 0
                tester_model.set_weights(w)
                nl, na = tester_model.evaluate(data, verbose=0, steps=38, workers=12, use_multiprocessing=True)
                print(f'Node {current_pos}: {ol - nl}')
                if ol - nl >= tol:
                    oa = na
                    ol = nl
                    size -= 1
                    nodes_removed += [current_pos]
                    nodes_to_estimate.remove(current_pos)
                    best_weights[weight_len - (2*layer+1)][..., current_pos] = 0
                    best_weights[weight_len - 2*layer][current_pos] = 0
                    bas += [oa]
                    bls += [ol]
                    num_removed += 1
                    idx = 0
                    if folder_path:
                        f_rem = open(folder_path+'/removed.txt', "a+")
                        f_loss = open(folder_path+'/loss.txt', "a+")
                        f_acc = open(folder_path+'/accuracy.txt', "a+")

                        f_rem.write(str(current_pos) + ' ')
                        f_loss.write(str(ol) + ' ')
                        f_acc.write(str(oa) + ' ')

                        f_rem.close()
                        f_loss.close()
                        f_acc.close()

                        tester_model.save_weights(folder_path+'/weights')
                elif ol - nl <= ignore_cutoff:
                    size -= 1
                    nodes_to_estimate.remove(current_pos)
                else:
                    idx += 1
                if idx >= size:
                    end_not_reached = False
                    if folder_path:
                        f_rem = open(folder_path+'/removed.txt', "a+")
                        f_rem.write('\n')
                        f_rem.close()
                else:
                    current_pos = nodes_to_estimate[idx]
        elif method == 'greedy_fast':
            improved = False
            nodes_to_estimate = list(np.arange(size))
            if folder_path and layer < len(temp_remove):
                for i in nodes_removed:
                    nodes_to_estimate.remove(i)
                size -= num_removed
            current_pos = nodes_to_estimate[0]
            idx = 0
            while end_not_reached:
                w = copy.deepcopy(best_weights)
                w[weight_len - (2*layer+1)][...,current_pos] = 0
                w[weight_len - 2*layer][current_pos] = 0
                tester_model.set_weights(w)
                nl, na = tester_model.evaluate(data, verbose=0, steps=38, workers=12, use_multiprocessing=True)
                print(f'Node {current_pos}: {ol - nl}')
                if ol - nl >= tol:
                    oa = na
                    ol = nl
                    size -= 1
                    nodes_removed += [current_pos]
                    nodes_to_estimate.remove(current_pos)
                    best_weights[weight_len - (2*layer+1)][..., current_pos] = 0
                    best_weights[weight_len - 2*layer][current_pos] = 0
                    bas += [oa]
                    bls += [ol]
                    num_removed += 1
                    if folder_path:
                        f_rem = open(folder_path+'/removed.txt', "a+")
                        f_loss = open(folder_path+'/loss.txt', "a+")
                        f_acc = open(folder_path+'/accuracy.txt', "a+")

                        f_rem.write(str(current_pos) + ' ')
                        f_loss.write(str(ol) + ' ')
                        f_acc.write(str(oa) + ' ')

                        f_rem.close()
                        f_loss.close()
                        f_acc.close()

                        tester_model.save_weights(folder_path+'/weights')
                elif ol - nl <= ignore_cutoff:
                    size -= 1
                    nodes_to_estimate.remove(current_pos)
                else:
                    idx += 1
                if idx >= size:
                    if improved:
                        idx = 0
                        current_pos = nodes_to_estimate[idx]
                    else:
                        end_not_reached = False
                        if folder_path:
                            f_rem = open(folder_path+'/removed.txt', "a+")
                            f_rem.write('\n')
                            f_rem.close()
                else:
                    current_pos = nodes_to_estimate[idx]
        print(f'Removed {num_removed} nodes for Layer {len(layer_sizes) - layer}')
        amounts.append(num_removed)
        places.append(nodes_removed)

    #if folder_path:
        #f_rem.close()
        #f_loss.close()
        #f_acc.close()

    return best_weights, bas, bls, amounts, places_____no_output_____reset_keras()
best_weights, bas, bls, amounts, places = node_pruning(vgg16, vgg16_test, val_ds, layer_sizes, tl, -5e-5, method='greedy', folder_path='.\Pruning')124
0
Starting removal in Layer 15
Removed 2334 nodes for Layer 15
Starting removal in Layer 14
Removed 1337 nodes for Layer 14
Starting removal in Layer 13
Removed 33 nodes for Layer 13
Starting removal in Layer 12
Removed 88 nodes for Layer 12
Starting removal in Layer 11
Removed 109 nodes for Layer 11
Starting removal in Layer 10
Removed 143 nodes for Layer 10
Starting removal in Layer 9
Removed 186 nodes for Layer 9
Starting removal in Layer 8
Removed 170 nodes for Layer 8
Starting removal in Layer 7
Removed 84 nodes for Layer 7
Starting removal in Layer 6
Removed 94 nodes for Layer 6
Starting removal in Layer 5
Removed 96 nodes for Layer 5
Starting removal in Layer 4
Removed 47 nodes for Layer 4
Starting removal in Layer 3
Removed 46 nodes for Layer 3
Starting removal in Layer 2
Removed 25 nodes for Layer 2
Starting removal in Layer 1
Removed 21 nodes for Layer 1
vgg16_test.load_weights('./Pruning/weights')
vgg16_test.evaluate(test_ds, steps=38)38/38 [==============================] - 3s 86ms/step - loss: 0.4103 - accuracy: 0.8467
vgg16.evaluate(test_ds, steps=38)38/38 [==============================] - 3s 86ms/step - loss: 0.4099 - accuracy: 0.8725
vgg16_test.evaluate(val_ds, steps=38)38/38 [==============================] - 3s 86ms/step - loss: 0.1920 - accuracy: 0.9625 1s - loss:
vgg16.evaluate(val_ds, steps=38)38/38 [==============================] - 3s 86ms/step - loss: 0.3570 - accuracy: 0.8817
vgg16_test.evaluate(train_ds, steps=175)175/175 [==============================] - 16s 89ms/step - loss: 0.2818 - accuracy: 0.9034
vgg16.evaluate(train_ds, steps=175)175/175 [==============================] - 16s 90ms/step - loss: 0.1371 - accuracy: 0.9554
amounts_____no_output_____bls_val = bls
bas_val = bas_____no_output_____otel, otea = vgg16.evaluate(test_ds, steps=38)
otrl, otra = vgg16.evaluate(train_ds, steps=175)

try:
    r = open('Evolution/loss_train.txt', 'r')
    bls_train = list(map(np.double,r.readline().split()))
    num_rem_train_loss = len(bls_train) - 1
    r.close()
except:
    print("No Train Loss File")
    f_loss = open('Evolution/loss_train.txt', "a+")
    bls_train = [otrl]
    f_loss.write(str(otrl)+' ')
    f_loss.close()
    num_rem_train_loss = 0
try:
    r = open('Evolution/loss_test.txt', 'r')
    bls_test = list(map(np.double,r.readline().split()))
    num_rem_test_loss = len(bls_test) - 1
    r.close()
except:
    print("No Test Loss File")
    f_loss = open('Evolution/loss_test.txt', "a+")
    bls_test = [otel]
    f_loss.write(str(otel)+' ')
    f_loss.close()
    num_rem_test_loss = 0
# Retriving accuracy
try:
    r = open('Evolution/accuracy_train.txt', 'r+')
    bas_train = r.readline().split()
    bas_train = list(map(np.double,bas_train))
    num_rem_train_acc = len(bas_train) - 1
    r.close()
except:
    print("No Train Accuracy File")
    f_acc = open('Evolution/accuracy_train.txt', "a+")
    bas_train = [otra]
    f_acc.write(str(otra)+' ')
    f_acc.close()
    num_rem_train_acc = 0
try:
    r = open('Evolution/accuracy_test.txt', 'r+')
    bas_test = r.readline().split()
    bas_test = list(map(np.double,bas_test))
    num_rem_test_acc = len(bas_test) - 1
    r.close()
except:
    print("No Test Accuracy File")
    f_acc = open('Evolution/accuracy_test.txt', "a+")
    bas_test = [otea]
    f_acc.write(str(otea)+' ')
    f_acc.close()
    num_rem_test_acc = 0

w = vgg16.get_weights()
weight_len = len(w) - 3
test = (num_rem_test_acc + num_rem_test_loss + num_rem_train_acc + num_rem_train_loss) - 4 * sum(amounts)
if test:
    for layer, layer_place in enumerate(places):
        print(f"Starting removal in layer {len(places) - layer}")
        for i, place in enumerate(layer_place):
            if i % 100 == 0:
                print(f'Removed {i} nodes!')
            w[weight_len - (2*layer+1)][..., place] = 0
            w[weight_len - 2*layer][place] = 0
            vgg16_test.set_weights(w)

            if num_rem_test_loss:
                if num_rem_test_acc:
                    num_rem_test_acc -= 1
                    #print('Double Test')
                else:
                    _, tea = vgg16_test.evaluate(test_ds, steps=38, verbose=0)
                    bas_test += [tea]
                    f = open('Evolution/accuracy_test.txt', "a")
                    f.write(str(tea)+' ')
                    f.close()
                    print('Loss Test only')
                num_rem_test_loss -= 1
            else:
                if num_rem_test_acc:
                    num_rem_test_acc -= 1
                    tel, _ = vgg16_test.evaluate(test_ds, steps=38, verbose=0)
                    bls_test += [tel]
                    print('Accuracy Test only')
                else:
                    tel, tea = vgg16_test.evaluate(test_ds, steps=38, verbose=0)
                    bls_test += [tel]
                    bas_test += [tea]
                    f = open('Evolution/accuracy_test.txt', "a")
                    f.write(str(tea)+' ')
                    f.close()
                f = open('Evolution/loss_test.txt', "a")
                f.write(str(tel)+' ')
                f.close()

            if num_rem_train_loss:
                if num_rem_train_acc:
                    num_rem_train_acc -= 1
                    #print('Double Train')
                else:
                    _, tra = vgg16_test.evaluate(train_ds, steps=175, verbose=0)
                    bas_train += [tra]
                    f = open('Evolution/accuracy_train.txt', "a")
                    f.write(str(tra)+' ')
                    f.close()
                    print('Loss Train only')
                num_rem_train_loss -= 1
            else:
                if num_rem_train_acc:
                    num_rem_train_acc -= 1
                    trl, _ = vgg16_test.evaluate(train_ds, steps=175, verbose=0)
                    bls_train += [trl]
                    print('Accuracy Train only')
                else:
                    trl, tra = vgg16_test.evaluate(train_ds, steps=175, verbose=0)
                    bls_train += [trl]
                    bas_train += [tra]
                    f = open('Evolution/accuracy_train.txt', "a")
                    f.write(str(tra)+' ')
                    f.close()
                f = open('Evolution/loss_train.txt', "a")
                f.write(str(trl)+' ')
                f.close()38/38 [==============================] - 3s 86ms/step - loss: 0.4099 - accuracy: 0.8725
175/175 [==============================] - 16s 90ms/step - loss: 0.1371 - accuracy: 0.9554
import matplotlib.pyplot as plt
changes = []
tot_rem = 0
for rem in amounts:
    tot_rem += rem
    changes.append(tot_rem)
colors = ["black", "blue", "red", "green", "lightseagreen", "purple", "cyan", "saddlebrown", "orange", "grey", "gold", "crimson", "navy", "forestgreen", "darkslategrey"]_____no_output_____plt.rcParams["figure.figsize"] = (16,8)
plt.plot(bls_val)
plt.title("Evolution of validation loss as nodes are removed", size=28)
plt.ylabel("Loss", size=20)
plt.yticks(fontsize=20)
plt.grid(True)
num_layers = len(changes)
for i, pos in enumerate(changes[:-1]):
    plt.axvline(x=pos, color=colors[i])
plt.legend(fontsize=20)
plt.savefig("../../../figures/Case_study/ev_loss_val.png")No handles with labels found to put in legend.
plt.plot(bls_train)
plt.title("Evolution of train loss as nodes are removed", size=28)
plt.ylabel("Loss", size=20)
plt.yticks(fontsize=20)
plt.grid(True)
num_layers = len(changes)
for i, pos in enumerate(changes[:-1]):
    plt.axvline(x=pos, color=colors[i])
plt.legend(fontsize=20)
plt.savefig("../../../figures/Case_study/ev_loss_train.png")No handles with labels found to put in legend.
plt.plot(bls_test)
plt.title("Evolution of test loss as nodes are removed", size=28)
plt.ylabel("Loss", size=20)
plt.yticks(fontsize=20)
plt.grid(True)
num_layers = len(changes)
for i, pos in enumerate(changes[:-1]):
    plt.axvline(x=pos, color=colors[i])
plt.legend(fontsize=20)
plt.savefig("../../../figures/Case_study/ev_loss_test.png")No handles with labels found to put in legend.
plt.plot(bls_val, label='Validation loss')
plt.plot(bls_train, label='Train loss')
plt.plot(bls_test, label='Test loss')
plt.title("Evolution of loss as nodes are removed", size=28)
plt.ylabel("Loss", size=20)
plt.yticks(fontsize=20)
plt.grid(True)
num_layers = len(changes)
for i, pos in enumerate(changes[:-1]):
    plt.axvline(x=pos, color=colors[i])
plt.legend(fontsize=20)
plt.savefig("../../../figures/Case_study/ev_loss_all.png")_____no_output_____plt.plot(bas_val)
plt.title("Evolution of validation accuracy as nodes are removed", size=28)
plt.ylabel("Accuracy", size=20)
plt.yticks(fontsize=20)
plt.grid(True)
num_layers = len(changes)
for i, pos in enumerate(changes[:-1]):
    plt.axvline(x=pos, color=colors[i])
plt.legend(fontsize=20)
plt.savefig("../../../figures/Case_study/ev_accuracy_val.png")No handles with labels found to put in legend.
plt.plot(bas_train)
plt.title("Evolution of train accuracy as nodes are removed", size=28)
plt.ylabel("Accuracy", size=20)
plt.yticks(fontsize=20)
plt.grid(True)
num_layers = len(changes)
for i, pos in enumerate(changes[:-1]):
    plt.axvline(x=pos, color=colors[i])
plt.legend(fontsize=20)
plt.savefig("../../../figures/Case_study/ev_accuracy_train.png")No handles with labels found to put in legend.
plt.plot(bas_test)
plt.title("Evolution of test accuracy as nodes are removed", size=28)
plt.ylabel("Accuracy", size=20)
plt.yticks(fontsize=20)
plt.grid(True)
num_layers = len(changes)
for i, pos in enumerate(changes[:-1]):
    plt.axvline(x=pos, color=colors[i])
plt.legend(fontsize=20)
plt.savefig("../../../figures/Case_study/ev_accuracy_test.png")No handles with labels found to put in legend.
plt.plot(bas_val, label='Validation accuracy')
plt.plot(bas_train, label='Train accuracy')
plt.plot(bas_test, label='Test accuracy')
plt.title("Evolution of accuracy as nodes are removed", size=28)
plt.ylabel("Accuracy", size=20)
plt.yticks(fontsize=20)
plt.grid(True)
num_layers = len(changes)
for i, pos in enumerate(changes[:-1]):
    plt.axvline(x=pos, color=colors[i])
plt.legend(fontsize=20)
plt.savefig("../../../figures/Case_study/ev_accuracy_all.png")_____no_output_____vgg16_test.load_weights('./Pruning/weights')_____no_output_____labels = np.array(list(test_ds.as_numpy_iterator()))[:,1]
y_test = np.zeros(1200)
i = 0
for label in labels:
    for l in label:
        y_test[i] = int(np.where(l)[0])
        i += 1_____no_output_____labels = np.array(list(val_ds.as_numpy_iterator()))[:,1]
y_val = np.zeros(1200)
i = 0
for label in labels:
    for l in label:
        y_val[i] = int(np.where(l)[0])
        i += 1_____no_output_____train_ds2 = train_ds.unbatch().take(5600)
data = np.array(list(train_ds2.as_numpy_iterator()))
labels = data[:,1]
xt = data[:,0]
x_train = np.zeros((5600, 224, 224, 3))
y_train = np.zeros(5600)
i = 0
for label in labels:
    y_train[i] = int(np.where(label)[0])
    i += 1
i = 0
for x in xt:
    x_train[i] = x
    i += 1_____no_output_____def confusion_matrix(true, pred, K):
    cm = np.zeros((K, K))
    for i in range(len(true)):
        cm[int(true[i]), int(pred[i])] += 1
    return cm_____no_output_____y_pred = vgg16.predict(test_ds)
K = 8
acc1 = []
yp = np.argmax(y_pred, axis=1)
for i in range(K):
    a = np.mean((yp[y_test == i] == y_test[y_test == i]))
    acc1.append(a)
acc1_____no_output_____cm_test_old = confusion_matrix(y_test, yp, K)_____no_output_____y_pred = vgg16_test.predict(test_ds)
K = 8
acc1 = []
yp = np.argmax(y_pred, axis=1)
for i in range(K):
    a = np.mean((yp[y_test == i] == y_test[y_test == i]))
    acc1.append(a)
acc1_____no_output_____cm_test_new = confusion_matrix(y_test, yp, K)_____no_output_____y_pred = vgg16.predict(val_ds)
K = 8
acc1 = []
yp = np.argmax(y_pred, axis=1)
for i in range(K):
    a = np.mean((yp[y_val == i] == y_val[y_val == i]))
    acc1.append(a)
acc1_____no_output_____cm_val_old = confusion_matrix(y_val, yp, K)_____no_output_____y_pred = vgg16_test.predict(val_ds)
K = 8
acc1 = []
yp = np.argmax(y_pred, axis=1)
for i in range(K):
    a = np.mean((yp[y_val == i] == y_val[y_val == i]))
    acc1.append(a)
acc1_____no_output_____cm_val_new = confusion_matrix(y_val, yp, K)_____no_output_____y_pred = vgg16.predict(x_train)
K = 8
acc1 = []
yp = np.argmax(y_pred, axis=1)
for i in range(K):
    a = np.mean((yp[y_train == i] == y_train[y_train == i]))
    acc1.append(a)
acc1_____no_output_____cm_train_old = confusion_matrix(y_train, yp, K)_____no_output_____y_pred = vgg16_test.predict(x_train)
K = 8
acc1 = []
yp = np.argmax(y_pred, axis=1)
for i in range(K):
    a = np.mean((yp[y_train == i] == y_train[y_train == i]))
    acc1.append(a)
acc1_____no_output_____cm_train_new = confusion_matrix(y_train, yp, K)_____no_output_____cm_test_old_____no_output_____cm_test_new_____no_output_____cm_val_old_____no_output_____cm_val_new_____no_output_____cm_train_old_____no_output_____cm_train_new_____no_output_____import tqdm
from hfunc import models
import pandas as pd
from sklearn.model_selection import train_test_split_____no_output_____np.random.seed(2020)_____no_output_____mnist = tf.keras.datasets.mnist
(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_train, x_test = x_train / 255.0, x_test / 255.0  # Converting interger values to floats (0 to 1)_____no_output_____trials = 100
accs = []
losses = []
for trial in tqdm.trange(trials):
    model, r = models.train_basic_ANN(x_train, y_train, 128, (x_test, y_test), epochs=5)
    l, a = model.evaluate(x_test, y_test, verbose=0)
    accs += [a]
    losses += [l]100%|████████████████████████████████████████████████████████████████████████████████| 100/100 [11:09<00:00,  6.70s/it]
A = pd.Series(accs)
A.to_csv('../../../results/train_vs_val/mnist_acc_train_ANN.csv')
A.describe()_____no_output_____L = pd.Series(losses)
L.to_csv('../../../results/train_vs_val/mnist_loss_train_ANN.csv')
L.describe()_____no_output_____x_train, x_val, y_train, y_val = train_test_split(x_train, y_train, train_size=0.85, stratify=None)_____no_output_____trials = 100
accs = []
losses = []
for trial in tqdm.trange(trials):
    model, r = models.train_basic_ANN(x_train, y_train, 128, (x_test, y_test), epochs=5)
    l, a = model.evaluate(x_test, y_test, verbose=0)
    accs += [a]
    losses += [l]100%|████████████████████████████████████████████████████████████████████████████████| 100/100 [12:47<00:00,  7.67s/it]
A = pd.Series(accs)
A.to_csv('../../../results/train_vs_val/mnist_acc_val_ANN.csv')
A.describe()_____no_output_____L = pd.Series(losses)
L.to_csv('../../../results/train_vs_val/mnist_loss_val_ANN.csv')
L.describe()_____no_output_____np.random.seed(2020)_____no_output_____fmnist = tf.keras.datasets.fashion_mnist
(x_train, y_train), (x_test, y_test) = fmnist.load_data()
x_train, x_test = x_train / 255.0, x_test / 255.0  # Converting interger values to floats (0 to 1)_____no_output_____trials = 100
accs = []
losses = []
for trial in tqdm.trange(trials):
    model = tf.keras.models.Sequential([
        tf.keras.layers.Flatten(input_shape=(28, 28)),
        tf.keras.layers.Dense(128, activation='relu'),
        tf.keras.layers.Dense(64, activation='relu'),
        tf.keras.layers.Dense(32, activation='relu'),
        tf.keras.layers.Dense(10, activation='softmax')
    ])
    model.compile(
        optimizer='adam',
        loss='sparse_categorical_crossentropy',
        metrics=['accuracy']
    )

    model.fit(x_train, y_train, epochs=5, verbose=0)
    l, a = model.evaluate(x_test, y_test, verbose=0)
    accs += [a]
    losses += [l]100%|████████████████████████████████████████████████████████████████████████████████| 100/100 [11:03<00:00,  6.64s/it]
A = pd.Series(accs)
A.to_csv('../../../results/train_vs_val/fmnist_acc_train_MLP.csv')
A.describe()_____no_output_____L = pd.Series(losses)
L.to_csv('../../../results/train_vs_val/fmnist_loss_train_MLP.csv')
L.describe()_____no_output_____x_train, x_val, y_train, y_val = train_test_split(x_train, y_train, train_size=0.85, stratify=None)_____no_output_____trials = 100
accs = []
losses = []
for trial in tqdm.trange(trials):
    model = tf.keras.models.Sequential([
        tf.keras.layers.Flatten(input_shape=(28, 28)),
        tf.keras.layers.Dense(128, activation='relu'),
        tf.keras.layers.Dense(64, activation='relu'),
        tf.keras.layers.Dense(32, activation='relu'),
        tf.keras.layers.Dense(10, activation='softmax')
    ])
    model.compile(
        optimizer='adam',
        loss='sparse_categorical_crossentropy',
        metrics=['accuracy']
    )

    model.fit(x_train, y_train, epochs=5, verbose=0)
    l, a = model.evaluate(x_test, y_test, verbose=0)
    accs += [a]
    losses += [l]100%|████████████████████████████████████████████████████████████████████████████████| 100/100 [11:51<00:00,  7.12s/it]
A = pd.Series(accs)
A.to_csv('../../../results/train_vs_val/fmnist_acc_val_MLP.csv')
A.describe()_____no_output_____L = pd.Series(losses)
L.to_csv('../../../results/train_vs_val/fmnist_loss_val_MLP.csv')
L.describe()_____no_output_____np.random.seed(2020)_____no_output_____cifar = tf.keras.datasets.cifar10
(x_train, y_train), (x_test, y_test) = cifar.load_data()
x_train, x_test = x_train / 255.0, x_test / 255.0  # Converting interger values to floats (0 to 1)_____no_output_____trials = 100
accs = []
losses = []
for trial in tqdm.trange(trials):
    model = tf.keras.models.Sequential([
        tf.keras.layers.Conv2D(32, 3, padding='same', activation='relu', input_shape=(32, 32, 3)),
        tf.keras.layers.MaxPool2D(),
        tf.keras.layers.Conv2D(64, 3, padding='same', activation='relu'),
        tf.keras.layers.MaxPool2D(),
        tf.keras.layers.Conv2D(128, 3, padding='same', activation='relu'),
        tf.keras.layers.MaxPool2D(),
        tf.keras.layers.Conv2D(256, 3, padding='same', activation='relu'),
        tf.keras.layers.Flatten(),
        tf.keras.layers.Dense(64, activation='relu'),
        tf.keras.layers.Dense(10, activation='softmax')
    ])
    model.compile(
        optimizer='adam',
        loss='sparse_categorical_crossentropy',
        metrics=['accuracy']
    )

    model.fit(x_train, y_train, epochs=5, verbose=0)
    l, a = model.evaluate(x_test, y_test, verbose=0)
    accs += [a]
    losses += [l]100%|████████████████████████████████████████████████████████████████████████████████| 100/100 [24:24<00:00, 14.65s/it]
A = pd.Series(accs)
A.to_csv('../../../results/train_vs_val/cifar_acc_train_CNN.csv')
A.describe()_____no_output_____L = pd.Series(losses)
L.to_csv('../../../results/train_vs_val/cifar_loss_train_CNN.csv')
L.describe()_____no_output_____x_train, x_val, y_train, y_val = train_test_split(x_train, y_train, train_size=0.85, stratify=None)_____no_output_____trials = 100
accs = []
losses = []
for trial in tqdm.trange(trials):
    model = tf.keras.models.Sequential([
        tf.keras.layers.Conv2D(32, 3, padding='same', activation='relu', input_shape=(32, 32, 3)),
        tf.keras.layers.MaxPool2D(),
        tf.keras.layers.Conv2D(64, 3, padding='same', activation='relu'),
        tf.keras.layers.MaxPool2D(),
        tf.keras.layers.Conv2D(128, 3, padding='same', activation='relu'),
        tf.keras.layers.MaxPool2D(),
        tf.keras.layers.Conv2D(256, 3, padding='same', activation='relu'),
        tf.keras.layers.Flatten(),
        tf.keras.layers.Dense(64, activation='relu'),
        tf.keras.layers.Dense(10, activation='softmax')
    ])
    model.compile(
        optimizer='adam',
        loss='sparse_categorical_crossentropy',
        metrics=['accuracy']
    )

    model.fit(x_train, y_train, epochs=5, verbose=0)
    l, a = model.evaluate(x_test, y_test, verbose=0)
    accs += [a]
    losses += [l]100%|████████████████████████████████████████████████████████████████████████████████| 100/100 [20:53<00:00, 12.54s/it]
A = pd.Series(accs)
A.to_csv('../../../results/train_vs_val/cifar_acc_val_CNN.csv')
A.describe()_____no_output_____L = pd.Series(losses)
L.to_csv('../../../results/train_vs_val/cifar_loss_val_CNN.csv')
L.describe()_____no_output_____def node_pruning2(model, tester_model, x, y, layer_sizes, tol, ignore_cutoff=0, method='exhaustive', folder_path=None):

    loss, acc = model.evaluate(x, y, verbose=0)
    original = model.get_weights()
    weight_len = len(original) - 3
    bas = [acc]
    bls = [loss]
    best_weights = model.get_weights()
    best_acc = 0
    best_loss = 1e20
    ol = loss
    oa = acc
    amounts = []
    places = []

    if folder_path:
        # Retriving Nodes removed
        try:
            r = open(folder_path+'/removed.txt', 'r')
            temp_remove = [list(map(int,line.split())) for line in r.readlines()]
            r.close()
        except:
            print('No node removal file')
            temp_remove = []
        # Retriving loss
        try:
            r = open(folder_path+'/loss.txt', 'r')
            bls = list(map(np.double,r.readline().split()))
            ol = bls[-1]
            r.close()
        except:
            print("No Loss File")
            f_loss = open(folder_path+'/loss.txt', "a+")
            bls = [loss]
            f_loss.write(str(loss)+' ')
            f_loss.close()
        # Retriving accuracy
        try:
            r = open(folder_path+'/accuracy.txt', 'r+')
            bas = r.readline().split()
            bas = list(map(np.double,bas))
            oa = bas[-1]
            r.close()
        except:
            print("No Accuracy File")
            f_acc = open(folder_path+'/accuracy.txt', "a+")
            bas = [acc]
            f_acc.write(str(acc)+' ')
            f_acc.close()
        # Retriving best weights
        try:
            tester_model.load_weights(folder_path+'/weights')
            best_weights = tester_model.get_weights()
        except:
            print('No weights found')

    for layer, size in enumerate(layer_sizes):
        if folder_path and layer < len(temp_remove):
            nodes_removed = temp_remove[layer]
            num_removed = len(nodes_removed)
            if layer + 1 < len(temp_remove):
                end_not_reached = False
            else:
                end_not_reached = True
        else:
            num_removed = 0
            nodes_removed = []
            end_not_reached = True
        print(f'Starting removal in Layer {len(layer_sizes) - layer}')
        if method == 'exhaustive':
            current_pos = 0
            best_change = tol
            best_pos = -1
            improved = False
            while end_not_reached or improved:
                if not(end_not_reached):
                    end_not_reached = True
                    improved = False
                    current_pos = 0
                    size -= 1
                    nodes_removed += [best_pos]
                    best_weights[weight_len - (2*layer+1)][...,best_pos] = 0
                    best_weights[weight_len - 2*layer][best_pos] = 0
                    best_pos = -1
                    ol = best_loss
                    oa = best_acc
                    bas += [best_acc]
                    bls += [best_loss]
                    best_change = tol
                    num_removed += 1
                if current_pos in nodes_removed:
                    current_pos += 1
                    if current_pos - num_removed >= size:
                        end_not_reached = False
                    continue
                w = copy.deepcopy(best_weights)
                w[weight_len - (2*layer+1)][...,current_pos] = 0
                w[weight_len - 2*layer][current_pos] = 0
                tester_model.set_weights(w)
                nl, na = tester_model.evaluate(x, y, verbose=0)
                if ol - nl >= best_change:
                    best_change = ol - nl
                    best_pos = current_pos
                    improved = True
                    best_acc = na
                    best_loss = nl
                current_pos += 1
                if current_pos - num_removed >= size:
                    end_not_reached = False
        elif method == 'greedy':
            nodes_to_estimate = list(np.arange(size))
            if folder_path and layer < len(temp_remove):
                for i in nodes_removed:
                    nodes_to_estimate.remove(i)
                size -= num_removed
            current_pos = nodes_to_estimate[0]
            idx = 0
            while end_not_reached:
                w = copy.deepcopy(best_weights)
                w[weight_len - (2*layer+1)][...,current_pos] = 0
                w[weight_len - 2*layer][current_pos] = 0
                tester_model.set_weights(w)
                nl, na = tester_model.evaluate(x, y, verbose=0)
                if ol - nl >= tol:
                    oa = na
                    ol = nl
                    size -= 1
                    nodes_removed += [current_pos]
                    nodes_to_estimate.remove(current_pos)
                    best_weights[weight_len - (2*layer+1)][..., current_pos] = 0
                    best_weights[weight_len - 2*layer][current_pos] = 0
                    bas += [oa]
                    bls += [ol]
                    num_removed += 1
                    idx = 0
                    if folder_path:
                        f_rem = open(folder_path+'/removed.txt', "a+")
                        f_loss = open(folder_path+'/loss.txt', "a+")
                        f_acc = open(folder_path+'/accuracy.txt', "a+")

                        f_rem.write(str(current_pos) + ' ')
                        f_loss.write(str(ol) + ' ')
                        f_acc.write(str(oa) + ' ')

                        f_rem.close()
                        f_loss.close()
                        f_acc.close()

                        tester_model.save_weights(folder_path+'/weights')
                elif ol - nl <= ignore_cutoff:
                    size -= 1
                    nodes_to_estimate.remove(current_pos)
                else:
                    idx += 1
                if idx >= size:
                    end_not_reached = False
                    if folder_path:
                        f_rem = open(folder_path+'/removed.txt', "a+")
                        f_rem.write('\n')
                        f_rem.close()
                else:
                    current_pos = nodes_to_estimate[idx]
        elif method == 'greedy_fast':
            improved = False
            nodes_to_estimate = list(np.arange(size))
            if folder_path and layer < len(temp_remove):
                for i in nodes_removed:
                    nodes_to_estimate.remove(i)
                size -= num_removed
            current_pos = nodes_to_estimate[0]
            idx = 0
            while end_not_reached:
                w = copy.deepcopy(best_weights)
                w[weight_len - (2*layer+1)][...,current_pos] = 0
                w[weight_len - 2*layer][current_pos] = 0
                tester_model.set_weights(w)
                nl, na = tester_model.evaluate(x, y, verbose=0)
                if ol - nl >= tol:
                    oa = na
                    ol = nl
                    size -= 1
                    nodes_removed += [current_pos]
                    nodes_to_estimate.remove(current_pos)
                    best_weights[weight_len - (2*layer+1)][..., current_pos] = 0
                    best_weights[weight_len - 2*layer][current_pos] = 0
                    bas += [oa]
                    bls += [ol]
                    num_removed += 1
                    improved = True
                    if folder_path:
                        f_rem = open(folder_path+'/removed.txt', "a+")
                        f_loss = open(folder_path+'/loss.txt', "a+")
                        f_acc = open(folder_path+'/accuracy.txt', "a+")

                        f_rem.write(str(current_pos) + ' ')
                        f_loss.write(str(ol) + ' ')
                        f_acc.write(str(oa) + ' ')

                        f_rem.close()
                        f_loss.close()
                        f_acc.close()

                        tester_model.save_weights(folder_path+'/weights')
                elif ol - nl <= ignore_cutoff:
                    size -= 1
                    nodes_to_estimate.remove(current_pos)
                else:
                    idx += 1
                if idx >= size:
                    if improved:
                        idx = 0
                        current_pos = nodes_to_estimate[idx]
                        improved = False
                    else:
                        end_not_reached = False
                        if folder_path:
                            f_rem = open(folder_path+'/removed.txt', "a+")
                            f_rem.write('\n')
                            f_rem.close()
                else:
                    current_pos = nodes_to_estimate[idx]
        print(f'Removed {num_removed} nodes for Layer {len(layer_sizes) - layer}')
        amounts.append(num_removed)
        places.append(nodes_removed)

    return best_weights, bas, bls, amounts, places_____no_output_____np.random.seed(2020)_____no_output_____mnist = tf.keras.datasets.mnist
(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_train, x_test = x_train / 255.0, x_test / 255.0  # Converting interger values to floats (0 to 1)_____no_output_____x_train, x_val, y_train, y_val = train_test_split(x_train, y_train, train_size=0.85, stratify=None)_____no_output_____tester_model = tf.keras.models.Sequential([
        tf.keras.layers.Flatten(input_shape=(28, 28)),
        tf.keras.layers.Dense(128, activation='relu'),
        tf.keras.layers.Dense(10, activation='softmax')
    ])
tester_model.compile(
    optimizer='adam',
    loss='sparse_categorical_crossentropy',
    metrics=['accuracy']
)_____no_output_____trials = 20

accs_or = np.zeros(trials)
losses_or = np.zeros(trials)
accs_val = np.zeros(trials)
losses_val = np.zeros(trials)
accs_train = np.zeros(trials)
losses_train = np.zeros(trials)
accs_train_val = np.zeros(trials)
losses_train_val = np.zeros(trials)

am_val = np.zeros(trials)
am_train = np.zeros(trials)
am_train_val = np.zeros(trials)

for trial in tqdm.trange(trials):
    model, r = models.train_basic_ANN(x_train, y_train, 128, (x_test, y_test), epochs=5)
    losses_or[trial], accs_or[trial] = model.evaluate(x_test, y_test, verbose=0)

    shrinked_weights, _, _, amval, pval = node_pruning2(model, tester_model, x_val, y_val, layer_sizes, tol, -1, 'greedy_fast')
    tester_model.set_weights(shrinked_weights)
    losses_val[trial], accs_val[trial] = tester_model.evaluate(x_test, y_test, verbose=0)
    am_val[trial] = amval[0]

    shrinked_weights, _, _, amtrain, ptrain = node_pruning2(model, tester_model, x_train, y_train, layer_sizes, tol, -1, 'greedy_fast')
    tester_model.set_weights(shrinked_weights)
    losses_train[trial], accs_train[trial] = tester_model.evaluate(x_test, y_test, verbose=0)
    am_train[trial] = amtrain[0]

    model.set_weights(shrinked_weights)
    shrinked_weights, _, _, amtrainval, ptrainval = node_pruning2(model, tester_model, x_val, y_val, layer_sizes, tol, -1, 'greedy_fast')
    tester_model.set_weights(shrinked_weights)
    losses_train_val[trial], accs_train_val[trial] = tester_model.evaluate(x_test, y_test, verbose=0)
    am_train_val[trial] = amtrainval[0]  0%|                                                                                | 0/20 [00:00<?, ?it/s]A = pd.DataFrame(np.array([accs_or, accs_val, accs_train, accs_train_val, accs_train - accs_val, accs_train - accs_train_val, accs_val - accs_train_val]).T, columns=['Before Pruning', 'Val Pruning', 'Train Pruning', 'Train + Val Pruning', 'Diff Train - Val',
                                                                                'Diff Train - Train + Val', 'Diff Val - Train + Val'])
A_____no_output_____A.describe()_____no_output_____L = pd.DataFrame(np.array([losses_or, losses_val, losses_train, losses_train_val, losses_train - losses_val, losses_train - losses_train_val, losses_val - losses_train_val]).T, columns=['Before Pruning', 'Val Pruning', 'Train Pruning', 'Train + Val Pruning', 'Diff Train - Val',
                                                                                'Diff Train - Train + Val', 'Diff Val - Train + Val'])
L_____no_output_____L.describe()_____no_output_____trials = 10

accs_or = np.zeros(trials)
losses_or = np.zeros(trials)
accs_val = np.zeros(trials)
losses_val = np.zeros(trials)
accs_train = np.zeros(trials)
losses_train = np.zeros(trials)
accs_train_val = np.zeros(trials)
losses_train_val = np.zeros(trials)

am_val = np.zeros(trials)
am_train = np.zeros(trials)
am_train_val = np.zeros(trials)

for trial in tqdm.trange(trials):
    model, r = models.train_basic_ANN(x_train, y_train, 128, (x_test, y_test), epochs=5)
    losses_or[trial], accs_or[trial] = model.evaluate(x_test, y_test, verbose=0)

    shrinked_weights, _, _, amval, pval = node_pruning2(model, tester_model, x_val, y_val, layer_sizes, tol)
    tester_model.set_weights(shrinked_weights)
    losses_val[trial], accs_val[trial] = tester_model.evaluate(x_test, y_test, verbose=0)
    am_val[trial] = amval[0]

    shrinked_weights, _, _, amtrain, ptrain = node_pruning2(model, tester_model, x_train, y_train, layer_sizes, tol)
    tester_model.set_weights(shrinked_weights)
    losses_train[trial], accs_train[trial] = tester_model.evaluate(x_test, y_test, verbose=0)
    am_train[trial] = amtrain[0]

    model.set_weights(shrinked_weights)
    shrinked_weights, _, _, amtrainval, ptrainval = node_pruning2(model, tester_model, x_val, y_val, layer_sizes, tol)
    tester_model.set_weights(shrinked_weights)
    losses_train_val[trial], accs_train_val[trial] = tester_model.evaluate(x_test, y_test, verbose=0)
    am_train_val[trial] = amtrainval[0]  0%|                                                                                | 0/10 [00:00<?, ?it/s]A = pd.DataFrame(np.array([accs_or, accs_val, accs_train, accs_train_val, accs_train - accs_val, accs_train - accs_train_val, accs_val - accs_train_val]).T, columns=['Before Pruning', 'Val Pruning', 'Train Pruning', 'Train + Val Pruning', 'Diff Train - Val',
                                                                                'Diff Train - Train + Val', 'Diff Val - Train + Val'])
A_____no_output_____A.describe()_____no_output_____L = pd.DataFrame(np.array([losses_or, losses_val, losses_train, losses_train_val, losses_train - losses_val, losses_train - losses_train_val, losses_val - losses_train_val]).T, columns=['Before Pruning', 'Val Pruning', 'Train Pruning', 'Train + Val Pruning', 'Diff Train - Val',
                                                                                'Diff Train - Train + Val', 'Diff Val - Train + Val'])
L_____no_output_____L.describe()_____no_output_____tol = -1e-5
layer_sizes = [128]
model, r = models.train_basic_ANN(x_train, y_train, 128, (x_test, y_test), epochs=5)
shrinked_weights, aval, lval, amval, pval = node_pruning2(model, tester_model, x_val, y_val, layer_sizes, tol, -1, 'greedy_fast')
print(model.evaluate(x_test, y_test))
tester_model.set_weights(shrinked_weights)
print(tester_model.evaluate(x_test, y_test))
shrinked_weights, atrain, ltrain, amtrain, ptrain = node_pruning2(model, tester_model, x_train, y_train, layer_sizes, tol, -1, 'greedy_fast')
tester_model.set_weights(shrinked_weights)
print(tester_model.evaluate(x_test, y_test))Starting removal in Layer 1
Node 0: -0.002056613564491272
Node 1: -0.0005248785018920898
Node 2: -0.002936147153377533
Node 3: -0.028483398258686066
Node 4: -0.003854818642139435
Node 5: -0.0025076866149902344
Node 6: -0.006499655544757843
Node 7: -0.005253158509731293
Node 8: -0.0016536861658096313
Node 9: -0.0002019926905632019
Node 10: -0.004399359226226807
Node 11: -0.00415029376745224
Node 12: 9.020417928695679e-05
Node 13: -0.0045790895819664
Node 14: -0.004175134003162384
Node 15: -0.0015794336795806885
Node 16: -0.0003643408417701721
Node 17: -0.0005084574222564697
Node 18: -0.0012055858969688416
Node 19: -0.0029353871941566467
Node 20: -0.0015078261494636536
Node 21: -0.00194501131772995
Node 22: -0.004516951739788055
Node 23: -5.781650543212891e-06
Node 24: -0.002385452389717102
Node 25: -0.0013469979166984558
Node 26: -0.0005596578121185303
Node 27: -0.0017693862318992615
Node 28: -0.0005755871534347534
Node 29: -0.0005262270569801331
Node 30: 0.00018386542797088623
Node 31: -0.002118118107318878
Node 32: 0.00012125074863433838
Node 33: -0.0031244978308677673
Node 34: -0.00325804203748703
Node 35: -0.0010466575622558594
Node 36: -0.0005731508135795593
Node 37: -0.0008754134178161621
Node 38: 0.0007355585694313049
Node 39: -0.0035538598895072937
Node 40: -0.0004794076085090637
Node 41: -0.0015714839100837708
Node 42: -0.002095632255077362
Node 43: 0.0013653486967086792
Node 44: -0.005522787570953369
Node 45: -0.00015190988779067993
Node 46: -0.0012622922658920288
Node 47: -0.0007906481623649597
Node 48: -0.0032567456364631653
Node 49: -0.0006779804825782776
Node 50: 0.00018739700317382812
Node 51: -0.002108737826347351
Node 52: -0.001623779535293579
Node 53: -0.002897031605243683
Node 54: -0.00137251615524292
Node 55: -0.0006108656525611877
Node 56: -0.01510234922170639
Node 57: -0.00018968433141708374
Node 58: -0.00020134449005126953
Node 59: 1.0445713996887207e-05
Node 60: 0.0
Node 61: -0.0011134371161460876
Node 62: -7.411837577819824e-05
Node 63: -0.0009087398648262024
Node 64: -0.0032318606972694397
Node 65: -0.0017628967761993408
Node 66: -0.00166349858045578
Node 67: -0.004573151469230652
Node 68: -0.0008500218391418457
Node 69: -0.0021081343293190002
Node 70: -0.001111060380935669
Node 71: -0.00047075748443603516
Node 72: -0.0007275640964508057
Node 73: -0.0002526119351387024
Node 74: -0.0022218450903892517
Node 75: -0.003677092492580414
Node 76: -0.0016372352838516235
Node 77: -0.001380130648612976
Node 78: -0.0006827488541603088
Node 79: 0.000569269061088562
Node 80: -0.00096169114112854
Node 81: -0.003182917833328247
Node 82: -0.0019749850034713745
Node 83: -0.0008460730314254761
Node 84: -0.004861526191234589
Node 85: -0.0019507110118865967
Node 86: -0.0012206658720970154
Node 87: -0.0009807273745536804
Node 88: -0.0026578158140182495
Node 89: -0.007044166326522827
Node 90: -0.011680647730827332
Node 91: -0.0006700083613395691
Node 92: -0.0015710294246673584
Node 93: -0.00040118396282196045
Node 94: -0.003338068723678589
Node 95: -0.0007098093628883362
Node 96: -0.012422583997249603
Node 97: -0.0009584426879882812
Node 98: -0.006305038928985596
Node 99: -0.001118779182434082
Node 100: 9.24915075302124e-05
Node 101: -0.0026741400361061096
Node 102: -0.001197710633277893
Node 103: -0.0020455047488212585
Node 104: -0.00027949362993240356
Node 105: -0.0023546069860458374
Node 106: 2.868473529815674e-06
Node 107: -5.967915058135986e-06
Node 108: -0.002745606005191803
Node 109: -0.0025080591440200806
Node 110: -0.000331781804561615
Node 111: -0.0020358338952064514
Node 112: -0.0032144784927368164
Node 113: -0.00165461003780365
Node 114: 6.858259439468384e-05
Node 115: -0.002045057713985443
Node 116: -0.0011416152119636536
Node 117: -0.0007076188921928406
Node 118: -0.0014028772711753845
Node 119: -0.00026845186948776245
Node 120: -0.0007951483130455017
Node 121: -0.001690894365310669
Node 122: -0.0010894760489463806
Node 123: -0.0010769441723823547
Node 124: -0.004610143601894379
Node 125: -0.0013167187571525574
Node 126: -0.004031702876091003
Node 127: -0.005125835537910461
Removed 14 nodes for Layer 1
313/313 [==============================] - 0s 1ms/step - loss: 0.0727 - accuracy: 0.9783
[0.07265818119049072, 0.9782999753952026]
313/313 [==============================] - 0s 1ms/step - loss: 0.0719 - accuracy: 0.9779
[0.07190108299255371, 0.9779000282287598]
Starting removal in Layer 1
Node 0: -0.0014621429145336151
Node 1: -0.000994991511106491
Node 2: -0.002102520316839218
Node 3: -0.027253154665231705
Node 4: -0.003981597721576691
Node 5: -0.0016839802265167236
Node 6: -0.00440574437379837
Node 7: -0.003233596682548523
Node 8: -0.0012741722166538239
Node 9: -0.0004980675876140594
Node 10: -0.003938667476177216
Node 11: -0.0037008076906204224
Node 12: 0.000440746545791626
Node 13: -0.003224264830350876
Node 14: -0.003402523696422577
Node 15: -0.0005972236394882202
Node 16: -0.0007471032440662384
Node 17: -0.0005104690790176392
Node 18: -0.0011023394763469696
Node 19: -0.001455608755350113
Node 20: -0.0010640285909175873
Node 21: -0.0019867196679115295
Node 22: -0.004469066858291626
Node 23: -0.00037911906838417053
Node 24: -0.002464640885591507
Node 25: -0.001569412648677826
Node 26: -0.0008091814815998077
Node 27: -0.0017103925347328186
Node 28: -0.0006191767752170563
Node 29: -0.0009939707815647125
Node 30: -3.5744160413742065e-05
Node 31: -0.0023779086768627167
Node 32: 0.00011369213461875916
Node 33: -0.001726679503917694
Node 34: -0.0028207339346408844
Node 35: -0.0016468018293380737
Node 36: -0.00046288594603538513
Node 37: -0.0010004714131355286
Node 38: -0.0004953891038894653
Node 39: -0.0008957050740718842
Node 40: -0.00041467323899269104
Node 41: -0.001062680035829544
Node 42: -0.0017747282981872559
Node 43: 0.0002661794424057007
Node 44: -0.0036482959985733032
Node 45: -0.00022394955158233643
Node 46: -0.001601632684469223
Node 47: -0.0007709823548793793
Node 48: -0.0031584985554218292
Node 49: -0.0007904432713985443
Node 50: -1.728534698486328e-05
Node 51: -0.0006052181124687195
Node 52: -0.0015573129057884216
Node 53: -0.002051856368780136
Node 54: -0.0016807913780212402
Node 55: -0.0025054626166820526
Node 56: -0.01622065156698227
Node 57: -0.00015731528401374817
Node 58: -0.0006881952285766602
Node 59: -0.00015199929475784302
Node 60: 0.0
Node 61: -0.0015308745205402374
Node 62: 3.3713877201080322e-06
Node 63: -0.00156310573220253
Node 64: -0.0035513751208782196
Node 65: -0.000956457108259201
Node 66: -0.003859143704175949
Node 67: -0.00495312362909317
Node 68: -0.0015446394681930542
Node 69: -0.0016912519931793213
Node 70: -0.0008102990686893463
Node 71: 0.0003291666507720947
Node 72: 0.00015431270003318787
Node 73: -0.00032183900475502014
Node 74: -0.000739172101020813
Node 75: -0.005619499832391739
Node 76: -0.003342069685459137
Node 77: -0.0013465732336044312
Node 78: -0.0006118565797805786
Node 79: -0.00024382397532463074
Node 80: -0.00026277080178260803
Node 81: -0.0006528496742248535
Node 82: -0.001418277621269226
Node 83: -0.0008380189538002014
Node 84: -0.003060951828956604
Node 85: -0.00030736997723579407
Node 86: -0.0007322803139686584
Node 87: -0.0009765028953552246
Node 88: -0.001891203224658966
Node 89: -0.0063582174479961395
Node 90: -0.009089749306440353
Node 91: -0.0004425905644893646
Node 92: -0.0004422776401042938
Node 93: -0.0006899572908878326
Node 94: -0.0022984780371189117
Node 95: -0.00026740506291389465
Node 96: -0.008946944028139114
Node 97: -0.0006350353360176086
Node 98: -0.0024785995483398438
Node 99: -0.0006970614194869995
Node 100: -0.00021098926663398743
Node 101: -0.0009399838745594025
Node 102: -0.0016544200479984283
Node 103: -0.0016166679561138153
Node 104: -0.0012787356972694397
Node 105: -0.001519523561000824
Node 106: -0.0005586631596088409
Node 107: -3.9305537939071655e-05
Node 108: -0.0017899498343467712
Node 109: -0.0012801140546798706
Node 110: -0.00034324079751968384
Node 111: -0.0022653378546237946
Node 112: -0.0031984560191631317
Node 113: -0.0010820701718330383
Node 114: -0.001109108328819275
Node 115: -0.0015748478472232819
Node 116: -0.001950737088918686
Node 117: -0.0010780245065689087
Node 118: -0.0010768845677375793
Node 119: -0.0012443847954273224
Node 120: -0.0005498230457305908
Node 121: -0.0010038577020168304
Node 122: -0.0008237361907958984
Node 123: -0.0009851157665252686
Node 124: -0.0039714016020298
Node 125: -0.002462513744831085
Node 126: -0.002207621932029724
Node 127: -0.003989178687334061
Removed 7 nodes for Layer 1
313/313 [==============================] - 0s 1ms/step - loss: 0.0685 - accuracy: 0.9797
[0.06846363842487335, 0.9797000288963318]
model.set_weights(shrinked_weights)
print(model.evaluate(x_test, y_test))
shrinked_weights, amix, lmix, ammix, pmix = node_pruning2(model, tester_model, x_val, y_val, layer_sizes, tol, -1, 'greedy_fast')
tester_model.set_weights(shrinked_weights)
print(tester_model.evaluate(x_test, y_test))313/313 [==============================] - 0s 1ms/step - loss: 0.0707 - accuracy: 0.9795
[0.07067720592021942, 0.9794999957084656]
Starting removal in Layer 1
Node 0: -0.001238018274307251
Node 1: -0.0010597482323646545
Node 2: -0.005421541631221771
Node 3: -0.020296096801757812
Node 4: -0.0012318938970565796
Node 5: -0.0021179169416427612
Node 6: -0.004004672169685364
Node 7: -0.005093805491924286
Node 8: -0.004712134599685669
Node 9: 0.0
Node 10: -0.002037063241004944
Node 11: -0.0036157220602035522
Node 12: 0.0
Node 13: -0.004125118255615234
Node 14: -0.0036092251539230347
Node 15: -0.0016311854124069214
Node 16: 6.421655416488647e-05
Node 17: -0.00017757713794708252
Node 18: -0.0012790411710739136
Node 19: -0.0025338754057884216
Node 20: -0.0016934201121330261
Node 21: -0.0013367459177970886
Node 22: -0.006222710013389587
Node 23: 0.0
Node 24: -0.0008446648716926575
Node 25: -0.0017002597451210022
Node 26: -0.0005596578121185303
Node 27: -0.00139705091714859
Node 28: -0.0008200034499168396
Node 29: -0.0015716999769210815
Node 30: -0.0009295865893363953
Node 31: -0.0031860917806625366
Node 32: 0.0
Node 33: -0.0023637935519218445
Node 34: -0.0034848004579544067
Node 35: -0.0035855546593666077
Node 36: -0.0004084184765815735
Node 37: -0.00174703449010849
Node 38: -0.0025942325592041016
Node 39: -0.0023965463042259216
Node 40: -0.000505693256855011
Node 41: -0.0012249425053596497
Node 42: -0.003207579255104065
Node 43: 0.0
Node 44: -0.0046379417181015015
Node 45: -0.0002160966396331787
Node 46: -0.0014315694570541382
Node 47: -0.0010339319705963135
Node 48: -0.002611108124256134
Node 49: -0.0008807405829429626
Node 50: 0.0
Node 51: -0.0006374344229698181
Node 52: -0.0022785142064094543
Node 53: -0.002912379801273346
Node 54: -0.0013090074062347412
Node 55: -0.00021533668041229248
Node 56: -0.013061858713626862
Node 57: -0.0003621801733970642
Node 58: -0.0003997683525085449
Node 59: 0.0
Node 60: 0.0
Node 61: -0.002451777458190918
Node 62: 0.0
Node 63: -0.0017140507698059082
Node 64: -0.004442684352397919
Node 65: -0.001799561083316803
Node 66: -0.004085637629032135
Node 67: -0.00722184032201767
Node 68: -0.001267760992050171
Node 69: -0.0022513121366500854
Node 70: -0.0009608641266822815
Node 71: 0.0
Node 72: 0.0
Node 73: -0.000311531126499176
Node 74: -0.0021324530243873596
Node 75: -0.00736226886510849
Node 76: -0.0016165673732757568
Node 77: -0.0015345066785812378
Node 78: -0.0003755614161491394
Node 79: 0.0
Node 80: -0.0004018619656562805
Node 81: -0.001368604600429535
Node 82: -0.0022562742233276367
Node 83: -0.000983797013759613
Node 84: -0.004088141024112701
Node 85: 0.0
Node 86: -0.0014631673693656921
Node 87: -0.0009323060512542725
Node 88: -0.0020443350076675415
Node 89: -0.0069674476981163025
Node 90: -0.010280132293701172
Node 91: -0.00015922635793685913
Node 92: -0.0005816519260406494
Node 93: -0.0004623532295227051
Node 94: -0.002064645290374756
Node 95: -0.0009511485695838928
Node 96: -0.008247390389442444
Node 97: -0.0011394694447517395
Node 98: -0.006580568850040436
Node 99: -0.00045409053564071655
Node 100: 4.781782627105713e-05
Node 101: -0.0013825595378875732
Node 102: -0.0008831173181533813
Node 103: -0.0021056383848190308
Node 104: -0.0014276877045631409
Node 105: -0.0022268667817115784
Node 106: 0.0
Node 107: 0.0
Node 108: -0.0037903189659118652
Node 109: -0.0023503378033638
Node 110: -0.0003764331340789795
Node 111: -0.0032049641013145447
Node 112: -0.003878548741340637
Node 113: -0.00132066011428833
Node 114: 0.0
Node 115: -0.002522088587284088
Node 116: -0.0019630342721939087
Node 117: -0.0013917908072471619
Node 118: -0.0015831142663955688
Node 119: -0.0007327795028686523
Node 120: -0.0009252950549125671
Node 121: -0.0020476877689361572
Node 122: -0.0013221725821495056
Node 123: -0.0008139312267303467
Node 124: -0.0046674758195877075
Node 125: -0.0013359934091567993
Node 126: -0.005092017352581024
Node 127: -0.005129687488079071
Node 0: -0.0015413090586662292
Node 1: -0.0007477551698684692
Node 2: -0.0052588507533073425
Node 3: -0.021121494472026825
Node 4: -0.0013561844825744629
Node 5: -0.0021743029356002808
Node 6: -0.006373219192028046
Node 7: -0.0045464783906936646
Node 8: -0.004826553165912628
Node 10: -0.002087622880935669
Node 11: -0.00402175635099411
Node 13: -0.005139097571372986
Node 14: -0.0035950839519500732
Node 15: -0.0010764077305793762
Node 17: -0.0003762245178222656
Node 18: -0.0014714822173118591
Node 19: -0.002415090799331665
Node 20: -0.0020790845155715942
Node 21: -0.0014439970254898071
Node 22: -0.0049410611391067505
Node 24: -0.0009167715907096863
Node 25: -0.0017276480793952942
Node 26: -0.00048666447401046753
Node 27: -0.0014690086245536804
Node 28: -0.0008166581392288208
Node 29: -0.001497015357017517
Node 30: -0.0011478587985038757
Node 31: -0.0030308738350868225
Node 33: -0.0026214197278022766
Node 34: -0.003788381814956665
Node 35: -0.0032904893159866333
Node 36: -0.0003712847828865051
Node 37: -0.001681983470916748
Node 38: -0.002043023705482483
Node 39: -0.0019762665033340454
Node 40: -0.0004523172974586487
Node 41: -0.0019457638263702393
Node 42: -0.0027996450662612915
Node 44: -0.0038294717669487
Node 45: -0.0002533644437789917
Node 46: -0.0013981461524963379
Node 47: -0.0010652169585227966
Node 48: -0.002714112401008606
Node 49: -0.0009399950504302979
Node 51: -0.0007532089948654175
Node 52: -0.002376049757003784
Node 53: -0.002454213798046112
Node 54: -0.0013169050216674805
Node 55: 0.0002540871500968933
Node 56: -0.012466780841350555
Node 57: 2.874433994293213e-05
Node 58: -0.0006937310099601746
Node 61: -0.0027161240577697754
Node 63: -0.0008234232664108276
Node 64: -0.006547160446643829
Node 65: -0.0016079694032669067
Node 66: -0.004984110593795776
Node 67: -0.006376318633556366
Node 68: -0.0009590312838554382
Node 69: -0.002919591963291168
Node 70: -0.0006689876317977905
Node 73: -0.0002971440553665161
Node 74: -0.0014815330505371094
Node 75: -0.008846715092658997
Node 76: -0.0017038434743881226
Node 77: -0.0016752257943153381
Node 78: -0.00016949325799942017
Node 80: -0.0003680288791656494
Node 81: -0.001339934766292572
Node 82: -0.0023316144943237305
Node 83: -0.0007940903306007385
Node 84: -0.0031349584460258484
Node 86: -0.0008252933621406555
Node 87: -0.0015402138233184814
Node 88: -0.002261906862258911
Node 89: -0.00872708112001419
Node 90: -0.014548659324645996
Node 91: -0.00019241869449615479
Node 92: -0.0008395016193389893
Node 93: -0.0004183053970336914
Node 94: -0.0031437277793884277
Node 95: -0.000705525279045105
Node 96: -0.007390521466732025
Node 97: -0.0009234398603439331
Node 98: -0.0066113099455833435
Node 99: -0.0007977858185768127
Node 101: -0.000843144953250885
Node 102: -0.0009497478604316711
Node 103: -0.002408333122730255
Node 104: -0.0016569122672080994
Node 105: -0.001103818416595459
Node 108: -0.003726877272129059
Node 109: -0.001270972192287445
Node 110: -0.00015725940465927124
Node 111: -0.0034189745783805847
Node 112: -0.0038361847400665283
Node 113: -0.000917389988899231
Node 115: -0.002475649118423462
Node 116: -0.001485481858253479
Node 117: -0.0016876086592674255
Node 118: -0.0014751628041267395
Node 119: -0.000933527946472168
Node 120: -0.001156918704509735
Node 121: -0.0007345229387283325
Node 122: -0.0013465136289596558
Node 123: -0.00090780109167099
Node 124: -0.004558145999908447
Node 125: -0.0014364048838615417
Node 126: -0.004987955093383789
Node 127: -0.004488155245780945
Node 0: -0.0017449930310249329
Node 1: -0.001583680510520935
Node 2: -0.00616232305765152
Node 3: -0.03449380397796631
Node 4: -0.0026144832372665405
Node 5: -0.002063460648059845
Node 6: -0.006350629031658173
Node 7: -0.004681974649429321
Node 8: -0.004855699837207794
Node 10: -0.0037477388978004456
Node 11: -0.0037630274891853333
Node 13: -0.005255885422229767
Node 14: -0.003428809344768524
Node 15: -0.0011020153760910034
Node 17: -0.000596262514591217
Node 18: -0.0015122294425964355
Node 19: -0.0024123862385749817
Node 20: -0.0017850324511528015
Node 21: -0.0014219656586647034
Node 22: -0.006127119064331055
Node 24: -0.0016154125332832336
Node 25: -0.0019425004720687866
Node 26: -0.000583745539188385
Node 27: -0.0023394599556922913
Node 28: -0.0008079409599304199
Node 29: -0.0011687353253364563
Node 30: -0.00010785460472106934
Node 31: -0.003367885947227478
Node 33: -0.002709329128265381
Node 34: -0.0037460699677467346
Node 35: -0.0047235265374183655
Node 36: -0.0002695545554161072
Node 37: -0.001698605716228485
Node 38: -0.0026044026017189026
Node 39: -0.001364283263683319
Node 40: -0.0005608722567558289
Node 41: -0.001984134316444397
Node 42: -0.0033158957958221436
Node 44: -0.0041336789727211
Node 45: -0.0002993643283843994
Node 46: -0.001471012830734253
Node 47: -0.0012098997831344604
Node 48: -0.0035054683685302734
Node 49: -0.000828668475151062
Node 51: -5.938112735748291e-05
Node 52: -0.002774149179458618
Node 53: -0.0032096579670906067
Node 54: -0.0015852898359298706
Node 56: -0.012483872473239899
Node 58: -0.0006937310099601746
Node 61: -0.0027161240577697754
Node 63: -0.0008234232664108276
Node 64: -0.006547160446643829
Node 65: -0.0016079694032669067
Node 66: -0.004984110593795776
Node 67: -0.006376318633556366
Node 68: -0.0009590312838554382
Node 69: -0.002919591963291168
Node 70: -0.0006689876317977905
Node 73: -0.0002971440553665161
Node 74: -0.0014815330505371094
Node 75: -0.008846715092658997
Node 76: -0.0017038434743881226
Node 77: -0.0016752257943153381
Node 78: -0.00016949325799942017
Node 80: -0.0003680288791656494
Node 81: -0.001339934766292572
Node 82: -0.0023316144943237305
Node 83: -0.0007940903306007385
Node 84: -0.0031349584460258484
Node 86: -0.0008252933621406555
Node 87: -0.0015402138233184814
Node 88: -0.002261906862258911
Node 89: -0.00872708112001419
Node 90: -0.014548659324645996
Node 91: -0.00019241869449615479
Node 92: -0.0008395016193389893
Node 93: -0.0004183053970336914
Node 94: -0.0031437277793884277
Node 95: -0.000705525279045105
Node 96: -0.007390521466732025
Node 97: -0.0009234398603439331
Node 98: -0.0066113099455833435
Node 99: -0.0007977858185768127
Node 101: -0.000843144953250885
Node 102: -0.0009497478604316711
Node 103: -0.002408333122730255
Node 104: -0.0016569122672080994
Node 105: -0.001103818416595459
Node 108: -0.003726877272129059
Node 109: -0.001270972192287445
Node 110: -0.00015725940465927124
Node 111: -0.0034189745783805847
Node 112: -0.0038361847400665283
Node 113: -0.000917389988899231
Node 115: -0.002475649118423462
Node 116: -0.001485481858253479
Node 117: -0.0016876086592674255
Node 118: -0.0014751628041267395
Node 119: -0.000933527946472168
Node 120: -0.001156918704509735
Node 121: -0.0007345229387283325
Node 122: -0.0013465136289596558
Node 123: -0.00090780109167099
Node 124: -0.004558145999908447
Node 125: -0.0014364048838615417
Node 126: -0.004987955093383789
Node 127: -0.004488155245780945
Removed 20 nodes for Layer 1
313/313 [==============================] - 0s 1ms/step - loss: 0.0719 - accuracy: 0.9784
[0.07192785292863846, 0.9783999919891357]
pval_____no_output_____ptrain_____no_output_____pmix_____no_output_____np.random.seed(2020)_____no_output_____mnist = tf.keras.datasets.fashion_mnist
(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_train, x_test = x_train / 255.0, x_test / 255.0  # Converting interger values to floats (0 to 1)_____no_output_____x_train, x_val, y_train, y_val = train_test_split(x_train, y_train, train_size=0.85, stratify=None)_____no_output_____tester_model = tf.keras.models.Sequential([
    tf.keras.layers.Flatten(input_shape=(28, 28)),
    tf.keras.layers.Dense(128, activation='relu'),
    tf.keras.layers.Dense(64, activation='relu'),
    tf.keras.layers.Dense(32, activation='relu'),
    tf.keras.layers.Dense(10, activation='softmax')
])
tester_model.compile(
    optimizer='adam',
    loss='sparse_categorical_crossentropy',
    metrics=['accuracy']
)_____no_output_____model = tf.keras.models.Sequential([
    tf.keras.layers.Flatten(input_shape=(28, 28)),
    tf.keras.layers.Dense(128, activation='relu'),
    tf.keras.layers.Dense(64, activation='relu'),
    tf.keras.layers.Dense(32, activation='relu'),
    tf.keras.layers.Dense(10, activation='softmax')
])
model.compile(
    optimizer='adam',
    loss='sparse_categorical_crossentropy',
    metrics=['accuracy']
)
model.fit(x_train, y_train, epochs=5)Epoch 1/5
1594/1594 [==============================] - 2s 1ms/step - loss: 0.5265 - accuracy: 0.8143
Epoch 2/5
1594/1594 [==============================] - 2s 1ms/step - loss: 0.3836 - accuracy: 0.8602
Epoch 3/5
1594/1594 [==============================] - 2s 1ms/step - loss: 0.3489 - accuracy: 0.8708
Epoch 4/5
1594/1594 [==============================] - 2s 1ms/step - loss: 0.3245 - accuracy: 0.8806
Epoch 5/5
1594/1594 [==============================] - 2s 1ms/step - loss: 0.3041 - accuracy: 0.8857
runs = 10
layer_sizes = [32, 64, 128]
accs = np.zeros(runs)
losses = np.zeros(runs)
ps = []
ams = []
for i in tqdm.trange(runs):
    shrinked_weights, _, _, am, p = node_pruning2(model, tester_model, x_val, y_val, layer_sizes, tol, -1, 'greedy_fast')
    model.set_weights(shrinked_weights)
    losses[i], accs[i] = model.evaluate(x_test, y_test)
    ps += [p]
    ams += [am]  0%|                                                                                | 0/10 [00:00<?, ?it/s]