# Notebook from wconnell/metrx
Path: .ipynb_checkpoints/2020.01.10_ActivationMax_rna-seq_pan_BRCA_tumor-v-normal-checkpoint.ipynb

import numpy as np
import pandas as pd

# viz
import matplotlib.pyplot as plt

# notebook settings
from IPython.core.interactiveshell import InteractiveShell
InteractiveShell.ast_node_interactivity = "all"

pd.set_option('display.max_columns', 1000)_____no_output_____

## Sample Prep_____no_output_____

samples = pd.read_csv('../data/TCGA/rna-seq_pan/meta/gdc_sample_sheet.2019-12-12.tsv', sep="\t")
# get file type
samples['data'] = [val[1] for i,val in samples['File Name'].str.split(".").items()]
samples.head()_____no_output_____

Samples with RNAseq adjacent normal tissue_____no_output_____

samples[samples['Sample Type']=='Solid Tissue Normal']['data'].value_counts()_____no_output_____samples['project'] = [val[1] for i,val in samples['Project ID'].str.split("-").items()]
samples['project'].value_counts()_____no_output_____# all cases with adjacent normal tissue
cases = samples[samples['Sample Type']=='Solid Tissue Normal']['Case ID']_____no_output_____# disparity in cases
samples[(samples['Case ID'].isin(cases)) & (samples['Sample Type']=='Primary Tumor')
        & (samples['data']=='FPKM') & (samples['project']=='BRCA')]['Case ID'].nunique()
samples[(samples['Case ID'].isin(cases)) & (samples['Sample Type']=='Solid Tissue Normal')
        & (samples['data']=='FPKM') & (samples['project']=='BRCA')]['Case ID'].nunique()_____no_output_____# divide, join, subset
case_tumor = samples[(samples['Case ID'].isin(cases)) & (samples['Sample Type']=='Primary Tumor') &
                     (samples['data']=='FPKM') & (samples['project']=='BRCA')]
case_norm = samples[(samples['Case ID'].isin(cases)) & (samples['Sample Type']=='Solid Tissue Normal') &
                    (samples['data']=='FPKM') & (samples['project']=='BRCA')]
cases = case_norm[case_norm['Case ID'].isin(case_tumor['Case ID'])]['Case ID']
cases.shape_____no_output_____case_tumor = case_tumor[case_tumor['Case ID'].isin(cases)]
case_norm = case_norm[case_norm['Case ID'].isin(cases)]
cases = pd.concat([case_tumor, case_norm])_____no_output_____case_tumor.shape
case_norm.shape
cases.shape_____no_output_____

## Dataset Prep_____no_output_____

from sklearn.model_selection import train_test_split

target = 'Sample Type'
cases[target] = cases[target].astype('category')

train, test = train_test_split(cases)
train[target].value_counts()
test[target].value_counts()_____no_output_____import torch
from torch.optim import lr_scheduler
import torch.optim as optim
from torch.autograd import Variable

from trainer import fit
import visualization as vis
import numpy as np
cuda = torch.cuda.is_available()
print("Cuda is available: {}".format(cuda))

classes = train[target].cat.categories.values
Cuda is available: True
from tcga_datasets import TCGA, SiameseTCGA
root_dir = "../data/TCGA/rna-seq_pan/"
batch_size = 1

train_dataset = TCGA(root_dir, samples=train, train=True, target=target)
test_dataset = TCGA(root_dir, samples=test, train=False, target=target)

kwargs = {'num_workers': 10, 'pin_memory': True} if cuda else {'num_workers': 10}
train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True, **kwargs)
test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False, **kwargs)_____no_output_____

## Siamese Network_____no_output_____

# Step 1 set up dataloader
root_dir = "../data/TCGA"
siamese_train_dataset = SiameseTCGA(train_dataset) # Returns pairs of images and target same/different
siamese_test_dataset = SiameseTCGA(test_dataset)
batch_size = 8
kwargs = {'num_workers': 10, 'pin_memory': True} if cuda else {}
siamese_train_loader = torch.utils.data.DataLoader(siamese_train_dataset, batch_size=batch_size, shuffle=True, **kwargs)
siamese_test_loader = torch.utils.data.DataLoader(siamese_test_dataset, batch_size=batch_size, shuffle=False, **kwargs)

# Set up the network and training parameters
from tcga_networks import EmbeddingNet, SiameseNet
from losses import ContrastiveLoss
from metrics import AccumulatedAccuracyMetric

# Step 2
embedding_net = EmbeddingNet()
# Step 3
model = SiameseNet(embedding_net)
if cuda:
    model.cuda()

# Step 4
margin = 1.
loss_fn = ContrastiveLoss(margin)
lr = 1e-3
optimizer = optim.Adam(model.parameters(), lr=lr)
scheduler = lr_scheduler.StepLR(optimizer, 8, gamma=0.1, last_epoch=-1)
n_epochs = 10
# print training metrics every log_interval * batch_size
log_interval = 30_____no_output_____train_loss, val_loss = fit(siamese_train_loader, siamese_test_loader, model, loss_fn, optimizer, scheduler,
    n_epochs, cuda, log_interval)Train: [0/170 (0%)]	Loss: 0.237146
Epoch: 1/10. Train set: Average loss: 24.8524
Epoch: 1/10. Validation set: Average loss: 4.1286
Train: [0/170 (0%)]	Loss: 3.122349
Epoch: 2/10. Train set: Average loss: 51.4417
Epoch: 2/10. Validation set: Average loss: 18.8456
Train: [0/170 (0%)]	Loss: 24.674990
Epoch: 3/10. Train set: Average loss: 52.2205
Epoch: 3/10. Validation set: Average loss: 5.4529
Train: [0/170 (0%)]	Loss: 18.828060
Epoch: 4/10. Train set: Average loss: 16.1846
Epoch: 4/10. Validation set: Average loss: 1.9008
Train: [0/170 (0%)]	Loss: 1.131958
Epoch: 5/10. Train set: Average loss: 6.2124
Epoch: 5/10. Validation set: Average loss: 0.3215
Train: [0/170 (0%)]	Loss: 5.370792
Epoch: 6/10. Train set: Average loss: 1.0096
Epoch: 6/10. Validation set: Average loss: 0.6983
Train: [0/170 (0%)]	Loss: 0.806775
Epoch: 7/10. Train set: Average loss: 0.9206
Epoch: 7/10. Validation set: Average loss: 0.1207
Train: [0/170 (0%)]	Loss: 0.086260
Epoch: 8/10. Train set: Average loss: 0.1156
Epoch: 8/10. Validation set: Average loss: 0.0930
Train: [0/170 (0%)]	Loss: 0.044900
Epoch: 9/10. Train set: Average loss: 0.1161
Epoch: 9/10. Validation set: Average loss: 0.0777
Train: [0/170 (0%)]	Loss: 0.082760
Epoch: 10/10. Train set: Average loss: 0.0652
Epoch: 10/10. Validation set: Average loss: 0.0642
plt.plot(range(0, n_epochs), train_loss, 'rx-')
plt.plot(range(0, n_epochs), val_loss, 'bx-')_____no_output_____train_embeddings_cl, train_labels_cl = vis.extract_embeddings(train_loader, model)
vis.plot_embeddings(train_embeddings_cl, train_labels_cl, classes)_____no_output_____val_embeddings_baseline, val_labels_baseline = vis.extract_embeddings(test_loader, model)
vis.plot_embeddings(val_embeddings_baseline, val_labels_baseline, classes)_____no_output_____

## Activation Maximization_____no_output_____

flat = pd.Series(train_dataset.data.values.ravel())
flat.describe()_____no_output_____

### From
https://github.com/MisaOgura/flashtorch/blob/master/flashtorch/activmax/gradient_ascent.py_____no_output_____

model()_____no_output_____for name_embed, embed in model._modules.items():
    for name_fc, fc in embed._modules.items():
        for name, layer in fc._modules.items():
            print(name, layer)0 Linear(in_features=60483, out_features=2000, bias=True)
1 PReLU(num_parameters=1)
2 Linear(in_features=2000, out_features=500, bias=True)
3 PReLU(num_parameters=1)
4 Linear(in_features=500, out_features=250, bias=True)
5 PReLU(num_parameters=1)
6 Linear(in_features=250, out_features=100, bias=True)
7 PReLU(num_parameters=1)
8 Linear(in_features=100, out_features=50, bias=True)
9 PReLU(num_parameters=1)
10 Linear(in_features=50, out_features=10, bias=True)
11 PReLU(num_parameters=1)
12 Linear(in_features=10, out_features=2, bias=True)
import torch.nn as nn_____no_output_____import copy
tmp_model = copy.deepcopy(model)_____no_output_____activations = {}
def hook_fn(m, i, o):
    activations[m] = o _____no_output_____def get_all_layers(net):
    for name, layer in net._modules.items():
    #If it is a sequential, don't register a hook on it
    # but recursively register hook on all it's module children
        if isinstance(layer, EmbeddingNet) or isinstance(layer, nn.Sequential):
            get_all_layers(layer)
#         if isinstance(layer, nn.Sequential):
#             get_all_layers(layer)
        else:
            print(layer)
            # it's a non sequential. Register a hook
            layer.register_forward_hook(hook_fn)_____no_output_____get_all_layers(tmp_model)Linear(in_features=60483, out_features=2000, bias=True)
PReLU(num_parameters=1)
Linear(in_features=2000, out_features=500, bias=True)
PReLU(num_parameters=1)
Linear(in_features=500, out_features=250, bias=True)
PReLU(num_parameters=1)
Linear(in_features=250, out_features=100, bias=True)
PReLU(num_parameters=1)
Linear(in_features=100, out_features=50, bias=True)
PReLU(num_parameters=1)
Linear(in_features=50, out_features=10, bias=True)
PReLU(num_parameters=1)
Linear(in_features=10, out_features=2, bias=True)
siamese_test_dataset.test_labels
siamese_test_dataset.labels_____no_output_____siamese_test_dataset[-1]
(dat1, dat2), targ = siamese_test_dataset[-1]_____no_output_____tmp_model.cuda()_____no_output_____out1, out2 = tmp_model.forward(dat1.cuda(), dat2.cuda())_____no_output_____for key,val in activations.items():
    print(key, val.shape, "\n\t\t", val)Linear(in_features=60483, out_features=2000, bias=True) torch.Size([2000])
		 tensor([-1959.8584,  -915.4311, -1742.4884,  ..., -2054.9080, -2072.6201,
        -1417.6904], device='cuda:0', grad_fn=<AddBackward0>)
PReLU(num_parameters=1) torch.Size([2000])
		 tensor([-422.1277, -197.1718, -375.3091,  ..., -442.6001, -446.4151,
        -305.3519], device='cuda:0', grad_fn=<PreluBackward>)
Linear(in_features=2000, out_features=500, bias=True) torch.Size([500])
		 tensor([  572.7196, -4703.2534, -2647.1064,  1574.2081, -1687.2845,  -792.9661,
        -9323.7090,  3969.4727, -6689.9917, -3620.5178, -4267.5493,   853.0196,
        -5019.8970,  1684.7874, -3057.4758, -1364.5518, -5555.0234, -3903.5127,
        -3745.5305,  -620.9858, -1022.8352,   945.7989, -4086.0432, -2508.0491,
         1977.3368, -5146.5083,  -262.2778, -3711.0894,  -966.7634, -3135.9282,
         5586.8315, -1975.5148,  2441.5288, -5492.5596, -3703.6726, -1214.9739,
        -2721.6372, -7003.0649, -1281.4534,   634.8958,   434.2592, -3529.6042,
        -2463.0474, -1849.0328, -6633.5923, -2117.5752, -3834.3367,  -917.3609,
        -3852.7646, -4978.9043, -3630.6792, -5027.5186,  4658.8481,  1470.2103,
        -1987.2683, -1952.0148, -7526.3989, -4117.9580, -3679.9265, -3883.7119,
         -898.9501,  -578.3416, -3910.0837, -1514.7126, -2664.0256, -4496.9365,
        -3305.5520, -6023.5815, -5503.1753, -4244.7520, -5382.3320,  1378.1962,
        -4772.9844, -2444.2891, -3597.9800, -6319.1909, -2721.1174, -4191.1118,
         -742.2968, -3615.5969, -1061.2701, -2949.0942,  -966.5499, -3741.2896,
        -2848.1953, -2858.8477, -3117.2366, -7910.5176,    49.6671, -2439.4683,
         2386.1875, -5531.1484,  5859.3467, -3585.2268, -4702.4141,  1944.2827,
         1517.1484, -4254.1260,  1836.0527, -3181.3989,  1056.5229, -4370.5625,
        -6303.0747, -5611.5557,  1257.0552, -1599.6628, -1021.2236,  3877.3760,
        -3449.9556, -4166.8047,  4651.4980,  3390.7949, -3203.1726, -8460.1230,
        -4764.5654, -7264.4155, -5125.1089, -4078.6379, -5767.0884, -3865.8689,
         3097.6362, -3998.6868, -6196.4658, -3435.4866, -1761.8948, -1970.7500,
        -2230.1418,  2726.6938, -4800.5825, -6882.7051, -3067.9453, -1138.9856,
        -3882.2065, -2341.2214,   689.5919,  2036.8890, -3317.8459, -2145.7458,
        -5327.3813, -4678.6113, -4136.0361, -3437.0117, -4410.3711,  1157.3915,
        -5969.5386,   -27.1888, -2603.2124, -2619.0916,   239.1495, -1873.5535,
        -3286.3845, -3846.8860, -4487.5581, -5967.7993, -3153.8809, -8179.6704,
        -5043.8389, -2735.2979, -5780.7817, -3450.7119,  1393.4111,  -880.0508,
        -5131.4570, -6474.3315, -1695.3650, -5158.4053, -2026.1177, -1095.6416,
        -5462.1567, -8578.8584,  -746.8298, -2000.0626,  3614.5671, -7838.6323,
         8830.0215, -2473.4790,  -306.2760, -1997.0825, -3028.9263, -1028.7603,
        -7019.7524,  2811.8728, -2610.8774, -8260.4648, -1006.9952, -2958.8171,
        -5253.6704, -1616.2411, -3304.6760, -1458.2065, -3352.6199, -1196.8135,
        -2688.4780, -4888.0884, -1693.2747, -3600.0894, -1083.6365, -7328.8262,
        -6014.1914,  3923.7756, -5146.3213, -3211.5620, -2071.3191, -3145.4292,
          385.6330, -2229.2559, -1884.2341, -1419.1515, -5055.0112, -3307.0168,
        -1458.7087, -1285.0502,  -664.7488, -5536.2026, -1766.4418,  2481.9329,
        -1464.1110, -6929.3354,  3454.9895,  6461.3452,  2360.5662, -6835.6177,
        -6485.3022, -4505.7256, -3427.2725,  1103.5564, -2946.3440, -2190.0732,
        -6097.6470, -2164.5093,  -827.6810, -2220.7441, -2988.8286,  7677.7275,
        -3781.5330, -2110.2852, -4183.8271, -2744.1973, -2496.4331, -1509.1364,
        -1055.7535, -1598.5815, -2338.8291, -1495.9375,  5460.8252, -4031.6606,
        -3940.3816,  8896.9932, -3814.5969, -6402.6016, -4074.7507,  -994.0394,
        -6112.5376, -3812.1479,  7304.5737,  4258.3394, -7784.0659, -6980.0996,
        -1890.6809, -3407.7253, -4115.0342, -6519.1797, -3528.9832, -3017.7195,
        -5070.1460, -6407.7124, -3460.6289, -1495.8878, -4096.2085, -2334.6504,
         3459.6548, -3671.3074,   419.9745, -1899.5361, -4279.9858, -5779.1318,
        -5510.7759, -9154.7393, -3525.4873, -1165.5781,  -392.7631, -1582.9685,
        -3781.3528, -1773.8490,  3394.4500, -1731.3486, -4230.0420, -6015.8193,
         -943.2582, -4256.4004, -1636.1223, -6698.9736,  2980.9951, -2767.1858,
        -1876.3340,  3757.5752, -5264.8506, -6697.5625, -4820.1748, -4622.5488,
        -1424.0354, -2710.3728, -1506.2888, -2842.1650, -2269.3152, -1386.7312,
        -1734.0021, -1762.5769, -4017.9614, -1073.9397, -3989.8591, -3666.6311,
        -2761.6577, -2870.0815, -1152.8917,  5619.5713, -1576.5283, -6360.6113,
        -1256.1010,  4417.3906, -1443.5077, -4755.1997, -2807.8582,  6974.4048,
        -1396.3140,   261.5494, -1619.2734, -2795.5930, -4244.8999, -3576.0891,
        -2440.4797, -3829.4016, -7066.5098, -5737.7471, -1071.1550, -5720.4331,
        -3055.4802, -5155.6753, -2408.0513, -6467.6289,  -681.0384,  2943.3315,
         1531.9037, -7139.0684, -1172.9774, -4157.4810, -1235.0043, -8219.3916,
        -6832.7212, -4049.3420, -2594.5164,  3063.0547, -2172.4336, -2867.4067,
           17.3359, -3940.8894, -3406.5027, -4780.9736, -7631.7725, -4353.4575,
          357.5221, -7992.1709,  -849.6646, -1958.6875, -1833.4469, -3609.6660,
         8066.9756, -3637.8862, -5483.9546, -3272.1265,  1661.6428, -4438.6099,
        -2215.4224, -1094.4885, -5596.5259, -2003.4888, -3457.6460,  4284.8838,
        -1741.0879, -3167.4509, -3863.2188, -4457.4023, -2216.6414, -1633.1488,
         5681.4189,  3202.1196, -6750.9346, -2477.0559,  8058.2261, -2893.5999,
        -4546.8486, -6685.2358, -6217.4868,   966.8078, -2763.7998, -1433.1746,
        -6443.2939,  8836.2441,  -707.0114, -7734.6328, -1608.2061,  -835.9663,
        -1327.8142, -6458.7007, -2187.8596, -3457.0803, -5360.7905,  2520.4939,
        -2819.6631, -3555.7258,  -762.7374,   918.0925, -2821.6318,  1153.3087,
        -4071.3914, -2614.0908, -4890.1812, -3183.0356, -3662.1648, -5097.9062,
        -4185.0137, -2626.1709, -1120.2168, -3203.3582, -3627.7629, -8262.9072,
        -4702.3643, -7498.3159, -1601.4357, -5405.0820,  4735.5854, -6062.3164,
        -1209.2518, -1866.2814, -3703.8950, -1382.6796,  -703.6142, -5092.9771,
        -1320.5242, -2018.4199, -4413.7544, -3381.4797, -4148.0186, -3601.5254,
         3079.5256, -1410.4429,  6020.3955, -4081.8740,  4544.2139, -6399.5283,
        -7055.1807,  -852.9716,  3472.3613,  1597.4303, -1290.3784,  -973.9455,
        -4938.2734, -5341.6235, -3510.5078,  -935.6163, -1945.1602, -4066.5027,
        -1531.3815, -1300.2158,  2256.7117,  -455.6976, -5105.1646, -4083.1997,
        -1496.2130, -6351.1182, -1705.6841, -4183.3018, -1716.4410, -3091.1448,
        -6911.1787, -2870.8232, -3900.3196,  3777.8298,  5110.2036, -5592.5176,
        -3183.8538, -5397.2588, -5693.8281,   601.0595, -4877.4897, -2706.0310,
        -2934.0435, -1788.6935, -4504.5679, -2635.4353, -2113.7251,  7768.5957,
         1322.3715, -5133.9316,  1096.0374, -4315.8809,  7212.5728, -8191.6147,
        -1187.5363,  2151.3101], device='cuda:0', grad_fn=<AddBackward0>)
PReLU(num_parameters=1) torch.Size([500])
		 tensor([ 5.7272e+02, -9.4286e+02, -5.3066e+02,  1.5742e+03, -3.3825e+02,
        -1.5897e+02, -1.8691e+03,  3.9695e+03, -1.3411e+03, -7.2580e+02,
        -8.5551e+02,  8.5302e+02, -1.0063e+03,  1.6848e+03, -6.1293e+02,
        -2.7355e+02, -1.1136e+03, -7.8254e+02, -7.5086e+02, -1.2449e+02,
        -2.0505e+02,  9.4580e+02, -8.1913e+02, -5.0279e+02,  1.9773e+03,
        -1.0317e+03, -5.2579e+01, -7.4396e+02, -1.9381e+02, -6.2866e+02,
         5.5868e+03, -3.9603e+02,  2.4415e+03, -1.1011e+03, -7.4247e+02,
        -2.4357e+02, -5.4561e+02, -1.4039e+03, -2.5689e+02,  6.3490e+02,
         4.3426e+02, -7.0758e+02, -4.9377e+02, -3.7067e+02, -1.3298e+03,
        -4.2451e+02, -7.6867e+02, -1.8390e+02, -7.7236e+02, -9.9812e+02,
        -7.2784e+02, -1.0079e+03,  4.6588e+03,  1.4702e+03, -3.9839e+02,
        -3.9132e+02, -1.5088e+03, -8.2552e+02, -7.3771e+02, -7.7857e+02,
        -1.8021e+02, -1.1594e+02, -7.8385e+02, -3.0365e+02, -5.3406e+02,
        -9.0150e+02, -6.6266e+02, -1.2075e+03, -1.1032e+03, -8.5094e+02,
        -1.0790e+03,  1.3782e+03, -9.5684e+02, -4.9001e+02, -7.2129e+02,
        -1.2668e+03, -5.4550e+02, -8.4019e+02, -1.4881e+02, -7.2482e+02,
        -2.1275e+02, -5.9120e+02, -1.9376e+02, -7.5001e+02, -5.7098e+02,
        -5.7311e+02, -6.2491e+02, -1.5858e+03,  4.9667e+01, -4.8904e+02,
         2.3862e+03, -1.1088e+03,  5.8593e+03, -7.1873e+02, -9.4269e+02,
         1.9443e+03,  1.5171e+03, -8.5282e+02,  1.8361e+03, -6.3777e+02,
         1.0565e+03, -8.7616e+02, -1.2636e+03, -1.1249e+03,  1.2571e+03,
        -3.2068e+02, -2.0472e+02,  3.8774e+03, -6.9161e+02, -8.3532e+02,
         4.6515e+03,  3.3908e+03, -6.4214e+02, -1.6960e+03, -9.5515e+02,
        -1.4563e+03, -1.0274e+03, -8.1764e+02, -1.1561e+03, -7.7499e+02,
         3.0976e+03, -8.0161e+02, -1.2422e+03, -6.8871e+02, -3.5321e+02,
        -3.9508e+02, -4.4708e+02,  2.7267e+03, -9.6237e+02, -1.3798e+03,
        -6.1503e+02, -2.2833e+02, -7.7826e+02, -4.6934e+02,  6.8959e+02,
         2.0369e+03, -6.6513e+02, -4.3016e+02, -1.0680e+03, -9.3792e+02,
        -8.2915e+02, -6.8902e+02, -8.8414e+02,  1.1574e+03, -1.1967e+03,
        -5.4505e+00, -5.2186e+02, -5.2505e+02,  2.3915e+02, -3.7559e+02,
        -6.5882e+02, -7.7118e+02, -8.9962e+02, -1.1964e+03, -6.3226e+02,
        -1.6398e+03, -1.0111e+03, -5.4834e+02, -1.1589e+03, -6.9176e+02,
         1.3934e+03, -1.7642e+02, -1.0287e+03, -1.2979e+03, -3.3987e+02,
        -1.0341e+03, -4.0617e+02, -2.1964e+02, -1.0950e+03, -1.7198e+03,
        -1.4972e+02, -4.0095e+02,  3.6146e+03, -1.5714e+03,  8.8300e+03,
        -4.9586e+02, -6.1399e+01, -4.0035e+02, -6.0721e+02, -2.0624e+02,
        -1.4072e+03,  2.8119e+03, -5.2340e+02, -1.6560e+03, -2.0187e+02,
        -5.9315e+02, -1.0532e+03, -3.2401e+02, -6.6249e+02, -2.9233e+02,
        -6.7210e+02, -2.3992e+02, -5.3896e+02, -9.7991e+02, -3.3945e+02,
        -7.2171e+02, -2.1724e+02, -1.4692e+03, -1.2057e+03,  3.9238e+03,
        -1.0317e+03, -6.4382e+02, -4.1524e+02, -6.3056e+02,  3.8563e+02,
        -4.4690e+02, -3.7773e+02, -2.8450e+02, -1.0134e+03, -6.6296e+02,
        -2.9243e+02, -2.5761e+02, -1.3326e+02, -1.1098e+03, -3.5412e+02,
         2.4819e+03, -2.9351e+02, -1.3891e+03,  3.4550e+03,  6.4613e+03,
         2.3606e+03, -1.3703e+03, -1.3001e+03, -9.0326e+02, -6.8706e+02,
         1.1036e+03, -5.9065e+02, -4.3904e+02, -1.2224e+03, -4.3392e+02,
        -1.6592e+02, -4.4519e+02, -5.9917e+02,  7.6777e+03, -7.5808e+02,
        -4.2305e+02, -8.3873e+02, -5.5013e+02, -5.0046e+02, -3.0254e+02,
        -2.1165e+02, -3.2047e+02, -4.6886e+02, -2.9989e+02,  5.4608e+03,
        -8.0822e+02, -7.8993e+02,  8.8970e+03, -7.6471e+02, -1.2835e+03,
        -8.1686e+02, -1.9927e+02, -1.2254e+03, -7.6422e+02,  7.3046e+03,
         4.2583e+03, -1.5605e+03, -1.3993e+03, -3.7902e+02, -6.8314e+02,
        -8.2494e+02, -1.3069e+03, -7.0745e+02, -6.0496e+02, -1.0164e+03,
        -1.2846e+03, -6.9375e+02, -2.9988e+02, -8.2116e+02, -4.6803e+02,
         3.4597e+03, -7.3599e+02,  4.1997e+02, -3.8080e+02, -8.5801e+02,
        -1.1585e+03, -1.1047e+03, -1.8352e+03, -7.0675e+02, -2.3366e+02,
        -7.8737e+01, -3.1734e+02, -7.5805e+02, -3.5560e+02,  3.3944e+03,
        -3.4708e+02, -8.4799e+02, -1.2060e+03, -1.8909e+02, -8.5328e+02,
        -3.2799e+02, -1.3429e+03,  2.9810e+03, -5.5474e+02, -3.7615e+02,
         3.7576e+03, -1.0554e+03, -1.3427e+03, -9.6630e+02, -9.2668e+02,
        -2.8548e+02, -5.4335e+02, -3.0196e+02, -5.6977e+02, -4.5493e+02,
        -2.7800e+02, -3.4761e+02, -3.5334e+02, -8.0548e+02, -2.1529e+02,
        -7.9984e+02, -7.3505e+02, -5.5363e+02, -5.7536e+02, -2.3112e+02,
         5.6196e+03, -3.1605e+02, -1.2751e+03, -2.5181e+02,  4.4174e+03,
        -2.8938e+02, -9.5327e+02, -5.6289e+02,  6.9744e+03, -2.7992e+02,
         2.6155e+02, -3.2461e+02, -5.6043e+02, -8.5097e+02, -7.1690e+02,
        -4.8924e+02, -7.6768e+02, -1.4166e+03, -1.1502e+03, -2.1473e+02,
        -1.1468e+03, -6.1253e+02, -1.0336e+03, -4.8274e+02, -1.2966e+03,
        -1.3653e+02,  2.9433e+03,  1.5319e+03, -1.4312e+03, -2.3515e+02,
        -8.3345e+02, -2.4758e+02, -1.6477e+03, -1.3698e+03, -8.1177e+02,
        -5.2012e+02,  3.0631e+03, -4.3551e+02, -5.7483e+02,  1.7336e+01,
        -7.9003e+02, -6.8290e+02, -9.5844e+02, -1.5299e+03, -8.7274e+02,
         3.5752e+02, -1.6022e+03, -1.7033e+02, -3.9266e+02, -3.6755e+02,
        -7.2363e+02,  8.0670e+03, -7.2929e+02, -1.0994e+03, -6.5596e+02,
         1.6616e+03, -8.8981e+02, -4.4412e+02, -2.1941e+02, -1.1219e+03,
        -4.0164e+02, -6.9315e+02,  4.2849e+03, -3.4903e+02, -6.3498e+02,
        -7.7446e+02, -8.9357e+02, -4.4437e+02, -3.2740e+02,  5.6814e+03,
         3.2021e+03, -1.3534e+03, -4.9657e+02,  8.0582e+03, -5.8008e+02,
        -9.1150e+02, -1.3402e+03, -1.2464e+03,  9.6681e+02, -5.5406e+02,
        -2.8731e+02, -1.2917e+03,  8.8362e+03, -1.4173e+02, -1.5506e+03,
        -3.2240e+02, -1.6759e+02, -2.6619e+02, -1.2948e+03, -4.3860e+02,
        -6.9304e+02, -1.0747e+03,  2.5205e+03, -5.6526e+02, -7.1281e+02,
        -1.5291e+02,  9.1809e+02, -5.6565e+02,  1.1533e+03, -8.1619e+02,
        -5.2405e+02, -9.8033e+02, -6.3810e+02, -7.3415e+02, -1.0220e+03,
        -8.3897e+02, -5.2647e+02, -2.2457e+02, -6.4218e+02, -7.2726e+02,
        -1.6565e+03, -9.4268e+02, -1.5032e+03, -3.2104e+02, -1.0836e+03,
         4.7356e+03, -1.2153e+03, -2.4242e+02, -3.7413e+02, -7.4252e+02,
        -2.7719e+02, -1.4105e+02, -1.0210e+03, -2.6472e+02, -4.0463e+02,
        -8.8482e+02, -6.7788e+02, -8.3155e+02, -7.2200e+02,  3.0795e+03,
        -2.8275e+02,  6.0204e+03, -8.1829e+02,  4.5442e+03, -1.2829e+03,
        -1.4143e+03, -1.7099e+02,  3.4724e+03,  1.5974e+03, -2.5868e+02,
        -1.9525e+02, -9.8997e+02, -1.0708e+03, -7.0375e+02, -1.8756e+02,
        -3.8995e+02, -8.1521e+02, -3.0700e+02, -2.6065e+02,  2.2567e+03,
        -9.1353e+01, -1.0234e+03, -8.1856e+02, -2.9995e+02, -1.2732e+03,
        -3.4194e+02, -8.3862e+02, -3.4409e+02, -6.1968e+02, -1.3855e+03,
        -5.7551e+02, -7.8189e+02,  3.7778e+03,  5.1102e+03, -1.1211e+03,
        -6.3827e+02, -1.0820e+03, -1.1414e+03,  6.0106e+02, -9.7779e+02,
        -5.4248e+02, -5.8819e+02, -3.5858e+02, -9.0303e+02, -5.2832e+02,
        -4.2374e+02,  7.7686e+03,  1.3224e+03, -1.0292e+03,  1.0960e+03,
        -8.6520e+02,  7.2126e+03, -1.6422e+03, -2.3806e+02,  2.1513e+03],
       device='cuda:0', grad_fn=<PreluBackward>)
Linear(in_features=500, out_features=250, bias=True) torch.Size([250])
		 tensor([-3576.2542, -2910.6213, -2125.1699,  3155.0562, -2926.7493,   234.3650,
        -1609.7311,  -692.2714,  1694.0576, -4118.9468,  4757.9541,   782.5114,
         6285.6094,  2360.9460, -2698.1560,  -525.4883, -2508.7910, -4126.7358,
         6795.5640, -2320.6067, -2490.3975,  3539.0317,  -348.1837, -3857.1233,
        -2363.3508,   -78.7871,  -596.4022, 10003.9336, -1156.2139,   862.2061,
        -2541.7043,  1507.2841, -2676.5820, -1763.0021, -2115.4900, -1256.7489,
         4448.0483, -1231.2062, -2893.5598, -2869.0737, -1486.8424,  1340.0104,
        -3424.5566, -2513.8801, -1402.1711,  1282.6060, -1172.5215, -2837.1487,
        -1770.3126, -2235.5146, -1908.9391,  6321.4814,  1217.0509, -4250.9229,
        -3049.0088, -4295.8633, -3033.8267,  -990.4301,  -358.1209,  -565.2449,
         -922.6882, -3315.8298, -3153.5635,   530.9604, -1598.6296, -1264.5537,
        -2482.9736, -1340.5741,  -988.0959,  -639.8603,  -571.6329, -1537.5465,
        -1185.6926,  1418.4871, -1285.5808, -3962.7244, -3325.3552, -2543.2729,
         4201.6973, -1464.6360,  5454.1045, -1600.0897,  3210.9438,  -748.6773,
          421.7250, -1445.6230,  1582.4493, -2434.7029, -1932.2507,   193.0994,
         -640.2206,   402.6122,  -859.0637, -1550.4822, -1698.7756,  -565.0869,
          242.8593,  -402.7460,  2667.9148, -1720.1403, -2955.3987,  3439.9131,
         4478.2915, -3398.5203, -1266.9277,  1403.1245, -3033.6790, -3094.9792,
        -2812.7866, -3124.5356,  3174.9778, -2743.7244, -1165.3910,  -984.7109,
        -4058.0657, -5195.4526, -4918.1895, -3945.2725, -3608.3694,  1033.9290,
        -2692.2446,   595.5238,  -365.4036, -1434.8191,  4301.7207, -2027.3506,
        -2825.5559,  -574.6487,  3822.7051,  -622.9424,   660.8259, -4379.8125,
        -1029.9341, -4616.6323, -2574.3848,  2041.7178, -1051.7408, -1385.4191,
        -2671.7029, -3325.7463, -4350.5586, -1387.2740,  4432.1470, -4337.4644,
        -2522.4285, -1656.1117,  1751.8319, -3580.4568, -1816.5787, -2481.3933,
        -3032.9915, -1039.4064, -3040.4062, -2457.6431, -3630.1846,  3650.2185,
        -2339.2065, -1277.6124, -2608.5535, -1840.8607,  1757.4323,  -726.6791,
        -4101.5381, -2722.4326, -4323.1479, -1789.5752, -4641.0278, -2394.6855,
        -3603.7688,  3030.0303, -1597.6635,  -636.5080, -3533.3501, -2063.2764,
        -1438.0204, -2853.5383,  2402.2520, -1021.9927, -2194.6677, -2898.5935,
         -861.2507,  7961.1958, -2383.0000, -1193.1178,  4001.6868,  3456.7163,
        -1908.3501, -4345.8765, -3320.1304, -3144.9717, -3122.7061,  1964.3755,
         -591.6462,  3028.5789,   229.7841, -1418.2717,  -146.5800, -2977.3801,
         5314.5381, -1279.7058,   -64.9887, -4214.6953, -2933.0024,  -198.8483,
        -4085.9988,    28.4915,  2463.7034, -3603.0361, -1394.9564,  1659.5216,
        -1792.0360, -3750.0310, -1538.4462, -1764.1428, -2796.0403,  3758.4829,
        -1265.9873, -1810.5266,  3898.9507, -1477.3784, -2724.8257, -1344.6112,
         6359.6914, -1211.9993, -1804.6315,  -976.2101,  5595.1328, -4255.0806,
         6044.4097, -1996.8929, -4474.2993, -3196.6504, -1875.7843,  1970.6863,
        -2506.8513,  2993.6648, -1643.4607, -2375.5789, -4214.8364, -4554.7197,
        -3783.0525, -2789.5024,   307.1060, -2297.0137, -3403.8171, -1377.3927,
        -1194.9435,  3157.4243, -2385.1162,   361.1021], device='cuda:0',
       grad_fn=<AddBackward0>)
PReLU(num_parameters=1) torch.Size([250])
		 tensor([ -710.7701,  -578.4775,  -422.3713,  3155.0562,  -581.6829,   234.3650,
         -319.9294,  -137.5869,  1694.0576,  -818.6287,  4757.9541,   782.5114,
         6285.6094,  2360.9460,  -536.2506,  -104.4393,  -498.6149,  -820.1767,
         6795.5640,  -461.2138,  -494.9592,  3539.0317,   -69.2005,  -766.5920,
         -469.7091,   -15.6587,  -118.5332, 10003.9336,  -229.7941,   862.2061,
         -505.1563,  1507.2841,  -531.9629,  -350.3915,  -420.4475,  -249.7752,
         4448.0483,  -244.6986,  -575.0865,  -570.2200,  -295.5056,  1340.0104,
         -680.6207,  -499.6263,  -278.6774,  1282.6060,  -233.0352,  -563.8750,
         -351.8445,  -444.3020,  -379.3961,  6321.4814,  1217.0509,  -844.8585,
         -605.9816,  -853.7903,  -602.9642,  -196.8451,   -71.1755,  -112.3408,
         -183.3816,  -659.0115,  -626.7615,   530.9604,  -317.7230,  -251.3264,
         -493.4838,  -266.4352,  -196.3812,  -127.1704,  -113.6104,  -305.5829,
         -235.6530,  1418.4871,  -255.5054,  -787.5799,  -660.9047,  -505.4681,
         4201.6973,  -291.0921,  5454.1045,  -318.0132,  3210.9438,  -148.7974,
          421.7250,  -287.3134,  1582.4493,  -483.8901,  -384.0292,   193.0994,
         -127.2420,   402.6122,  -170.7364,  -308.1538,  -337.6267,  -112.3094,
          242.8593,   -80.0446,  2667.9148,  -341.8729,  -587.3769,  3439.9131,
         4478.2915,  -675.4460,  -251.7982,  1403.1245,  -602.9348,  -615.1181,
         -559.0331,  -620.9924,  3174.9778,  -545.3072,  -231.6181,  -195.7084,
         -806.5287, -1032.5811,  -977.4758,  -784.1114,  -717.1529,  1033.9290,
         -535.0757,   595.5238,   -72.6229,  -285.1661,  4301.7207,  -402.9300,
         -561.5710,  -114.2097,  3822.7051,  -123.8080,   660.8259,  -870.4750,
         -204.6964,  -917.5422,  -511.6515,  2041.7178,  -209.0304,  -275.3480,
         -530.9932,  -660.9824,  -864.6608,  -275.7167,  4432.1470,  -862.0584,
         -501.3253,  -329.1474,  1751.8319,  -711.6053,  -361.0397,  -493.1697,
         -602.7982,  -206.5790,  -604.2719,  -488.4494,  -721.4886,  3650.2185,
         -464.9105,  -253.9217,  -518.4424,  -365.8657,  1757.4323,  -144.4254,
         -815.1688,  -541.0756,  -859.2131,  -355.6729,  -922.3907,  -475.9368,
         -716.2385,  3030.0303,  -317.5309,  -126.5041,  -702.2430,  -410.0702,
         -285.8024,  -567.1324,  2402.2520,  -203.1181,  -436.1838,  -576.0870,
         -171.1711,  7961.1958,  -473.6143,  -237.1287,  4001.6868,  3456.7163,
         -379.2790,  -863.7303,  -659.8662,  -625.0540,  -620.6287,  1964.3755,
         -117.5880,  3028.5789,   229.7841,  -281.8774,   -29.1323,  -591.7456,
         5314.5381,  -254.3378,   -12.9163,  -837.6584,  -582.9257,   -39.5205,
         -812.0803,    28.4915,  2463.7034,  -716.0929,  -277.2435,  1659.5216,
         -356.1619,  -745.3077,  -305.7617,  -350.6183,  -555.7048,  3758.4829,
         -251.6113,  -359.8369,  3898.9507,  -293.6247,  -541.5511,  -267.2375,
         6359.6914,  -240.8813,  -358.6653,  -194.0189,  5595.1328,  -845.6849,
         6044.4097,  -396.8766,  -889.2540,  -635.3250,  -372.8067,  1970.6863,
         -498.2294,  2993.6648,  -326.6330,  -472.1394,  -837.6865,  -905.2372,
         -751.8707,  -554.4055,   307.1060,  -456.5248,  -676.4987,  -273.7528,
         -237.4915,  3157.4243,  -474.0349,   361.1021], device='cuda:0',
       grad_fn=<PreluBackward>)
Linear(in_features=250, out_features=100, bias=True) torch.Size([100])
		 tensor([-1653.6443,  1007.6517, -1194.8662, -2894.0132,  -551.9333,  1158.0707,
         -688.5219, -2029.1406, -2135.4185,  3222.4788,  2530.6819, -2827.0669,
        -3485.2664,  1205.4763,  -565.0276,  -847.1330, -1849.6372,  1894.7200,
        -3538.7864, -2439.6365, -1002.6965, -2332.8909,  3924.0120, -1812.5946,
         2605.0972, -1659.3237, -1586.3112, -2844.9050, -1314.1910,  1148.8649,
         2833.4773,  1812.9878,   867.6544, -2151.7883, -4159.5483,  -994.9314,
        -1092.6354, -2287.0857, -1229.7512,   248.3469, -1187.3593,  3228.2585,
        -2208.2290,    84.8848, -2165.7849,  6877.3872,  1181.1155,  -258.9266,
        -2166.1892,  5251.7427, -1536.0157, -2065.2339, -4615.0591,  4799.0513,
        -1327.2201,   840.9239, -1007.0834, -1510.3427,  -471.8027, -2688.6685,
         -492.5772, -1227.5703, -3053.3794,  2346.8669, -1073.2665, -2941.7827,
         1653.0297,  1388.4874,  -848.5579, -3933.7385,   815.1813, -1221.2501,
         -405.9483,  -404.0185, -1081.6826, -2863.3420, -1027.6982, -1455.8401,
        -3204.7798, -3209.2834, -1273.0697, -2048.8652, -1971.8081, -1554.9803,
        -2574.4036, -1111.8971, -2759.6423, -3095.9006, -2750.4365, -3159.5396,
        -1639.4934, -1557.6448, -3562.9126, -1343.4330,  -859.3905, -2237.8057,
        -2261.2957,  1138.1794,   981.4890, -3586.1248], device='cuda:0',
       grad_fn=<AddBackward0>)
PReLU(num_parameters=1) torch.Size([100])
		 tensor([ -363.7857,  1007.6517,  -262.8590,  -636.6549,  -121.4200,  1158.0707,
         -151.4682,  -446.3913,  -469.7714,  3222.4788,  2530.6819,  -621.9274,
         -766.7249,  1205.4763,  -124.3006,  -186.3611,  -406.9023,  1894.7200,
         -778.4988,  -536.6964,  -220.5835,  -513.2134,  3924.0120,  -398.7533,
         2605.0972,  -365.0352,  -348.9731,  -625.8516,  -289.1093,  1148.8649,
         2833.4773,  1812.9878,   867.6544,  -473.3726,  -915.0604,  -218.8753,
         -240.3692,  -503.1367,  -270.5334,   248.3469,  -261.2076,  3228.2585,
         -485.7890,    84.8848,  -476.4517,  6877.3872,  1181.1155,   -56.9614,
         -476.5407,  5251.7427,  -337.9086,  -454.3315, -1015.2684,  4799.0513,
         -291.9756,   840.9239,  -221.5486,  -332.2608,  -103.7920,  -591.4811,
         -108.3622,  -270.0536,  -671.7139,  2346.8669,  -236.1082,  -647.1638,
         1653.0297,  1388.4874,  -186.6745,  -865.3844,   815.1813,  -268.6632,
          -89.3047,   -88.8802,  -237.9597,  -629.9075,  -226.0837,  -320.2708,
         -705.0206,  -706.0113,  -280.0630,  -450.7305,  -433.7787,  -342.0806,
         -566.3439,  -244.6066,  -607.0946,  -681.0682,  -605.0695,  -695.0681,
         -360.6727,  -342.6668,  -783.8063,  -295.5423,  -189.0576,  -492.2956,
         -497.4632,  1138.1794,   981.4890,  -788.9128], device='cuda:0',
       grad_fn=<PreluBackward>)
Linear(in_features=100, out_features=50, bias=True) torch.Size([50])
		 tensor([-2725.3862,  1302.4177,   140.1605,  -838.4193, -1461.0621,  2508.7908,
          708.8267,  -704.0532,  -379.7896,  -400.9212,   276.0836, -1020.0711,
         -688.1002,  3036.7478, -2571.3633,   487.7722, -1608.2797,  -588.9832,
        -1918.1100, -1596.2020,   741.2000,   292.3058, -1038.4480,   751.3865,
        -1742.6245, -1366.7766,   359.9605,   996.9689, -1342.4202, -1244.7686,
        -1373.9982,  -438.1609, -2013.5051, -1792.4235, -1209.4238,   502.9578,
        -1346.5151, -1716.4722,  -158.0659,  -742.0252,  1100.9381,  -912.7789,
        -1132.2008, -1998.2372, -1206.1855,  -694.7863,  -800.3167,   156.7986,
          167.7683,  -715.1786], device='cuda:0', grad_fn=<AddBackward0>)
PReLU(num_parameters=1) torch.Size([50])
		 tensor([-618.1693, 1302.4177,  140.1605, -190.1694, -331.3966, 2508.7908,
         708.8267, -159.6926,  -86.1435,  -90.9365,  276.0836, -231.3715,
        -156.0742, 3036.7478, -583.2339,  487.7722, -364.7883, -133.5926,
        -435.0637, -362.0489,  741.2000,  292.3058, -235.5397,  751.3865,
        -395.2603, -310.0109,  359.9605,  996.9689, -304.4864, -282.3371,
        -311.6488,  -99.3832, -456.7011, -406.5556, -274.3203,  502.9578,
        -305.4152, -389.3284,  -35.8524, -168.3054, 1100.9381, -207.0356,
        -256.8046, -453.2381, -273.5858, -157.5907, -181.5270,  156.7986,
         167.7683, -162.2161], device='cuda:0', grad_fn=<PreluBackward>)
Linear(in_features=50, out_features=10, bias=True) torch.Size([10])
		 tensor([-663.2064,  293.6829, -500.8496, -581.4441, -106.3747, -116.6517,
        -122.2263, -787.7101, -979.7127, -460.2240], device='cuda:0',
       grad_fn=<AddBackward0>)
PReLU(num_parameters=1) torch.Size([10])
		 tensor([-159.5633,  293.6829, -120.5013, -139.8918,  -25.5931,  -28.0657,
         -29.4069, -189.5181, -235.7127, -110.7270], device='cuda:0',
       grad_fn=<PreluBackward>)
Linear(in_features=10, out_features=2, bias=True) torch.Size([2])
		 tensor([-6.6892,  3.6589], device='cuda:0', grad_fn=<AddBackward0>)
out2_____no_output_____# need to record activations at final layer
# then optimize random_____no_output_____