# Notebook from Jared-Luxton/Fukushima-Nuclear-Disaster-Humans
Path: workflow & analyses notebooks/01_Fukushima_Project_Data_Extraction_and_Cleaning.ipynb

# for grabbing/handling data
import numpy as np
import pandas as pd
import fukushima_telomere_methods as f_tm

# incase needing to reload modules
import importlib
%load_ext autoreload
%autoreload 2
%reload_ext autoreload_____no_output_____

---

The Fukushima Project seeks to determine whether regions contaminated by radiation from the Fukushima Daiichi Nuclear Disaster can be considered safe for human repopulation. Here, we use free-roaming >200 wild boar trapped from the contaminated regions as mammalian proxies for humans. On the boar we've collected >10 types of data from chemistry, physics (radiation), and molecular biology techniques to ascertain whether boar are negatively impacted by radiation in terms of various biometrics. We have data on snake as well.

The data was generated over >4 years and exists as tabular data in various formats. Our first aim is to read in all of the data, then clean/reformat it to enable combination of the various datasets. We've focused on standardizing the format for boar sample identifications (Sample ID) to enable combination of the data.

---_____no_output_____# Aryn's Data (qPCR)
---_____no_output_____## Reading Aryn's snake data to dataframe_____no_output_____

snake_df = pd.read_excel('../excel data/raw/snake dose data-15Nov19 (2).xlsx', skiprows=4)
# snake_df.drop([20, 21, 22, 23], axis=0, inplace=True)
snake_df.rename({'Telomere Length (qPCR)':'Mean Telomere Length (qPCR)'}, axis=1, inplace=True)
snake_df.head(4)_____no_output_____snake_dose = pd.read_excel('../excel data/raw/Gerke_SupplementaryTable1_revised (1).xlsx', skiprows=3)
snake_dose
snake_dose = snake_dose[['Unnamed: 1', 'Unnamed: 6', 'Unnamed: 7', 'Unnamed: 21', 'Unnamed: 25']].copy()
snake_dose.rename({'Unnamed: 1': 'Sample ID',
                   'Unnamed: 6': 'Total length (cm)',
                   'Unnamed: 7': 'Mass (g)',
                   'Unnamed: 21': 'Total dose rates (μGy h-1) meth1',
                   'Unnamed: 25': 'Total dose rates (μGy h-1) meth2'},
                   axis=1, inplace=True)_____no_output_____snake_clean = snake_df[['Sample ID', 'Sex', 'Mean Telomere Length (qPCR)']].merge(snake_dose, on='Sample ID')
snake_clean.drop(list(range(20, 39)), inplace=True)
snake_clean.head(4)_____no_output_____# import re

# def natural_key(string_):
#     return [int(s) if s.isdigit() else s for s in re.split(r'(\d+)', string_) if s]

# list_snake_ID = list(snake_df['Sample ID'].unique())
# sorted(list_snake_ID, key=natural_key)

# # making column for radiation exposure status and encoded dummy variables from the new column
# aryn_snake_df = f_tm.readable_snake_df_dummy_variables(aryn_snake_df)_____no_output_____

## Saving Aryn's Snake dataframe for later retrieval_____no_output_____

snake_clean.to_csv('../excel data/cleaned/aryn_snake_df.csv', index=False)_____no_output_____

## Reading Aryn's Boar data to dataframe_____no_output_____

aryn_boar_dose = pd.read_excel('../excel data/raw/Kelly& Aryn boar_1Dec2018-edited-TH-14Jun20.xlsx',
                               sheet_name=0, skiprows=70)

aryn_boar_dose_raw = aryn_boar_dose.copy()
aryn_boar_dose.drop([46, 47], axis=0, inplace=True)

aryn_boar_dose['Age (months)'] = aryn_boar_dose['hours'].apply(lambda x: int(x / (24 * 30)))
aryn_boar_dose['Sex'] = aryn_boar_dose['Sex'].apply(lambda row: f_tm.male_or_female(row))

# dropping empty columns & rows, unneccessary cols
# aryn_boar_dose.drop([
#     'Unnamed: 0', 'Unnamed: 2', 'Unnamed: 3', 'Unnamed: 7',
#                      'Unnamed: 9', 'Unnamed: 11', 'Unnamed: 38', 'Cs-134 Bq/kg (dry)',
#                      'Cs-137 Bq/kg (dry)', 'EXTERNAL Dose Maximum Life Time, mGy (134+137)',
#                      'Maximum Total Life Time Dose mGy (Int+Ext,134+137)',
#                      'hours', 'Age', 'tissue', 'dry wt.'
#                     ],
#                      axis=1, inplace=True)

aryn_boar_dose = aryn_boar_dose[['Sample ID', 'Age (months)', 'Sex',
                                 'INTERNAL Life time dose, mGy 134+137',
                                 'Reasonable Total Life Time Dose mGy (Int+Ext,134+137)',
                                  'Dose rate (uGy/h) at time of capture (Int+Ext; 134+137)'
                                ]].copy()_____no_output_____aryn_boar_qPCR = pd.read_excel('../excel data/raw/boar snake master template.xlsx', sheet_name=0)
aryn_boar_qPCR.head(4)_____no_output_____

## Cleaning Aryn's boar dose/qPCR dataframes_____no_output_____

# changing errant values to np.NaN
aryn_boar_qPCR = aryn_boar_qPCR.replace('-', np.NaN)

# cleaning Sample ID of dashes & spaces, + 'GPS' & 'collar' terms
aryn_boar_qPCR['Sample ID'] = aryn_boar_qPCR['Sample ID'].apply(lambda row: f_tm.remove_dashes_space_sampleIDs(row))
aryn_boar_qPCR['Sample ID'] = aryn_boar_qPCR['Sample ID'].astype('str')

aryn_boar_dose['Sample ID'] = aryn_boar_dose['Sample ID'].apply(lambda row: f_tm.remove_dashes_space_sampleIDs(row))
aryn_boar_dose['Sample ID'] = aryn_boar_dose['Sample ID'].astype('str')_____no_output_____

## Saving Aryn's boar dose/qPCR data for later retrieval_____no_output_____

aryn_boar_dose.to_csv('../excel data/cleaned/aryn_boar_dose.csv', index=False)
aryn_boar_qPCR.to_csv('../excel data/cleaned/aryn_boar_qPCR.csv', index=False)_____no_output_____

# Cortisol Data
---_____no_output_____## Reading the boar cortisol data_____no_output_____

boar_cortisol_df = pd.read_excel('../excel data/raw/boar cortisol.xlsx')
boar_cortisol_df_raw = boar_cortisol_df.copy()
boar_cortisol_df.drop([47], axis=0, inplace=True)
boar_cortisol_df.head(4)_____no_output_____

## Prep boar cortisol data for merging with other dataframes on Sample ID_____no_output_____

# grabbing Sample IDs column in new dataframe, preserving index
prep_boar_cortisol_df = boar_cortisol_df[['Sample ID', 'Sample Number (electronic sheet)', 'Cortisol (pg/mg)']].copy()
prep_boar_cortisol_df['Sample ID'] = prep_boar_cortisol_df['Sample ID'].astype('str')

# removing dashes/spaces & terms 'GPS' + 'collar' from sample IDs
prep_boar_cortisol_df['Sample ID'] = prep_boar_cortisol_df['Sample ID'].apply(lambda row: f_tm.remove_dashes_space_sampleIDs(row))
prep_boar_cortisol_df['Sample ID'] = prep_boar_cortisol_df['Sample ID'].astype('str')_____no_output_____

## Saving prepped cortisol data for later retrieval_____no_output_____

prep_boar_cortisol_df.to_csv('../excel data/cleaned/prep_boar_cortisol_df.csv', index=False)_____no_output_____

# Kelly's Data (teloFISH, dicentrics)
---_____no_output_____## Extracting Kelly's boar telomere FISH data_____no_output_____

boar_teloFISH_list = f_tm.extract_boar_teloFISH_as_list('../excel data/raw/')Handling Telomeres - Fukushima Hyb 4 complete.xlsx...

## Formatting telomere FISH data_____no_output_____

# making a dataframe from the list
kelly_boar_teloFISH_df = pd.DataFrame(boar_teloFISH_list, columns=['Sample ID', 'teloFISH data', 'teloFISH means'])

# turning the teloFISH data, which is a series in each row containing 4800 individial telomere lengths,
# into a list in each row so that it's saved & loaded properly
kelly_boar_teloFISH_df['teloFISH data'] = kelly_boar_teloFISH_df['teloFISH data'].apply(lambda row: row.tolist())
kelly_boar_teloFISH_df['Sample ID'] = kelly_boar_teloFISH_df['Sample ID'].apply(lambda row: f_tm.remove_dashes_space_sampleIDs(row))_____no_output_____

## Saving kelly boar teloFISH dataframe for later retrieval_____no_output_____

kelly_boar_teloFISH_df.to_csv('../excel data/cleaned/kelly_boar_teloFISH_df.csv', index=False)_____no_output_____

## Loading Kelly boar dicentric data_____no_output_____

kelly_boar_dicentrics = pd.read_excel('../excel data/raw/Fukushima Dicentrics.xlsx')_____no_output_____kelly_boar_dicentrics.rename(columns={'Pig ID': 'Sample ID'}, inplace=True)
kelly_boar_dicentrics['Sample ID'] = kelly_boar_dicentrics['Sample ID'].apply(lambda row: f_tm.remove_dashes_space_sampleIDs(row))
kelly_boar_dicentrics['Average Dicentrics per cell'] = kelly_boar_dicentrics['Dicentrics']/kelly_boar_dicentrics['Total Scored']
kelly_boar_dicentrics.drop(['Dicentric Images', 'Notes', 'Dicentrics', 'Total Scored'], axis=1, inplace=True)

kelly_boar_dicentrics.head(4)_____no_output_____

## Saving Kelly boar dicentrics dataframe for later retrieval_____no_output_____

kelly_boar_dicentrics.to_csv('../excel data/cleaned/kelly_boar_dicentrics_df.csv', index=False)_____no_output_____

## Loading new dose exposure data for Kelly's boar _____no_output_____

kelly_boar_teloFISH_df = pd.read_csv('../excel data/cleaned/kelly_boar_teloFISH_df.csv')_____no_output_____kelly_boar_dose = pd.read_excel('../excel data/raw/Kelly& Aryn boar_1Dec2018-edited-TH-14Jun20.xlsx',
                               sheet_name=0, skiprows=7, nrows=40)
kelly_boar_dose.head(3)_____no_output_____kelly_boar_dose['Age (months)'] = kelly_boar_dose['Age (hours)'].apply(lambda x: int(x / (24 * 30)))
kelly_boar_dose['Sample ID'] = kelly_boar_dose['Sample ID'].apply(lambda row: f_tm.remove_dashes_space_sampleIDs(row))
# kelly_boar_dose.drop(['QA/AC', 'Unnamed: 3', 'Unnamed: 6', 'Unnamed: 7', 'Unnamed: 8', ' '], axis=1, inplace=True)

kelly_boar_dose = kelly_boar_dose[['Sample ID', 'Age (months)', 'Sex',
                                 'INTERNAL Life time dose, mGy 134+137',
                                 'Reasonable Total Life Time Dose mGy (Int+Ext,134+137)',
                                  'Dose rate (uGy/h) at time of capture (Int+Ext; 134+137)'
                                ]].copy()_____no_output_____

## Saving Kelly's boar dose exposure data for later retrieval_____no_output_____

kelly_boar_dose.to_csv('../excel data/cleaned/kelly_boar_dose.csv', index=False)_____no_output_____

## Briefly looking at sharing samples between kelly's boar w/ teloFISH data vs. the exposure data & cortisol data_____no_output_____

df_list = [trim_kelly_new_exposure_df, prep_boar_cortisol_df]

for df in df_list:
    f_tm.count_shared_sample_IDs(kelly_boar_teloFISH_df, df, print_names='no')_____no_output_____kelly_teloFISH_IDs = list(kelly_boar_teloFISH_df['Sample ID'].unique())
kelly_exposure_IDs = list(trim_kelly_new_exposure_df['Sample ID'].unique())

a = kelly_teloFISH_IDs
b = kelly_exposure_IDs

mismatched = [sample_ID for sample_ID in a if sample_ID not in b]
print(mismatched)_____no_output_____

## Pulling data Tom H request week of 4/27/20_____no_output_____

one = aryn_boar_dose_raw[['Sample ID']].copy()
one.shape

two = boar_cortisol_df_raw[['Sample ID']].copy()
two.rename({'Sample Number (electronic sheet)':'Sample ID'}, axis=1, inplace=True)

three = pd.read_csv('../excel data/cleaned/kelly_boar_teloFISH_df.csv')
three = three[['Sample ID']].copy()_____no_output_____cat = pd.concat([one, two, three], axis=0)

def replace_row(row):
    row = row.replace('-', '')
    row = row.replace(' ', '')
    return row

cat = cat.dropna()
cat['Sample ID'] = cat['Sample ID'].apply(lambda row: replace_row(row))

cat = (cat
 .dropna()
 .drop_duplicates()
 .reset_index(drop=True))_____no_output_____# two_____no_output_____cat.to_excel('kelly_aryn_cortisol_ids_fukushima_boar_ids_minimal_cleaning.xlsx')_____no_output_____

## Combining size data from Tom w/ cortisol data_____no_output_____

tom_size_data = pd.read_excel("../excel data/cleaned/Jared's_ids_minimal_cleaning-th-da-10may20_READY.xlsx",
                              skiprows=list(range(0,6)), usecols=list(range(0,10)))
tom_size_data.drop(['Unnamed: 0', 'Entry', 'Sex', 'Age (wk)'], axis=1, inplace=True)
tom_size_data = tom_size_data.dropna(axis=0, how='any').copy()
tom_size_data.reset_index(drop=True, inplace=True)
tom_size_data['Sample ID'] = tom_size_data['Sample ID'].apply(lambda row: f_tm.remove_dashes_space_sampleIDs(row))
tom_size_data.head(4)_____no_output_____total_dose_cortisol = pd.read_csv('../excel data/cleaned/total_dose_cortisol.csv')
merge_tom_data_cortisol = total_dose_cortisol.merge(tom_size_data, on=['Sample ID'])
merge_tom_data_cortisol.drop(3, inplace=True)
merge_tom_data_cortisol.reset_index(drop=True, inplace=True)
merge_tom_data_cortisol.to_csv('../excel data/cleaned/merge_tom_data_cortisol.csv', index=False)_____no_output_____merge_tom_data_cortisol_____no_output_____


# Notebook from ShravyaRNadig/Matplot
Path: Matplot practice.ipynb

import matplotlib.pyplot as plt
%matplotlib inline_____no_output_____x=[10,20,30,40,50]
y=[22,24,26,28,30]_____no_output_____plt.plot(x)
plt.show()_____no_output_____plt.plot(x,"o")
plt.show()_____no_output_____plt.plot(x,"-")
plt.show()_____no_output_____plt.plot(x,"-.")
plt.show()_____no_output_____plt.plot(x,"--")
plt.show()_____no_output_____plt.plot(x,y,"--")
plt.show()_____no_output_____x=[10,20,30,40,50]
y=[22,24,26,28,30]
z=[35,45,50,60,70]_____no_output_____plt.plot(x,y,"o")
plt.plot(x,z,"^")_____no_output_____plt.plot(x,y,"o")
plt.plot(x,z,"^")
plt.show()_____no_output_____plt.plot(x,y,"o")
plt.show()
plt.plot(x,z,"^")
plt.show()_____no_output_____plt.plot(x,y,"o")
plt.title("line graph")#Giving tirle
plt.xlabel("Age")#x value (axis)
plt.ylabel("Height")#y value(axis)_____no_output_____plt.plot(x,y,"r")#giving color r-red
plt.title("line graph")#Giving title n convays info being shown in graph
plt.xlabel("Age")#x value (axis)
plt.ylabel("Height")#y value(axis)_____no_output_____plt.plot(x,y,"r")#giving color r-red
plt.title("line graph")
plt.xlabel("Age")
plt.ylabel("Height")
plt.show()_____no_output_____plt.plot(x,y,"g")#giving color g-green
plt.title("line graph")
plt.xlabel("Age")
plt.ylabel("Height")
## matlotlib automatically selects the range of axis depending on data,
##but sometimes might have to set to specific values
## can set for individual axes
## setting axis limts
plt.xlim(0,50)
plt.ylim(0,30)
plt.show()_____no_output_____plt.plot(x,y,"g")#giving color g-green
plt.title("line graph")
plt.xlabel("Age")
plt.ylabel("Height")
plt.xlim(0,50)
plt.ylim(25,30)
plt.show()_____no_output_____plt.plot(x,y,"orange")
plt.title("line graph")
plt.xlabel("Age")
plt.ylabel("Height")
plt.show()_____no_output_____plt.plot(x,y,"turquoise")
plt.plot(x,z,"indigo")
plt.title("line graph")
plt.xlabel("Age")
plt.ylabel("Height")
plt.show()_____no_output_____plt.plot(x,y,"brown")
plt.plot(x,z,"violet")
plt.title("line graph")
plt.xlabel("Age")
plt.ylabel("Height")
plt.text(0,40,"X vs Z")
plt.text(45,20,"X vs Y")
plt.show()_____no_output_____plt.plot(x,y,"magenta")
plt.plot(x,z,"gold")
plt.title("line graph")
plt.xlabel("Age")
plt.ylabel("Height")
plt.text(10,40,"X vs Z")
plt.text(45,20,"X vs Y")
plt.show()_____no_output_____

##Markers
. point marker
O circle
v triangle_down
^ triangle_up
< triangle_left
> triangle_right
1 tri_down
2 tri_up
3 tri_left
4 tri_right
s square
p pentagon
* star
h hexagon1
H hexagon2
+ plus
x x
D diamond
d thin_diamond
| vline
_ hline_____no_output_____

plt.plot(x,y,"o")
plt.plot(x,z,"p")
plt.title("line graph")
plt.xlabel("Age")
plt.ylabel("Height")
plt.text(10,40,"X vs Z")
plt.text(45,20,"X vs Y")
plt.show()_____no_output_____plt.plot(x,y,"-.")
plt.plot(x,z,"-")
plt.title("line graph")
plt.xlabel("Age")
plt.ylabel("Height")
plt.text(10,40,"X vs Z")
plt.text(45,20,"X vs Y")
plt.show()_____no_output_____plt.plot(x,y,"r")
plt.plot(x,z,"-.")
plt.title("line graph")
plt.xlabel("Age")
plt.ylabel("Height")
plt.text(10,40,"X vs Z")
plt.text(45,20,"X vs Y")
plt.show()_____no_output_____


# Notebook from deisenroth/prysm
Path: docs/source/tutorials/Image Simulation.ipynb

# Image Simulation

In this tutorial, we will show how to use prysm to perform very accurate image simulation.  This combines previously shown concepts in the [Lens MTF Model](./Lens-MTF-Model.ipynb) with a few additional ones and is a more advanced tutorial.  The reader is assumed to understand basic propagation for computing PSFs.

Any image chain model or image simulation begins with defining parameters, so we'll choose some fairly arbitrary ones for our system:

Detector:
- pixel pitch: 4.5 microns
- Output resolution: 512x512
- Read Noise: 10 e-
- Full-well capacity: 50,000 e-
- Dark current: 30 e-/px/s
- bias: 800 e-
- conversion gain 5e-/DN
- Bit depth: 12-bit

Optics:
- lens F/#: 2.8
- lens EFL: 100 mm
- aperture: circular
- Optical path error? yes - sum of 2D-Q polynomials
- Fully achromatic (constant OPD over all wavelengths)

Object/Scene:
- Object: Siemens' Star
- Spectrum: Gaussian about 550 nm, 10% fractional bandwidth

From these, we begin to determine the parameters of the forward model.  The model _must_ be at least Nyquist sampled, or due to aliasing it will be invalid.  We define:

$$
Q = \frac{\lambda \text{F\#}}{pp}
$$
where $pp$ is the pixel pitch, and compute $Q = \tfrac{2.8 * .495}{4.5} = 0.306$.  495 nm is 10% below 550 nm.  Since we require $Q>=2$ in the forward model, and assume at the moment that we will use an integer level of oversampling, then the forward model is run at $Q=\text{roundup}\{2/.306\}=7x$ oversampling, or $Q=2.156$.  A notable problem is that this equation for $Q$ contains the wavelength; in other words, $Q$ is chromatic.  To get around this, we'll use matrix triple product DFTs to propagate directly to the oversampled version of the detector grid, which will be 3584x3584 samples across._____no_output_____

%load_ext autoreload
%autoreload 2

import numpy as np

from scipy import stats

from prysm import (
    coordinates,
    convolution,
    detector,
    geometry,
    propagation,
    polynomials,
    objects,
)

from matplotlib import pyplot as plt_____no_output_____

First, we define a bunch of parameters and set up the basic representation of the pupil:_____no_output_____

pp = 4.5
res = 512
fno = 2.8
efl = 100
epd = efl/fno
r_aper = epd / 2

xi, eta = coordinates.make_xy_grid(1800, diameter=epd)
r, t = coordinates.cart_to_polar(xi,eta)
dx = xi[0,1] - xi[0,0]

r_aber = r / r_aper

amp = geometry.circle(r_aper, r)

nms = [polynomials.noll_to_nm(j) for j in range(1,11)]
basis = list(polynomials.Q2d_sequence(nms, r_aber, t))

phs_coefs = np.random.rand(len(basis)) * 2000 # 200/sqrt(12) nm per mode, per uniform distribution statistics

phs = polynomials.sum_of_2d_modes(basis, phs_coefs)

# only used for plotting
mask = amp == 0
phs2 = phs.copy()
phs2[mask] = np.nan
im = plt.imshow(phs2, cmap='inferno')
plt.colorbar(im)_____no_output_____

Note carefully the use of r_aber when computing the basis -- the polynomials are defined on $r \in [0,1]$, which necessitates this normalization.  We've used the smallest possible grid that can contain the pupil with no padding, and a large number of samples.  A requirement of the matrix triple product DFT is that the output resolution, divided by Q, must not exceed the input resolution.  If this is not honored, then Dirichlet clones of the PSF will be visible in the edge of the array, which is unphysical and incorrect.  Since we will have 3584 px on the output and Q~=2.1, we need about 1800 px on the input,_____no_output_____

wvl0 = .550
halfbw = 0.1
wvls = np.linspace(wvl0*(1-halfbw), wvl0*(1+halfbw), 7)

def gauss(x, mu, sigma):
    num = (x-mu)**2
    den = 2 * sigma ** 2
    return np.exp(-num/den)

spectral_weights = gauss(wvls, .550, .550*.05)
plt.stem(wvls, spectral_weights)_____no_output_____

Having now completed the bulk of the preparatory work, we can now compute the PSF associated with this OPD and this spectrum.  We'll propagate each wavelength to the oversampled grid:_____no_output_____

psfs = []
for wvl in wvls:
    pup = propagation.Wavefront.from_amp_and_phase(amp, phs, wvl, dx)
    tmp = pup.focus_fixed_sampling(efl, pp/7, 3584)
    psfs.append(tmp.intensity.data)

# re-use this function, identical behavior
psf = polynomials.sum_of_2d_modes(psfs, spectral_weights)

# norm to sum of 1, no loss or creation of energy from convolution
psf /= psf.sum()_____no_output_____

After producing the PSF, we need to rasterize the target.  For this, we need a new grid:_____no_output_____

x, y = coordinates.make_xy_grid(tmp.data.shape, dx=tmp.dx)
r, t = coordinates.cart_to_polar(x,y)
obj = objects.siemensstar(r,t, 100, oradius=x.max()*.8)

plt.figure(figsize=(15,15))
plt.imshow(obj, cmap='gray')_____no_output_____

Now in posession of the object and PSF, we can synthesize the aerial image, which has been blurred by the optical system:_____no_output_____

img = convolution.conv(obj, psf)_____no_output_____

The blur by the detector is modeled in this case as a 100% fill factor pixel.  Binning will bring us to the output reoslution and simultaneously:_____no_output_____

img2 = detector.bindown(img, 7)_____no_output_____

To treat noise, we build a detector model, and capture the image:_____no_output_____

peak_eminus = 45_000 # heavy underexposure_____no_output_____dark_current = 10
fwc = 50_000
rn = 10
bias = 800
kgain = 5
bit_depth = 12
texp = 1/60 # 1/60 sec
cam = detector.Detector(dark_current, rn, bias, fwc, kgain, bit_depth, texp)

im = cam.expose(img2*peak_eminus)_____no_output_____plt.figure(figsize=(15,15))
plt.imshow(im, cmap='gray')_____no_output_____

With this, we have fully modeled the image chain, including optical and electrical blurs and noise.  _____no_output_____


# Notebook from Switham1/PromoterArchitecture
Path: src/plotting/luminescence/28.6.21/luminescence_plots_28.6.21.ipynb

<code>
import pandas as pd
import shutil
import numpy as np
import skbio
from collections import Counter
import seaborn as sns
import matplotlib.pyplot as plt

from scipy import stats
from statsmodels.formula.api import ols
import researchpy as rp
from matplotlib import rcParams
#from statsmodels.stats.multicomp import pairwise_tukeyhsd
#from statsmodels.stats.multicomp import MultiComparison
#dunns
from itertools import combinations
from statsmodels.stats.multitest import multipletests
from statsmodels.stats.libqsturng import psturng
import warnings

import plotly.express as px
import plotly.graph_objects as go
import os
import re
from plotly.subplots import make_subplots

from statsmodels.stats.multicomp import pairwise_tukeyhsd
from statsmodels.stats.multicomp import MultiComparison
import statsmodels.api as sm

import statsmodels.api as sa
import scikit_posthocs as sp

from pingouin import kruskal

from statannot import add_stat_annotation

rcParams.update({'figure.autolayout': True})/home/witham/opt/anaconda3/envs/PromoterArchitecturePipeline/lib/python3.7/site-packages/outdated/utils.py:18: OutdatedPackageWarning: The package outdated is out of date. Your version is 0.2.0, the latest is 0.2.1.
Set the environment variable OUTDATED_IGNORE=1 to disable these warnings.
  **kwargs
/home/witham/opt/anaconda3/envs/PromoterArchitecturePipeline/lib/python3.7/site-packages/outdated/utils.py:18: OutdatedPackageWarning: The package pingouin is out of date. Your version is 0.3.7, the latest is 0.3.12.
Set the environment variable OUTDATED_IGNORE=1 to disable these warnings.
  **kwargs
date1 = '28.6.21'
date2 = '28.6.21'
finaldate = '28.6.21'_____no_output_____if not os.path.exists(f'../../../../data/plots/luminescence/{finaldate}'):
    os.mkdir(f'../../../../data/plots/luminescence/{finaldate}')_____no_output_____if not os.path.exists(f'../../../../data/luminescence/to_be_sorted/{finaldate}'):
    os.mkdir(f'../../../../data/luminescence/to_be_sorted/{finaldate}')_____no_output_____# #copy files to new folder
# src1 = f'../../../../data/luminescence/to_be_sorted/{date1}/output_raw.csv'
# dst1 = f'../../../../data/luminescence/to_be_sorted/{finaldate}/output_raw_{date1}.csv'
# src2 = f'../../../../data/luminescence/to_be_sorted/{date2}/output_raw.csv'
# dst2 = f'../../../../data/luminescence/to_be_sorted/{finaldate}/output_raw_{date2}.csv'
# src3 = f'../../../../data/luminescence/to_be_sorted/{date2}/output_raw.csv'
# dst3 = f'../../../../data/luminescence/to_be_sorted/{finaldate}/output_raw_{date2}.csv'
# shutil.copyfile(src1,dst1)
# shutil.copyfile(src2,dst2)
# shutil.copyfile(src3,dst3)_____no_output_____luminescence_raw_date1 = f'../../../../data/luminescence/to_be_sorted/{finaldate}/plate1_output_raw.csv'
luminescence_raw_date2 = f'../../../../data/luminescence/to_be_sorted/{finaldate}/plate2_output_raw.csv'
luminescence_raw_date3 = f'../../../../data/luminescence/to_be_sorted/{finaldate}/plate3_output_raw.csv'_____no_output_____luminescence_raw_df_date1 = pd.read_csv(luminescence_raw_date1, header=0)
luminescence_raw_df_date2 = pd.read_csv(luminescence_raw_date2, header=0)
luminescence_raw_df_date3 = pd.read_csv(luminescence_raw_date3, header=0)_____no_output_____luminescence_raw_df_date1_____no_output_____luminescence_raw_df_date2_____no_output_____# #add promoter names column
# luminescence_raw_df_date1['Promoter'] = luminescence_raw_df_date1.name _____no_output_____# luminescence_raw_df.loc[luminescence_raw_df.name == '71 + 72', 'Promoter'] = 'UBQ10'
# luminescence_raw_df.loc[luminescence_raw_df.name == '71+72', 'Promoter'] = 'UBQ10'
# luminescence_raw_df.loc[luminescence_raw_df.name == '25+72', 'Promoter'] = 'NIR1'
# luminescence_raw_df.loc[luminescence_raw_df.name == '35+72', 'Promoter'] = 'NOS'
# luminescence_raw_df.loc[luminescence_raw_df.name == '36+72', 'Promoter'] = 'STAP4'
# luminescence_raw_df.loc[luminescence_raw_df.name == '92+72', 'Promoter'] = 'NRP'
# luminescence_raw_df.loc[luminescence_raw_df.name == '92+29+72', 'Promoter'] = 'NRP+NLP7_TF'
# luminescence_raw_df_date1.loc[luminescence_raw_df_date1.name == '25 + 34', 'Promoter'] = 'NIR1(1000bp)'
# luminescence_raw_df_date1.loc[luminescence_raw_df_date1.name == '35 + 34', 'Promoter'] = 'NOS'
# luminescence_raw_df_date1.loc[luminescence_raw_df_date1.name == '36 + 34', 'Promoter'] = 'STAP4'
# luminescence_raw_df_date1.loc[luminescence_raw_df_date1.name == '70 + 34', 'Promoter'] = '35S'
# luminescence_raw_df_date1.loc[luminescence_raw_df_date1.name == '14 + 34', 'Promoter'] = 'ANAC032(1000bp)'
# luminescence_raw_df_date1.loc[luminescence_raw_df_date1.name == '13 + 34', 'Promoter'] = 'ANAC032(500bp)'
# luminescence_raw_df_date1.loc[luminescence_raw_df_date1.name == '15 + 34', 'Promoter'] = 'ANR1(500bp)'
# luminescence_raw_df_date1.loc[luminescence_raw_df_date1.name == '16 + 34', 'Promoter'] = 'ANR1(1000bp)'
# luminescence_raw_df_date1.loc[luminescence_raw_df_date1.name == '17 + 34', 'Promoter'] = 'ARF18(500bp)'
# luminescence_raw_df_date1.loc[luminescence_raw_df_date1.name == '18 + 34', 'Promoter'] = 'ARF18(1000bp)'
# luminescence_raw_df_date1.loc[luminescence_raw_df_date1.name == '21 + 34', 'Promoter'] = 'DREB26(500bp)'
# luminescence_raw_df_date1.loc[luminescence_raw_df_date1.name == '22 + 34', 'Promoter'] = 'DREB26(1000bp)'
# luminescence_raw_df_date1.loc[luminescence_raw_df_date1.name == '23 + 34', 'Promoter'] = 'NLP7(500bp)'
# luminescence_raw_df_date1.loc[luminescence_raw_df_date1.name == '24 + 34', 'Promoter'] = 'NLP7(1000bp)'
# # luminescence_raw_df_date1.loc[luminescence_raw_df_date1.name == '14+30+34', 'Promoter'] = 'ANAC032)'

# luminescence_raw_df_date1.loc[luminescence_raw_df_date1.name.str.match(r'(^14.*)')==True, 'Promoter'] = 'ANAC032'
# luminescence_raw_df_date1.loc[luminescence_raw_df_date1.name.str.match(r'(^16.*)')==True, 'Promoter'] = 'ANR1'
# luminescence_raw_df_date1.loc[luminescence_raw_df_date1.name.str.match(r'(^18.*)')==True, 'Promoter'] = 'ARF18'
# luminescence_raw_df_date1.loc[luminescence_raw_df_date1.name.str.match(r'(^20.*)')==True, 'Promoter'] = 'NLP6'
# luminescence_raw_df_date1.loc[luminescence_raw_df_date1.name.str.match(r'(^22.*)')==True, 'Promoter'] = 'DREB26'
# luminescence_raw_df_date1.loc[luminescence_raw_df_date1.name.str.match(r'(^24.*)')==True, 'Promoter'] = 'NLP7'
# luminescence_raw_df_date1.loc[luminescence_raw_df_date1.name.str.match(r'(^25.*)')==True, 'Promoter'] = 'NIR1(1000bp)'
# luminescence_raw_df_date1.loc[luminescence_raw_df_date1.name.str.match(r'(^35.*)')==True, 'Promoter'] = 'NOS'
# luminescence_raw_df_date1.loc[luminescence_raw_df_date1.name.str.match(r'(^36.*)')==True, 'Promoter'] = 'STAP4'
# luminescence_raw_df_date1.loc[luminescence_raw_df_date1.name.str.match(r'(^70.*)')==True, 'Promoter'] = '35S'

#luminescence_raw_df.loc[luminescence_raw_df.name == '71 + 34', 'Promoter'] = 'UBQ10:LucN/35S:LucF'_____no_output_____def add_promoter_column(df,promoter_number,promoter):
    df.loc[df.name.str.match(fr'(^{promoter_number}.*)')==True, 'Promoter'] = promoter
    _____no_output_____#add TF coexpression column
def add_tf_column(df, TF_number, TF):
    """function to add TF added column based on an input df, a TF_number and a TF name"""
    df.loc[df.name.str.contains(fr'.*{TF_number}.*')&df.TF_added.isnull()==True, 'TF_added'] = TF
    for index, value in df.loc[df.name.str.contains(fr'.*{TF_number}.*')&df.TF_added.notnull()].TF_added.items():
        if re.match(fr'.*{TF}.*', value):
            pass
        else:
            df.iloc[index, df.columns.get_loc('TF_added')] = f'{value}+{TF}'
_____no_output_____#divide by NOS calibrator (mean nos-LucN/35SLucF luminescence)
def batch_correction(df,calibrator_promoter_name):
   #print(df[df.Promoter==calibrator_promoter_name])
    calibrator = df[df.Promoter==calibrator_promoter_name]
    calibrator_no_NAA = calibrator[calibrator.condition=='nitrogen']
    calibrator_mean = calibrator_no_NAA['nluc/fluc'].mean(axis=0)
    df['corrected_luminescence'] = df['nluc/fluc']/calibrator_mean*100 _____no_output_____#add promoters to date1 df
add_promoter_column(luminescence_raw_df_date1, '14', 'ANAC032')
add_promoter_column(luminescence_raw_df_date1, '16', 'ANR1')
add_promoter_column(luminescence_raw_df_date1, '18', 'ARF18')
add_promoter_column(luminescence_raw_df_date1, '20', 'NLP6')
add_promoter_column(luminescence_raw_df_date1, '22', 'DREB26')
add_promoter_column(luminescence_raw_df_date1, '24', 'NLP7')
add_promoter_column(luminescence_raw_df_date1, '25', 'NIR1')
add_promoter_column(luminescence_raw_df_date1, '35', 'NOS')
add_promoter_column(luminescence_raw_df_date1, '36', 'STAP4')
add_promoter_column(luminescence_raw_df_date1, '102', 'minimal_STAP4')
add_promoter_column(luminescence_raw_df_date1, '103', '1xANR1')
add_promoter_column(luminescence_raw_df_date1, '104', '1xNLP7')
add_promoter_column(luminescence_raw_df_date1, '105', '1xTCP20+NLP7')
add_promoter_column(luminescence_raw_df_date1, '106', '4xANR1')
add_promoter_column(luminescence_raw_df_date1, '107', '4xNLP7')
add_promoter_column(luminescence_raw_df_date1, '108', '4xTCP20+NLP7')
add_promoter_column(luminescence_raw_df_date1, '109', '4xNRE')
add_promoter_column(luminescence_raw_df_date1, '71', 'UBQ10')
add_promoter_column(luminescence_raw_df_date1, '372', '4x[NRE-TATA]')
add_promoter_column(luminescence_raw_df_date1, '373', '4x[NRE-TATA]+spacing')
add_promoter_column(luminescence_raw_df_date1, '374', '4x[NRE-random]')
add_promoter_column(luminescence_raw_df_date1, '375', '4x[NRE-random]+spacing')
add_promoter_column(luminescence_raw_df_date1, '376', '3xHHO2-minsyn_105')
add_promoter_column(luminescence_raw_df_date1, '377', 'minsyn_105-3xHHO2')
add_promoter_column(luminescence_raw_df_date1, '378', '4x[bZIP3-random]')
add_promoter_column(luminescence_raw_df_date1, '379', '4x[TGA1-random]')
add_promoter_column(luminescence_raw_df_date1, '380', '2x[bZIP3-random-NRE-random]')
add_promoter_column(luminescence_raw_df_date1, '381', '2x[TGA1-random-NRE-random]')
1_____no_output_____#add promoters to date2 df
add_promoter_column(luminescence_raw_df_date2, '14', 'ANAC032')
add_promoter_column(luminescence_raw_df_date2, '16', 'ANR1')
add_promoter_column(luminescence_raw_df_date2, '18', 'ARF18')
add_promoter_column(luminescence_raw_df_date2, '20', 'NLP6')
add_promoter_column(luminescence_raw_df_date2, '22', 'DREB26')
add_promoter_column(luminescence_raw_df_date2, '24', 'NLP7')
add_promoter_column(luminescence_raw_df_date2, '25', 'NIR1')
add_promoter_column(luminescence_raw_df_date2, '35', 'NOS')
add_promoter_column(luminescence_raw_df_date2, '36', 'STAP4')
add_promoter_column(luminescence_raw_df_date2, '102', 'minimal_STAP4')
add_promoter_column(luminescence_raw_df_date2, '103', '1xANR1')
add_promoter_column(luminescence_raw_df_date2, '104', '1xNLP7')
add_promoter_column(luminescence_raw_df_date2, '105', '1xTCP20+NLP7')
add_promoter_column(luminescence_raw_df_date2, '106', '4xANR1')
add_promoter_column(luminescence_raw_df_date2, '107', '4xNLP7')
add_promoter_column(luminescence_raw_df_date2, '108', '4xTCP20+NLP7')
add_promoter_column(luminescence_raw_df_date2, '109', '4xNRE')
add_promoter_column(luminescence_raw_df_date2, '71', 'UBQ10')
add_promoter_column(luminescence_raw_df_date2, '372', '4x[NRE-TATA]')
add_promoter_column(luminescence_raw_df_date2, '373', '4x[NRE-TATA]+spacing')
add_promoter_column(luminescence_raw_df_date2, '374', '4x[NRE-random]')
add_promoter_column(luminescence_raw_df_date2, '375', '4x[NRE-random]+spacing')
add_promoter_column(luminescence_raw_df_date2, '376', '3xHHO2-minsyn_105')
add_promoter_column(luminescence_raw_df_date2, '377', 'minsyn_105-3xHHO2')
add_promoter_column(luminescence_raw_df_date2, '378', '4x[bZIP3-random]')
add_promoter_column(luminescence_raw_df_date2, '379', '4x[TGA1-random]')
add_promoter_column(luminescence_raw_df_date2, '380', '2x[bZIP3-random-NRE-random]')
add_promoter_column(luminescence_raw_df_date2, '381', '2x[TGA1-random-NRE-random]')
2_____no_output_____#add promoters to date2 df
add_promoter_column(luminescence_raw_df_date3, '14', 'ANAC032')
add_promoter_column(luminescence_raw_df_date3, '16', 'ANR1')
add_promoter_column(luminescence_raw_df_date3, '18', 'ARF18')
add_promoter_column(luminescence_raw_df_date3, '20', 'NLP6')
add_promoter_column(luminescence_raw_df_date3, '22', 'DREB26')
add_promoter_column(luminescence_raw_df_date3, '24', 'NLP7')
add_promoter_column(luminescence_raw_df_date3, '25', 'NIR1')
add_promoter_column(luminescence_raw_df_date3, '35', 'NOS')
add_promoter_column(luminescence_raw_df_date3, '36', 'STAP4')
add_promoter_column(luminescence_raw_df_date3, '102', 'minimal_STAP4')
add_promoter_column(luminescence_raw_df_date3, '103', '1xANR1')
add_promoter_column(luminescence_raw_df_date3, '104', '1xNLP7')
add_promoter_column(luminescence_raw_df_date3, '105', '1xTCP20+NLP7')
add_promoter_column(luminescence_raw_df_date3, '106', '4xANR1')
add_promoter_column(luminescence_raw_df_date3, '107', '4xNLP7')
add_promoter_column(luminescence_raw_df_date3, '108', '4xTCP20+NLP7')
add_promoter_column(luminescence_raw_df_date3, '109', '4xNRE')
add_promoter_column(luminescence_raw_df_date3, '71', 'UBQ10')
add_promoter_column(luminescence_raw_df_date3, '372', '4x[NRE-TATA]')
add_promoter_column(luminescence_raw_df_date3, '373', '4x[NRE-TATA]+spacing')
add_promoter_column(luminescence_raw_df_date3, '374', '4x[NRE-random]')
add_promoter_column(luminescence_raw_df_date3, '375', '4x[NRE-random]+spacing')
add_promoter_column(luminescence_raw_df_date3, '376', '3xHHO2-minsyn_105')
add_promoter_column(luminescence_raw_df_date3, '377', 'minsyn_105-3xHHO2')
add_promoter_column(luminescence_raw_df_date3, '378', '4x[bZIP3-random]')
add_promoter_column(luminescence_raw_df_date3, '379', '4x[TGA1-random]')
add_promoter_column(luminescence_raw_df_date3, '380', '2x[bZIP3-random-NRE-random]')
add_promoter_column(luminescence_raw_df_date3, '381', '2x[TGA1-random-NRE-random]')
2_____no_output_____#add TF column to date1 df
luminescence_raw_df_date1['TF_added'] = np.nan
add_tf_column(luminescence_raw_df_date1,'27','NLP6')
add_tf_column(luminescence_raw_df_date1,'29','NLP7')
add_tf_column(luminescence_raw_df_date1,'30','DREB26')
add_tf_column(luminescence_raw_df_date1,'31','ANAC032')
add_tf_column(luminescence_raw_df_date1,'32','ARF18')
add_tf_column(luminescence_raw_df_date1,'33','ANR1')
add_tf_column(luminescence_raw_df_date1,'111','TCP20')
add_tf_column(luminescence_raw_df_date1,'348','HHO2')
add_tf_column(luminescence_raw_df_date1,'349','bZIP3')
add_tf_column(luminescence_raw_df_date1,'351','TGA1')_____no_output_____#add TF column to date1 df
luminescence_raw_df_date2['TF_added'] = np.nan
add_tf_column(luminescence_raw_df_date2,'27','NLP6')
add_tf_column(luminescence_raw_df_date2,'29','NLP7')
add_tf_column(luminescence_raw_df_date2,'30','DREB26')
add_tf_column(luminescence_raw_df_date2,'31','ANAC032')
add_tf_column(luminescence_raw_df_date2,'32','ARF18')
add_tf_column(luminescence_raw_df_date2,'33','ANR1')
add_tf_column(luminescence_raw_df_date2,'111','TCP20')
add_tf_column(luminescence_raw_df_date2,'348','HHO2')
add_tf_column(luminescence_raw_df_date2,'349','bZIP3')
add_tf_column(luminescence_raw_df_date2,'351','TGA1')_____no_output_____#add TF column to date1 df
luminescence_raw_df_date3['TF_added'] = np.nan
add_tf_column(luminescence_raw_df_date3,'27','NLP6')
add_tf_column(luminescence_raw_df_date3,'29','NLP7')
add_tf_column(luminescence_raw_df_date3,'30','DREB26')
add_tf_column(luminescence_raw_df_date3,'31','ANAC032')
add_tf_column(luminescence_raw_df_date3,'32','ARF18')
add_tf_column(luminescence_raw_df_date3,'33','ANR1')
add_tf_column(luminescence_raw_df_date3,'111','TCP20')
add_tf_column(luminescence_raw_df_date3,'348','HHO2')
add_tf_column(luminescence_raw_df_date3,'349','bZIP3')
add_tf_column(luminescence_raw_df_date3,'351','TGA1')_____no_output_____#divide by NOS calibrator (mean nos-LucN/35SLucF luminescence)
batch_correction(luminescence_raw_df_date1, 'NOS')
batch_correction(luminescence_raw_df_date2, 'NOS')
batch_correction(luminescence_raw_df_date3, 'NOS')_____no_output_____luminescence_raw_df_date1[:60]_____no_output_____#merge the dfs with different dates
luminescence_raw_df = pd.concat([luminescence_raw_df_date1,luminescence_raw_df_date2,luminescence_raw_df_date3],ignore_index=True)
luminescence_raw_df_____no_output_____#Change TF_added NaN values to none
luminescence_raw_df.loc[luminescence_raw_df.TF_added.isnull()==True, 'TF_added'] = 'No TF'_____no_output_____#if NAA condition, add that name to the TF name
#luminescence_raw_df_date3.TF_added = pd.np.where(luminescence_raw_df_date3.condition.str.contains('1μM_NAA'), 'ARF18_1μM_NAA')
luminescence_raw_df.loc[luminescence_raw_df['condition'].str.contains('1μM_NAA'),'TF_added' ] = luminescence_raw_df.loc[luminescence_raw_df['condition'].str.contains('1μM_NAA'), 'TF_added'] + '_1μM_NAA'

_____no_output_____print(luminescence_raw_df)    well content  fluc_luminescence  nluc_luminescence well_row  well_col  \
0    A01  Sample           382739.0           729907.0        A         1
1    A02  Sample            45726.0           106113.0        A         2
2    A03  Sample            84139.0           270668.0        A         3
3    A04  Sample            30215.0            37681.0        A         4
4    A05  Sample             4174.0             5637.0        A         5
..   ...     ...                ...                ...      ...       ...
261  H03  Sample            20594.0            13299.0        H         3
262  H04  Sample             6946.0             4799.0        H         4
263  H05  Sample            13282.0            19037.0        H         5
264  H07  Sample            10445.0             8760.0        H         7
265  H08  Sample             2991.0              793.0        H         8

         name condition  nluc/fluc     date Promoter       TF_added  \
0       22+34  nitrogen   1.907062  28.6.21   DREB26          No TF
1       22+34  nitrogen   2.320627  28.6.21   DREB26          No TF
2       22+34  nitrogen   3.216915  28.6.21   DREB26          No TF
3       24+34  nitrogen   1.247096  28.6.21     NLP7          No TF
4       24+34  nitrogen   1.350503  28.6.21     NLP7          No TF
..        ...       ...        ...      ...      ...            ...
261  22+32+34   1μM_NAA   0.645771  28.6.21   DREB26  ARF18_1μM_NAA
262  24+32+34   1μM_NAA   0.690901  28.6.21     NLP7  ARF18_1μM_NAA
263  24+32+34   1μM_NAA   1.433293  28.6.21     NLP7  ARF18_1μM_NAA
264     18+34  nitrogen   0.838679  28.6.21    ARF18          No TF
265     18+34  nitrogen   0.265129  28.6.21    ARF18          No TF

     corrected_luminescence
0                  2.024013
1                  2.462941
2                  3.414193
3                  1.323574
4                  1.433323
..                      ...
261                3.150592
262                3.370776
263                6.992765
264                4.091755
265                1.293513

[266 rows x 13 columns]
#sort rows by the custom promoter order I want so plots are in the correct order
luminescence_raw_df.Promoter = pd.Categorical(luminescence_raw_df.Promoter,
                                              ['NOS','STAP4','ANAC032','ARF18','DREB26','NLP7'])
luminescence_raw_df = luminescence_raw_df.sort_values("Promoter")_____no_output_____#save as file
location = f'../../../../data/luminescence/to_be_sorted/{finaldate}/combined_output.tsv'

luminescence_raw_df.to_csv(location,sep='\t',header=True)_____no_output_____# luminescence_raw_df_renamed = luminescence_raw_df.copy()
# luminescence_raw_df_renamed.gene_type.replace('DREB26+ARF18','+DREB26+ARF18', inplace=True)
# luminescence_raw_df_renamed.gene_type.replace('ARF18','+ARF18', inplace=True)
# luminescence_raw_df_renamed.gene_type.replace('randCont','random', inplace=True)
_____no_output_____#get TF names
pd.Categorical(luminescence_raw_df.TF_added)
names = luminescence_raw_df.TF_added.unique()
#Change TF names to +TF
luminescence_raw_df_renamed = luminescence_raw_df.copy()
for TF in names:
    if TF != 'No TF':
        luminescence_raw_df_renamed.TF_added.replace(TF,f'+{TF}', inplace=True)
luminescence_raw_df = luminescence_raw_df_renamed_____no_output_____
</code>

### get names of each TF_added for later_____no_output_____

<code>
pd.Categorical(luminescence_raw_df.TF_added)
names = luminescence_raw_df.TF_added.unique()
for name in names:
    print(name)+No TF_1μM_NAA
No TF
+DREB26
+ANAC032
+NLP6
+NLP7
+ARF18
+ARF18_1μM_NAA
+ANR1
#get list of promoters
pd.Categorical(luminescence_raw_df.Promoter)
prom_names = luminescence_raw_df.Promoter.unique()
for name in prom_names:
    print(name)NOS
STAP4
ANAC032
ARF18
DREB26
NLP7
#create colours
from random import randint
color = []
n = len(names)
for i in range(n):
    color.append('#%06X' % randint(0, 0xFFFFFF))
print(color)
['#E125A2', '#FAB2AF', '#64C393', '#DC4F90', '#CAC86A', '#5FEF4F', '#9F2EB0', '#E99DF5', '#580F49']
#make dictionary with TF_added names as key and the randomly generated colors as values
new_color = dict(zip(names, color))
new_color_____no_output_____# #layout = go.Layout(plot_bgcolor='rgba(0,0,0,0)', paper_bgcolor='rgba(0,0,0,0)' )
# #make graphs for each promoter
# #color_discrete_map = {'red': 'red', 'blue': 'blue', 'orange': 'orange', 'purple':'purple','green':'green','yellow':'yellow','brown':'brown','black':'black'}

# for prom in prom_names:
#     if prom == 'NOS' or prom == 'STAP4':
#         pass
#     else:
#         df = luminescence_raw_df[luminescence_raw_df.Promoter == prom].sort_values("TF_added", ascending=False)

#         fig = px.violin(df, x='TF_added',
#                                   y='corrected_luminescence', box=True,
#                                 color='TF_added', color_discrete_map=new_color,violinmode='overlay') #points="all"

#         fig.update_layout(plot_bgcolor='rgba(0,0,0,0)', paper_bgcolor='rgba(0,0,0,0)', title=prom,autosize=True)
#         fig.update_xaxes(showline=True, linewidth=1, linecolor='black',ticks="outside")
#         fig.update_yaxes(showline=True, linewidth=1, linecolor='black',ticks="outside", title_text='Mean luminescence (a.u.)')
#         #hide legend
#         fig.layout.update(showlegend=False)
#         #write to file static image and interactive one
#         fig.write_image(f'../../../../data/plots/luminescence/{finaldate}/violin{prom}.pdf', width=500, height=350, scale=5)
#         fig.write_html(f'../../../../data/plots/luminescence/{finaldate}/violin{prom}.html')
#         fig.show()_____no_output_____for TF in names:
    print(TF)+No TF_1μM_NAA
No TF
+DREB26
+ANAC032
+NLP6
+NLP7
+ARF18
+ARF18_1μM_NAA
+ANR1
# #layout = go.Layout(plot_bgcolor='rgba(0,0,0,0)', paper_bgcolor='rgba(0,0,0,0)' )
# fig = go.Figure(go.Violin(y=luminescence_raw_df[luminescence_raw_df.Promoter == 'ANR1'].corrected_luminescence, x=luminescence_raw_df[luminescence_raw_df.Promoter == 'ANR1'].TF_added,
#                           fillcolor='#840122', box_visible=True, meanline_visible=True))
#               #points="all"

# fig.update_layout(plot_bgcolor='rgba(0,0,0,0)', paper_bgcolor='rgba(0,0,0,0)')
# fig.update_xaxes(showline=True, linewidth=1, linecolor='black',ticks="outside")
# fig.update_yaxes(showline=True, linewidth=1, linecolor='black',ticks="outside", title_text='Mean luminescence (a.u.)')
# #write to file static image and interactive one
# #fig.write_image(f'../../../../data/plots/luminescence/{date}/violin23.9.19.pdf')
# #fig.write_html(f'../../../../data/plots/luminescence/{date}/violin23.9.19.html')

# fig.show()_____no_output_____def dunn_posthoc_test(df, dependent_variable, between):
    """dunn_posthoc tests with bonferroni multiple correction"""
    return sp.posthoc_dunn(
        df,
        val_col=dependent_variable,
        group_col=between,
        p_adjust="bonferroni",
    )_____no_output_____#layout = go.Layout(plot_bgcolor='rgba(0,0,0,0)', paper_bgcolor='rgba(0,0,0,0)' )
#make graphs for each promoter
#color_discrete_map = {'red': 'red', 'blue': 'blue', 'orange': 'orange', 'purple':'purple','green':'green','yellow':'yellow','brown':'brown','black':'black'}
#plots for promoters 2 Tf categories

for prom in prom_names:
    if prom == 'NOS' or prom == 'STAP4':
        pass
    else:
        df = luminescence_raw_df[luminescence_raw_df.Promoter == prom].sort_values("TF_added", ascending=False)
        TFs =  df.TF_added.unique()
        print(prom)
        print(TFs)
        if len(TFs) is not 2:
            pass
        else:
            plt.figure()
            order = [TFs[0],TFs[1]]
            sns.swarmplot(x='TF_added', y='corrected_luminescence', data=df, color=".25", order=order)

            # fig = sns.catplot(
            #     x='TF_added',
            #     y='corrected_luminescence',
            #     data=df,
            #     kind='violin',
            #     order=order,
            #     cut=0,
            #     legend=False,
            #     #color=new_color
            # )

            #fig = px.violin(df, x='TF_added',
            #                         y='corrected_luminescence', box=True,
            #                      color='TF_added', color_discrete_map=new_color,violinmode='overlay',order=order) #points="all"

            #fig.update_layout(plot_bgcolor='rgba(0,0,0,0)', paper_bgcolor='rgba(0,0,0,0)', title=prom,autosize=True)
            #fig.update_xaxes(showline=True, linewidth=1, linecolor='black',ticks="outside")
            #fig.update_yaxes(showline=True, linewidth=1, linecolor='black',ticks="outside", title_text='Mean luminescence (a.u.)')
            #hide legend
            #fig.layout.update(showlegend=False)
            # add significance if necessary - dunn's posthocs with multiple Bonferroni correction
            #stat = dunn_posthoc_test(df, 'corrected_luminescence', 'TF_added')
            # label box pairs
            box_pairs = [(TFs[0],TFs[1])]
            # make empty list of p_values
            p_values = []
            # populate the list of p_values according to the box_pairs
            # for pair in box_pairs:
            #     print(pair)
            #     # select p value for each pair
            #     p = stat.loc[pair[0], pair[1]]
            #     p_values.append(p)

            # add stats annotation to the plot
            add_stat_annotation(
                fig,
                data=df,
                x='TF_added',
                y='corrected_luminescence',
                order=order,
                box_pairs=box_pairs,
                text_format="star",
                loc="outside",
                verbose=2,
                #perform_stat_test=False,
                #pvalues=p_values,
                test='Mann-Whitney',
            )
            # change axes labels
            plt.ylabel('Mean luminescence (a.u.)')
            #plt.xlabel(x_label)
            # tight layout
            plt.tight_layout()
            # save figure

            plt.savefig(
                f'../../../../data/plots/luminescence/{finaldate}/scatter{prom}.pdf',
                format="pdf",
                bbox_inches="tight",
            )

        #write to file static image and interactive one
       # fig.write_image(f'../../../../data/plots/luminescence/{finaldate}/violin{prom}.pdf', width=500, height=350, scale=5)
      #  fig.write_html(f'../../../../data/plots/luminescence/{finaldate}/violin{prom}.html')
      #  fig.show()ANAC032
['No TF' '+DREB26' '+ANAC032']
ARF18
['No TF' '+NLP7' '+NLP6' '+DREB26' '+ANAC032']
DREB26
['No TF' '+NLP7' '+NLP6' '+DREB26' '+ARF18_1μM_NAA' '+ARF18' '+ANR1'
 '+ANAC032']
NLP7
['No TF' '+NLP7' '+NLP6' '+DREB26' '+ARF18_1μM_NAA' '+ARF18' '+ANR1'
 '+ANAC032']
#layout = go.Layout(plot_bgcolor='rgba(0,0,0,0)', paper_bgcolor='rgba(0,0,0,0)' )
#make graphs for each promoter
#color_discrete_map = {'red': 'red', 'blue': 'blue', 'orange': 'orange', 'purple':'purple','green':'green','yellow':'yellow','brown':'brown','black':'black'}
#plots for promoters with more than 2 Tf categories
#prom == 'minimal_STAP4'
for prom in prom_names:
    if prom == 'NOS' or prom == 'STAP4':
        pass
    else:
        df1 = luminescence_raw_df[luminescence_raw_df.Promoter == prom].sort_values("TF_added", ascending=False)
        df2 = luminescence_raw_df[luminescence_raw_df.Promoter == 'STAP4'].sort_values("TF_added", ascending=False)
        #change name of TF added in df2 to minimal_stap4
        df2['TF_added'] = 'pSTAP4 no TF'
        df = pd.merge(df1,df2, how='outer').sort_values("TF_added", ascending=False)
        TFs =  df.TF_added.unique()
        #print(df)
        #get length of number of TFs
        length_tf = len(TFs)
        #create order and box pairs based on the length of TFs
        order = []
        box_pairs = []
        for x in range (0, (length_tf)):
            order.append(TFs[x])
            if TFs[x] == "No TF":
                pass
            else:
                box_pairs.append(('No TF',TFs[x]))
        #add every combination of box pairs of length 2
        #box_pairs=list(combinations(order, 2))
        box_pairs_all_pvalues = {}
        for pair in box_pairs:
            #create series objects
            TF1 = df.query(f'TF_added == "{pair[0]}"')['corrected_luminescence']
            TF2 = df.query(f'TF_added == "{pair[1]}"')['corrected_luminescence']
        # probe1 = df.loc[df['probe'] == pair[0], corrected_luminescence]
        # probe2 = df.loc[df['probe'] == pair[1], corrected_luminescence]
        #equal variance = false so do Welch's T-test
            results = stats.ttest_ind(TF1, TF2, equal_var=False)
            #append pvalue to dict
            box_pairs_all_pvalues[pair] = results.pvalue
        # box_pairs_significant.append(pair)
        # p_values.append(results.pvalue)
        print(f'all pvalues={box_pairs_all_pvalues}')

        # #filter dict by significance and put in a new dictionary
        box_pairs_significant = {}
        for k,v in box_pairs_all_pvalues.items():
            if v <0.05:
                box_pairs_significant[k] = v

        # if len(TFs) is 3:
        #     order = [TFs[0],TFs[1],TFs[2]]
        #     # label box pairs
        #     box_pairs = [(TFs[0],TFs[1]), (TFs[0],TFs[2]),(TFs[1],TFs[2])]
        # elif len(TFs) is 4:
        #     order = [TFs[0],TFs[1],TFs[2],TFs[3]]
        #     # label box pairs
        #     box_pairs = [(TFs[0],TFs[1]),(TFs[0],TFs[2]),(TFs[0],TFs[3]), (TFs[1],TFs[2]),(TFs[1],TFs[3]),(TFs[2],TFs[3])]
        # elif len(TFs) is 5:
        #     order = [TFs[0],TFs[1],TFs[2],TFs[3],TFs[4]]
        #     # label box pairs
        #     box_pairs = [(TFs[0],TFs[1]), (TFs[0],TFs[2]),(TFs[0],TFs[3]),(TFs[0],TFs[4]), (TFs[1],TFs[2]),(TFs[1],TFs[3]),(TFs[1],TFs[4]),(TFs[2],TFs[3]),(TFs[2],TFs[4]),(TFs[3],TFs[4])]
        # elif len(TFs) is 6:
        #     order = [TFs[0],TFs[1],TFs[2],TFs[3],TFs[4],TFs[5]]
        #     # label box pairs
        #     box_pairs = [(TFs[0],TFs[1]), (TFs[0],TFs[2]),(TFs[0],TFs[3]),(TFs[0],TFs[4]),(TFs[0],TFs[5]), (TFs[1],TFs[2]),(TFs[1],TFs[3]),(TFs[1],TFs[4]),(TFs[1],TFs[5]),(TFs[2],TFs[3]),(TFs[2],TFs[4]),(TFs[2],TFs[5]),(TFs[3],TFs[4]),(TFs[3],TFs[5]), (TFs[4],TFs[5])]
        # elif len(TFs) is 7:
        #     order = [TFs[0],TFs[1],TFs[2],TFs[3],TFs[4],TFs[5],TFs[6]]
        #     # label box pairs
        #     box_pairs = [(TFs[0],TFs[1]), (TFs[0],TFs[2]),(TFs[0],TFs[3]),(TFs[0],TFs[4]),(TFs[0],TFs[5]),(TFs[0],TFs[6]), (TFs[1],TFs[2]),(TFs[1],TFs[3]),(TFs[1],TFs[4]),(TFs[1],TFs[5]),(TFs[1],TFs[6]),(TFs[2],TFs[3]),(TFs[2],TFs[4]),(TFs[2],TFs[5]),(TFs[2],TFs[6]),(TFs[3],TFs[4]),(TFs[3],TFs[5]), (TFs[3],TFs[6]),(TFs[4],TFs[5]),(TFs[4],TFs[6]),(TFs[5],TFs[6])]
        # elif len(TFs) is 8:
        #     order = [TFs[0],TFs[1],TFs[2],TFs[3],TFs[4],TFs[5],TFs[6],TFs[7]]
        #     # label box pairs
        #     box_pairs = [(TFs[0],TFs[1]), (TFs[0],TFs[2]),(TFs[0],TFs[3]),(TFs[0],TFs[4]),(TFs[0],TFs[5]),(TFs[0],TFs[6]),(TFs[0],TFs[7]), (TFs[1],TFs[2]),(TFs[1],TFs[3]),(TFs[1],TFs[4]),(TFs[1],TFs[5]),(TFs[1],TFs[6]),(TFs[1],TFs[7]),(TFs[2],TFs[3]),(TFs[2],TFs[4]),(TFs[2],TFs[5]),(TFs[2],TFs[6]),(TFs[2],TFs[7]),(TFs[3],TFs[4]),(TFs[3],TFs[5]), (TFs[3],TFs[6]),(TFs[3],TFs[7]),(TFs[4],TFs[5]),(TFs[4],TFs[6]),(TFs[4],TFs[7]),(TFs[5],TFs[6]),(TFs[5],TFs[7]),(TFs[6],TFs[7])]

        if len(TFs) < 3:
            pass
        else:
            print(prom)
            print(TFs)
            plt.figure(figsize=((length_tf+2),5))
            # fig = sns.catplot(
            #     x='TF_added',
            #     y='corrected_luminescence',
            #     data=df,
            #     kind='violin',
            #     order=order,
            #     cut=0,
            #     legend=False,
            #     #color=new_color
            # )
            fig = sns.swarmplot(x='TF_added', y='corrected_luminescence', data=df, order=order, hue='condition')
            #fig = px.violin(df, x='TF_added',
            #                         y='corrected_luminescence', box=True,
            #                      color='TF_added', color_discrete_map=new_color,violinmode='overlay',order=order) #points="all"

            #fig.update_layout(plot_bgcolor='rgba(0,0,0,0)', paper_bgcolor='rgba(0,0,0,0)', title=prom,autosize=True)
            #fig.update_xaxes(showline=True, linewidth=1, linecolor='black',ticks="outside")
            #fig.update_yaxes(showline=True, linewidth=1, linecolor='black',ticks="outside", title_text='Mean luminescence (a.u.)')
            #hide legend
            #fig.layout.update(showlegend=False)
            # add significance if necessary - dunn's posthocs with multiple Bonferroni correction
            # stat = dunn_posthoc_test(df, 'corrected_luminescence', 'TF_added')
            # # label box pairs
            # #box_pairs = [(TFs[0],TFs[1]), (TFs[0],TFs[2]),(TFs[0],TFs[3]), (TFs[1],TFs[2]),(TFs[1],TFs[3]),(TFs[2],TFs[3])]
            # # make empty list of p_values
            # p_values = []
            # # populate the list of p_values according to the box_pairs
            # for pair in box_pairs:
            # #    print(pair)
            #     # select p value for each pair
            #     p = stat.loc[pair[0], pair[1]]
            #     p_values.append(p)

            # add stats annotation to the plot
            add_stat_annotation(
                fig,
                data=df,
                x='TF_added',
                y='corrected_luminescence',
                order=order,
                box_pairs=list(box_pairs_significant.keys()),
                text_format="star",
                loc="outside",
                verbose=2,
                perform_stat_test=False,
                pvalues=list(box_pairs_significant.values()),
                #test_short_name="Dunn",
            )
            # change axes labels
            plt.ylabel('Luminescence (a.u.)')
            #add title
            plt.title(prom)
            #plt.xlabel(x_label)
            # tight layout
            plt.tight_layout()
            # save figure
            plt.savefig(
                f'../../../../data/plots/luminescence/{finaldate}/scatter{prom}.pdf',
                format="pdf",
                bbox_inches="tight",
            )

            #write to file static image and interactive one
        # fig.write_image(f'../../../../data/plots/luminescence/{finaldate}/violin{prom}.pdf', width=500, height=350, scale=5)
        #  fig.write_html(f'../../../../data/plots/luminescence/{finaldate}/violin{prom}.html')
        #  fig.show()all pvalues={('No TF', 'pSTAP4 no TF'): 0.0006122488359758388, ('No TF', '+DREB26'): 0.16905211450583213, ('No TF', '+ANAC032'): 0.22627710813920637}
ANAC032
['pSTAP4 no TF' 'No TF' '+DREB26' '+ANAC032']
p-value annotation legend:
ns: 5.00e-02 < p <= 1.00e+00
*: 1.00e-02 < p <= 5.00e-02
**: 1.00e-03 < p <= 1.00e-02
***: 1.00e-04 < p <= 1.00e-03
****: p <= 1.00e-04

pSTAP4 no TF v.s. No TF: Custom statistical test, P_val:6.122e-04
all pvalues={('No TF', 'pSTAP4 no TF'): 5.994758769737356e-06, ('No TF', '+NLP7'): 0.2509273536241101, ('No TF', '+NLP6'): 0.28882665494525717, ('No TF', '+DREB26'): 0.29125686818637814, ('No TF', '+ANAC032'): 0.02291062926054166}
ARF18
['pSTAP4 no TF' 'No TF' '+NLP7' '+NLP6' '+DREB26' '+ANAC032']
p-value annotation legend:
ns: 5.00e-02 < p <= 1.00e+00
*: 1.00e-02 < p <= 5.00e-02
**: 1.00e-03 < p <= 1.00e-02
***: 1.00e-04 < p <= 1.00e-03
****: p <= 1.00e-04

pSTAP4 no TF v.s. No TF: Custom statistical test, P_val:5.995e-06
No TF v.s. +ANAC032: Custom statistical test, P_val:2.291e-02
all pvalues={('No TF', 'pSTAP4 no TF'): 0.04029302161571522, ('No TF', '+NLP7'): 0.0007319020140811448, ('No TF', '+NLP6'): 0.0007710516903631507, ('No TF', '+DREB26'): 2.169175093952461e-05, ('No TF', '+ARF18_1μM_NAA'): 0.09140580321879474, ('No TF', '+ARF18'): 0.05824959029035842, ('No TF', '+ANR1'): 0.06702987828693956, ('No TF', '+ANAC032'): 6.665215673784396e-05}
DREB26
['pSTAP4 no TF' 'No TF' '+NLP7' '+NLP6' '+DREB26' '+ARF18_1μM_NAA'
 '+ARF18' '+ANR1' '+ANAC032']
p-value annotation legend:
ns: 5.00e-02 < p <= 1.00e+00
*: 1.00e-02 < p <= 5.00e-02
**: 1.00e-03 < p <= 1.00e-02
***: 1.00e-04 < p <= 1.00e-03
****: p <= 1.00e-04

pSTAP4 no TF v.s. No TF: Custom statistical test, P_val:4.029e-02
No TF v.s. +NLP7: Custom statistical test, P_val:7.319e-04
No TF v.s. +NLP6: Custom statistical test, P_val:7.711e-04
No TF v.s. +DREB26: Custom statistical test, P_val:2.169e-05
No TF v.s. +ANAC032: Custom statistical test, P_val:6.665e-05
all pvalues={('No TF', 'pSTAP4 no TF'): 0.7487199735795765, ('No TF', '+NLP7'): 0.01274400242348921, ('No TF', '+NLP6'): 0.018650899722359268, ('No TF', '+DREB26'): 0.04736978436885365, ('No TF', '+ARF18_1μM_NAA'): 0.006268335701138679, ('No TF', '+ARF18'): 0.021783601035969147, ('No TF', '+ANR1'): 0.053235488484844426, ('No TF', '+ANAC032'): 0.20503017552584446}
NLP7
['pSTAP4 no TF' 'No TF' '+NLP7' '+NLP6' '+DREB26' '+ARF18_1μM_NAA'
 '+ARF18' '+ANR1' '+ANAC032']
p-value annotation legend:
ns: 5.00e-02 < p <= 1.00e+00
*: 1.00e-02 < p <= 5.00e-02
**: 1.00e-03 < p <= 1.00e-02
***: 1.00e-04 < p <= 1.00e-03
****: p <= 1.00e-04

No TF v.s. +NLP7: Custom statistical test, P_val:1.274e-02
No TF v.s. +NLP6: Custom statistical test, P_val:1.865e-02
No TF v.s. +DREB26: Custom statistical test, P_val:4.737e-02
No TF v.s. +ARF18_1μM_NAA: Custom statistical test, P_val:6.268e-03
No TF v.s. +ARF18: Custom statistical test, P_val:2.178e-02

</code>

### test normality_____no_output_____

<code>
#returns test statistic, p-value
for name1 in prom_names:
    for name in names:
        print(name1,'{}: {}'.format(name, stats.shapiro(luminescence_raw_df['nluc/fluc'][luminescence_raw_df.TF_added == name])))
NOS +No TF_1μM_NAA: ShapiroResult(statistic=0.7646836042404175, pvalue=0.000960038392804563)
NOS No TF: ShapiroResult(statistic=0.49370336532592773, pvalue=1.1646506513102867e-14)
NOS +DREB26: ShapiroResult(statistic=0.6407406330108643, pvalue=5.032755368006292e-08)
NOS +ANAC032: ShapiroResult(statistic=0.7635840773582458, pvalue=4.252855433151126e-06)
NOS +NLP6: ShapiroResult(statistic=0.9308120012283325, pvalue=0.0723535493016243)
NOS +NLP7: ShapiroResult(statistic=0.9029814600944519, pvalue=0.01826162450015545)
NOS +ARF18: ShapiroResult(statistic=0.944246768951416, pvalue=0.3720642030239105)
NOS +ARF18_1μM_NAA: ShapiroResult(statistic=0.9050775766372681, pvalue=0.08264098316431046)
NOS +ANR1: ShapiroResult(statistic=0.785582959651947, pvalue=0.0009571603732183576)
STAP4 +No TF_1μM_NAA: ShapiroResult(statistic=0.7646836042404175, pvalue=0.000960038392804563)
STAP4 No TF: ShapiroResult(statistic=0.49370336532592773, pvalue=1.1646506513102867e-14)
STAP4 +DREB26: ShapiroResult(statistic=0.6407406330108643, pvalue=5.032755368006292e-08)
STAP4 +ANAC032: ShapiroResult(statistic=0.7635840773582458, pvalue=4.252855433151126e-06)
STAP4 +NLP6: ShapiroResult(statistic=0.9308120012283325, pvalue=0.0723535493016243)
STAP4 +NLP7: ShapiroResult(statistic=0.9029814600944519, pvalue=0.01826162450015545)
STAP4 +ARF18: ShapiroResult(statistic=0.944246768951416, pvalue=0.3720642030239105)
STAP4 +ARF18_1μM_NAA: ShapiroResult(statistic=0.9050775766372681, pvalue=0.08264098316431046)
STAP4 +ANR1: ShapiroResult(statistic=0.785582959651947, pvalue=0.0009571603732183576)
ANAC032 +No TF_1μM_NAA: ShapiroResult(statistic=0.7646836042404175, pvalue=0.000960038392804563)
ANAC032 No TF: ShapiroResult(statistic=0.49370336532592773, pvalue=1.1646506513102867e-14)
ANAC032 +DREB26: ShapiroResult(statistic=0.6407406330108643, pvalue=5.032755368006292e-08)
ANAC032 +ANAC032: ShapiroResult(statistic=0.7635840773582458, pvalue=4.252855433151126e-06)
ANAC032 +NLP6: ShapiroResult(statistic=0.9308120012283325, pvalue=0.0723535493016243)
ANAC032 +NLP7: ShapiroResult(statistic=0.9029814600944519, pvalue=0.01826162450015545)
ANAC032 +ARF18: ShapiroResult(statistic=0.944246768951416, pvalue=0.3720642030239105)
ANAC032 +ARF18_1μM_NAA: ShapiroResult(statistic=0.9050775766372681, pvalue=0.08264098316431046)
ANAC032 +ANR1: ShapiroResult(statistic=0.785582959651947, pvalue=0.0009571603732183576)
ARF18 +No TF_1μM_NAA: ShapiroResult(statistic=0.7646836042404175, pvalue=0.000960038392804563)
ARF18 No TF: ShapiroResult(statistic=0.49370336532592773, pvalue=1.1646506513102867e-14)
ARF18 +DREB26: ShapiroResult(statistic=0.6407406330108643, pvalue=5.032755368006292e-08)
ARF18 +ANAC032: ShapiroResult(statistic=0.7635840773582458, pvalue=4.252855433151126e-06)
ARF18 +NLP6: ShapiroResult(statistic=0.9308120012283325, pvalue=0.0723535493016243)
ARF18 +NLP7: ShapiroResult(statistic=0.9029814600944519, pvalue=0.01826162450015545)
ARF18 +ARF18: ShapiroResult(statistic=0.944246768951416, pvalue=0.3720642030239105)
ARF18 +ARF18_1μM_NAA: ShapiroResult(statistic=0.9050775766372681, pvalue=0.08264098316431046)
ARF18 +ANR1: ShapiroResult(statistic=0.785582959651947, pvalue=0.0009571603732183576)
DREB26 +No TF_1μM_NAA: ShapiroResult(statistic=0.7646836042404175, pvalue=0.000960038392804563)
DREB26 No TF: ShapiroResult(statistic=0.49370336532592773, pvalue=1.1646506513102867e-14)
DREB26 +DREB26: ShapiroResult(statistic=0.6407406330108643, pvalue=5.032755368006292e-08)
DREB26 +ANAC032: ShapiroResult(statistic=0.7635840773582458, pvalue=4.252855433151126e-06)
DREB26 +NLP6: ShapiroResult(statistic=0.9308120012283325, pvalue=0.0723535493016243)
DREB26 +NLP7: ShapiroResult(statistic=0.9029814600944519, pvalue=0.01826162450015545)
DREB26 +ARF18: ShapiroResult(statistic=0.944246768951416, pvalue=0.3720642030239105)
DREB26 +ARF18_1μM_NAA: ShapiroResult(statistic=0.9050775766372681, pvalue=0.08264098316431046)
DREB26 +ANR1: ShapiroResult(statistic=0.785582959651947, pvalue=0.0009571603732183576)
NLP7 +No TF_1μM_NAA: ShapiroResult(statistic=0.7646836042404175, pvalue=0.000960038392804563)
NLP7 No TF: ShapiroResult(statistic=0.49370336532592773, pvalue=1.1646506513102867e-14)
NLP7 +DREB26: ShapiroResult(statistic=0.6407406330108643, pvalue=5.032755368006292e-08)
NLP7 +ANAC032: ShapiroResult(statistic=0.7635840773582458, pvalue=4.252855433151126e-06)
NLP7 +NLP6: ShapiroResult(statistic=0.9308120012283325, pvalue=0.0723535493016243)
NLP7 +NLP7: ShapiroResult(statistic=0.9029814600944519, pvalue=0.01826162450015545)
NLP7 +ARF18: ShapiroResult(statistic=0.944246768951416, pvalue=0.3720642030239105)
NLP7 +ARF18_1μM_NAA: ShapiroResult(statistic=0.9050775766372681, pvalue=0.08264098316431046)
NLP7 +ANR1: ShapiroResult(statistic=0.785582959651947, pvalue=0.0009571603732183576)

</code>

#### not normal_____no_output_____

<code>
#test variance
stats.levene(luminescence_raw_df['nluc/fluc'][luminescence_raw_df.TF_added == names[0]],
               luminescence_raw_df['nluc/fluc'][luminescence_raw_df.TF_added == names[1]])_____no_output_____test = luminescence_raw_df.groupby('Promoter')_____no_output_____test_____no_output_____
</code>

# one-way ANOVA: scipy.stats.kruskal_____no_output_____

<code>
for name in names:
    print(name)+No TF_1μM_NAA
No TF
+DREB26
+ANAC032
+NLP6
+NLP7
+ARF18
+ARF18_1μM_NAA
+ANR1
#luminescence anova
for promoter, value in test:
    if promoter == 'ANAC032':
        print(promoter)
        print(stats.f_oneway(value['corrected_luminescence'][value.TF_added == 'None'],
                value['corrected_luminescence'][value.TF_added == 'DREB26'],
                value['corrected_luminescence'][value.TF_added == 'ARF18'],
                value['corrected_luminescence'][value.TF_added == 'DREB26+ARF18']))
    if promoter == 'ANR1':
        print(promoter)
        print(stats.f_oneway(value['corrected_luminescence'][value.TF_added == 'None'],
                value['corrected_luminescence'][value.TF_added == 'ANR1'],
                value['corrected_luminescence'][value.TF_added == 'NLP7'],
                value['corrected_luminescence'][value.TF_added == 'ARF18'],
                value['corrected_luminescence'][value.TF_added == 'DREB26']))
    if promoter == 'ARF18':
        print(promoter)
        print(stats.f_oneway(value['corrected_luminescence'][value.TF_added == 'None'],
                value['corrected_luminescence'][value.TF_added == 'NLP7'],
                value['corrected_luminescence'][value.TF_added == 'DREB26']))
    if promoter == 'NLP6':
        print(promoter)
        print(stats.f_oneway(value['corrected_luminescence'][value.TF_added == 'None'],
                value['corrected_luminescence'][value.TF_added == 'ANR1'],
                value['corrected_luminescence'][value.TF_added == 'ARF18'],
                value['corrected_luminescence'][value.TF_added == 'DREB26']))
    if promoter == 'DREB26':
        print(promoter)
        print(stats.f_oneway(value['corrected_luminescence'][value.TF_added == 'None'],
                value['corrected_luminescence'][value.TF_added == 'ARF18'],
               ))
    if promoter == 'NLP7':
        print(promoter)
        print(stats.f_oneway(value['corrected_luminescence'][value.TF_added == 'None'],
                value['corrected_luminescence'][value.TF_added == 'ANR1'],
                value['corrected_luminescence'][value.TF_added == 'NLP7'],
                value['corrected_luminescence'][value.TF_added == 'DREB26'],
                value['corrected_luminescence'][value.TF_added == 'ANAC032'],))

    if promoter == 'NIR1':

            print(promoter)
            print(stats.f_oneway(value['corrected_luminescence'][value.TF_added == 'None'],
                value['corrected_luminescence'][value.TF_added == 'NLP6+NLP7'],
                value['corrected_luminescence'][value.TF_added == 'NLP7'],
                value['corrected_luminescence'][value.TF_added == 'DREB26'],
                value['corrected_luminescence'][value.TF_added == 'NLP6+NLP7+DREB26'],
                value['corrected_luminescence'][value.TF_added == 'NLP6']))
ANAC032
F_onewayResult(statistic=nan, pvalue=nan)
ARF18
F_onewayResult(statistic=nan, pvalue=nan)
DREB26
F_onewayResult(statistic=nan, pvalue=nan)
NLP7
F_onewayResult(statistic=nan, pvalue=nan)
df = luminescence_raw_df
#First rerun NIR1 it as a GLM
NIR1 = df[df.Promoter == 'NIR1']
results = ols('corrected_luminescence ~ C(TF_added)', data=NIR1).fit()
results.summary()

_____no_output_____# #make anova table for NIR1
# aov_table = sm.stats.anova_lm(results, typ=2)
# aov_table_____no_output_____# #pairwise tukey posthoc test for NIR1

# mc = MultiComparison(NIR1['corrected_luminescence'], NIR1['TF_added'])
# mc_results = mc.tukeyhsd()
# print(mc_results)_____no_output_____rp.summary_cont(NIR1['corrected_luminescence'].groupby(NIR1['TF_added']))

ANR1 = df[df.Promoter == 'ANR1']
rp.summary_cont(ANR1['corrected_luminescence'].groupby(ANR1['TF_added']))

for promoter, value in test:
    print('\033[1m' +promoter + '\033[0m')
    print(kruskal(data=df[df.Promoter == promoter], dv='corrected_luminescence', between='TF_added'))
    print('')NOS
           Source  ddof1    H
Kruskal  TF_added      0  0.0

STAP4
           Source  ddof1    H
Kruskal  TF_added      0  0.0

ANAC032
           Source  ddof1         H     p-unc
Kruskal  TF_added      3  2.424242  0.489138

ANR1
           Source  ddof1          H     p-unc
Kruskal  TF_added      4  14.875661  0.004966

ARF18
           Source  ddof1         H     p-unc
Kruskal  TF_added      2  0.297143  0.861938

NLP6
           Source  ddof1         H    p-unc
Kruskal  TF_added      3  5.814737  0.12098

NLP7
           Source  ddof1         H     p-unc
Kruskal  TF_added      4  9.275132  0.054579

DREB26
           Source  ddof1         H     p-unc
Kruskal  TF_added      1  0.066667  0.796253

NIR1
           Source  ddof1          H     p-unc
Kruskal  TF_added      5  20.823656  0.000875

#run Dunn's post-hoc test
#ANR1
sp.posthoc_dunn(df[df.Promoter == 'ANR1'], val_col='corrected_luminescence', group_col='TF_added', p_adjust='bonferroni')_____no_output_____#NIR1 post-hoc test
sp.posthoc_dunn(df[df.Promoter == 'NIR1'], val_col='corrected_luminescence', group_col='TF_added', p_adjust='bonferroni')_____no_output_____
</code>


# Notebook from basiralab/survey-multigraph-integration-methods
Path: Reproducibility of the plots for single-view fusion methods/Hamming distance.ipynb

<code>
"""
@author: nadachaari.nc@gmail.com
"""

# This code generate the average Hamming distance across 5 folds cross-validation
# between all combinations of the single-view fusion methods (SNF, SM-netFusion, and NAGFS)

import pickle
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt

# load data for 4 populations: female LH, male LH, female RH, male RH)
# LH means left hemisphere
# RH means right hemisphere
# GSP is Brain Genomics Superstrcut Project dataset which consist of healthy female and male populations

args_dataset = 'LH_GSP'

with open('CBT_male_' + args_dataset + '_' + 'SNF', 'rb') as f:
    CBT_SNF_LH_male = pickle.load(f)
with open('CBT_male_' + args_dataset + '_' + 'NAGFS', 'rb') as f:
    CBT_NAGFS_LH_male = pickle.load(f)
with open('CBT_male_' + args_dataset + '_' + 'SM-netFusion', 'rb') as f:
    CBT_SMnetFusion_LH_male = pickle.load(f)

with open('CBT_female_' + args_dataset + '_' + 'SNF', 'rb') as f:
    CBT_SNF_LH_female = pickle.load(f)
with open('CBT_female_' + args_dataset + '_' + 'NAGFS', 'rb') as f:
    CBT_NAGFS_LH_female = pickle.load(f)
with open('CBT_female_' + args_dataset + '_' + 'SM-netFusion', 'rb') as f:
    CBT_SMnetFusion_LH_female = pickle.load(f)

args_dataset = 'RH_GSP'

with open('CBT_male_' + args_dataset + '_' + 'SNF', 'rb') as f:
    CBT_SNF_RH_male = pickle.load(f)
with open('CBT_male_' + args_dataset + '_' + 'NAGFS', 'rb') as f:
    CBT_NAGFS_RH_male = pickle.load(f)
with open('CBT_male_' + args_dataset + '_' + 'SM-netFusion', 'rb') as f:
    CBT_SMnetFusion_RH_male = pickle.load(f)

with open('CBT_female_' + args_dataset + '_' + 'SNF', 'rb') as f:
    CBT_SNF_RH_female = pickle.load(f)
with open('CBT_female_' + args_dataset + '_' + 'NAGFS', 'rb') as f:
    CBT_NAGFS_RH_female = pickle.load(f)
with open('CBT_female_' + args_dataset + '_' + 'SM-netFusion', 'rb') as f:
    CBT_SMnetFusion_RH_female = pickle.load(f)

def heat_map(CBT_SNF,CBT_SMnetFusion,CBT_NAGFS):
    CBT_singleview = []
    CBT_singleview.append(CBT_SNF)
    CBT_singleview.append(CBT_SMnetFusion)
    CBT_singleview.append(CBT_NAGFS)

    columns = ['SNF', 'SMnetFusion', 'NAGFS']
    index = ['SNF', 'SMnetFusion', 'NAGFS']
    H_singleview = pd.DataFrame(index=index, columns=columns)
    for i in range(len(CBT_singleview)):
        for j in range(len(CBT_singleview)):
            humm_dist = mean_humm_dist(CBT_singleview[i],CBT_singleview[j])
            H_singleview.iloc[i, j] = humm_dist
    H_singleview = H_singleview.astype(float)

    return H_singleview

def hamming_dist(G1,G2):
    summ = 0
    N = G1.shape[0]
    for i in range (N):
        for j in range(N):
            summ = summ + abs(G1[i,j]-G2[i,j])
    hamm = summ/(N*(N-1))
    return (hamm)

def mean_humm_dist (G1,G2):
    summ = 0
    for i in range(len(G1)):
        summ = summ + hamming_dist(G1[i], G2[i])
    mean = summ / 5
    return(mean)

H_singleview_LH_female = heat_map(CBT_SNF_LH_female,CBT_SMnetFusion_LH_female,CBT_NAGFS_LH_female)
H_singleview_RH_female = heat_map(CBT_SNF_RH_female,CBT_SMnetFusion_RH_female,CBT_NAGFS_RH_female)
H_singleview_LH_male = heat_map(CBT_SNF_LH_male,CBT_SMnetFusion_LH_male,CBT_NAGFS_LH_male)
H_singleview_RH_male = heat_map(CBT_SNF_RH_male,CBT_SMnetFusion_RH_male,CBT_NAGFS_RH_male)

plt.rcParams['figure.dpi'] = 300

fig = plt.figure(figsize=(13, 3))
cmap = sns.cm.rocket_r
ax1 = fig.add_subplot(141)
ax2 = fig.add_subplot(142)
ax3 = fig.add_subplot(143)
ax4 = fig.add_subplot(144)
sns.set(font_scale=1)

sns.heatmap(H_singleview_LH_female, ax=ax1, fmt=".3f", vmin=0, cmap=cmap)
sns.heatmap(H_singleview_LH_male, ax=ax2, fmt=".3f", vmin=0, cmap=cmap)
sns.heatmap(H_singleview_RH_female, ax=ax3, fmt=".3f", vmin=0, cmap=cmap)
sns.heatmap(H_singleview_RH_male, ax=ax4, fmt=".3f", vmin=0, cmap=cmap)

ax1.set_title('LH_female', y=1.1, fontsize=10)
ax2.set_title('LH_male', y=1.1, fontsize=10)
ax3.set_title('RH_female', y=1.1, fontsize=10)
ax4.set_title('RH_male', y=1.1, fontsize=10)
plt.rcParams['figure.dpi'] = 300
#fig.suptitle('Hamming distance between all pairs singleview fusion methods', y=1.1)
plt.yticks(rotation=0)
plt.tight_layout()_____no_output_____
</code>


# Notebook from PacktPublishing/Apache-Spark-in-7-Days
Path: Section 3/3.3_code_DataFrameRowOperations.ipynb

<code>
# In the first cell, type:
import findspark
findspark.init()
_____no_output_____# In the second cell, type:
import pyspark # only run after findspark.init()
from pyspark.sql import SparkSession
spark = SparkSession.builder.getOrCreate()
_____no_output_____# Create a dataframe, use toDF()
df = spark.range(500).toDF("number")
df.show()
+------+
|number|
+------+
|     0|
|     1|
|     2|
|     3|
|     4|
|     5|
|     6|
|     7|
|     8|
|     9|
|    10|
|    11|
|    12|
|    13|
|    14|
|    15|
|    16|
|    17|
|    18|
|    19|
+------+
only showing top 20 rows

# alter values  in rows
df.select(df["number"] + 10).show()
+-------------+
|(number + 10)|
+-------------+
|           10|
|           11|
|           12|
|           13|
|           14|
|           15|
|           16|
|           17|
|           18|
|           19|
|           20|
|           21|
|           22|
|           23|
|           24|
|           25|
|           26|
|           27|
|           28|
|           29|
+-------------+
only showing top 20 rows

# Filter row such that number module 2 is not equal to 0.
df.filter(df['number'] % 2 != 0).show()+------+
|number|
+------+
|     1|
|     3|
|     5|
|     7|
|     9|
|    11|
|    13|
|    15|
|    17|
|    19|
|    21|
|    23|
|    25|
|    27|
|    29|
|    31|
|    33|
|    35|
|    37|
|    39|
+------+
only showing top 20 rows

# use rdd to convert to rdd and perform rdd operations
df.rdd.count()
_____no_output_____# take the first 10 elements
df.rdd.take(10)_____no_output_____# create a row object
spark.range(10).collect()
_____no_output_____# creating a row
from pyspark.sql import Row
myRow = Row("Hello", None, 1, False)
type(myRow)_____no_output_____# return the first element
myRow[0]_____no_output_____# return the third element
myRow[2]_____no_output_____from pyspark.sql import Row

from pyspark.sql.types import StructField, StructType, StringType, LongType

myManualSchema =StructType([
	StructField("some", StringType(), True),
	StructField("col", StringType(), True),
	StructField("names", LongType(), False)
])_____no_output_____# Create a row
myRow = Row("Hello", None, 1)_____no_output_____# Create dataframe from a row
myDf = spark.createDataFrame([myRow], myManualSchema)_____no_output_____myDf.show()+-----+----+-----+
| some| col|names|
+-----+----+-----+
|Hello|null|    1|
+-----+----+-----+

# Example of using Row function to create a dataframe
from pyspark.sql import Row
cats = Row("Name", "Nickname", "Location", "Treat")_____no_output_____cat1 = Row('Dakota', 'Sweetie', "house", "salmon")
cat2 = Row('George', 'Grumpy', "apt", "liver")
cat3 = Row('Karrot', 'BiggieK', "condo", "chicken")
cat4 = Row('Tigress', 'Claw', "street", "trout")
cat5 = Row('Kitty', 'Meow', "house", "salmon")_____no_output_____print(cat3)<Row(Karrot, BiggieK, condo, chicken)>
# Create Row elements
shelter1 = Row(id='23456', name='CatColony')
shelter2 = Row(id='11111', name='Mauhaus')
shelter3 = Row(id='98765', name='BigCatHouse')
shelter4 = Row(id='56789', name='WindowCats')_____no_output_____print(shelter2)Row(id='11111', name='Mauhaus')
# Create Row elements
shelterWithCats1 = Row(shelter=shelter1, cats=[cat1, cat2])
shelterWithCats2 = Row(shelter=shelter2, cats=[cat3, cat4])
shelterWithCats3 = Row(shelter=shelter3, cats=[cat5, cat4, cat1])
shelterWithCats4 = Row(shelter=shelter4, cats=[cat2, cat3])_____no_output_____shelterWithCats = [shelterWithCats1, shelterWithCats2, shelterWithCats3, shelterWithCats4]_____no_output_____# Create dataframe
dframe = spark.createDataFrame(shelterWithCats)_____no_output_____# Show dataframe
dframe.show()+--------------------+--------------------+
|                cats|             shelter|
+--------------------+--------------------+
|[[Dakota, Sweetie...|  [23456, CatColony]|
|[[Karrot, BiggieK...|    [11111, Mauhaus]|
|[[Kitty, Meow, ho...|[98765, BigCatHouse]|
|[[George, Grumpy,...| [56789, WindowCats]|
+--------------------+--------------------+

# Stop the Spark Context
spark.stop()_____no_output_____
</code>


# Notebook from sazio/NMAs
Path: src/Data_Loader_W&BFinetuning.ipynb

_____no_output_____# Exploratory Data Analysis of Stringer Dataset
@authors: Simone Azeglio, Chetan Dhulipalla , Khalid Saifullah

Part of the code here has been taken from [Neuromatch Academy's Computational Neuroscience Course](https://compneuro.neuromatch.io/projects/neurons/README.html), and specifically from [this notebook](https://colab.research.google.com/github/NeuromatchAcademy/course-content/blob/master/projects/neurons/load_stringer_spontaneous.ipynb)_____no_output_____# to do list

1. custom normalization: dividing by mean value per neuron
1a. downsampling: convolve then downsample by 5
2. training validation split: withhold last 20 percent of time series for testing
3. RNN for each layer: a way to capture the dynamics inside each layer instead of capturing extra dynamics from inter-layer interactions. it will be OK to compare the different RNNs. maintain same neuron count in each layer to reduce potential bias
4. layer weight regularization: L2
5. early stopping , dropout?_____no_output_____## Loading of Stringer spontaneous data

_____no_output_____

#@title Data retrieval
import os, requests

fname = "stringer_spontaneous.npy"
url = "https://osf.io/dpqaj/download"

if not os.path.isfile(fname):
    try:
        r = requests.get(url)
    except requests.ConnectionError:
        print("!!! Failed to download data !!!")
    else:
        if r.status_code != requests.codes.ok:
            print("!!! Failed to download data !!!")
        else:
            with open(fname, "wb") as fid:
                fid.write(r.content)_____no_output_____#@title Import matplotlib and set defaults
from matplotlib import rcParams
from matplotlib import pyplot as plt
rcParams['figure.figsize'] = [20, 4]
rcParams['font.size'] =15
rcParams['axes.spines.top'] = False
rcParams['axes.spines.right'] = False
rcParams['figure.autolayout'] = True_____no_output_____

## Exploratory Data Analysis (EDA)_____no_output_____

#@title Data loading
import numpy as np
dat = np.load('stringer_spontaneous.npy', allow_pickle=True).item()
print(dat.keys())_____no_output_____# functions

def moving_avg(array, factor = 5):
    """Reducing the number of compontents by averaging of N = factor
    subsequent elements of array"""
    zeros_ = np.zeros((array.shape[0], 2))
    array = np.hstack((array, zeros_))

    array = np.reshape(array, (array.shape[0],  int(array.shape[1]/factor), factor))
    array = np.mean(array, axis = 2)

    return array_____no_output_____

## Extracting Data for RNN (or LFADS)
The first problem to address is that for each layer we don't have the exact same number of neurons. We'd like to have a single RNN encoding all the different layers activities, to make it easier we can take the number of neurons ($N_{neurons} = 1131$ of the least represented class (layer) and level out each remaining class. _____no_output_____

# Extract labels from z - coordinate
from sklearn import preprocessing
x, y, z = dat['xyz']

le = preprocessing.LabelEncoder()
labels = le.fit_transform(z)
### least represented class (layer with less neurons)
n_samples = np.histogram(labels, bins=9)[0][-1]_____no_output_____### Data for LFADS / RNN
import pandas as pd
dataSet = pd.DataFrame(dat["sresp"])
dataSet["label"] = labels _____no_output_____# it can be done in one loop ...
data_ = []
for i in range(0, 9):
    data_.append(dataSet[dataSet["label"] == i].sample(n = n_samples).iloc[:,:-1].values)

dataRNN = np.zeros((n_samples*9, dataSet.shape[1]-1))
for i in range(0,9):

    # dataRNN[n_samples*i:n_samples*(i+1), :] = data_[i]
    ## normalized by layer
    dataRNN[n_samples*i:n_samples*(i+1), :] = data_[i]/np.mean(np.asarray(data_)[i,:,:], axis = 0)

## shuffling for training purposes

#np.random.shuffle(dataRNN)_____no_output_____#unshuffled = np.array(data_)_____no_output_____#@title Convolutions code

# convolution moving average

# kernel_length = 50
# averaging_kernel = np.ones(kernel_length) / kernel_length

# dataRNN.shape

# avgd_dataRNN = list()

# for neuron in dataRNN:
#   avgd_dataRNN.append(np.convolve(neuron, averaging_kernel))

# avg_dataRNN = np.array(avgd_dataRNN)

# print(avg_dataRNN.shape)_____no_output_____# @title Z Score Code

# from scipy.stats import zscore

# neuron = 500

# scaled_all = zscore(avg_dataRNN)
# scaled_per_neuron = zscore(avg_dataRNN[neuron, :])

# scaled_per_layer = list()

# for layer in unshuffled:
#   scaled_per_layer.append(zscore(layer))

# scaled_per_layer = np.array(scaled_per_layer)

# plt.plot(avg_dataRNN[neuron, :])
# plt.plot(avg_dataRNN[2500, :])
# plt.figure()
# plt.plot(dataRNN[neuron, :])
# plt.figure()
# plt.plot(scaled_all[neuron, :])
# plt.plot(scaled_per_neuron)
# plt.figure()
# plt.plot(scaled_per_layer[0,neuron,:])
_____no_output_____# custom normalization

normed_dataRNN = list()
for neuron in dataRNN:
    normed_dataRNN.append(neuron)# / neuron.mean())
normed_dataRNN = np.array(normed_dataRNN)

# downsampling and averaging
#avgd_normed_dataRNN = dataRNN#
avgd_normed_dataRNN = moving_avg(dataRNN, factor=2)_____no_output_____plt.plot(avgd_normed_dataRNN[7,:])_____no_output_____

issue: does the individual scaling by layer introduce bias that may artificially increase performance of the network?_____no_output_____## Data Loader
_____no_output_____

import torch
import torch.nn as nn
import torch.nn.functional as F
torch.cuda.empty_cache()_____no_output_____device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')_____no_output_____# set the seed
np.random.seed(42)

# number of neurons
NN = dataRNN.shape[0]_____no_output_____# swapping the axes to maintain consistency with seq2seq notebook in the following code - the network takes all the neurons at a time step as input, not just one neuron

# avgd_normed_dataRNN = np.swapaxes(avgd_normed_dataRNN, 0, 1)
avgd_normed_dataRNN.shape_____no_output_____frac = 4/5

#x1 = torch.from_numpy(dataRNN[:,:int(frac*dataRNN.shape[1])]).to(device).float().unsqueeze(0)
#x2 = torch.from_numpy(dataRNN[:,int(frac*dataRNN.shape[1]):]).to(device).float().unsqueeze(0)
#x1 = torch.from_numpy(avgd_normed_dataRNN[:1131,:]).to(device).float().unsqueeze(2)
#x2 = torch.from_numpy(avgd_normed_dataRNN[:1131,:]).to(device).float().unsqueeze(2)

n_neurs = 1131
# let's use n_neurs/10 latent components
ncomp = int(n_neurs/5)

x1_train = torch.from_numpy(avgd_normed_dataRNN[:n_neurs,:int(frac*avgd_normed_dataRNN.shape[1])]).to(device).float().unsqueeze(2)
x2_train = torch.from_numpy(avgd_normed_dataRNN[:n_neurs,:int(frac*avgd_normed_dataRNN.shape[1])]).to(device).float().unsqueeze(2)

x1_valid = torch.from_numpy(avgd_normed_dataRNN[:n_neurs,int(frac*avgd_normed_dataRNN.shape[1]):]).to(device).float().unsqueeze(2)
x2_valid = torch.from_numpy(avgd_normed_dataRNN[:n_neurs,int(frac*avgd_normed_dataRNN.shape[1]):]).to(device).float().unsqueeze(2)

NN1 = x1_train.shape[0]
NN2 = x2_train.shape[0]
_____no_output_____lass Net(nn.Module):
    def __init__(self, ncomp, NN1, NN2, bidi=True):
        super(Net, self).__init__()

        # play with some of the options in the RNN!

        self.rnn = nn.LSTM(NN1, ncomp, num_layers = 3, dropout = 0.3,
                         bidirectional = bidi)
        """
        self.rnn = nn.RNN(NN1, ncomp, num_layers = 1, dropout = 0,
                    bidirectional = bidi, nonlinearity = 'tanh')
        self.rnn = nn.GRU(NN1, ncomp, num_layers = 1, dropout = 0,
                         bidirectional = bidi)
        """

        self.mlp = nn.Sequential(
                    nn.Linear(ncomp, ncomp*2),
                    nn.Mish(),
                    nn.Linear(ncomp*2, ncomp*2),
                    nn.Mish(),
                    nn.Dropout(0.25),
                    nn.Linear(ncomp*2, ncomp),
                    nn.Mish())

        self.fc = nn.Linear(ncomp, NN2)

    def forward(self, x):
        x = x.permute(1, 2, 0)
        #print(x.shape)
        # h_0 = torch.zeros(2, x.size()[1], self.ncomp).to(device)

        y, h_n = self.rnn(x)

        #print(y.shape)
        #print(h_n.shape)
        if self.rnn.bidirectional:
          # if the rnn is bidirectional, it concatenates the activations from the forward and backward pass
          # we want to add them instead, so as to enforce the latents to match between the forward and backward pass
            q = (y[:, :, :ncomp] + y[:, :, ncomp:])/2
        else:
            q = y

        q = self.mlp(q)

        # the softplus function is just like a relu but it's smoothed out so we can't predict 0
        # if we predict 0 and there was a spike, that's an instant Inf in the Poisson log-likelihood which leads to failure
        #z = F.softplus(self.fc(q), 10)
        #print(q.shape)
        z = self.fc(q).permute(2, 0, 1)
        # print(z.shape)
        return z, q_____no_output_____# we initialize the neural network
net = Net(ncomp, NN1, NN2, bidi = True).to(device)

# special thing:  we initialize the biases of the last layer in the neural network
# we set them as the mean firing rates of the neurons.
# this should make the initial predictions close to the mean, because the latents don't contribute much
#net.fc.bias.data[:] = x1_train.mean(axis = (0,1))

# we set up the optimizer. Adjust the learning rate if the training is slow or if it explodes.
optimizer1 = torch.optim.Adam(net.parameters(), lr=0.0005798, weight_decay = 1e-5)
# optimizer2 = torch.optim.SGD(net.parameters(), lr = 0.0001, momentum = 0.9, weight_decay = 0.01, )
# optimizer3 = torch.optim._____no_output_____# forward check
# net(x1)
net(x1_train)_____no_output_____

## Training _____no_output_____

from tqdm import tqdm_____no_output_____#valid_save = []
#train_save = []

## Original training
# you can keep re-running this cell if you think the cost might decrease further

cost = nn.MSELoss()

niter = 50000
# rnn_loss = 0.2372, lstm_loss = 0.2340, gru_lstm = 0.2370
for k in tqdm(range(niter)):
    net.train()
    # the network outputs the single-neuron prediction and the latents
    z, y = net(x1_train)

    # our cost
    loss = cost(z, x2_train)

    # train the network as usual
    loss.backward()
    optimizer1.step()
    optimizer1.zero_grad()

    with torch.no_grad():
        net.eval()
        train_save.append(loss.item())
        valid_loss = cost(net(x1_valid)[0], x2_valid)
        valid_save.append(valid_loss.item())

    if k % 50 == 0:
        with torch.no_grad():
            net.eval()
            valid_loss = cost(net(x1_valid)[0], x2_valid)

            print(f' iteration {k}, train cost {loss.item():.4f}, valid cost {valid_loss.item():.4f}')
 _____no_output_____plt.plot(valid_save)
plt.plot(train_save)_____no_output_____

## Validation from same neurons_____no_output_____

test, hidden = net(x1_valid)_____no_output_____plt.plot(x2_valid[3,:,0].cpu().detach().numpy())
plt.plot(test[3,:,0].cpu().detach().numpy())_____no_output_____

## Testing neurons from same layer_____no_output_____

test, hidden = net(torch.from_numpy(avgd_normed_dataRNN[n_neurs:2*n_neurs,:]).unsqueeze(2).to(device).float())_____no_output_____test.shape_____no_output_____n_n = 15
plt.plot(test[n_n,:,0].cpu().detach().numpy())
plt.plot(avgd_normed_dataRNN[n_neurs + n_n,:])_____no_output_____

## Testing neurons from another layer (#9)_____no_output_____

test, hidden = net(torch.from_numpy(avgd_normed_dataRNN[10000:10100,:]).unsqueeze(2).to(device).float())_____no_output_____plt.plot(test[10,:,0].cpu().detach().numpy())
plt.plot(avgd_normed_dataRNN[10010,:])_____no_output_____

# Training 9 Networks
Each Network corresponds to a different layer in V1_____no_output_____

# you can keep re-running this cell if you think the cost might decrease further

cost = nn.MSELoss()

niter = 10000
# rnn_loss = 0.2372, lstm_loss = 0.2340, gru_lstm = 0.2370
for k in tqdm(range(niter)):
    net.train()
    # the network outputs the single-neuron prediction and the latents
    z, y = net(x1_train)

    # our cost
    loss = cost(z, x2_train)

    # train the network as usual
    loss.backward()
    optimizer1.step()
    optimizer1.zero_grad()

    if k % 50 == 0:
        with torch.no_grad():
            net.eval()
            valid_loss = cost(net(x1_valid)[0], x2_valid)

            print(f' iteration {k}, train cost {loss.item():.4f}, valid cost {valid_loss.item():.4f}')

    _____no_output_____