# Notebook from shrestha-bikash/disteval-1
Path: LDDT.ipynb

import numpy as np
from math import sqrt
import os_____no_output_____valid_amino_acids = {
    'LLP': 'K', 'TPO': 'T', 'CSS': 'C', 'OCS': 'C', 'CSO': 'C', 'PCA': 'E', 'KCX': 'K', \
    'CME': 'C', 'MLY': 'K', 'SEP': 'S', 'CSX': 'C', 'CSD': 'C', 'MSE': 'M', \
    'ALA': 'A', 'ASN': 'N', 'CYS': 'C', 'GLN': 'Q', 'HIS': 'H', 'LEU': 'L', \
    'MET': 'M', 'MHO': 'M', 'PRO': 'P', 'THR': 'T', 'TYR': 'Y', 'ARG': 'R', 'ASP': 'D', \
    'GLU': 'E', 'GLY': 'G', 'ILE': 'I', 'LYS': 'K', 'PHE': 'F', 'SER': 'S', \
    'TRP': 'W', 'VAL': 'V', 'SEC': 'U'
    }_____no_output_____def check_pdb_valid_row(valid_amino_acids, l):
    if (get_pdb_rname(l) in valid_amino_acids.keys()) and (l.startswith('ATOM') or l.startswith('HETA')):
        return True
    return False

def get_pdb_atom_name(l):
    return l[12: 16].strip()

def get_pdb_rnum(l):
    return int(l[22: 27].strip())

def get_pdb_rname(l):
    return l[17: 20].strip()

def get_pdb_xyz_cb(lines):
    xyz = {}
    for l in lines:
        if get_pdb_atom_name(l) == 'CB':
            xyz[get_pdb_rnum(l)] = (float(l[30:38].strip()), float(l[38:46].strip()), float(l[46:54].strip()))
    for l in lines:
        if (get_pdb_rnum(l) not in xyz) and get_pdb_atom_name(l) == 'CA':
            xyz[get_pdb_rnum(l)] = (float(l[30:38].strip()), float(l[38:46].strip()), float(l[46:54].strip()))
    return xyz

def get_pdb_xyz_ca(lines):
    xyz = {}
    for l in lines:
        if get_pdb_atom_name(l) == 'CA':
            xyz[get_pdb_rnum(l)] = (float(l[30:38].strip()), float(l[38:46].strip()), float(l[46:54].strip()))
    return xyz_____no_output_____def pdb2dmap(pdbfile):
    f = open(pdbfile, mode = 'r')
    flines = f.read()
    f.close()
    lines = flines.splitlines()
    templines = flines.splitlines()
    for l in templines:
        if not l.startswith('ATOM'): lines.remove(l)
    # We have filtered out all non ATOMs at this point
    rnum_rnames = {}
    for l in lines:
        atom = get_pdb_atom_name(l)
        if atom != 'CA': continue
        if not get_pdb_rname(l) in valid_amino_acids.keys():
            print ('' + get_pdb_rname(l) + ' is unknown amino acid in ' + l)
            return
        rnum_rnames[int(get_pdb_rnum(l))] = valid_amino_acids[get_pdb_rname(l)]
    seq = ""
    for i in range(max(rnum_rnames.keys())):
        if i+1 not in rnum_rnames:
            #print (rnum_rnames)
            #print ('Warning! residue not defined for rnum = ' + str(i+1))
            seq += '-'
        else:
            seq += rnum_rnames[i+1]
    L = len(seq)
    xyz_cb = get_pdb_xyz_cb(lines)
    total_valid_residues = len(xyz_cb)
    if len(xyz_cb) != L:
        print(rnum_rnames)
        for i in range(L):
            if i+1 not in xyz_cb: print('XYZ not defined for ' + str(i+1))
        print ('Warning! Something went wrong - len of cbxyz != seqlen!! ' + str(len(xyz_cb)) + ' ' +  str(L))
    cb_map = np.full((L, L), np.nan)
    for r1 in sorted(xyz_cb):
        (a, b, c) = xyz_cb[r1]
        for r2 in sorted(xyz_cb):
            (p, q, r) = xyz_cb[r2]
            cb_map[r1 - 1, r2 - 1] = sqrt((a-p)**2+(b-q)**2+(c-r)**2)
    return (total_valid_residues, cb_map, rnum_rnames)_____no_output_____

# Open files_____no_output_____## PDB files_____no_output_____

pdb_d = "test"_____no_output_____

### True_____no_output_____

refer_pdb_f = "1guuA.pdb"
refer_pdb_p = os.path.join(pdb_d, refer_pdb_f)
ref_L, ref_dmap, ref_rnames = pdb2dmap(refer_pdb_p)_____no_output_____

### Predicted_____no_output_____

# model_pdb_f = ""
# model_pdb_p = os.path.join(pdb_d, model_pdb_f)
# mod_L, mod_dmap, mod_rnames = pdb2dmap(model_pdb_p)_____no_output_____

## NPY files_____no_output_____

npy_d = "test"_____no_output_____

### True_____no_output_____

# refer_npy_f = "1guuA.true.npy"
# refer_npy_p = os.path.join(npy_d, refer_npy_f)
# ref_dmap = np.load(refer_npy_p)
# ref_dmap = np.squeeze(ref_dmap)_____no_output_____

### Predicted_____no_output_____

model_npy_f = "1guuA.predicted.npy"
model_npy_p = os.path.join(npy_d, model_npy_f)
mod_dmap = np.load(model_npy_p)
mod_dmap = np.squeeze(mod_dmap)_____no_output_____# Helpers for metrics calculated using numpy scheme
def get_flattened(dmap):
  if dmap.ndim == 1:
    return dmap
  elif dmap.ndim == 2:
    return dmap[np.triu_indices_from(dmap, k=1)]
  else:
    assert False, "ERROR: the passes array has dimension not equal to 2 or 1!"

def get_separations(dmap):
  t_indices = np.triu_indices_from(dmap, k=1)
  separations = np.abs(t_indices[0] - t_indices[1])
  return separations

# return a 1D boolean array indicating where the sequence separation in the
# upper triangle meets the threshold comparison
def get_sep_thresh_b_indices(dmap, thresh, comparator):
  assert comparator in {'gt', 'lt', 'ge', 'le'}, "ERROR: Unknown comparator for thresholding!"
  dmap_flat = get_flattened(dmap)
  separations = get_separations(dmap)
  if comparator == 'gt':
    threshed = separations > thresh
  elif comparator == 'lt':
    threshed = separations < thresh
  elif comparator == 'ge':
    threshed = separations >= thresh
  elif comparator == 'le':
    threshed = separations <= thresh

  return threshed

# return a 1D boolean array indicating where the distance in the
# upper triangle meets the threshold comparison
def get_dist_thresh_b_indices(dmap, thresh, comparator):
  assert comparator in {'gt', 'lt', 'ge', 'le'}, "ERROR: Unknown comparator for thresholding!"
  dmap_flat = get_flattened(dmap)
  if comparator == 'gt':
    threshed = dmap_flat > thresh
  elif comparator == 'lt':
    threshed = dmap_flat < thresh
  elif comparator == 'ge':
    threshed = dmap_flat >= thresh
  elif comparator == 'le':
    threshed = dmap_flat <= thresh
  return threshed

# Calculate lDDT using numpy scheme
def get_LDDT(true_map, pred_map, R=15, sep_thresh=-1, T_set=[0.5, 1, 2, 4], precision=4):
    '''
    Mariani V, Biasini M, Barbato A, Schwede T.
    lDDT: a local superposition-free score for comparing protein structures and models using distance difference tests.
    Bioinformatics. 2013 Nov 1;29(21):2722-8.
    doi: 10.1093/bioinformatics/btt473.
    Epub 2013 Aug 27.
    PMID: 23986568; PMCID: PMC3799472.
    '''

    # Helper for number preserved in a threshold
    def get_n_preserved(ref_flat, mod_flat, thresh):
        err = np.abs(ref_flat - mod_flat)
        n_preserved = (err < thresh).sum()
        return n_preserved

    # flatten upper triangles
    true_flat_map = get_flattened(true_map)
    pred_flat_map = get_flattened(pred_map)

    # Find set L
    S_thresh_indices = get_sep_thresh_b_indices(true_map, sep_thresh, 'gt')
    R_thresh_indices = get_dist_thresh_b_indices(true_flat_map, R, 'lt')

    L_indices = S_thresh_indices & R_thresh_indices

    true_flat_in_L = true_flat_map[L_indices]
    pred_flat_in_L = pred_flat_map[L_indices]

    # Number of pairs in L
    L_n = L_indices.sum()

    # Calculated lDDT
    preserved_fractions = []
    for _thresh in T_set:
        _n_preserved = get_n_preserved(true_flat_in_L, pred_flat_in_L, _thresh)
        _f_preserved = _n_preserved / L_n
        preserved_fractions.append(_f_preserved)

    lDDT = np.mean(preserved_fractions)
    if precision > 0:
        lDDT = round(lDDT, precision)
    return lDDT_____no_output_____get_LDDT(ref_dmap, mod_dmap)_____no_output_____


# Notebook from yelabucsf/scrna-parameter-estimation
Path: analysis/sciplex/drug_variability.ipynb

# Variability

Investigate the differences in transcriptome variability between drugs that target the same system_____no_output_____

import scanpy as sc
import seaborn as sns
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
import itertools
import scipy.stats as stats_____no_output_____import sys
sys.path.append('/home/ssm-user/Github/scrna-parameter-estimation/dist/memento-0.0.6-py3.8.egg')
sys.path.append('/home/ssm-user/Github/misc-seq/miscseq/')
import encode
import memento_____no_output_____import gseapy as gp
from gseapy.plot import gseaplot_____no_output_____pd.set_option('display.max_columns', None)
pd.set_option('display.max_rows', None)_____no_output_____data_path = '/data_volume/memento/sciplex/'_____no_output_____ct = 'A549'_____no_output_____adata = sc.read(data_path + 'h5ad/{}.h5ad'.format(ct))_____no_output_____

### Define the drug classes_____no_output_____

target_list = [
    'Aurora Kinase',
    'DNA/RNA Synthesis',
    'HDAC',
    'Histone Methyltransferase',
    'JAK',
    'PARP',
    'Sirtuin']

target_to_dir = {
    'Aurora Kinase':'aurora',
    'DNA/RNA Synthesis':'dna_rna',
    'HDAC':'hdac',
    'Histone Methyltransferase':'histmeth',
    'JAK':'jak',
    'PARP':'parp',
    'Sirtuin':'sirt'}_____no_output_____

### Setup memento_____no_output_____

adata.obs['q'] = 0.05_____no_output_____memento.setup_memento(adata, q_column='q', filter_mean_thresh=0.07)Version 0.0.6

### control vs every dose in strata_____no_output_____

target = 'HDAC'_____no_output_____drug_counts = adata.obs.query('target == "{}"'.format(target)).product_name.value_counts()
drug_list = drug_counts[drug_counts > 0].index.tolist()
num_drugs = len(drug_list)_____no_output_____names = ['Vehicle', 'Pracinostat (SB939)']
names = ['Vehicle', target]_____no_output_____subset = adata[adata.obs.target.isin(names)].copy().copy()_____no_output_____num_vehicle = (subset.obs.product_name =='Vehicle').sum()_____no_output_____subset.obs.loc[(subset.obs.product_name =='Vehicle'), 'dose'] = np.random.choice([10, 100, 1000, 10000], size=num_vehicle)
subset.obs['is_drug'] = (subset.obs.product_name != 'Vehicle').astype(int)_____no_output_____memento.create_groups(subset, label_columns=['is_drug', 'dose', 'product_name'])
memento.compute_1d_moments(subset, min_perc_group=.9)
memento.ht_1d_moments(
    subset,
    formula_like='1 + is_drug + dose',
    treatment_col='is_drug',
    num_boot=20000,
    verbose=1,
    num_cpus=92,
    resampling='permutation',
    approx=False)/home/ssm-user/Github/scrna-parameter-estimation/dist/memento-0.0.6-py3.8.egg/memento/estimator.py:105: RuntimeWarning: invalid value encountered in log
[Parallel(n_jobs=92)]: Using backend LokyBackend with 92 concurrent workers.
[Parallel(n_jobs=92)]: Done  16 tasks      | elapsed:   19.9s
[Parallel(n_jobs=92)]: Done 266 tasks      | elapsed:  1.1min
[Parallel(n_jobs=92)]: Done 616 tasks      | elapsed:  2.2min
[Parallel(n_jobs=92)]: Done 1066 tasks      | elapsed:  3.7min
[Parallel(n_jobs=92)]: Done 1616 tasks      | elapsed:  5.4min
[Parallel(n_jobs=92)]: Done 2266 tasks      | elapsed:  7.5min
[Parallel(n_jobs=92)]: Done 2646 out of 2646 | elapsed: 13.4min finished
df = memento.get_1d_ht_result(subset)_____no_output_____df['dv_fdr'] = memento.util._fdrcorrect(df['dv_pval'])
df['de_fdr'] = memento.util._fdrcorrect(df['de_pval'])_____no_output_____df.query('dv_fdr < 0.1 & dv_coef > 0')_____no_output_____enr = gp.enrichr(gene_list=df.query('dv_fdr < 0.1').gene,
                 gene_sets=['GO_Biological_Process_2021'],
                 organism='Human', # don't forget to set organism to the one you desired! e.g. Yeast
                 outdir=None,
                 cutoff=0.5 # test dataset, use lower value from range(0,1)
                )
enr.results.head(10)
_____no_output_____enr = gp.enrichr(gene_list=df.query('dv_fdr < 0.1').gene,
                 gene_sets=['WikiPathways_2019_Human'],
                 organism='Human', # don't forget to set organism to the one you desired! e.g. Yeast
                 outdir=None,
                 cutoff=0.5 # test dataset, use lower value from range(0,1)
                )
enr.results.head(10)
_____no_output_____WikiPathways_2019_Human_____no_output_____

### Summarize DE/DV_____no_output_____

target = 'HDAC'_____no_output_____drug_counts = adata.obs.query('target == "{}"'.format(target)).product_name.value_counts()
drug_list = drug_counts[drug_counts > 0].index.tolist()
num_drugs = len(drug_list)_____no_output_____heatmap = pd.DataFrame(np.zeros((num_drugs, num_drugs)), index=drug_list, columns=drug_list)_____no_output_____for drug1, drug2 in itertools.combinations(drug_list,2):

    subset = sc.read(data_path + '{}/{}_vs_{}_stratified_perm.h5ad'.format(target_to_dir[target], drug1, drug2))
    df = memento.get_1d_ht_result(subset)
    df['dv_fdr'] = memento.util._fdrcorrect(df['dv_pval'])
    df['de_fdr'] = memento.util._fdrcorrect(df['de_pval'])

    print(drug1, drug2, df.query('dv_fdr < 0.1').shape[0], df.query('dv_fdr < 0.1').shape[0])

    break
TMP195 MC1568 0 0
plt.hist(df['de_pval'])_____no_output_____def get_combined_1d_result(target, col='dv_pval'):

    drug_counts = adata.obs.query('target == "{}"'.format(target)).product_name.value_counts()
    drug_list = drug_counts[drug_counts > 0].index.tolist()

    result = pd.DataFrame()
    start = True
    for drug1, drug2 in itertools.combinations(drug_list,2):

        subset = sc.read(data_path + '{}/{}_vs_{}_10000.h5ad'.format(target_to_dir[target], drug1, drug2))
        df = memento.get_1d_ht_result(subset)
        df['dv_fdr'] = memento.util._fdrcorrect(df['dv_pval'])
        df['de_fdr'] = memento.util._fdrcorrect(df['de_pval'])
        if start:

            result['gene'] = df['gene']
            result['{}_{}'.format(drug1, drug2)] = df[col]
            start = False
        else:
            result = result.merge(df[['gene', col]].rename(columns={col:'{}_{}'.format(drug1, drug2)}), how='outer')

        result['num_sig'] = (result.iloc[:, 1:] < 0.1).sum()

    combined_pvals = []
    for g in result.gene:
        pvalues = result.query('gene == "{}"'.format(g)).iloc[0, 1:].values.astype(float)
        chisq, pv = stats.combine_pvalues(pvalues[np.isfinite(pvalues)])
        combined_pvals.append((g, chisq, pv))

    combined_pvals = pd.DataFrame(combined_pvals, columns=['gene', 'chisq', 'pv'])
    combined_pvals['fdr'] = memento.util._fdrcorrect(combined_pvals['pv'])

    return combined_pvals_____no_output_____target = 'HDAC'_____no_output_____combined_dv = get_combined_1d_result(target, col='dv_pval')_____no_output_____combined_de = get_combined_1d_result(target, col='de_pval')_____no_output_____combined_dv['bonf'] = combined_dv['pv']*combined_dv.shape[0]
combined_de['bonf'] = combined_de['pv']*combined_de.shape[0]_____no_output_____combined_dv.query('bonf < 0.001').sort_values('fdr').gene.shape_____no_output_____combined_de.query('bonf < 0.001').sort_values('fdr').gene.shape_____no_output_____enr = gp.enrichr(gene_list=combined_de.query('bonf < 0.001').sort_values('fdr').gene,
                 gene_sets=['GO_Biological_Process_2021'],
                 organism='Human', # don't forget to set organism to the one you desired! e.g. Yeast
                 outdir=None,
                 cutoff=0.5 # test dataset, use lower value from range(0,1)
                )
enr.results.head(10)
_____no_output_____enr = gp.enrichr(gene_list=combined_dv.query('bonf < 0.001').sort_values('fdr').gene,
                 gene_sets=['GO_Biological_Process_2021'],
                 organism='Human', # don't forget to set organism to the one you desired! e.g. Yeast
                 outdir=None,
                 cutoff=0.5 # test dataset, use lower value from range(0,1)
                )
enr.results.head(10)
_____no_output_____

### GSEA + Enrichr_____no_output_____

import gseapy as gp
from gseapy.plot import gseaplot
_____no_output_____combined_pvals.query('fdr < 0.01').shape_____no_output_____enr = gp.enrichr(gene_list=combined_pvals.query('fdr < 0.005').sort_values('fdr').gene,
                 gene_sets=['GO_Biological_Process_2021'],
                 organism='Human', # don't forget to set organism to the one you desired! e.g. Yeast
                 outdir=None,
                 cutoff=0.5 # test dataset, use lower value from range(0,1)
                )
enr.results.head(10)
_____no_output_____enr.results.head(10)
_____no_output_____enr = gp.enrichr(gene_list=combined_pvals.query('fdr < 0.01').sort_values('fdr').gene,
                 gene_sets=['WikiPathways_2019_Human'],
                 organism='Human', # don't forget to set organism to the one you desired! e.g. Yeast
                 outdir=None,
                 cutoff=0.5 # test dataset, use lower value from range(0,1)
                )_____no_output_____df = enr.results.sort_values('Odds Ratio', ascending=False)
df[df['Adjusted P-value'] < 0.1].head(30)_____no_output_____

### Run 1D test against log10(dosage)

Use different drugs as strata_____no_output_____

for target in target_list:

    print('Starting ', target, '...')

#     target = 'HDAC'

    subset = adata[((adata.obs.target==target)) | (adata.obs.vehicle)].copy().copy()
    subset.obs['dose_indicator'] = subset.obs.dose.apply(lambda x: {0:0.0 ,10:1.0, 100:2.0, 1000:3.0, 10000:1.0}[x])
    subset.obs['dose_level'] = 'dose_' + subset.obs['dose'].astype(str)

    memento.create_groups(subset, label_columns=['dose_indicator'])
    memento.compute_1d_moments(subset, min_perc_group=.9)
    memento.ht_1d_moments(
        subset,
        formula_like='1 + dose_indicator',
        treatment_col='dose_indicator',
        num_boot=20000,
        verbose=0,
        num_cpus=92,
        resampling='permutation',
        approx=False)

    subset.write(data_path + 'dosage/{}.h5ad'.format(target_to_dir[target]))

#     breakStarting  Aurora Kinase ...

### AK_____no_output_____

for target in target_list:

#     target = 'DNA/RNA Synthesis'
#     target = 'PARP'
    subset = sc.read(data_path + 'dosage/{}.h5ad'.format(target_to_dir[target]))

    df = memento.get_1d_ht_result(subset)
    df['de_fdr'] = memento.util._fdrcorrect(df['de_pval'])

    df = df.query('de_fdr < 0.05')
    df['dv_fdr'] = memento.util._fdrcorrect(df['dv_pval'])

    print(target)

    if df.query('dv_fdr <0.1').shape[0] > 10:
        enr = gp.enrichr(gene_list=df.query('dv_fdr <0.1 & dv_coef < 0').sort_values('dv_coef', ascending=False).gene,
             gene_sets=['WikiPathways_2019_Human'],
             organism='Human', # don't forget to set organism to the one you desired! e.g. Yeast
             outdir=None,
             cutoff=0.5 # test dataset, use lower value from range(0,1)
                    )
        print(enr.results.sort_values('Adjusted P-value').head(10)[['Term', 'Adjusted P-value']])

Aurora Kinase
                                                 Term  Adjusted P-value
0                                 NRF2 pathway WP2884          0.216304
1   Corticotropin-releasing hormone signaling path...          0.216304
2   Transcription co-factors SKI and SKIL protein ...          0.285740
3   miRNA targets in ECM and membrane receptors WP...          0.318287
27                      Leptin and adiponectin WP3934          0.329234
26  Interleukin-1 Induced Activation of NF-kappa-B...          0.329234
25                        Vitamin D Metabolism WP1531          0.329234
24                        Genotoxicity pathway WP4286          0.329234
23          MECP2 and Associated Rett Syndrome WP3584          0.329234
22                    Ectoderm Differentiation WP2858          0.329234
df.sort_values('dv_pval')_____no_output_____# df = memento.get_1d_ht_result(subset)_____no_output_____plt.hist(df.dv_pval, bins=40)_____no_output_____


# Notebook from EdwardCatoiu/cobrame
Path: docs/MEModelFundamentals.ipynb

# ME-Model Fundamentals

Models of metabolism and expression (ME-models) are unique in that they are capable
of predicting the optimal macromolecular expression
required to sustain a metabolic phenotype. In other words, they are capable of making novel
predictions of the amount of individual protein, nucleotides, cofactors, etc. that the cell must
synthesize in order to grow optimally. To enable these types of predictions, ME-models differ
from metabolic models (M-models) in a few key ways:

1. ME-models are multi-scale in nature so they require the addition of *coupling constraints* to
couple cellular processes to each other.

2. ME-models predict the biomass composition of a growing cell thus forgoing much of the
M-model biomass composition function. For this reason, the function representing growth
needs to be updated.

These two ME-model features are briefly described below. Their practical implementation is further outlined in **ME-model Reaction Building**._____no_output_____## Coupling Constraints

Coupling constraints are required in an ME-model in order to couple a reaction flux to the synthesis of the macromolecule catalyzing the flux. The easiest example of this is for the coupling of metabolic enzymes to metabolic reactions. This has the form:_____no_output_____

where $\mu$ is the growth rate and $k_{eff}$ is an approximation of the effective turnover rate for the metabolic process. The coupling of enzyme synthesis cost to metabolic flux scales with $\mu$ to represent the dilution of macromolecules as they are passed on the daughter cells. More macromolecules are diluted at faster growth rates. Enzyme turnover rates determine the efficiency of of an enzyme *in vivo* and are largly unknown for a majority of metabolic and expression enzymes. Optimizing the vector of $k_{eff}s$ for the cellular processes modeled in the ME-model is an ongoing area of research. Currently, the $k_{eff}s$ are set with an average of 65 $s^{-1}$ and scaled by their solvent accessible surface area (approximated as $protein\_moleculare\_weight^{\frac{3}{4}}$). A set ~125 metabolic $k_{eff}s$ were found by [Ebrahim et. al. 2016](https://www.ncbi.nlm.nih.gov/pubmed/27782110?dopt=Abstract) to be particularly important in *E. coli* for computing an accurate metabolic/proteomic state using proteomics data. We suspect similar observations would be seen in other organisms.

For non metabolic macromolecules such as ribosome, mRNA, tRNA and RNA polymerase, the coupling constrains coefficients ('$k_{eff}s$') are derived by essentially back-calculating the individual rates using a measured RNA-to-Protein ratio from [Scott et. al. 2010](https://www.ncbi.nlm.nih.gov/pubmed/21097934) and measured mRNA, tRNA and rRNA fractions. The coupling constraints coefficients for these macromolecules are derived in detail in [O'Brien et. al. 2013](https://www.ncbi.nlm.nih.gov/pubmed/24084808). Applying these constraints results in a final nonlinear optimization problem (NLP) shown below. COBRAme reformulates these coupling constraints to embed them directly into the reaction which they are used as follows:_____no_output_____

The biomass_dilution constraint is discussed below.

### Previous ME-Model Coupling Constraint Implementation
The previous iterations of ME models applied coupling constraints using the following:_____no_output_____

Where $\alpha$ is the coupling coefficient applied to "coupling", a metabolite (constraint) that can be given a "constraint_sense" in COBRApy of 'L' which effectively allows the second reaction to proceed at a rate equal to or greater than $v2 \geq \alpha \cdot v1$.

### COBRAme Coupling Constraint Implementation
With COBRAme it is assumed that the optimal ME solution will never dilute more enzyme than is required by the coupling constraint thus constraining $v2 = \alpha \cdot v1$ and allowing us to combine the implementation of the constraint with the reaction that uses the enzyme to give._____no_output_____

The coupling constraints and coefficients were derived as in [O'Brien et. al. 2013](https://www.ncbi.nlm.nih.gov/pubmed/24084808).  As stated above, however, these were implemented in the current study as equality constraints . Effectively, this means that each ME-model solution will give the computed optimal proteome allocation for the *in silico* conditions. Previous ME-model formulations have applied the constraints as inequalities thus allowing the simulation to overproduce macromolecule components. While overproduction is seen *in vivo* in cells, this phenomenon would not be selected as the optimal solution. Furthermore, using inequality constraints greatly expands the size of the possible solution space significantly increasing the time required to solve the optimization.  Reformulating the model using equality constraints thus resulted in a reduced ME-matrix with the coupling constraints embedded directly into the reaction in which they are used._____no_output_____## Biomass Dilution Constraints
For metabolic models (M-models), the biomass objective function has been used to represent the amount of biomass
production that is required for the cell to double. The metabolites represented in the biomass function are typically building blocks for major macromolecules (e.g. amino acids and nucleotides), cell wall components and cofactors. The coefficients of the biomass objective function are determined from empirical measurements from a cell growing at a measured rate. Since ME-models explicitly compute the predicted amount of RNA, protein, cofactors, etc. necessary for growth, this concept has to be modified for ME models.

This is accomplished via the *biomass_dilution* variable (reaction), which contains the *biomass* constraint (pseudo metabolite) that represents the mass produced by the synthesis of each functional RNA or protein. This reaction essentially ensured that the ME-model can only produce biomass at the rate it is being diluted (via growth and division)._____no_output_____### Implementation
For each ME-model transcription or translation reaction in an ME-model an amount of a *biomass* constraint (pseudo
metabolite) is created with a stoichiometry equal to the molecular weight of the mRNA or protein being made (in
*kDA*). The below figure shows an example of this where a translation reaction produces both the catalytic protein
as well as the protein biomass constraint. The formed *protein_biomass* constraint is a participant in the overall ME *Biomas_ Dilution* reaction which restricts the total production of the major biomass components to equal the rate at which biomass is diluted (eg. the cell’s growth rate, $\mu$).

Some biomsas constituents do not have a mechanistic function in the ME-model (e.g. cell wall components, DNA
and glycogen). These metabolites are included in the *biomass_dilution* reaction indentical to the M-model biomass
reaction .

![Biomass Dilution for iJO1366 and for a ME-Model](./_static/Biomass_constraint_figure.png)

The units for this contraint work out as follows:

The units of a given reaction in the ME-mdoel are in molecules per hour._____no_output_____

The individual components of the biomass dilution constrain are in units of *kDa*._____no_output_____

Therefore, when the *biomass dilution* variable (reaction) carries flux it gives units of $hr^{-1}$ representing the growth rateof the cell, $\mu$_____no_output_____


# Notebook from jedbrown/numerical-computation
Path: DifferentialEquations.ipynb

#### Jupyter notebooks

This is a [Jupyter](http://jupyter.org/) notebook using Python.  You can install Jupyter locally to edit and interact with this notebook.

# Differential Equations

We now consider the problem of computing $y(t)$ from an *ordinary differential equation* (ODE)

$$ y'(t) = f(t,y) $$

and initial condition $y(0)$.
In the following, $y$ may be a scalar or vector value.
For convenience of notation, we may suppress the time dependence by writing $y$ instead of $y(t)$.

### Second Order Equations

A second order system such as

$$ y'' = f(t, y, y') $$

may always be converted to a first order system by introducing a new variable

$$ \begin{bmatrix} y_0 \\ y_1 \end{bmatrix}' = \begin{bmatrix} y_1 \\ f(t, y_0, y_1) \end{bmatrix} . $$
Therefore, without loss of generality, we will focus on first order systems.

### Implicit formulations

We have chosen the explicit representation $y' = f(t,y)$, but it is more general to write $h(t,y,y') = 0$.
If $\partial h/\partial y'$ is singular, then this describes a *differential algebraic equation* (DAE).  DAE are more challenging to solve and beyond the scope of this course.

## Linear Equations

If $f(y,t)$ is a linear function of $y$ then we have a linear ODE
$$ y' = A(t) y + \text{source}(t) . $$
If $A(t)$ is independent of $t$ and $\text{source}(t) = 0$ then we have a linear, constant, autonomous ODE.

When $y$ is a scalar then $A = a$ is a scalar and the solution is
$$ y = y(0) e^{at} . $$

#### Question
What qualitative dynamics does this imply for
* $a > 0$?
* $a < 0$?
* $a$ imaginary?

#### Question
* What if $A$ is diagonal?
* What if $A = X \Lambda X^{-1}$?

#### Matrix exponential
The general solution can be written in terms of the matrix exponential.
$$ y(t) = e^{At} y(0) . $$
The matrix exponential is defined by its Taylor series
$$ e^A = 1 + A + \frac{A^2}{2} + \frac{A^3}{3!} + \dotsb $$
and there are many [practical ways to compute it](http://www.cs.cornell.edu/cv/ResearchPDF/19ways+.pdf).

#### Question
Suppose that the diagonalization $A = X \Lambda X^{-1}$ exists and derive a finite expression for the matrix exponential using the scalar `exp` function.

## Forward Euler Method

The simplest method for solving $y'(t) = f(t,y)$ is
$$ \tilde y(h) = y(0) + h f(0, y(0)) . $$_____no_output_____

%matplotlib notebook
import numpy
from matplotlib import pyplot

def ode_euler(f, y0, tfinal=1, h=0.1):
    y = numpy.array(y0)
    t = 0
    thist = [t]
    yhist = [y0]
    while t < tfinal:
        h = min(h, tfinal - t)
        y += h * f(t, y)
        t += h
        thist.append(t)
        yhist.append(y.copy())
    return numpy.array(thist), numpy.array(yhist)

tests = []

class cosine:
    def __init__(self, k=5):
        self.k = k
    def __repr__(self):
        return 'cosine(k={:d})'.format(self.k)
    def f(self, t, y):
        return -self.k * (y - numpy.cos(t))
    def y(self, t, y0):
        k2p1 = self.k**2+1
        return (y0 - self.k**2/k2p1) * numpy.exp(-self.k*t) + self.k*(numpy.sin(t) + self.k*numpy.cos(t))/k2p1

tests.append(cosine(k=3))
tests.append(cosine(k=20))    _____no_output_____pyplot.style.use('ggplot')
pyplot.rcParams['figure.max_open_warning'] = False

y0 = numpy.array([.2])
pyplot.figure()
for test in tests:
    thist, yhist = ode_euler(test.f, y0, h=.05, tfinal=20)
    pyplot.plot(thist, yhist, '*', label=repr(test)+' Forward Euler')
    pyplot.plot(thist, test.y(thist, y0), label=repr(test)+' exact')
pyplot.legend(loc='upper right')_____no_output_____

#### Question

* What happens when $h$ is increased?_____no_output_____

def expm(A):
    """Compute the matrix exponential"""
    L, X = numpy.linalg.eig(A)
    return X.dot(numpy.diag(numpy.exp(L))).dot(numpy.linalg.inv(X))

class linear:
    def __init__(self, A):
        self.A = A.copy()
    def f(self, t, y):
        return self.A.dot(y)
    def y(self, t, y0):
        return [numpy.real(expm(self.A*s).dot(y0)) for s in t]

test = linear(numpy.array([[0, 1],[-1, 0]]))
y0 = numpy.array([.5, 0])
thist, yhist = ode_euler(test.f, y0, h=.05, tfinal=20)
pyplot.figure()
pyplot.plot(thist, yhist, '*', label='Euler')
pyplot.plot(thist, test.y(thist, y0), label='exact')
pyplot.legend(loc='upper right')_____no_output_____

#### Questions
* Does shrinking $h$ make this more accurate?
* What if `tfinal` is extended?
* What are the eigenvalues of $A$?

#### Runge-Kutta 4
Let's try a different method.  (We'll look back at where this comes from later.)_____no_output_____

def ode_rk4(f, y0, tfinal=1, h=0.1):
    y = numpy.array(y0)
    t = 0
    thist = [t]
    yhist = [y0]
    while t < tfinal:
        h = min(h, tfinal - t)
        k1 = f(t, y)
        k2 = f(t+h/2, y + k1*h/2)
        k3 = f(t+h/2, y + k2*h/2)
        k4 = f(t+h, y + k3*h)
        y += h/6 * (k1 + 2*k2 + 2*k3 + k4)
        t += h
        thist.append(t)
        yhist.append(y.copy())
    return numpy.array(thist), numpy.array(yhist)

thist, yhist = ode_rk4(test.f, y0, h=.1, tfinal=5)
pyplot.figure()
pyplot.plot(thist, yhist, '*', label=repr(test)+' RK4')
pyplot.plot(thist, test.y(thist, y0), label=repr(test)+' exact')_____no_output_____

### Linear Stability Analysis

Why did Euler diverge (even if slowly) while RK4 solved this problem accurately?
And why do both methods diverge if the step size is too large?
We can understand the convergence of methods by analyzing the test problem
$$ y' = \lambda y $$
for different values of $\lambda$ in the complex plane.
One step of the Euler method with step size $h$ maps
$$ y \to y + h \lambda y = (1 + h \lambda) y =: R(h \lambda) y $$
where we have introduced the complex-valued function $R(z)$.

* When is this map stable?_____no_output_____

pyplot.figure()
x = numpy.linspace(-2,2)
xx, yy = numpy.meshgrid(x, x)
zz = xx + 1j*yy
R = 1 + zz
C = pyplot.contour(xx, yy, numpy.abs(R))
pyplot.clabel(C)_____no_output_____

Evidently the forward Euler method is stable if $h\lambda$ is in the unit circle centered at $z=-1$, but not stable otherwise.

#### Implicit methods
Recall that forward Euler is the step
$$ \tilde y(h) = y(0) + h f(0, y(0)) . $$
This can be evaluated **explicitly**; all the terms on the right hand side are known so the approximation $\tilde y(h)$ is computed merely by evaluating the right hand side.
Let's consider an alternative, **backward Euler** (or "implicit Euler"),
$$ \tilde y(h) = y(0) + h f(h, \tilde y(h)) . $$
This is a (generally) nonlinear equation for $\tilde y(h)$.
For the test equation $y' = \lambda y$, the backward Euler method is
$$ \tilde y(h) = y(0) + h \lambda \tilde y(h) $$
or
$$ \tilde y(h) = \frac{y(0)}{1 - h \lambda} . $$
Evidently the stability function is
$$ R(z) = \frac{1}{1-z} .$$_____no_output_____

pyplot.figure()
C = pyplot.contour(xx, yy, numpy.abs(1/(1-zz)), [.5, 1, 1.5, 2])
pyplot.clabel(C)_____no_output_____

Evidently backward Euler is stable in the entire left half plane (where the exact solution is also stable) and also in some significant portions of the right half plane.  Let's test it on the oscillator problem._____no_output_____

def solve_newtonfd(f, x0):
    def fdjacobian(x):
        J = numpy.eye(len(x),len(x))
        base = f(x)
        for col in J.T:
            col[:] = (f(x + 1e-8*col) - base) / 1e-8
        return J
    x = x0.copy()
    while True:
        res = f(x)
        if numpy.linalg.norm(res) < 1e-6:
            return x
        x -= numpy.linalg.solve(fdjacobian(x), res)

def ode_beuler(f, y0, tfinal=1, h=0.1):
    y = numpy.array(y0)
    t = 0
    thist = [t]
    yhist = [y0]
    while t < tfinal:
        h = min(h, tfinal - t)
        # Solve x = y + h f(x)
        def residual(x):
            return x - h * f(t+h, x) - y
        y = solve_newtonfd(residual, y)
        t += h
        thist.append(t)
        yhist.append(y.copy())
    return numpy.array(thist), numpy.array(yhist)

thist, yhist = ode_beuler(test.f, y0, h=.01, tfinal=50)
pyplot.figure()
pyplot.plot(thist, yhist, '*')
pyplot.plot(thist, test.y(thist, y0))
pyplot.title('Backward Euler')_____no_output_____

#### Observations
* We need to solve a system of equations on each time step.
* The cost to solve the linear system is $O(n^3)$ for a system of $n$ equations.
* The Jacobian matrix is $n\times n$, so requires $n^2$ storage when everything else requires $O(n)$.
* I used finite differencing to compute the Jacobian, but many problems have efficient ways to compute an exact Jacobian (it just takes some programming).
* Backward Euler is stable (as predicted by theory), but not very accurate.
* What happens when you make the time step smaller or larger?

### Midpoint Method

What if instead of evaluating the function at the end of the time step, we evaluated in the middle of the time step using the average of the endpoint values.  After all, something similar improved accuracy for our numerical integration...

$$ \tilde y(h) = y(0) + h f\left(\frac h 2, \frac{\tilde y(h) + y(0)}{2} \right) $$_____no_output_____

def ode_midpoint(f, y0, tfinal=1, h=0.1):
    y = y0.copy()
    t = 0
    thist = [t]
    yhist = [y0]
    while t < tfinal:
        h = min(h, tfinal - t)
        # Solve x = y + h f(x)
        def residual(x):
            return x - h * f(t+h/2, (x + y)/2) - y
        y = solve_newtonfd(residual, y)
        t += h
        thist.append(t)
        yhist.append(y.copy())
    return numpy.array(thist), numpy.array(yhist)

thist, yhist = ode_midpoint(test.f, y0, h=.2, tfinal=50)
pyplot.figure()
pyplot.plot(thist, yhist, '*')
pyplot.plot(thist, test.y(thist, y0))
pyplot.title('Midpoint')_____no_output_____y0 = numpy.array([.2])
pyplot.figure()
for test in tests:
    thist, yhist = ode_midpoint(test.f, y0, h=1, tfinal=20)
    pyplot.plot(thist, yhist, '*', label=repr(test)+' Midpoint')
    pyplot.plot(thist, test.y(thist, y0), label=repr(test)+' exact')
pyplot.legend(loc='upper right')
pyplot.title('Cosine')_____no_output_____

#### Observations/Questions

* This appears to be quite accurate.
* We still have to solve a system of equations on each time step.
* What happens when the step size changes?
* What is the stability function $R(z)$ for the midpoint rule?_____no_output_____### Runge-Kutta Methods

All of the methods we have seen thus far can be represented as Runge-Kutta methods, which can be arranged as a series of $s$ "stage" equations (possibly coupled) and a completion formula.

$$\begin{split}
Y_i = y(0) + h \sum_j a_{ij} f(t+c_j h, Y_j) \\
y(h) = y(0) + h \sum_j b_j f(t+c_j h, Y_j)
\end{split}$$

where $c$ is a vector of *abscissa*, $A$ is a table of coefficients, and $b$ is a vector of completion weights.
These coefficients are typically expressed in a Butcher Table
$$ \left[ \begin{array}{c|cc}
c_0 & a_{00} & a_{01} \\
c_1 & a_{10} & a_{11} \\
\hline
& b_0 & b_1
\end{array} \right] . $$

If the matrix $A$ is strictly lower triangular, then the method is **explicit** (does not require solving equations).  We have seen forward Euler

$$ \left[ \begin{array}{c|cc}
0 & 0 & 0 \\
1 & 1 & 0 \\
\hline
& 1 & 0
\end{array} \right] ,$$
backward Euler
$$ \left[ \begin{array}{c|c}
1 & 1 \\
\hline
& 1
\end{array} \right] ,$$
and Midpoint
$$ \left[ \begin{array}{c|c}
\frac 1 2 & \frac 1 2 \\
\hline
& \frac 1 2
\end{array} \right], $$
and that explicit method we called RK4
$$ \left[ \begin{array}{c|cccc}
0 & 0 & 0 & 0 & 0 \\
\frac 1 2 & \frac 1 2 & 0 & 0 & 0 \\
\frac 1 2 & 0 & \frac 1 2 & 0 & 0 \\
1 & 0 & 0 & 1 & 0 \\
\hline
& \frac 1 6 & \frac 1 3 & \frac 1 3 & \frac 1 6
\end{array} \right] . $$_____no_output_____## Ballistics (for HW7)

The velocity of a particle in the $x$-$z$ plane is the time derivative of its position,
$$ \begin{bmatrix} v_x \\ v_z \end{bmatrix} = \begin{bmatrix} p_x \\ p_z \end{bmatrix}' . $$
The acceleration (derivative of velocity) depends on the force applied,
$$ \begin{bmatrix} v_x \\ v_z \end{bmatrix}' = \frac 1 m \begin{bmatrix} F_x \\ F_z \end{bmatrix} $$
where $m$ is the mass of the particle.
The gravitational force is
$$ \mathbf F^{\text{grav}} = m \begin{bmatrix} 0 \\ -g \end{bmatrix} $$
where $g = 9.8\ \text{meter}/\text{second}^2 $.
The drag force is often approximated as
$$ F^{\text{drag}} = \frac 1 2 \rho v^2 c_d A $$
where $\rho$ is the density of the fluid (assume $1\ \text{kilogram}/\text{meter}^3$),
$v =\sqrt{v_x^2  + v_z^2}$ is the velocity, $c_d$ is the drag coefficient (assume 0.2 for this projectile), and $A = \pi r^2$  is the cross-sectional area of the projectile (assume the radius is $r = .05\ \text{meter}$).
The direction of the drag force is opposite the velocity, thus the vector drag is
$$ \mathbf F^{\text{drag}}(v_x, v_z) = - \frac{\pi}{2} v c_d r^2 \begin{bmatrix} v_x \\ v_z \end{bmatrix} . $$
This produces the system of equations

$$ \begin{bmatrix} p_x \\ p_z \\ v_x \\ v_z \end{bmatrix}' =
\begin{bmatrix} v_x \\ v_z \\ \frac 1 m F_x^{\text{drag}}  \\ \frac 1 m F_z^{\text{drag}} - 9.8 \end{bmatrix} . $$

For the ballistics computation, you should solve this differential equation with an $m=8$ kilogram projectile and initial condition of a velocity  $v_0 = 300\ \text{meter}/\text{second}$ at angle $\theta$ above the horizontal, i.e.,
$$ \begin{bmatrix} 0 \\ 0 \\ v_0 \cos \theta \\ v_0 \sin \theta \end{bmatrix} . $$

The homework question is to compute the angle needed to hit a target on a distant slope of angle $\phi = 40^\circ = 40 \pi / 180$.  For that, you should adapt one of the ODE solvers to determine where the projectile crosses the plane of the target slope and adjust the angle (either by hand or using a rootfinder) to hit the target.  Don't aim too high or your explosive will detonate on the other side of the mountain!  Make sure you use a time step that you are confident is accurate enough.

![Ballistics diagram](figures/Ballistics.png)

Your code should print the angle $\theta$ and the time at which your projectile hits the slope._____no_output_____### Chemical reactions (Take-home Final)

The [Oregonator](http://www.scholarpedia.org/article/Oregonator) mechanism in chemical kinetics describes an oscillatory chemical system.  It consists of three species with concentrations $\mathbf x = [x_0,x_1,x_2]^T$ (scaled units) and the evolution equations
$$ \mathbf {x'} = \begin{bmatrix} 77.27 \big(x_1 + x_0 (1 - 8.375\cdot 10^{-6} x_0 - x_1) \big) \\
                    \frac{1}{77.27} \big(x_2 - (1 + x_0) x_1 \big) \\
                    0.161 (x_0 - x_2)
                    \end{bmatrix} . $$
Starting with the initial conditions $\mathbf x_0 = [1, 2, 3]^T$, this produces the time evolution below.
![Oregonator time evolution](figures/oregonator.png)
This calculation was performed using 287 steps of an adaptive implicit time integrator, where the time step size was adapted to preserve accuracy.  The time step size in shown in the first panel and the other three are the evolution of concentrations over time.  The take-home final asks you to write code and to explain questions.  You may work together (with attribution, as always) on the coding portion, but please write your answers entirely in your own words.  Put your code in the `final` directory and write in [`final/README.md`](https://git.cs.colorado.edu/csci-3656/csci-3656/blob/master/final/README.md).  (I suggest `git pull upstream master` to get my template before starting, but you may copy it over.)

#### 1. How to measure accuracy?

A differential equation solver produces a time history of chemical concentrations
$$ x(t), \quad 0 \le t \le T_{\text{final}} .$$
We would like a measure of accuracy that does not explicitly depend on the solver or time steps that are used.  If a different method produces the evolution $\tilde x(t)$, then we would like to be able to measure the difference as
$$ \lVert \phi(\tilde x) - \phi(x) \rVert . $$
The function $\phi(\cdot)$ could return the solution at a particular time, it could measure the time between key events, or could be something else.
Choose a metric and explain why you chose it.

#### 2. Implicit versus Explicit

Is an implicit method better than an explicit method for solving the Oregonator?
Solve the Oregonator problem using an explicit method and using an implicit method (you can adapt code from the notebook or implement a method that is not in the notebook).
The Oregonator system has no known analytic solution, so you'll have to numerically compute a reference solution.
Do this using a method you believe to be highly accurate and explain why you believe it is very accurate.
(A common justification would be to refine the time step and compare using the metric $\phi$ above.)

Different methods (implicit versus explicit, Euler versus RK4, etc.) have different costs per time step.  Choose a way to measure cost and explain your choice and any shortcomings that it may have.

Compare the cost (using the metric you chose above) and accuracy (using $\phi$) of the implicit method to the cost and accuracy of the explicit method.  (This is often plotted as *accuracy versus cost* with a log-log scale.)  Explain what factors might go into your decision.

#### 3. Eigenvalues

Calculate the eigenvalues of the Jacobian matrix at different times.  (You can compute it analytically or using the `fdjacobian` code in this notebook.)  Do you notice any qualitative differences?  For example, are there any times when there are eigenvalues with positive real part?

#### 4. Generalization

Realistic chemical mechanisms can have hundreds or thousands of species (the Oregonator has only three).  Explain how the choice of methods might change when solving a large reaction mechanism._____no_output_____


# Notebook from edjuaro/tcga
Path: backups/old_notebook.ipynb

*To do: use the metadata to select patients based on recurrence? Maybe 5-year survival cutoff*_____no_output_____

import cuzcatlan as cusca
import pandas as pd
import numpy as np
import pickle
import os
SEED = 19920116_____no_output_____DATA_ROOT = 'raw_data'

SITE = 'BRCA'
DATA_LOC = 'gdac.broadinstitute.org_BRCA.Merge_rnaseqv2__illuminahiseq_rnaseqv2__unc_edu__Level_3__RSEM_genes_normalized__data.Level_3.2016012800.0.0'
DATA = 'BRCA.rnaseqv2__illuminahiseq_rnaseqv2__unc_edu__Level_3__RSEM_genes_normalized__data.data.txt'
METADATA_LOC = 'gdac.broadinstitute.org_BRCA.Clinical_Pick_Tier1.Level_4.2016012800.0.0'
METADATA = 'BRCA.clin.merged.picked.txt'

# SITE = 'LUAD'
# DATA_LOC = 'gdac.broadinstitute.org_LUAD.Merge_rnaseqv2__illuminahiseq_rnaseqv2__unc_edu__Level_3__RSEM_genes_normalized__data.Level_3.2016012800.0.0'
# DATA = 'LUAD.rnaseqv2__illuminahiseq_rnaseqv2__unc_edu__Level_3__RSEM_genes_normalized__data.data.txt'
# METADATA_LOC = 'gdac.broadinstitute.org_LUAD.Clinical_Pick_Tier1.Level_4.2016012800.0.0'
# METADATA = 'LUAD.clin.merged.picked.txt'

# SITE = 'LUSC'
# DATA_LOC = 'gdac.broadinstitute.org_LUSC.Merge_rnaseqv2__illuminahiseq_rnaseqv2__unc_edu__Level_3__RSEM_genes_normalized__data.Level_3.2016012800.0.0'
# DATA = 'LUSC.rnaseqv2__illuminahiseq_rnaseqv2__unc_edu__Level_3__RSEM_genes_normalized__data.data.txt'
# METADATA_LOC = 'gdac.broadinstitute.org_LUSC.Clinical_Pick_Tier1.Level_4.2016012800.0.0'
# METADATA = 'LUSC.clin.merged.picked.txt'

# SITE = 'KIPAN'
# DATA_LOC = 'gdac.broadinstitute.org_KIPAN.Merge_rnaseqv2__illuminahiseq_rnaseqv2__unc_edu__Level_3__RSEM_genes_normalized__data.Level_3.2016012800.0.0'
# DATA = 'KIPAN.rnaseqv2__illuminahiseq_rnaseqv2__unc_edu__Level_3__RSEM_genes_normalized__data.data.txt'
# METADATA_LOC = 'gdac.broadinstitute.org_KIPAN.Clinical_Pick_Tier1.Level_4.2016012800.0.0'
# METADATA = 'KIPAN.clin.merged.picked.txt'_____no_output_____meta_df = pd.read_table(os.path.join(DATA_ROOT, SITE, METADATA_LOC, METADATA), index_col=0, header=0)
data_df = pd.read_table(os.path.join(DATA_ROOT, SITE, DATA_LOC, DATA), index_col=0, header=0)

data_df.drop(['gene_id'], inplace=True)
data_df.index.names = ['gene_id']
print(len(list(data_df)))
print(len(list(meta_df)))

pickle.dump(file=open(os.path.join(DATA_ROOT, SITE, 'raw_metadata.p'), 'wb'), obj=meta_df)
pickle.dump(file=open(os.path.join(DATA_ROOT, SITE, 'raw_data.p'), 'wb'), obj=data_df)/Users/edjuaro/miniconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py:2698: DtypeWarning: Columns (1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,285,286,287,288,289,290,291,292,293,294,295,296,297,298,299,300,301,302,303,304,305,306,307,308,309,310,311,312,313,314,315,316,317,318,319,320,321,322,323,324,325,326,327,328,329,330,331,332,333,334,335,336,337,338,339,340,341,342,343,344,345,346,347,348,349,350,351,352,353,354,355,356,357,358,359,360,361,362,363,364,365,366,367,368,369,370,371,372,373,374,375,376,377,378,379,380,381,382,383,384,385,386,387,388,389,390,391,392,393,394,395,396,397,398,399,400,401,402,403,404,405,406,407,408,409,410,411,412,413,414,415,416,417,418,419,420,421,422,423,424,425,426,427,428,429,430,431,432,433,434,435,436,437,438,439,440,441,442,443,444,445,446,447,448,449,450,451,452,453,454,455,456,457,458,459,460,461,462,463,464,465,466,467,468,469,470,471,472,473,474,475,476,477,478,479,480,481,482,483,484,485,486,487,488,489,490,491,492,493,494,495,496,497,498,499,500,501,502,503,504,505,506,507,508,509,510,511,512,513,514,515,516,517,518,519,520,521,522,523,524,525,526,527,528,529,530,531,532,533,534,535,536,537,538,539,540,541,542,543,544,545,546,547,548,549,550,551,552,553,554,555,556,557,558,559,560,561,562,563,564,565,566,567,568,569,570,571,572,573,574,575,576,577,578,579,580,581,582,583,584,585,586,587,588,589,590,591,592,593,594,595,596,597,598,599,600,601,602,603,604,605,606,607,608,609,610,611,612,613,614,615,616,617,618,619,620,621,622,623,624,625,626,627,628,629,630,631,632,633,634,635,636,637,638,639,640,641,642,643,644,645,646,647,648,649,650,651,652,653,654,655,656,657,658,659,660,661,662,663,664,665,666,667,668,669,670,671,672,673,674,675,676,677,678,679,680,681,682,683,684,685,686,687,688,689,690,691,692,693,694,695,696,697,698,699,700,701,702,703,704,705,706,707,708,709,710,711,712,713,714,715,716,717,718,719,720,721,722,723,724,725,726,727,728,729,730,731,732,733,734,735,736,737,738,739,740,741,742,743,744,745,746,747,748,749,750,751,752,753,754,755,756,757,758,759,760,761,762,763,764,765,766,767,768,769,770,771,772,773,774,775,776,777,778,779,780,781,782,783,784,785,786,787,788,789,790,791,792,793,794,795,796,797,798,799,800,801,802,803,804,805,806,807,808,809,810,811,812,813,814,815,816,817,818,819,820,821,822,823,824,825,826,827,828,829,830,831,832,833,834,835,836,837,838,839,840,841,842,843,844,845,846,847,848,849,850,851,852,853,854,855,856,857,858,859,860,861,862,863,864,865,866,867,868,869,870,871,872,873,874,875,876,877,878,879,880,881,882,883,884,885,886,887,888,889,890,891,892,893,894,895,896,897,898,899,900,901,902,903,904,905,906,907,908,909,910,911,912,913,914,915,916,917,918,919,920,921,922,923,924,925,926,927,928,929,930,931,932,933,934,935,936,937,938,939,940,941,942,943,944,945,946,947,948,949,950,951,952,953,954,955,956,957,958,959,960,961,962,963,964,965,966,967,968,969,970,971,972,973,974,975,976,977,978,979,980,981,982,983,984,985,986,987,988,989,990,991,992,993,994,995,996,997,998,999,1000,1001,1002,1003,1004,1005,1006,1007,1008,1009,1010,1011,1012,1013,1014,1015,1016,1017,1018,1019,1020,1021,1022,1023,1024,1025,1026,1027,1028,1029,1030,1031,1032,1033,1034,1035,1036,1037,1038,1039,1040,1041,1042,1043,1044,1045,1046,1047,1048,1049,1050,1051,1052,1053,1054,1055,1056,1057,1058,1059,1060,1061,1062,1063,1064,1065,1066,1067,1068,1069,1070,1071,1072,1073,1074,1075,1076,1077,1078,1079,1080,1081,1082,1083,1084,1085,1086,1087,1088,1089,1090,1091,1092,1093,1094,1095,1096,1097,1098,1099,1100,1101,1102,1103,1104,1105,1106,1107,1108,1109,1110,1111,1112,1113,1114,1115,1116,1117,1118,1119,1120,1121,1122,1123,1124,1125,1126,1127,1128,1129,1130,1131,1132,1133,1134,1135,1136,1137,1138,1139,1140,1141,1142,1143,1144,1145,1146,1147,1148,1149,1150,1151,1152,1153,1154,1155,1156,1157,1158,1159,1160,1161,1162,1163,1164,1165,1166,1167,1168,1169,1170,1171,1172,1173,1174,1175,1176,1177,1178,1179,1180,1181,1182,1183,1184,1185,1186,1187,1188,1189,1190,1191,1192,1193,1194,1195,1196,1197,1198,1199,1200,1201,1202,1203,1204,1205,1206,1207,1208,1209,1210,1211,1212) have mixed types. Specify dtype option on import or set low_memory=False.
  interactivity=interactivity, compiler=compiler, result=result)
meta_df = pickle.load(file=open(os.path.join(DATA_ROOT, SITE, 'raw_metadata.p'), 'rb'))
data_df = pickle.load(file=open(os.path.join(DATA_ROOT, SITE, 'raw_data.p'), 'rb'))_____no_output_____print("These two must match the two above:")
print(len(list(data_df)))
print(len(list(meta_df)))These two must match the two above:
1212
1097

---
Checking if there are any normal samples, these are in the positions number 13 and 14. "Tumor types range from 01 - 09, normal types from 10 - 19 and control samples from 20 - 29" -- for more info look at:

https://gdc.cancer.gov/resources-tcga-users/tcga-code-tables/sample-type-codes

(barcode annotation here: https://wiki.nci.nih.gov/display/TCGA/TCGA+barcode)_____no_output_____

# This cell is for sanity check!
tissue_type = []
for patient in list(data_df):
    tissue_type.append(patient[13:15])
unique_tissue_types, counts = np.unique(tissue_type, return_counts=True)
for i in range(len(counts)):
    print("Tissue type:", unique_tissue_types[i], " -- Number of samples:", counts[i])Tissue type: 01  -- Number of samples: 1093
Tissue type: 06  -- Number of samples: 7
Tissue type: 11  -- Number of samples: 112
normal_df = pd.DataFrame()
cancer_df = pd.DataFrame()
for patient in list(data_df):
    if int(patient[13:15]) > 9:  #We only expect type 11, but just in case
#         normal_df[patient[5:15]] = data_df[patient]
        normal_df[patient[:15]] = data_df[patient]
    elif int(patient[13:15]) == 1 :  #For now take onle type 01 "Primary Solid Tumor"
#         cancer_df[patient[5:15]] = data_df[patient]
        cancer_df[patient[:15]] = data_df[patient]
print("These two numbers should match type 01 and type 11 above:")
print(len(list(cancer_df)), len(list(normal_df)))These two numbers should match type 01 and type 11 above:
1093 112
normal_df.head()_____no_output_____print(list(normal_df))['TCGA-A7-A0CE-11', 'TCGA-A7-A0CH-11', 'TCGA-A7-A0D9-11', 'TCGA-A7-A0DB-11', 'TCGA-A7-A13E-11', 'TCGA-A7-A13F-11', 'TCGA-A7-A13G-11', 'TCGA-AC-A23H-11', 'TCGA-AC-A2FB-11', 'TCGA-AC-A2FF-11', 'TCGA-AC-A2FM-11', 'TCGA-BH-A0AU-11', 'TCGA-BH-A0AY-11', 'TCGA-BH-A0AZ-11', 'TCGA-BH-A0B3-11', 'TCGA-BH-A0B5-11', 'TCGA-BH-A0B7-11', 'TCGA-BH-A0B8-11', 'TCGA-BH-A0BA-11', 'TCGA-BH-A0BC-11', 'TCGA-BH-A0BJ-11', 'TCGA-BH-A0BM-11', 'TCGA-BH-A0BQ-11', 'TCGA-BH-A0BS-11', 'TCGA-BH-A0BT-11', 'TCGA-BH-A0BV-11', 'TCGA-BH-A0BW-11', 'TCGA-BH-A0BZ-11', 'TCGA-BH-A0C0-11', 'TCGA-BH-A0C3-11', 'TCGA-BH-A0DD-11', 'TCGA-BH-A0DG-11', 'TCGA-BH-A0DH-11', 'TCGA-BH-A0DK-11', 'TCGA-BH-A0DL-11', 'TCGA-BH-A0DO-11', 'TCGA-BH-A0DP-11', 'TCGA-BH-A0DQ-11', 'TCGA-BH-A0DT-11', 'TCGA-BH-A0DV-11', 'TCGA-BH-A0DZ-11', 'TCGA-BH-A0E0-11', 'TCGA-BH-A0E1-11', 'TCGA-BH-A0H5-11', 'TCGA-BH-A0H7-11', 'TCGA-BH-A0H9-11', 'TCGA-BH-A0HA-11', 'TCGA-BH-A0HK-11', 'TCGA-BH-A18J-11', 'TCGA-BH-A18K-11', 'TCGA-BH-A18L-11', 'TCGA-BH-A18M-11', 'TCGA-BH-A18N-11', 'TCGA-BH-A18P-11', 'TCGA-BH-A18Q-11', 'TCGA-BH-A18R-11', 'TCGA-BH-A18S-11', 'TCGA-BH-A18U-11', 'TCGA-BH-A18V-11', 'TCGA-BH-A1EN-11', 'TCGA-BH-A1EO-11', 'TCGA-BH-A1ET-11', 'TCGA-BH-A1EU-11', 'TCGA-BH-A1EV-11', 'TCGA-BH-A1EW-11', 'TCGA-BH-A1F0-11', 'TCGA-BH-A1F2-11', 'TCGA-BH-A1F6-11', 'TCGA-BH-A1F8-11', 'TCGA-BH-A1FB-11', 'TCGA-BH-A1FC-11', 'TCGA-BH-A1FD-11', 'TCGA-BH-A1FE-11', 'TCGA-BH-A1FG-11', 'TCGA-BH-A1FH-11', 'TCGA-BH-A1FJ-11', 'TCGA-BH-A1FM-11', 'TCGA-BH-A1FN-11', 'TCGA-BH-A1FR-11', 'TCGA-BH-A1FU-11', 'TCGA-BH-A203-11', 'TCGA-BH-A204-11', 'TCGA-BH-A208-11', 'TCGA-BH-A209-11', 'TCGA-E2-A153-11', 'TCGA-E2-A158-11', 'TCGA-E2-A15I-11', 'TCGA-E2-A15K-11', 'TCGA-E2-A15M-11', 'TCGA-E2-A1BC-11', 'TCGA-E2-A1IG-11', 'TCGA-E2-A1L7-11', 'TCGA-E2-A1LB-11', 'TCGA-E2-A1LH-11', 'TCGA-E2-A1LS-11', 'TCGA-E9-A1N4-11', 'TCGA-E9-A1N5-11', 'TCGA-E9-A1N6-11', 'TCGA-E9-A1N9-11', 'TCGA-E9-A1NA-11', 'TCGA-E9-A1ND-11', 'TCGA-E9-A1NF-11', 'TCGA-E9-A1NG-11', 'TCGA-E9-A1R7-11', 'TCGA-E9-A1RB-11', 'TCGA-E9-A1RC-11', 'TCGA-E9-A1RD-11', 'TCGA-E9-A1RF-11', 'TCGA-E9-A1RH-11', 'TCGA-E9-A1RI-11', 'TCGA-GI-A2C8-11', 'TCGA-GI-A2C9-11']

# Taking random (the first) 15 normal samples_____no_output_____

# normal_df.sample(n=25, axis=1, random_state=SEED)
normal_df = normal_df.ix[:,:15]
normal_df.head()/Users/edjuaro/miniconda3/lib/python3.6/site-packages/ipykernel_launcher.py:2: DeprecationWarning:
.ix is deprecated. Please use
.loc for label based indexing or
.iloc for positional indexing

See the documentation here:
http://pandas.pydata.org/pandas-docs/stable/indexing.html#ix-indexer-is-deprecated

cancer_df.head()_____no_output_____

# Taking random (the first) 15 tumor samples_____no_output_____

cancer_df = cancer_df.ix[:,:15]
cancer_df.head()/Users/edjuaro/miniconda3/lib/python3.6/site-packages/ipykernel_launcher.py:1: DeprecationWarning:
.ix is deprecated. Please use
.loc for label based indexing or
.iloc for positional indexing

See the documentation here:
http://pandas.pydata.org/pandas-docs/stable/indexing.html#ix-indexer-is-deprecated
  """Entry point for launching an IPython kernel.

## Making CLS file for differential expression analysis_____no_output_____

labels = list(np.zeros(len(list(normal_df)))) + list(np.ones(len(list(cancer_df))))_____no_output_____print(labels)[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]
cusca.list2cls(in_list=np.array(labels).astype(int), name_of_out='BRCA_workshop.cls', sep=' ')_____no_output_____

## Making GCT file for differential expression analysis_____no_output_____

print(normal_df.shape, cancer_df.shape)
print("Desired shape is: (",len(normal_df),",",normal_df.shape[1]+cancer_df.shape[1],")")(20531, 15) (20531, 15)
Desired shape is: ( 20531 , 30 )
df = normal_df
df = df.join(cancer_df)

df.shape_____no_output_____

##### Keep only top 40% (~10k) most variable genes_____no_output_____

variances = np.nanvar(df.as_matrix().astype(float), axis=1)_____no_output_____# variance_dict = {}
# for index, row in df.iterrows():
#     variance_dict[index] = row.var()

# print(sorted(variances))
# print(len(variances))
# how_many = 60
min_var = np.percentile(variances,50)
# print(min_var)
how_many=(variances > min_var).sum()
print(how_many)10265
df['variance'] = variances
df.sort_values(by='variance', ascending=False, inplace=True)
df.drop('variance',1, inplace=True)
df_____no_output_____

# Renaming Gene IDs_____no_output_____

genes = df.index
names = []
IDs = []
for gene in genes:
    g_list = gene.split('|')
    names.append(g_list[0])
    IDs.append('NCBI Gene ID: ' + g_list[1])
df.index = names
df.index.names = ['Name']
df['Description'] = IDs
description = df['Description']
df.drop(labels=['Description'], axis=1,inplace = True)
df.insert(0, 'Description', description)_____no_output_____

# Removing genes with low variance_____no_output_____

filtered_df = df.head(how_many)
filtered_df_____no_output_____print(filtered_df.shape)(10265, 31)

## Enforcing no-duplicate-rows rule_____no_output_____

# duplicates = filtered_df.loc[filtered_df.index.duplicated()]
# for temp in np.unique(duplicates.index):
#     new_names = filtered_df.loc[temp].index.values+'-'+filtered_df.loc[temp, 'Description']
#     filtered_df.loc[temp].index = new_names.values
#     print(filtered_df.loc[temp].index)
duplicates = filtered_df.loc[filtered_df.index.duplicated()]
print(duplicates.shape)

filtered_df['to_drop'] = filtered_df.index
filtered_df.drop_duplicates(subset='to_drop', keep='first', inplace=True)
filtered_df.drop('to_drop', axis=1, inplace=True)

duplicates = filtered_df.loc[filtered_df.index.duplicated()]
print(duplicates.shape)

(4, 31)
(0, 31)

# Saving the GCT file_____no_output_____

cusca.df2gct(df=filtered_df, extra_columns=1, use_index=True, name='BRCA_workshop.gct', add_dummy_descriptions=False)_____no_output_____test = pd.read_table('BRCA_workshop.gct', skiprows=2, index_col=0)_____no_output_____test.shape_____no_output_____test.head()_____no_output_____

# Testing CMS -- DiffEx is only available in dev-ami_____no_output_____

# Requires GenePattern Notebook: pip install genepattern-notebook
import gp
import genepattern

# Username and password removed for security reasons.
genepattern.GPAuthWidget(genepattern.register_session("https://genepattern.broadinstitute.org/gp", "", ""))_____no_output_____comparativemarkerselection_task = gp.GPTask(genepattern.get_session(0), 'urn:lsid:broad.mit.edu:cancer.software.genepattern.module.analysis:00044')
comparativemarkerselection_job_spec = comparativemarkerselection_task.make_job_spec()
comparativemarkerselection_job_spec.set_parameter("input.file", "https://genepattern.broadinstitute.org/gp/users/edjuaro/tmp/run2501218938884100201.tmp/BRCA_workshop.gct")
comparativemarkerselection_job_spec.set_parameter("cls.file", "https://genepattern.broadinstitute.org/gp/users/edjuaro/tmp/run1225806758956640435.tmp/BRCA_workshop.cls")
comparativemarkerselection_job_spec.set_parameter("confounding.variable.cls.file", "")
comparativemarkerselection_job_spec.set_parameter("test.direction", "2")
comparativemarkerselection_job_spec.set_parameter("test.statistic", "0")
comparativemarkerselection_job_spec.set_parameter("min.std", "")
comparativemarkerselection_job_spec.set_parameter("number.of.permutations", "10000")
comparativemarkerselection_job_spec.set_parameter("log.transformed.data", "false")
comparativemarkerselection_job_spec.set_parameter("complete", "false")
comparativemarkerselection_job_spec.set_parameter("balanced", "false")
comparativemarkerselection_job_spec.set_parameter("random.seed", "779948241")
comparativemarkerselection_job_spec.set_parameter("smooth.p.values", "true")
comparativemarkerselection_job_spec.set_parameter("phenotype.test", "one versus all")
comparativemarkerselection_job_spec.set_parameter("output.filename", "<input.file_basename>.comp.marker.odf")
genepattern.GPTaskWidget(comparativemarkerselection_task)_____no_output_____job1562608 = gp.GPJob(genepattern.get_session(0), 1562608)
genepattern.GPJobWidget(job1562608)_____no_output_____comparativemarkerselectionviewer_task = gp.GPTask(genepattern.get_session(0), 'urn:lsid:broad.mit.edu:cancer.software.genepattern.module.visualizer:00045')
comparativemarkerselectionviewer_job_spec = comparativemarkerselectionviewer_task.make_job_spec()
comparativemarkerselectionviewer_job_spec.set_parameter("comparative.marker.selection.filename", "https://genepattern.broadinstitute.org/gp/jobResults/1562608/BRCA_workshop.comp.marker.odf")
comparativemarkerselectionviewer_job_spec.set_parameter("dataset.filename", "https://genepattern.broadinstitute.org/gp/users/edjuaro/tmp/run1684698323379922516.tmp/BRCA_workshop.gct")
genepattern.GPTaskWidget(comparativemarkerselectionviewer_task)_____no_output_____job1562612 = gp.GPJob(genepattern.get_session(0), 1562612)
genepattern.GPJobWidget(job1562612)_____no_output_____

# Testing GSEA_____no_output_____** How do I validate these outputs? **_____no_output_____


# Notebook from g2nb/workshop-notebooks
Path: stanford_2019_03_07/2019-03-07_04-02 Stanford Differential Analysis 2.ipynb

# Differential Expression part 2: Triple Negative BRCA_____no_output_____Compute differentially expressed genes of triple negative BRCA vs non triple negative BRCA_____no_output_____## Before you begin_____no_output_____You must log in to a GenePattern server, in this notebook we will use **```GenePattern Cloud``` **

 Instructions

Sign in to GenePattern by entering your username and password into the form below.

_____no_output_____

# Requires GenePattern Notebook: pip install genepattern-notebook
import gp
import genepattern

# Username and password removed for security reasons.
genepattern.display(genepattern.session.register("https://cloud.genepattern.org/gp", "", ""))_____no_output_____

# Identify markers for triple negative breast cancer_____no_output_____## Load data into the notebook_____no_output_____

- **Note:** For this excersise, we are using all of the BRCA primary tumor samples available at TCGA and separating them (using the provided CLS file) according to whether or not they are triple negative. We provide the necessary files in the cell below.

_____no_output_____

file_list = ['https://datasets.genepattern.org/data/TCGA_BRCA/GSEA/brca_primary_all.gct',
             'https://datasets.genepattern.org/data/TCGA_BRCA/GSEA/triple_negative.cls']

genepattern.GPUIOutput(files=file_list)_____no_output_____

## Compute differentially expressed genes_____no_output_____

**Note:** Because of the size of this dataset, running this function will take some time (especially the plotting portion). Remember to look at the status of your kernel represented in the dot in the upper right corner (next to "Python 3.6")

_____no_output_____

 Instructions

- For the gene expression dataset parameter, select from the dropdown menu the file: `brca_primary_all.gct`
- For the phenotype file parameter, select from the dropdown menu the file: `BRCA_Dataset.cls`
- Leave the rest of the parameters as default.
- Click Run

_____no_output_____

import genepattern
import ccalnoir as ccal

from ccalnoir import differential_gene_expression
# import pandas as pd
# import urllib.request
from ccalnoir import compute_information_coefficient
from ccalnoir import custom_pearson_corr
RANDOM_SEED = 20180314

genepattern.GPUIBuilder(differential_gene_expression, name="Differential gene expression, discrete pheotype.",
                        description="Sort genes according to their association with a discrete phenotype or class vector.",
                        parameters={
                                "gene_expression":{"name":"gene_expression_dataset",
                                                   "type": "file",
                                                   "kinds": ["gct"]},
                                "phenotype_file":{"type": "file",
                                                  "kinds": ["cls"]},
                                "ranking_method":{
                                                  "default": "custom_pearson_corr",
                                                  "choices":{'Pearson Correlation Matching':"custom_pearson_corr",
                                                             'Information Coefficient Matching':"compute_information_coefficient",
                                                            }
                                                     },
                                "title":{"default":"Differential Expression Results"},
                                "output_filename":{"default": "diffex_output",},
                                "ramdon_seed":{"default":20180314,},
                                "output_var":{"default": "gene_scores",},
                        })_____no_output_____

## Validate experimental results_____no_output_____We will corroborate the results summarized in the abstract of "A New Gene Expression Signature for Triple-Negative Breast Cancer Using Frozen Fresh Tissue before Neoadjuvant Chemotherapy" by Santuario-Facio et al https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5469719/ :

> Forty genes showed differential expression pattern in TNBC tumors. Among these, nine overexpressed genes (PRKX/PRKY, UGT8, HMGA1, LPIN1, HAPLN3, FAM171A1, BCL141A, FOXC1, and ANKRD11), and one underexpressed gene (ANX9) are involved in general metabolism. Based on this biochemical peculiarity and the overexpression of BCL11A and FOXC1 (involved in tumor growth and metastasis, respectively)[...]
_____no_output_____

# From https://www.ncbi.nlm.nih.gov/pmc/articles/PMC5469719/
genes_of_interest = ['PRKX','PRKY', 'UGT8','HMGA1','LPIN1','HAPLN3','FAM171A1','BCL141A','FOXC1','ANKRD11','BCL11A','ANX9']_____no_output_____tn_genes = pd.DataFrame()
ranks = gene_scores.rank(ascending=False)['Score']
for gene in genes_of_interest:
    if gene in gene_scores.index:
        tn_genes = tn_genes.append(gene_scores.loc[gene])
        tn_genes.loc[gene,'rank'] = ranks[gene]
    else:
        print(f"Gene {gene} not present.")
tn_genes = tn_genes.sort_values(by='Score',ascending=False)
tn_genes_____no_output_____import numpy as np
import pandas as pd

def make_clickable(url):
    to_display = url.split('=')[1]
    return '{}'.format(url, to_display)

def make_table(df,max_number_of_genes_to_show, actual_ranking=False):
    df['Rank'] = range(1,len(df)+1)
#     pd.options.display.max_colwidth = 100
    if max_number_of_genes_to_show > len(df):
        max_number_of_genes_to_show = len(df)
        print("You want to show more genes than your data contains ಠ_ಠ")
        print("Showing you only {} instead (i.e., all the genes you have provided)".format(max_number_of_genes_to_show))

    if actual_ranking:
        df['abs-Score'] = np.absolute(df['Score'])
        sorted_df = df.sort_values(by=["abs-Score"], ascending=False)
        sorted_df = sorted_df.head(max_number_of_genes_to_show)
    else:
        to_show = int(np.floor(max_number_of_genes_to_show/2))
        sorted_df = df.head(to_show).append(df.tail(to_show))

    sorted_df['GeneCards link']=pd.Series(["http://www.genecards.org/cgi-bin/carddisp.pl?gene={}".format(gene)
                                     for gene in sorted_df.index.values], index=sorted_df.index)

    cols = list(sorted_df)
    # move the column to head of list using index, pop and insert
    cols.insert(0, cols.pop(cols.index('GeneCards link')))
    # use ix to reorder
    sorted_df = sorted_df.ix[:, cols]
    sorted_df.set_index('Rank', inplace=True)
    styled_table = sorted_df.style.format({'GeneCards link': make_clickable})
    styled_table
    return styled_table

genepattern.GPUIBuilder(make_table, name="Table Creator",
                        description="Show differentially expressed genes with links to genecards.org",
                        parameters={
                                "df":{"name": "differential_expression_results",
                                      "description":'The output from differerential expression (for this exercise, leave it as "gene_scores")',
                                      "default":"gene_scores"},
                                "max_number_of_genes_to_show":{"description":"Maximum number of genes to show in the heatmap (half will be overexpressed in one class and half in the other)",
                                                               "default":20},
                                "actual_ranking":{"hide":True},
                                "output_var":{"hide":True},
                        })_____no_output_____

What if we want to look at these genes *as a set*?

_____no_output_____


# Notebook from Jennie8594/OCPhelp
Path: Beginner-notebooks/03_pandas.ipynb

# Pandas

[Pandas](http://pandas.pydata.org/) is a an open source library providing high-performance, easy-to-use data structures and data analysis tools. Pandas is particularly suited to the analysis of _tabular_ data, i.e. data that can can go into a table. In other words, if you can imagine the data in an Excel spreadsheet, then Pandas is the tool for the job.

A [recent analysis](https://stackoverflow.blog/2017/09/06/incredible-growth-python/) of questions from Stack Overflow showed that python is the fastest growing and most widely used programming language in the world (in developed countries).

![python growth](https://zgab33vy595fw5zq-zippykid.netdna-ssl.com/wp-content/uploads/2017/09/growth_major_languages-1-1024x878.png)

A [follow-up analysis](https://stackoverflow.blog/2017/09/14/python-growing-quickly/) showed that this growth is driven by the data science packages such as numpy, matplotlib, and especially pandas.

![pandas growth](https://zgab33vy595fw5zq-zippykid.netdna-ssl.com/wp-content/uploads/2017/09/related_tags_over_time-1-1024x1024.png)

The exponential growth of pandas is due to the fact that it _just works_. It saves you time and helps you do science more efficiently and effictively.

### Pandas capabilities (from the Pandas website):

* A fast and efficient DataFrame object for data manipulation with integrated indexing;
* Tools for reading and writing data between in-memory data structures and different formats: CSV and text files, Microsoft Excel, SQL databases, and the fast HDF5 format;
* Intelligent data alignment and integrated handling of missing data: gain automatic label-based alignment in computations and easily manipulate messy data into an orderly form;
* Flexible reshaping and pivoting of data sets;
* Intelligent label-based slicing, fancy indexing, and subsetting of large data sets;
* Columns can be inserted and deleted from data structures for size mutability;
* Aggregating or transforming data with a powerful group by engine allowing split-apply-combine operations on data sets;
* High performance merging and joining of data sets;
* Hierarchical axis indexing provides an intuitive way of working with high-dimensional data in a lower-dimensional data structure;
* Time series-functionality: date range generation and frequency conversion, moving window statistics, moving window linear regressions, date shifting and lagging. Even create domain-specific time offsets and join time series without losing data;
* Highly optimized for performance, with critical code paths written in Cython or C.
* Python with pandas is in use in a wide variety of academic and commercial domains, including Finance, Neuroscience, Economics, Statistics, Advertising, Web Analytics, and more.

In this lecture, we will go over the basic capabilities of Pandas. It is a very deep library, and you will need to dig into the [documentation](http://pandas.pydata.org/pandas-docs/stable/) for more advanced usage.

Pandas was created by [Wes McKinney](http://wesmckinney.com/). Many of the examples here are drawn from Wes McKinney's book [Python for Data Analysis](http://shop.oreilly.com/product/0636920023784.do), which includes a github repo of [code samples](https://github.com/wesm/pydata-book)._____no_output_____

import pandas as pd
import numpy as np
from matplotlib import pyplot as plt
%matplotlib inline_____no_output_____

## Pandas Data Structures: Series

A Series represents a one-dimensional array of data. The main difference between a Series and numpy array is that a Series has an _index_. The index contains the labels that we use to access the data.

There are many ways to [create a Series](https://pandas.pydata.org/pandas-docs/stable/dsintro.html#series). We will just show a few._____no_output_____

names = ['Ryan', 'Chiara', 'Johnny']
values = [35, 36, 1.8]
ages = pd.Series(values, index=names)
ages_____no_output_____

Series have built in plotting methods._____no_output_____

ages.plot(kind='bar')_____no_output_____

Arithmetic operations and most numpy function can be applied to Series.
An important point is that the Series keep their index during such operations._____no_output_____

np.log(ages) / ages**2_____no_output_____

We can access the underlying index object if we need to:_____no_output_____

ages.index_____no_output_____

We can get values back out using the index via the `.loc` attribute_____no_output_____

ages.loc['Johnny']_____no_output_____

Or by raw position using `.iloc`_____no_output_____

ages.iloc[2]_____no_output_____

If we need to, we can always get the raw data back out as well_____no_output_____

ages.values_____no_output_____ages.index_____no_output_____

## Pandas Data Structures: DataFrame

There is a lot more to Series, but they are limit to a single "column". A more useful Pandas data structure is the DataFrame. A DataFrame is basically a bunch of series that share the same index. It's a lot like a table in a spreadsheet.

Below we create a DataFrame._____no_output_____

# first we create a dictionary
data = {'age': [35, 36, 1.8],
        'height': [180, 155, 83],
        'weight': [72.5, np.nan, 11.3]}
df = pd.DataFrame(data, index=['Ryan', 'Chiara', 'Johnny'])
df_____no_output_____

Pandas handles missing data very elegantly, keeping track of it through all calculations._____no_output_____

df.info()_____no_output_____

A wide range of statistical functions are available on both Series and DataFrames._____no_output_____

df.min()_____no_output_____df.mean()_____no_output_____df.std()_____no_output_____df.describe()_____no_output_____

We can get a single column as a Series using python's getitem syntax on the DataFrame object._____no_output_____

df['height']_____no_output_____

...or using attribute syntax._____no_output_____

df.height_____no_output_____

New columns can easily be added to DataFrames_____no_output_____

df['density'] = df.weight / df.height
df_____no_output_____

## Merging Data

Pandas supports a wide range of methods for merging different datasets. These are described extensively in the [documentation](https://pandas.pydata.org/pandas-docs/stable/merging.html). Here we just give a few examples._____no_output_____

education = pd.Series(['PhD', 'PhD', None, 'masters'],
                     index=['Ryan', 'Chiara', 'Johnny', 'Takaya'],
                     name='education')
# returns a new DataFrame
df.join(education)_____no_output_____# returns a new DataFrame
df.join(education, how='right')_____no_output_____# returns a new DataFrame
df.reindex(['Ryan', 'Chiara', 'Johnny', 'Takaya', 'Kerry'])_____no_output_____

We can also index using a boolean series. This is very useful_____no_output_____

adults = df[df.age > 18]
adults_____no_output_____df['is_adult'] = df.age > 18
df_____no_output_____

## Plotting

DataFrames have all kinds of [useful plotting](https://pandas.pydata.org/pandas-docs/stable/visualization.html) built in._____no_output_____

df.plot(kind='scatter', x='age', y='height', grid=True)_____no_output_____df.plot(kind='bar')_____no_output_____

## Time Indexes

Indexes are very powerful. They are a big part of why Pandas is so useful. There are different indices for different types of data. Time Indexes are especially great!_____no_output_____

two_years = pd.date_range(start='2014-01-01', end='2016-01-01', freq='D')
timeseries = pd.Series(np.sin(2 *np.pi *two_years.dayofyear / 365),
                       index=two_years)
timeseries.plot()_____no_output_____

We can use python's slicing notation inside `.loc` to select a date range._____no_output_____

timeseries.loc['2015-01-01':'2015-07-01'].plot()_____no_output_____

## Stock Market Data

* oops - Ryan's links don't work, so we will use static files

Now we read some stock market data from Google finance. I have created direct links to Google and Apple stock price data._____no_output_____

#!curl -L -o goog.csv http://tinyurl.com/rces-goog
#!curl -L -o aapl.csv http://tinyurl.com/rces-aapl-csv
! cp /home/pangeo/notebooks/GOOGL.csv goog.csv
! cp /home/pangeo/notebooks/AAPL.csv aapl.csv_____no_output_____! head goog.csv_____no_output_____

We can see that this is well-formated, tidy CSV data, ready for immediate ingestion into Pandas.
We use Pandas' amazing [read_csv](https://pandas.pydata.org/pandas-docs/stable/generated/pandas.read_csv.html) function to do this._____no_output_____

goog = pd.read_csv('goog.csv')
goog.head()_____no_output_____

Not bad! But we can do better by giving read_csv some hints._____no_output_____

goog = pd.read_csv('goog.csv', parse_dates=[0], index_col=0)
goog.head()_____no_output_____goog.info()_____no_output_____goog.Close.plot()_____no_output_____aapl = pd.read_csv('aapl.csv', parse_dates=[0], index_col=0)
aapl.info()_____no_output_____aapl_close = aapl.Close.rename('aapl')
goog_close = goog.Close.rename('goog')
stocks = pd.concat([aapl_close, goog_close], axis=1)
stocks.head()_____no_output_____stocks.plot()_____no_output_____

Pandas knows how to take correlations. And [tons of other computations](https://pandas.pydata.org/pandas-docs/stable/computation.html)._____no_output_____

stocks.corr()_____no_output_____

Because it understands times, it can do really cool stuff like resampling._____no_output_____

# resample by taking the mean over each month

fig, ax = plt.subplots()
stocks.resample('MS').mean().plot(ax=ax, colors=['r', 'b'])
# and each year
stocks.resample('AS').mean().plot(ax=ax, colors=['r', 'b'])_____no_output_____

The string `QS` means "month start. The string `AS` mean "year start". There is a long list of possible [frequency aliases](https://pandas.pydata.org/pandas-docs/stable/timeseries.html#timeseries-offset-aliases).

We can also apply other reduction operations with resample. These are described in the [resample docs](https://pandas.pydata.org/pandas-docs/stable/timeseries.html#resampling)._____no_output_____

# get resample object
rs = stocks.goog.resample('MS')
# standard deviation of each month
rs.std().plot()_____no_output_____

## Temperature Data

We download some timeseries data from the [Berkeley Earth(http://berkeleyearth.org/) surface temperature dataset. This is timeseries data from various locations around earth. Let's get our local temperatures._____no_output_____

! curl -o nyc_temp.txt http://berkeleyearth.lbl.gov/auto/Local/TAVG/Text/40.99N-74.56W-TAVG-Trend.txt_____no_output_____

If we examine this data, we see it is NOT a well formated CSV file. Loading it will be a bit painful, but Pandas makes the job retatively easy._____no_output_____

! head -72 nyc_temp.txt | tail -8_____no_output_____##### http://berkeleyearth.lbl.gov/locations/40.99N-74.56W
# http://berkeleyearth.lbl.gov/auto/Local/TAVG/Text/40.99N-74.56W-TAVG-Trend.txt

#temp = pd.read_csv('nyc_temp.txt')

col_names = ['year', 'month', 'monthly_anom'] + 10*[]
temp = pd.read_csv('nyc_temp.txt',
                   header=None, usecols=[0, 1, 2], names=col_names,
                   delim_whitespace=True, comment='%')

temp.head()_____no_output_____# need a day
date_df = temp.drop('monthly_anom', axis=1)
date_df['day'] = 1
date_index = pd.DatetimeIndex(pd.to_datetime(date_df))
temp = temp.set_index(date_index).drop(['year', 'month'], axis=1)
temp.head()_____no_output_____temp.plot()_____no_output_____fig, ax = plt.subplots()
temp.plot(ax=ax)
temp.resample('AS').mean().plot(ax=ax)
temp.resample('10AS').mean().plot(ax=ax)_____no_output_____

Pandas can do both time-based resampling and operation over fixed-length rolling windows. These are very similar but distinct; see [discussion in Pandas docs](https://pandas.pydata.org/pandas-docs/stable/computation.html#time-aware-rolling-vs-resampling)._____no_output_____

# more advanced operation on rolling windows
def difference_max_min(data):
    return data.max() - data.min()

rw = temp.rolling('365D')
rw.apply(difference_max_min).plot()_____no_output_____

To create a "climatology" (i.e. the average of all same months), we can use Pandas' [groupby](https://pandas.pydata.org/pandas-docs/stable/groupby.html) functionality._____no_output_____

# diurnal cycle has been removed!
temp.groupby(temp.index.month).mean().plot()_____no_output_____# find the hottest years
temp.groupby(temp.index.year).mean().sort_values('monthly_anom', ascending=False).head(10)_____no_output_____

## Groupby

Now we will explore groupby's capabilities more in a public dataset from the City of New York: the [Rat Information Portal](The Rat Information Portal)!_____no_output_____

# https://data.cityofnewyork.us/Health/Rats/amyk-xiv9
rats = pd.read_csv('https://data.cityofnewyork.us/api/views/amyk-xiv9/rows.csv',
                  parse_dates=['APPROVED_DATE', 'INSPECTION_DATE'])_____no_output_____rats.info()_____no_output_____rats.head()_____no_output_____

Let's do some grouping to explore the data._____no_output_____

rats.groupby('INSPECTION_TYPE')['INSPECTION_TYPE'].count()_____no_output_____rats.groupby('BORO_CODE')['BORO_CODE'].count().head()_____no_output_____rats.groupby('STREET_NAME')['STREET_NAME'].count().head(20)_____no_output_____

This dataset clearly needs some cleaning. We can Pandas' [text features](https://pandas.pydata.org/pandas-docs/stable/text.html) to strip the whitespace out of the data._____no_output_____

# clean up street name
street_names_cleaned = rats.STREET_NAME.str.strip()
street_names_cleaned.groupby(street_names_cleaned).count().head(20)_____no_output_____count = street_names_cleaned.groupby(street_names_cleaned).count()
count.sort_values(ascending=False).head(20)_____no_output_____

To get a better idea of the geography, let's plot the locations of the inspections. But first let's look at the statistics._____no_output_____

rats[['LATITUDE', 'LONGITUDE']].describe()_____no_output_____

There are clearly some weird outliers in the location data. We need to strip these out before plotting._____no_output_____

valid_latlon = rats[(rats.LATITUDE > 30) & (rats.LONGITUDE < -70)]
valid_latlon.plot.hexbin('LONGITUDE', 'LATITUDE', C='BORO_CODE', cmap='Set1')_____no_output_____# https://github.com/pandas-dev/pandas/issues/10678
valid_latlon.plot.hexbin('LONGITUDE', 'LATITUDE', sharex=False)_____no_output_____valid_latlon.plot.hexbin('LONGITUDE', 'LATITUDE', sharex=False, bins='log', cmap='magma')_____no_output_____manhattan_rats = valid_latlon[valid_latlon.BORO_CODE==1]
manhattan_rats.plot.hexbin('LONGITUDE', 'LATITUDE', sharex=False, bins='log', cmap='magma')_____no_output_____inspection_date = pd.DatetimeIndex(rats.INSPECTION_DATE)

fig, ax = plt.subplots()
rats.groupby(inspection_date.weekday)['JOB_ID'].count().plot(kind='bar', ax=ax)
ax.set_xlabel('weekday');_____no_output_____fig, ax = plt.subplots()
rats.groupby(inspection_date.hour)['JOB_ID'].count().plot(kind='bar', ax=ax)
ax.set_xlabel('hour');_____no_output_____fig, ax = plt.subplots()
rats.groupby(inspection_date.month)['JOB_ID'].count().plot(kind='bar', ax=ax)
ax.set_xlabel('month')_____no_output_____


# Notebook from richstoner/cshl-synbio-binder
Path: CSHLSynBio2016/FinalProject/An interactive model of TXTL CRISPRi-mediated knockdown of p70a-deGFP expression.ipynb

Modeling gene expression with differential equations_____no_output_____constiutive gene expression -> constant, no regulation

reaction kinetics, rate * concentration _____no_output_____

import numpy as np
from scipy.integrate import odeint
import dnaplotlib
import matplotlib.pyplot as plt
from matplotlib import gridspec
from IPython.display import display
import ipywidgets as widgets
from ipywidgets import interactive, interact
import pandas as pd

%pylab inlinePopulating the interactive namespace from numpy and matplotlib

## Modeling the CRISPR TXTL system_____no_output_____

txtl_crispr = pd.read_excel('supplemental_data/experimental_baseline.xlsx')

# clean in up a bit
def time_in_seconds(s):
    hour = s['Time'].hour
    minute = s['Time'].minute
    second = s['Time'].second

    total_seconds = hour * 3600 + minute * 60 + second
    return total_seconds

txtl_crispr['ts'] = txtl_crispr.apply(time_in_seconds, axis=1)
txtl_crispr.set_index('ts', inplace=True)
txtl_crispr = txtl_crispr.drop('Time', axis=1)
_____no_output_____txtl_crispr.columns

txtl_crispr['g2_nM'] = 1000*txtl_crispr['g2']/544.0
txtl_crispr['g21_nM'] = 1000*txtl_crispr['g2.1']/544.0
txtl_crispr['NT_nM'] = 1000*txtl_crispr['NT']/544.0
txtl_crispr['NT1_nM'] = 1000*txtl_crispr['NT.1']/544.0
_____no_output_____g2_group = ['g2_nM', 'g21_nM']
nt_group = ['NT_nM', 'NT1_nM']
control_group = ['cont1', 'cont1.1', 'cont2']_____no_output_____txtl_crispr[g2_group + nt_group ].plot()

# this was done with 1 nM of reporter
# this was done with 2 nM of guide_____no_output_____txtl_crispr[g2_group + nt_group ].iloc[0:60].plot()_____no_output_____# lets model the txtl systems!

# variables:
"""
all concentrations in µM

d_dcas9 = dCas9 dna
m_dcas9 = dCas9 mRNA transcript
p_dcas9 = dCas9 protein

d_guide = guide dna
m_guide = guide rna transcript

p_rnp = dCas9 protein + guide RNA

d_gfp = gfp dna
d_gfp_bound = gfp dna bound to transcript
m_gfp = gfp mRNA transcript
p_gfpd = gfp protein, unfolded (immature)
p_gfpf = gfp protein, folded (glowing)

k_tx = rate of transcription (shared)
k_tl = rate of translation (shared)

b_rna = rate of rna degradation (shared) -> this is equal to our 1/tau_m
b_protein = rate of protein degradation (shared)
b_dna = rate of dna degradation (shared)

k_rnpform = rate of complex formation (E + S -> S), dC/dt = k_rnpform * [p_dcas9] * [m_guide]
k_targetbinding = rate of CRISPR binding

"""
# rate constants

k_tx = 0.5 # transcription kinetic
k_tx_guide = 0.5 # transcription of the guide plasmid

k_tl = 0.0003 # translation kinetic
k_mat = 0.001 # this is the maturation of the GFP protein

b_rna = 1.0/1000 # mRNA rate degradation
b_protein = 0.0 # protein degradation
b_dna = 0.0 # DNA degradation

k_rnpform = 0.000001 # best guess
k_targetbinding = 0.00001 # best guess
_____no_output_____

### Interactive simulation_____no_output_____

t = np.arange(0, 3600*6, 60*3) # simulate over 10minutes, sampling every second

d_dcas9, m_dcas9, p_dcas9, p_rnp = initial_dcas9_conditions = [0, 0, 75, 0]

d_guide, m_guide = initial_guide_conditions = [0,0]

d_gfp, d_gfp_bound, m_gfp, p_gfpd, p_gfpf = initial_reporter_conditions = [0,0,0,0,0]

initial_conditions = initial_dcas9_conditions + initial_guide_conditions + initial_reporter_conditions

def crispr_txtl_system(y,t, new_target_binding):

    d_dcas9, m_dcas9, p_dcas9, p_rnp = y[0:4]
    d_guide, m_guide = y[4:6]
    d_gfp, d_gfp_bound, m_gfp, p_gfpd, p_gfpf = y[6:11]

    dt_d_dcas9 = - b_dna * d_dcas9
    # degradation only

    dt_m_dcas9 = k_tx * d_dcas9 - b_rna * m_dcas9
    # degradation, transcription

    dt_p_dcas9 = k_tl * m_dcas9 - b_protein * p_dcas9 - k_rnpform * m_guide * p_dcas9
    # degradation, translation

    dt_p_rnp = + k_rnpform * m_guide * p_dcas9 - b_protein * p_rnp
    # formation of RNP, degradation

    dt_d_guide = -b_dna * d_guide
    # degradation only

    dt_m_guide = k_tx_guide * d_guide - b_rna * m_guide - k_rnpform * m_guide * p_dcas9
    # transcription, degradation

    dt_d_gfp = -b_dna * d_gfp - new_target_binding * d_gfp * p_rnp
    # degradation, binding

    dt_d_gfp_bound = new_target_binding * d_gfp * p_rnp
    # binding only, assume no degradation here

    dt_m_gfp = k_tx * d_gfp  - b_rna * m_gfp
    # translation, degradation

    dt_p_gfpd = k_tl * m_gfp - b_protein * p_gfpd - k_mat * p_gfpd
    # translation, degradation, folding

    dt_p_gfpf = k_mat * p_gfpd - b_protein * p_gfpf

    return [dt_d_dcas9, dt_m_dcas9, dt_p_dcas9, dt_p_rnp,
            dt_d_guide, dt_m_guide,
            dt_d_gfp, dt_d_gfp_bound, dt_m_gfp, dt_p_gfpd, dt_p_gfpf]_____no_output_____
f,ax = plt.subplots(1,1, figsize=(12, 6))

def pltfig(dcas9_start=0, d_reporter_start=0.75, d_guide_start=2, m_guide_start=0,k_binding_start=10 ):

    fig = matplotlib.pyplot.gcf()
    fig.set_size_inches(18.5, 10.5)

    d_dcas9, m_dcas9, p_dcas9, p_rnp = initial_dcas9_conditions = [0, 0, dcas9_start, 0]
    d_guide, m_guide = initial_guide_conditions = [d_guide_start,m_guide_start]
    d_gfp, d_gfp_bound, m_gfp, p_gfpd, p_gfpf = initial_reporter_conditions = [d_reporter_start,0,0,0,0]

    initial_conditions = initial_dcas9_conditions + initial_guide_conditions + initial_reporter_conditions

    # update the k's for rnp formation and binding

    k_targetbinding = 0.000001 * k_binding_start

    y = zip(*odeint(crispr_txtl_system, initial_conditions, t, (k_targetbinding,)))

    d_dcas9, m_dcas9, p_dcas9, p_rnp = y[0:4]
    d_guide, m_guide = y[4:6]
    d_gfp, d_gfp_bound, m_gfp, p_gfpd, p_gfpf = y[6:11]

    ax = gca()

    ax.plot(t, p_gfpf, label='deGFP (folded)')

    txtl_crispr[g2_group + nt_group ].iloc[0:120].plot(ax=ax)
    ax.set_xlim((0, 3600*6))
    ax.set_ylim((0, 3000))

    plt.show()

# things I'd like to interact with
# starting concentration of p_dcas9
# starting concentration of guide plasmid dna
# starting concentration of guide RNA

interact(pltfig,
         dcas9_start=(0,100, 0.1),
         d_reporter_start=(0, 2, 0.1),
         d_guide_start=(0, 2, 0.1),
         m_guide_start=(0, 5000, 0.1),
         k_binding_start=(0, 20, 0.1)
        )
_____no_output_____