# Notebook from farhat-lab/in-host-Mtbc-dynamics
Path: Short Read Simulations/(C) Call Variants in Reference Genomes and Simulated Samples against H37Rv.ipynb

#### This notebook runs a smallPipe script for all of the Reference Genomes that have corresponding FeatureCounts tables and have had variants introduced_____no_output_____

from IPython.core.display import display, HTML
display(HTML(""))
%matplotlib inline_____no_output_____import os
import pandas as pd
import numpy as np
import subprocess
import vcf
import shutil
from slurmpy import Slurm_____no_output_____

#### Get the number of Reference Genomes we want to run Small Pipe on_____no_output_____

#number of Reference Genome folders with simulated fastq files for both Unaltered and Altered Reference Genomes
Number_of_Reference_Genomes = len( os.listdir('/n/data1/hms/dbmi/farhat/Roger/inhost_TB_dynamics_project/simulated_reads/reference_sequence_info_and_calls_against_H37Rv/') )_____no_output_____

### [1] UN-ALTERED RefGenome_____no_output_____

def Launch_MUMmer_Unaltered_RefGenome(RefGenome_i):

    ###############################################################################################################
    ###################################### Create Directories & Get File Paths ####################################
    ###############################################################################################################

    RefGenome_Directory = '/n/data1/hms/dbmi/farhat/Roger/inhost_TB_dynamics_project/simulated_reads/reference_sequence_info_and_calls_against_H37Rv/'
    os.chdir(RefGenome_Directory)

    #Good Reference Genomes (determined from from gene-length sliding window analyses)
    RefGenome_folders = os.listdir(RefGenome_Directory)
    RefGenome_folder = RefGenome_folders[RefGenome_i]

    #SPECIFY REFERENCE GENOME
    RefGenome_Directory_with_files = RefGenome_Directory + RefGenome_folder

    #paths & name for complete genome
    RefGenome_fasta_file = RefGenome_Directory_with_files + '/' + RefGenome_folder + '_genomic.fasta'

    #tag ID for use in naming files
    tag = RefGenome_folder

    #H37Rv Reference Genome Path
    H37Rv_fasta_file = '/n/data1/hms/dbmi/farhat/bin/work-horse/bin/h37rv.fasta' #H37Rv reference

    #create directory to store MUMmer output files
    out_dir = RefGenome_Directory_with_files + '/' + 'MUMmer_output_RefGenome'

    if os.path.exists(out_dir):
        shutil.rmtree(out_dir)
        os.makedirs(out_dir)
    elif not os.path.exists(out_dir):
        os.makedirs(out_dir)

    #create directory to store O2 SLURM logs
    SLURM_log_dir = out_dir + '/' + 'O2_SLURM_logs'

    if os.path.exists(SLURM_log_dir):
        shutil.rmtree(SLURM_log_dir)
        os.makedirs(SLURM_log_dir)
    elif not os.path.exists(SLURM_log_dir):
        os.makedirs(SLURM_log_dir)

    ###############################################################################################################
    ######################################## Construct job to submit to O2 ########################################
    ###############################################################################################################

    #store all commands in a list
    commands_list = []

    ###################################
    ### Load Necessary Modules ########
    ###################################

    #load perl
    commands_list.append( 'module load perl/5.24.0' )

    ###################################
    #### Run NUCMER & collect SNPs ####
    ###################################

    #change working directory to output directory
    commands_list.append( 'cd {}'.format(out_dir) )

    #run Nucmer
    commands_list.append( 'nucmer -mum --prefix=H37Rv_{0} {1} {2}'.format(tag, H37Rv_fasta_file, RefGenome_fasta_file) )

    #attempt to determine 'correct' repeat copies in repeat regions, eliminates conflicting repeat copies
    commands_list.append( 'delta-filter -r -q H37Rv_{0}.delta > H37Rv_{0}.filter'.format(tag) )

    #call SNPs, -C option assures only SNPs in uniquely aligned sequence will be reported (excludes SNPs in repeats)
    commands_list.append( 'show-snps -Clr -T H37Rv_{0}.filter > H37Rv_{0}.snps'.format(tag) )

    ###############################################################################################################
    ######################################## SUBMIT the job to O2 #################################################
    ###############################################################################################################

    #append all commands in a single string to be submitted as a job
    MUMmer_job = ''
    for command_i in commands_list:

        MUMmer_job = MUMmer_job + '\n' + command_i

    #directory where you want output + error files
    os.chdir(SLURM_log_dir)

    job_name = 'U_RG_' + str(RefGenome_i)

    #submit MUMmer/nucmer job via SLURM
    s = Slurm(job_name , {'partition':'short' , 'n':'1' , 't':'0-0:20:00' , 'mem-per-cpu':'8G' , 'mail-type':'FAIL' , 'mail-user':'roger_vargas@g.harvard.edu'})

    #submits the job
    job_id = s.run(MUMmer_job)

    print job_name  + ' : ' +  str(job_id)_____no_output_____#submit a job for each of the RefGenomes (that have a feature annotation file & a good H37Rv gene - RefGenome CDS mapping) ; 0-53
for RefGenome_i in range(0 , Number_of_Reference_Genomes):

    Launch_MUMmer_Unaltered_RefGenome(RefGenome_i)submitted: Submitted batch job 2971854

### [2] ALTERED RefGenome_____no_output_____

def Launch_SmallPipe_Altered_RefGenome(RefGenome_i):

    ###############################################################################################################
    ###################################### Create Directories & Get File Paths ####################################
    ###############################################################################################################

    RefGenome_Directory = '/n/data1/hms/dbmi/farhat/Roger/inhost_TB_dynamics_project/simulated_reads/reference_sequence_info_and_calls_against_H37Rv/'
    os.chdir(RefGenome_Directory)

    #Good Reference Genomes (determined from from gene-length sliding window analyses)
    RefGenome_folders = os.listdir(RefGenome_Directory)
    RefGenome_folder = RefGenome_folders[RefGenome_i]

    #SPECIFY REFERENCE GENOME
    RefGenome_Directory_with_files = RefGenome_Directory + RefGenome_folder

    #paths & names for simulated fastq files
    fqf1 = RefGenome_Directory_with_files + '/' + 'fastq_files_from_ART_for_ALT_RefGenome/' + RefGenome_folder + '_ART_1.fq'
    fqf2 = RefGenome_Directory_with_files + '/' + 'fastq_files_from_ART_for_ALT_RefGenome/' + RefGenome_folder + '_ART_2.fq'

    #tag ID for use in naming files
    tag = RefGenome_folder

    #H37Rv Reference Genome Path
    RefGen = RefGenome_Directory_with_files + '/RefGen/TBRefGen.fasta' #H37Rv reference

    #create directory to store Reference Genome and corresponding Index Files
    RefGen_dir = RefGenome_Directory_with_files + '/' + 'RefGen'

    if os.path.exists(RefGen_dir):
        shutil.rmtree(RefGen_dir)
        os.makedirs(RefGen_dir)
    elif not os.path.exists(RefGen_dir):
        os.makedirs(RefGen_dir)

    #create directory to store pilon VCF, prinseq & Qualimap logs
    out_dir = RefGenome_Directory_with_files + '/' + 'SmPipe_output_ALT_RefGenome'

    if os.path.exists(out_dir):
        shutil.rmtree(out_dir)
        os.makedirs(out_dir)
    elif not os.path.exists(out_dir):
        os.makedirs(out_dir)

    #create directory to store intermediary files (trimmed fastq, SAM, sorted BAM, sorted BAM w/ duplicates removed)
    temp_dir = RefGenome_Directory_with_files + '/' + 'temp_for_intermediary_steps'

    if os.path.exists(temp_dir):
        shutil.rmtree(temp_dir)
        os.makedirs(temp_dir)
    elif not os.path.exists(temp_dir):
        os.makedirs(temp_dir)

    #create directory to store O2 SLURM logs
    SLURM_log_dir = out_dir + '/' + 'O2_SLURM_logs'

    if os.path.exists(SLURM_log_dir):
        shutil.rmtree(SLURM_log_dir)
        os.makedirs(SLURM_log_dir)
    elif not os.path.exists(SLURM_log_dir):
        os.makedirs(SLURM_log_dir)

    ###############################################################################################################
    ######################################## Construct job to submit to O2 ########################################
    ###############################################################################################################

    #store all commands in a list
    commands_list = []

    #change directory to 1 with fastq files
    commands_list.append( 'cd ' + RefGenome_Directory_with_files )

    ###################################
    ### Load Necessary Modules ########
    ###################################

    #load perl
    commands_list.append( 'module load perl/5.24.0' )

    #load java
    commands_list.append( 'module load java/jdk-1.8u112' )

    #load BWA
    commands_list.append( 'module load bwa/0.7.15' )

    #load Samtools
    commands_list.append( 'module load samtools/1.3.1' )

    #load BCFtools
    commands_list.append( 'module load bcftools/1.3.1' )

    #load Picard
    commands_list.append( 'module load picard/2.8.0' )

    #copy reference genome over to RefGen folder
    commands_list.append( 'cp /n/data1/hms/dbmi/farhat/bin/work-horse/bin/h37rv.fasta RefGen/TBRefGen.fasta' )

    #change directory to RefGen folder
    commands_list.append( 'cd RefGen' )

    ###################################
    ### Create Index Files for H37Rv ##
    ###################################
    commands_list.append( 'samtools faidx TBRefGen.fasta' )
    commands_list.append( 'bwa index TBRefGen.fasta' )

    #Go back up a directory
    commands_list.append( 'cd ..' )

    ####################################
    ### PRINSEQ (trim reads) ##########
    ###################################

    #create directory for prinseq in output directory
    commands_list.append( 'mkdir ' + out_dir + '/prinseq' )

    commands_list.append( 'perl /n/data1/hms/dbmi/farhat/bin/prinseq-lite-0.20.4/prinseq-lite.pl -fastq {0} -fastq2 {1} -out_format 3 -out_good {2}/{3}-trimmed -out_bad null -log {4}/{3}-trimmed.log -min_qual_mean 20 -verbose'.format(fqf1, fqf2, temp_dir, tag , out_dir+'/prinseq') )

    #use newly trimmed fastq files now
    fqf1 = temp_dir + '/{}'.format(tag) + '-trimmed_1.fastq'
    fqf2 = temp_dir + '/{}'.format(tag) + '-trimmed_2.fastq'

    ######################################
    ### BWA (align reads to reference) ###
    ######################################

    #create SAM file
    samfile = temp_dir + '/{}.sam'.format(tag)

    #run BWA
    commands_list.append( 'bwa mem -M {3} {0} {1} > {2}'.format(fqf1 , fqf2 , samfile , RefGen) )

    #####################################
    ### PICARD (sort & convert to BAM) ##
    #####################################

    #create BAM file
    bamfile = temp_dir + '/{0}.sorted.bam'.format(tag)

    commands_list.append( 'java -Xmx16G -jar /n/data1/hms/dbmi/farhat/bin/picard/picard/build/libs/picard.jar SortSam INPUT={0} OUTPUT={1} SORT_ORDER=coordinate'.format(samfile, bamfile) )

    ####################################
    ### PICARD (remove duplicates) ####
    ###################################

    #create BAM file with removed duplicates
    drbamfile = bamfile.replace(".bam", ".duprem.bam")

    #remove duplicates from BAM file
    commands_list.append( "java -Xmx32G -jar /n/data1/hms/dbmi/farhat/bin/picard/picard/build/libs/picard.jar MarkDuplicates I={0} O={1} REMOVE_DUPLICATES=true M={2} ASSUME_SORT_ORDER=coordinate".format(bamfile, drbamfile, drbamfile[:-4]+'.metrics') )

    ####################################
    ### SAMTOOLS (to index BAM file) ###
    ####################################

    commands_list.append( "samtools index {0}".format(drbamfile) )

    ######################################
    ### QUALIMAP (quality of BAM file) ###
    ######################################

    #store quality report, pilon VCF & lineage call information all in Output directory
    commands_list.append( 'cd ' + out_dir )
    commands_list.append( 'mkdir QualiMap' ) #make a folder for pilon output in output directory
    commands_list.append( 'unset DISPLAY' ) #unset JAVA virtual machine variable [http://qualimap.bioinfo.cipf.es/doc_html/faq.html]
    commands_list.append( "/n/data1/hms/dbmi/farhat/bin/qualimap_v2.2.1/qualimap bamqc -bam {0} --outdir {1} --outfile {2}.pdf --outformat PDF".format(drbamfile, out_dir+'/QualiMap', tag+'_stats') )

    ###################################
    ### PILON (call variants) #########
    ###################################

    #store quality report, pilon VCF & lineage call information all in Output directory
    commands_list.append( 'mkdir pilon' ) #make a folder for pilon output in output directory
    out_pilon_dir = out_dir + '/pilon/' #variable for pilon output path

    commands_list.append( 'java -Xmx32G -jar /n/data1/hms/dbmi/farhat/bin/pilon/pilon-1.22.jar --genome {0} --bam {1} --output {2} --outdir {3} --variant'.format(RefGen, drbamfile, tag, out_pilon_dir) )

    ###################################
    ### DELETE UNNECESSSRY FILES ######
    ###################################

    #delete directory containing following files
    # trimmed fastq files
    # fastq trimming log
    # SAM file produced by running BWA on original or trimmed fastq
    # sorted BAM file
    # sorted BAM file with removed duplicates
    # output text from picard after discarding duplicates
    # index file corresponding to sorted BAM file with or without removed duplicates
    commands_list.append( 'rm -rf {}'.format(temp_dir) )

    #delete RefGen
    commands_list.append( 'rm -rf {}'.format(RefGen_dir) )

    #delete fastq files simulate off of Altered Reference Genome
    commands_list.append( 'rm -rf {}'.format(RefGenome_Directory_with_files + '/' + 'fastq_files_from_ART_for_ALT_RefGenome/') )

    #delete PASS CALLS (supporting Reference) and AMBIGUOUS CALLS from VCF file that came from Pilon variant calling
    commands_list.append( 'python /n/data1/hms/dbmi/farhat/Roger/inhost_TB_dynamics_project/python_scripts/reduce-pilon-vcf-size.py ' + out_dir + '/pilon/' + tag + '.vcf' )

    ###############################################################################################################
    ######################################## SUBMIT the job to O2 #################################################
    ###############################################################################################################

    #append all commands in a single string to be submitted as a job
    SmallPipe_job = ''
    for command_i in commands_list:

        SmallPipe_job = SmallPipe_job + '\n' + command_i

    #directory where you want output + error files
    os.chdir(SLURM_log_dir)

    job_name = 'A_RG_' + str(RefGenome_i)

    #make sure to set run-time much shorter since we're only dealing with 80x coverage
    s = Slurm(job_name , {'partition':'short' , 'n':'1' , 't':'0-3:00:00' , 'mem-per-cpu':'32G' , 'mail-type':'FAIL' , 'mail-user':'roger_vargas@g.harvard.edu'})

    #submits the job
    job_id = s.run(SmallPipe_job)

    print job_name  + ' : ' +  str(job_id)_____no_output_____#submit a job for each of the RefGenomes (that have a feature annotation file & a good H37Rv gene - RefGenome CDS mapping) ; 0-53
for RefGenome_i in range(0 , Number_of_Reference_Genomes):

    Launch_SmallPipe_Altered_RefGenome(RefGenome_i)submitted: Submitted batch job 2971908


# Notebook from Old-Shatterhand/LectinOracle
Path: LectinOracle_GitHub.ipynb

## Code for LectinOracle_____no_output_____

import matplotlib
import matplotlib.pyplot as plt
import matplotlib.patheffects as path_effects
%matplotlib inline
import seaborn as sns
from google.colab import drive
drive.mount('/content/drive')
import sys, os
from collections import Counter
import itertools
from itertools import compress
import operator
import pickle
import pandas as pd
import numpy as np
import networkx as nx
import re
import copy
import time
import random
from sklearn.model_selection import train_test_split
from sklearn.manifold import TSNE
from sklearn.metrics import mean_squared_error, mean_absolute_error, max_error
from functools import reduce
from scipy.stats import wilcoxon
!pip install torch==1.8.0 -f https://download.pytorch.org/whl/cu101/torch_stable.html
import torch
from torch import nn
import torch.nn.functional as F

fp = 'data/'Mounted at /content/drive
Looking in links: https://download.pytorch.org/whl/cu101/torch_stable.html
Collecting torch==1.8.0
  Downloading https://download.pytorch.org/whl/cu101/torch-1.8.0%2Bcu101-cp37-cp37m-linux_x86_64.whl (763.5 MB)
     |████████████████████████████████| 763.5 MB 13 kB/s
Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from torch==1.8.0) (1.19.5)
Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from torch==1.8.0) (3.7.4.3)
Installing collected packages: torch
  Attempting uninstall: torch
    Found existing installation: torch 1.9.0+cu102
    Uninstalling torch-1.9.0+cu102:
      Successfully uninstalled torch-1.9.0+cu102
!pip install torch-scatter -f https://pytorch-geometric.com/whl/torch-1.8.0+cu101.html
!pip install torch-sparse -f https://pytorch-geometric.com/whl/torch-1.8.0+cu101.html
!pip install torch-cluster -f https://pytorch-geometric.com/whl/torch-1.8.0+cu101.html
!pip install torch-spline-conv -f https://pytorch-geometric.com/whl/torch-1.8.0+cu101.html
!pip install torch-geometric

!pip install glycowork
import glycowork_____no_output_____from torch_geometric.data import Data, DataLoader
from torch_geometric.nn import TopKPooling, GraphConv
from torch_geometric.nn import global_mean_pool as gap, global_max_pool as gmp

from glycowork.ml.processing import dataset_to_graphs
from glycowork.motif.processing import get_lib, min_process_glycans
from glycowork.motif.graph import glycan_to_graph, graph_to_string, glycan_to_nxGraph, try_string_conversion, subgraph_isomorphism
from glycowork.motif.annotate import annotate_dataset
from glycowork.motif.analysis import get_pvals_motifs
from glycowork.glycan_data.loader import unwrap, motif_list
from glycowork.network.biosynthesis import safe_max, create_neighbors_____no_output_____

# Data Loading_____no_output_____

all_meta = pd.read_csv(fp+'all_meta.csv')
train = pd.read_csv(fp+'all_arrays.csv')
data_min = min([k for k in train.iloc[:,:-1].min() if np.isfinite(k)])
data_max = np.nanmax(train.iloc[:,:-1].values)
train, val = train_test_split(train, test_size = 0.1, random_state = 1, shuffle = True)
with open(fp+'stored_protein_embeddings.pkl', 'rb') as file:
  old_dic = pickle.load(file)_____no_output_____

# Legacy: Getting ESM-1b representations for new lectins_____no_output_____

##LEGACY - for getting new ESM-1b representations if wanting to apply LectinOracle to new proteins
## Preparing protein representations via ESM-1b
#!pip install fair-esm
#import torch
#import esm

## Load ESM-1b model
#model, alphabet = esm.pretrained.esm1b_t33_650M_UR50S()
#batch_converter = alphabet.get_batch_converter()_____no_output_____##LEGACY - for getting new ESM-1b representations if wanting to apply LectinOracle to new proteins
## Get sequences for which we do not have a stored representation yet
#with open(fp+'stored_protein_embeddings.pkl', 'rb') as file:
#  old_dic = pickle.load(file)
#new_seqs = [k for k in list(set(train.target.values.tolist()+val.target.values.tolist())) if k not in list(old_dic.keys())]
#new_seqs = [k for k in all_meta.sequence.values.tolist() if k not in list(old_dic.keys())]
#new_seqs = list(set(new_seqs))
#new_seqs = [k for k in new_seqs if isinstance(k, str)]
#len(new_seqs)_____no_output_____##LEGACY - for getting new ESM-1b representations if wanting to apply LectinOracle to new proteins
## Get representation for protein sequences in new_seqs
#data_list = []
#for k in range(0,len(new_seqs)):
#  if len(new_seqs[k])<1000:
#    data_list.append(('protein'+str(k), new_seqs[k][:np.min([len(new_seqs[k]),1000])]))
#  else:
#    data_list.append(('protein'+str(k), new_seqs[k][:1000]))
#batch_labels, batch_strs, batch_tokens = batch_converter(data_list)
#with torch.no_grad():
#    results = model(batch_tokens, repr_layers=[33], return_contacts=False)
#token_representations = results["representations"][33]
#sequence_representations = []
#for i, (_, seq) in enumerate(data_list):
#    sequence_representations.append(token_representations[i, 1 : len(seq) + 1].mean(0))_____no_output_____##LEGACY - for getting new ESM-1b representations if wanting to apply LectinOracle to new proteins
## Update representation dictionary and save
#prot_dic =  {new_seqs[k]:sequence_representations[k].tolist() for k in range(len(sequence_representations))}
#prot_dic = {**old_dic, **prot_dic}
#with open(fp+'stored_protein_embeddings.pkl', 'wb') as file:
#  pickle.dump(prot_dic, file)_____no_output_____

# Data Preparation_____no_output_____

#retrieve representations for protein sequences
train.target = [old_dic[k] if k in list(old_dic.keys()) else np.nan for k in train.target.values.tolist()]
train = train[train.target.notna()].reset_index(drop = True)
val.target = [old_dic[k] if k in list(old_dic.keys()) else np.nan for k in val.target.values.tolist()]
val = val[val.target.notna()].reset_index(drop = True)
train_____no_output_____#converts data into format (protein, glycan, binding)
def generate_pair_data(df):
  """creates 3-tuples of lectin, glycan, and the corresponding binding value"""
  out = []
  for k in range(len(df)):
    for j in range(df.shape[1]-1):
      if ~np.isnan(df.iloc[k,j]) and np.isfinite(df.iloc[k,j]):
        out.append((df.loc[k,'target'], df.columns.values.tolist()[j], df.iloc[k,j]))
  return out

train = generate_pair_data(train)
val = generate_pair_data(val)
len(train+val)_____no_output_____#split into train/validation sets
train2 = pd.DataFrame(train, columns = ['seq', 'glycan', 'match'])
val2 = pd.DataFrame(val, columns = ['seq', 'glycan', 'match'])
#the extension of lib was done for unrelated downstream analyses and is needed to use the trained LectinOracle file but does not need to be there if you train a new LectinOracle model
lib = get_lib(list(set(train2.glycan.values.tolist()+val2.glycan.values.tolist()))) + ['GlcNAcOS', 'GalOS', 'HexNAc']
X_train = list(zip(train2.seq.values.tolist(), [glycan_to_graph(k, libr = lib) for k in train2.glycan.values.tolist()]))
y_train = train2.match.values.tolist()
X_val = list(zip(val2.seq.values.tolist(), [glycan_to_graph(k, libr = lib) for k in val2.glycan.values.tolist()]))
y_val = val2.match.values.tolist()_____no_output_____#make dataloaders for deep learning models
trainset = [Data(x = torch.tensor(k[0][1][0], dtype = torch.long), y = torch.tensor(k[1], dtype = torch.float),
                 edge_index = torch.tensor([k[0][1][1][0], k[0][1][1][1]], dtype = torch.long)) for k in list(zip(X_train, y_train))]
for k in range(len(trainset)):
  trainset[k].train_idx = torch.tensor(X_train[k][0], dtype = torch.float)
valset = [Data(x = torch.tensor(k[0][1][0], dtype = torch.long), y = torch.tensor(k[1], dtype = torch.float),
                 edge_index = torch.tensor([k[0][1][1][0], k[0][1][1][1]], dtype = torch.long)) for k in list(zip(X_val, y_val))]
for k in range(len(valset)):
  valset[k].train_idx = torch.tensor(X_val[k][0], dtype=torch.float)

train_loader = DataLoader(trainset, batch_size = 128, shuffle = True)
val_loader = DataLoader(valset, batch_size = 128, shuffle = False)
dataloaders = {'train':train_loader, 'val':val_loader}_____no_output_____

# Model Preparation_____no_output_____

#defining LectinOracle

def sigmoid_range(x, low, high):
    "Sigmoid function with range `(low, high)`"
    return torch.sigmoid(x) * (high - low) + low

class SigmoidRange(nn.Module):
    "Sigmoid module with range `(low, x_max)`"
    def __init__(self, low, high):
      super(SigmoidRange, self).__init__()
      self.low, self.high = low,high
    def forward(self, x): return sigmoid_range(x, self.low, self.high)

class LectinOracle(nn.Module):
  def __init__(self, input_size_glyco, hidden_size, num_classes, data_min,
               data_max, input_size_prot = 1280, n_layers = 1):
    super(LectinOracle,self).__init__()
    self.input_size_prot = input_size_prot
    self.input_size_glyco = input_size_glyco
    self.hidden_size = hidden_size
    self.num_classes = num_classes
    self.n_layers = n_layers

    self.conv1 = GraphConv(self.hidden_size, self.hidden_size)
    self.pool1 = TopKPooling(self.hidden_size, ratio = 0.8)
    self.conv2 = GraphConv(self.hidden_size, self.hidden_size)
    self.pool2 = TopKPooling(self.hidden_size, ratio = 0.8)
    self.conv3 = GraphConv(self.hidden_size, self.hidden_size)
    self.pool3 = TopKPooling(self.hidden_size, ratio = 0.8)
    self.item_embedding = torch.nn.Embedding(num_embeddings = self.input_size_glyco, embedding_dim = self.hidden_size)
    self.prot_encoder1 = nn.Linear(self.input_size_prot, 400)
    self.prot_encoder2 = nn.Linear(400, 128)
    self.glyco_encoder = nn.Embedding(self.input_size_glyco, self.hidden_size, padding_idx = self.input_size_glyco - 1)
    self.dp1 = nn.Dropout(0.5)
    self.dp_prot1 = nn.Dropout(0.2)
    self.dp_prot2 = nn.Dropout(0.1)
    self.fc1 = nn.Linear(128+2*self.hidden_size, int(np.round(self.hidden_size/2)))
    self.fc2 = nn.Linear(int(np.round(self.hidden_size/2)), num_classes)
    self.bn1 = nn.BatchNorm1d(int(np.round(self.hidden_size/2)))
    self.bn_prot1 = nn.BatchNorm1d(400)
    self.bn_prot2 = nn.BatchNorm1d(128)
    self.act1 = torch.nn.LeakyReLU()
    self.act_prot1 = torch.nn.LeakyReLU()
    self.act_prot2 = torch.nn.LeakyReLU()
    self.sigmoid = SigmoidRange(data_min, data_max)

  def forward(self, prot, nodes, edge_index, batch, inference = False):
    embedded_prot = self.bn_prot1(self.act_prot1(self.dp_prot1(self.prot_encoder1(prot))))
    embedded_prot = self.bn_prot2(self.act_prot2(self.dp_prot2(self.prot_encoder2(embedded_prot))))

    x = self.item_embedding(nodes)
    x = x.squeeze(1)

    x = F.leaky_relu(self.conv1(x, edge_index))

    x, edge_index, _, batch, _, _= self.pool1(x, edge_index, None, batch)
    x1 = torch.cat([gmp(x, batch), gap(x, batch)], dim = 1)

    x = F.leaky_relu(self.conv2(x, edge_index))

    x, edge_index, _, batch, _, _ = self.pool2(x, edge_index, None, batch)
    x2 = torch.cat([gmp(x, batch), gap(x, batch)], dim = 1)

    x = F.leaky_relu(self.conv3(x, edge_index))

    x, edge_index, _, batch, _, _ = self.pool3(x, edge_index, None, batch)
    x3 = torch.cat([gmp(x, batch), gap(x, batch)], dim = 1)

    x = x1 + x2 + x3

    h_n = torch.cat((embedded_prot, x), 1)

    h_n = self.bn1(self.act1(self.fc1(h_n)))

    #1
    x1 = self.fc2(self.dp1(h_n))
    #2
    x2 = self.fc2(self.dp1(h_n))
    #3
    x3 = self.fc2(self.dp1(h_n))
    #4
    x4 = self.fc2(self.dp1(h_n))
    #5
    x5 = self.fc2(self.dp1(h_n))
    #6
    x6 = self.fc2(self.dp1(h_n))
    #7
    x7 = self.fc2(self.dp1(h_n))
    #8
    x8 = self.fc2(self.dp1(h_n))

    out =  self.sigmoid(torch.mean(torch.stack([x1, x2, x3, x4, x5, x6, x7, x8]), dim = 0))

    if inference:
      return out, embedded_prot, x
    else:
      return out_____no_output_____#training loop function

def train_model(model, criterion, optimizer, scheduler, num_epochs = 25):
  """training loop for LectinOracle"""
  since = time.time()
  best_model_wts = copy.deepcopy(model.state_dict())
  best_mse = 100.0
  val_losses = []
  val_mse = []

  for epoch in range(num_epochs):
    print('Epoch {}/{}'.format(epoch, num_epochs - 1))
    print('-'*10)

    for phase in ['train', 'val']:
      if phase == 'train':
        model.train()
      else:
        model.eval()

      running_loss = []
      running_mse = []
      running_mae = []
      running_max = []
      for data in dataloaders[phase]:
        x, y, edge_index, prot, batch = data.x, data.y, data.edge_index, data.train_idx, data.batch
        x = x.cuda()
        y = y.cuda()
        prot = prot.view(max(batch)+1, -1).cuda()
        edge_index = edge_index.cuda()
        batch = batch.cuda()
        optimizer.zero_grad()

        with torch.set_grad_enabled(phase == 'train'):
          pred = model(prot, x, edge_index, batch)
          loss = criterion(pred, y.view(-1,1))
          if phase == 'train':
            loss.backward()
            optimizer.step()

        running_loss.append(loss.item())
        running_mse.append(mean_squared_error(y.detach().cpu().numpy(), pred.detach().cpu().numpy()))
        running_mae.append(mean_absolute_error(y.detach().cpu().numpy(), pred.detach().cpu().numpy()))
        running_max.append(max_error(y.detach().cpu().numpy(), pred.detach().cpu().numpy()))

      epoch_loss = np.mean(running_loss)
      epoch_mse = np.mean(running_mse)
      epoch_mae = np.mean(running_mae)
      epoch_max = np.mean(running_max)
      print('{} Loss: {:.4f} MSE: {:.4f} MAE: {:.4f} MaxE: {:.4f}'.format(
          phase, epoch_loss, epoch_mse, epoch_mae,
          epoch_max))

      if phase == 'val' and epoch_mse <= best_mse:
        best_mse = epoch_mse
        best_model_wts = copy.deepcopy(model.state_dict())
      if phase == 'val':
        val_losses.append(epoch_loss)
        val_mse.append(epoch_mse)
        early_stopping(epoch_loss, model)

      scheduler.step()

    if early_stopping.early_stop:
      print("Early stopping")
      break
    print()

  time_elapsed = time.time() - since
  print('Training complete in {:.0f}m {:.0f}s'.format(
      time_elapsed // 60, time_elapsed % 60))
  print('Best val MSE: {:4f}'.format(best_mse))
  model.load_state_dict(best_model_wts)

  ## plot loss & MSE over the course of training
  fig, ax = plt.subplots(nrows = 2, ncols = 1)
  plt.subplot(2, 1, 1)
  plt.plot(range(epoch+1), val_losses)
  plt.title('Training of Model')
  plt.ylabel('Validation Loss')
  plt.legend(['Validation Loss'], loc = 'best')

  plt.subplot(2, 1, 2)
  plt.plot(range(epoch+1), val_mse)
  plt.ylabel('Validation MSE')
  plt.xlabel('Number of Epochs')
  plt.legend(['Validation MSE'], loc = 'best')
  return model_____no_output_____#early stopping function if training stalls
class EarlyStopping:
    """Early stops the training if validation loss doesn't improve after a given patience."""
    def __init__(self, patience = 7, verbose = False):
        """
        Args:
            patience (int): How long to wait after last time validation loss improved.
                            Default: 7
            verbose (bool): If True, prints a message for each validation loss improvement.
                            Default: False
        """
        self.patience = patience
        self.verbose = verbose
        self.counter = 0
        self.best_score = None
        self.early_stop = False
        self.val_loss_min = 0

    def __call__(self, val_loss, model):

        score = -val_loss

        if self.best_score is None:
            self.best_score = score
            self.save_checkpoint(val_loss, model)
        elif score < self.best_score:
            self.counter += 1
            print(f'EarlyStopping counter: {self.counter} out of {self.patience}')
            if self.counter >= self.patience:
                self.early_stop = True
        else:
            self.best_score = score
            self.save_checkpoint(val_loss, model)
            self.counter = 0

    def save_checkpoint(self, val_loss, model):
        '''Saves model when validation loss decrease.'''
        if self.verbose:
            print(f'Validation loss decreased ({self.val_loss_min:.6f} --> {val_loss:.6f}).  Saving model ...')
        self.val_loss_min = val_loss_____no_output_____

# LectinOracle_____no_output_____

#specifiying the model and training it

def init_weights(m):
    if type(m) == nn.Linear:
        torch.nn.init.xavier_uniform(m.weight)

## instantiating the model itself, initializing it and putting it on the GPU
LeOr = LectinOracle(input_size_prot = 1280, input_size_glyco = len(lib) + 1, hidden_size = 128,
          num_classes = 1, data_min = data_min, data_max = data_max)
LeOr.apply(init_weights)
LeOr.cuda()

early_stopping = EarlyStopping(patience = 20, verbose = True)
optimizer_ft = torch.optim.Adam(LeOr.parameters(), lr = 0.0005, weight_decay = 0.0)
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer_ft, 80)
criterion = nn.MSELoss().cuda()
#model_ft = train_model(LeOr, criterion, optimizer_ft, scheduler,
#                   num_epochs = 100)_____no_output_____

# Get Lectin Binding Specificity_____no_output_____

#loading a trained LectinOracle model
model_ft = LectinOracle(input_size_prot = 1280, input_size_glyco = len(lib)+1, hidden_size = 128,
          num_classes = 1, data_min = data_min, data_max = data_max)
model_ft.load_state_dict(torch.load(fp+'LectinOracle_565.pt'))
model_ft = model_ft.cuda()_____no_output_____#functions to get predicted lectin binding with a trained LectinOracle model

df_corr = pd.read_csv(fp+'background_correction_for_565.csv')

def get_pred(prot, glycan, model_ft = model_ft, prot_dic = old_dic, batch_size = 128):
  try:
    repr = prot_dic[prot]
  except:
    print('new protein, no stored embedding')
  ggraph = glycan_to_graph(glycan, libr = lib)
  graph_data = [Data(x = torch.tensor(ggraph[0], dtype = torch.long), y = torch.tensor(0.99, dtype = torch.float),
                    edge_index = torch.tensor([ggraph[1][0], ggraph[1][1]], dtype = torch.long))]*batch_size
  for k in range(len(graph_data)):
    graph_data[k].train_idx = torch.tensor(repr, dtype = torch.float)
  train_loader = DataLoader(graph_data, batch_size = batch_size, shuffle = False)
  model_ft = model_ft.eval()
  res = []
  for k in train_loader:
    x, y, edge_index, prot, batch = k.x, k.y, k.edge_index, k.train_idx, k.batch
    x = x.cuda()
    y = y.cuda()
    prot = prot.view(max(batch)+1, -1).cuda()
    edge_index = edge_index.cuda()
    batch = batch.cuda()
    pred = model_ft(prot, x, edge_index, batch)
    res.append(pred)
  res = [res[k].detach().cpu().numpy() for k in range(len(res))][0][0].tolist()[0]
  return res

def get_multi_pred(prot, glycans, model_ft = model_ft, prot_dic = old_dic, batch_size = 128,
                   background_correction = False, correction_df = df_corr):
  try:
    repr = prot_dic[prot]
  except:
    print('new protein, no stored embedding')
  ggraph = [glycan_to_graph(k, libr = lib) for k in glycans]
  graph_data = [Data(x = torch.tensor(ggraph[k][0], dtype = torch.long), y = torch.tensor(0.99, dtype = torch.float),
                    edge_index = torch.tensor([ggraph[k][1][0], ggraph[k][1][1]], dtype = torch.long)) for k in range(len(ggraph))]
  for k in range(len(graph_data)):
    graph_data[k].train_idx = torch.tensor(repr, dtype=torch.float)
  train_loader = DataLoader(graph_data, batch_size = batch_size, shuffle = False)
  model_ft = model_ft.eval()
  res = []
  for k in train_loader:
    x, y, edge_index, prot, batch = k.x, k.y, k.edge_index, k.train_idx, k.batch
    x = x.cuda()
    y = y.cuda()
    prot = prot.view(max(batch)+1, -1).cuda()
    edge_index = edge_index.cuda()
    batch = batch.cuda()
    pred = model_ft(prot, x, edge_index, batch)
    res.append(pred)
  res = unwrap([res[k].detach().cpu().numpy() for k in range(len(res))])
  res = [k.tolist()[0] for k in res]
  if background_correction:
    correction_df = pd.Series(correction_df.pred.values, index = correction_df.motif).to_dict()
    bg_res = [correction_df[j] if j in list(correction_df.keys()) else 0 for j in glycans]
    if 0 in bg_res:
      print("Warning: not all glycans are in the correction_df; consider adding their background to correction_df")
    res = [a_i - b_i for a_i, b_i in zip(res, bg_res)]
  return res_____no_output_____with open(fp+'glycan_trisaccharide_motifs.pkl', 'rb') as file:
  old_motifs = pickle.load(file)
trisaccharides = list(set(unwrap(list(old_motifs.values()))))

def get_motif_predictions(protein, model = model_ft, more_motifs = False, prot_dic = old_dic,
                          trisaccharides = False, trisaccharide_list = trisaccharides, background_correction = False,
                          correction_df = df_corr, batch_size = 128, add_in_full = False, sorted = True):
  glycans = list(set(train2.glycan.values.tolist() + val2.glycan.values.tolist()))
  glycan_motifs = annotate_dataset(glycans, libr = lib, feature_set = ['exhaustive'], condense = True)
  glycan_motifs = glycan_motifs.columns.values.tolist()
  glycan_motifs = [k for k in glycan_motifs if '(' in k]
  if more_motifs:
    glycan_motifs = [glycan_motifs]
    forbidden_list = ['2(', '3(', '4(', '6(', ')(', 'b1-4GlcNAc']
    for k in glycans:
      ggraph = glycan_to_nxGraph(k, libr = lib)
      ggraph_nb = create_neighbors(ggraph, libr = lib)
      g_nb = [try_string_conversion(j, libr = lib) for j in ggraph_nb]
      g_nb = [j for j in g_nb if j is not None]
      g_nb = [j if j[0] != '[' else j.replace('[','',1).replace(']','',1) for j in g_nb]
      g_nb = [j for j in g_nb if j[0] != '(']
      g_nb = [j for j in g_nb if not any([m in j for m in forbidden_list])]
      glycan_motifs.append(g_nb)
    glycan_motifs = list(set(unwrap(glycan_motifs)))
  if trisaccharides:
    glycan_motifs = glycan_motifs + trisaccharide_list
  if add_in_full:
    glycan_motifs = glycan_motifs + glycans
  preds = get_multi_pred(protein, glycan_motifs, model, prot_dic = prot_dic,
                         batch_size = batch_size)
  df_pred = pd.DataFrame(glycan_motifs, columns = ['motif'])
  df_pred['pred'] = preds
  if background_correction:
    correction_df = pd.Series(correction_df.pred.values, index = correction_df.motif).to_dict()
    for j in df_pred.motif.values.tolist():
      motif_idx = df_pred.motif.values.tolist().index(j)
      df_pred.at[motif_idx, 'pred'] = df_pred.iloc[motif_idx, 1] - correction_df[j]
  if sorted:
    df_pred.sort_values('pred', ascending=True, inplace=True)
  return df_pred

ConA = 'ADTIVAVELDTYPNTDIGDPSYPHIGIDIKSVRSKKTAKWNMQNGKVGTAHIIYNSVDKRLSAVVSYPNADSATVSYDVDLDNVLPEWVRVGLSASTGLYKETNTILSWSFTSKLKSNSTHETNALHFMFNQFSKDQKDLILQGDATTGTDGNLELTRVSSNGSPQGSSVGRALFYAPVHIWESSAVVASFEATFTFLIKSPDSHPADGIAFFISNIDSSIPSGSTGRLLGLFPDAN'
cona_predictions = get_motif_predictions(ConA, trisaccharides = True, background_correction = True, add_in_full = True)_____no_output_____SNA = all_meta[all_meta.name=='SNA'].sequence.values.tolist()[0]
SNA_pred = get_motif_predictions(SNA, more_motifs = True, trisaccharides = True, background_correction = True)
SNA_predMRLVAKLLYLAVLAICGLGIHGALTHPRVTPPVYPSVSFNLTGADTYEPFLRALQEKVILGNHTAFDLPVLNPESQVSDSNRFVLVPLTNPSGDTVTLAIDVVNLYVVAFSSNGKSYFFSGSTAVQRDNLFVDTTQEELNFTGNYTSLERQVGFGRVYIPLGPKSLDQAISSLRTYTLTAGDTKPLARGLLVVIQMVSEAARFRYIELRIRTSITDASEFTPDLLMLSMENNWSSMSSEIQQAQPGGIFAGVVQLRDERNNSIEVTNFRRLFELTYIAVLLYGCAPVTSSSYSNNAIDAQIIKMPVFRGGEYEKVCSVVEVTRRISGWDGLCVDVRYGHYIDGNPVQLRPCGNECNQLWTFRTDGTIRWLGKCLTASSSVMIYDCNTVPPEATKWVVSIDGTITNPHSGLVLTAPQAAEGTALSLENNIHAARQGWTVGDVEPLVTFIVGYKQMCLRENGENNFVWLEDCVLNRVQQEWALYGDGTIRVNSNRSLCVTSEDHEPSDLIVILKCEGSGNQRWVFNTNGTISNPNAKLLMDVAQRDVSLRKIILYRPTGNPNQQWITTTHPA
def test_motif_enrichment(df_pred, motif):
  """testing for significant enrichment of motifs in glycans predicted to be bound"""
  df_pred = df_pred.sort_values(by = 'pred', ascending = False).reset_index(drop = True)
  if motif[-1] == ')':
    rank_order = [k for k in range(len(df_pred)) if motif in df_pred.motif.values.tolist()[k]]
  else:
    rank_order = [k for k in range(len(df_pred)) if motif in df_pred.motif.values.tolist()[k]]
    #rank_order = [k for k in range(len(df_pred)) if df_pred.motif.values.tolist()[k].startswith(motif)]
    #rank_order = [k for k in range(len(df_pred)) if subgraph_isomorphism(df_pred.motif.values.tolist()[k], motif, libr = lib)]
  bg = random.sample(list(range(len(df_pred))), len(rank_order))
  print(wilcoxon(rank_order, bg, alternative = 'less'))_____no_output_____

# Protein Space Plots_____no_output_____

#get protein representations from trained LectinOracle model

def get_multi_pred_prot(prots, glycan, model_ft, mode = 'rep', prot_dic = old_dic, batch_size = 128):
  try:
    repr = [prot_dic[k] for k in prots]
  except:
    print('new protein, no stored embedding')
  glycans = [glycan]*len(prots)
  ggraph = [glycan_to_graph(k, libr = lib) for k in glycans]
  graph_data = [Data(x = torch.tensor(ggraph[k][0], dtype = torch.long), y = torch.tensor(0.99, dtype = torch.float),
                    edge_index = torch.tensor([ggraph[k][1][0], ggraph[k][1][1]], dtype = torch.long)) for k in range(len(ggraph))]
  for k in range(len(graph_data)):
    graph_data[k].train_idx = torch.tensor(repr[k], dtype = torch.float)
  train_loader = DataLoader(graph_data, batch_size = batch_size, shuffle = False)
  model_ft = model_ft.eval()
  res = []
  for k in train_loader:
    x, y, edge_index, prot, batch = k.x, k.y, k.edge_index, k.train_idx, k.batch
    x = x.cuda()
    y = y.cuda()
    prot = prot.view(max(batch)+1, -1).cuda()
    edge_index = edge_index.cuda()
    batch = batch.cuda()
    pred, rep_prot, rep_glyc = model_ft(prot, x, edge_index, batch, inference = True)
    if mode == 'rep':
      res.append(rep_prot)
    else:
      res.append(pred)
  res = unwrap([res[k].detach().cpu().numpy() for k in range(len(res))])
  return {prots[k]:res[k] for k in range(len(prots))}

prot_specs = [get_motif_predictions(k, more_motifs = False, trisaccharides = False, background_correction = True) for k in list(old_dic.keys())]
prot_specs_freq = [k if prot_specs.count(k) > 20 else 'misc' for k in prot_specs]_____no_output_____#getting learned protein represenations from LectinOracle
prot_reps = get_multi_pred_prot(list(old_dic.keys()), 'Man(b1-4)GlcNAc(b1-4)GlcNAc', model_ft)
df_rep = pd.DataFrame(prot_reps)
df_rep = df_rep.T
tsne_emb = TSNE(random_state = 42).fit_transform(df_rep)_____no_output_____#motif_pred_dict is not in data folder so this won't run but you can just predict binding to core fucose for all lectins and get a list of values to color this plot
motif = 'core_fucose'
plt.figure(figsize = (12, 9))
sns.scatterplot(x = tsne_emb[:,0], y = tsne_emb[:,1], s = 50, alpha = 0.6,
                hue = motif_pred_dict[motif], palette = 'Greens')
plt.legend(loc = 'center left', bbox_to_anchor = (1, 0.5))
plt.xlabel('t-SNE Dim1')
plt.ylabel('t-SNE Dim2')
plt.title(motif)
plt.tight_layout()_____no_output_____#getting protein representations from ESM-1b (based on sequence similarity)
pre_preds = list(old_dic.values())
df_pre = pd.DataFrame(pre_preds)
tsne_emb_pre = TSNE(random_state = 42).fit_transform(df_pre)_____no_output_____plt.figure(figsize = (12, 9))
sns.scatterplot(tsne_emb_pre[:,0], tsne_emb_pre[:,1], s = 50, alpha = 0.6,
                hue = prot_specs_freq, palette = 'colorblind')
plt.legend(loc = 'center left', bbox_to_anchor = (1, 0.5))
plt.xlabel('t-SNE Dim1')
plt.ylabel('t-SNE Dim2')
plt.title('Proteins (only sequence)')
plt.tight_layout()/usr/local/lib/python3.7/dist-packages/seaborn/_decorators.py:43: FutureWarning: Pass the following variables as keyword args: x, y. From version 0.12, the only valid positional argument will be `data`, and passing other arguments without an explicit keyword will result in an error or misinterpretation.
  FutureWarning

# Vaginal Microbiome Lectome_____no_output_____

##data from https://www.nature.com/articles/s41522-021-00220-9
#vm_lec = pd.read_csv(fp+'vaginal_microbiome_lectins.csv')
#with open(fp+'vaginal_microbiome_lectins_stored_protein_embeddings.pkl', 'rb') as file:
#  vm_dic = pickle.load(file)
#prot_specs = [get_motif_predictions(k, more_motifs=False, prot_dic = vm_dic, background_correction=True).motif.values.tolist()[-1] for k in list(vm_dic.keys())]_____no_output_____lectin_class = [vm_lec.Lectin_class.values.tolist()[vm_lec.Sequence.values.tolist().index(k)] \
                if k in vm_lec.Sequence.values.tolist() else np.nan for k in list(vm_dic.keys())]
lectin_class_freq = [k if lectin_class.count(k)>5 else 'misc' for k in lectin_class]
bac_type = [vm_lec.Bac_Type.values.tolist()[vm_lec.Sequence.values.tolist().index(k)] \
                if k in vm_lec.Sequence.values.tolist() else np.nan for k in list(vm_dic.keys())]_____no_output_____def get_binding_repertoire(strain, df = vm_lec, prot_dic = vm_dic, thresh = 0, more_motifs = False):
  lectins = df[df.Species_and_strain == strain].Sequence.values.tolist()
  motif_pool = []
  for k in lectins:
    df_motif = get_motif_predictions(k, more_motifs = more_motifs, trisaccharides = True,
                                     prot_dic = prot_dic, background_correction = True)
    motif_pool.append(df_motif)
  return motif_pool

#bp_ecoli_UMB0731 = get_binding_repertoire('Escherichia_coli_UMB0731')_____no_output_____vm_lec2 = vm_lec[~vm_lec.Lectin_class.isin(['LysM-like', 'CBM50'])].reset_index(drop = True)
only_lec = vm_lec2[vm_lec2.Type == 'LEC'].reset_index(drop = True)(1551, 8)
(1194, 8)
species_list = list(sorted(list(set(vm_lec.Species_and_strain.values.tolist()))))
species_repertoires = {species:get_binding_repertoire(species, df = only_lec, thresh = 0.0, more_motifs = True) for species in species_list}
species_repertoires2 = {k:(pd.concat(species_repertoires[k], axis = 0).reset_index(drop = True) if len(species_repertoires[k])>0 else pd.DataFrame(columns = ['motif', 'pred'])) for k in species_list}
species_repertoires_0 = [len(list(set(species_repertoires2[k][species_repertoires2[k].pred>0].motif.values.tolist()))) for k in species_list]_____no_output_____#finding enriched motifs in pathobionts & commensals
comm = only_lec[only_lec['Bac_Type'] == 'Commensal'].reset_index(drop = True)
comm_preds = [get_motif_predictions(k, trisaccharides = True, prot_dic = vm_dic,
                                     background_correction = True, add_in_full = True, sorted = False) for k in comm.Sequence.values.tolist()]
temp = pd.concat([pd.Series(k.pred.values.tolist()) for k in comm_preds], axis = 1).T
temp.columns = comm_preds[0].motif.values.tolist()
lib2 = list(set(lib + ['GalNAc6S', 'GlcN2S6S', 'D-IdoA', 'NeuNAc', 'Kdo', 'a2-4', 'GalA', 'Araf', 'GalN', 'Fuc4S',
                      'L-GulA', 'ManA']))

comm_pvals = get_pvals_motifs(temp, glycan_col_name = 'motif', label_col_name = 'pred', thresh = 0, feature_set = ['known', 'exhaustive'],
                               multiple_samples = True, libr = lib2)_____no_output_____patho = only_lec[only_lec['Bac_Type'] == 'Pathobiont'].reset_index(drop = True)
patho_preds = [get_motif_predictions(k, trisaccharides = True, prot_dic = vm_dic,
                                     background_correction = True, add_in_full = True, sorted = False) for k in patho.Sequence.values.tolist()]
temp2 = pd.concat([pd.Series(k.pred.values.tolist()) for k in patho_preds], axis = 1).T
temp2.columns = patho_preds[0].motif.values.tolist()
patho_pvals = get_pvals_motifs(temp2, glycan_col_name = 'motif', label_col_name = 'pred', thresh = 0, feature_set = ['exhaustive', 'known'],
                               multiple_samples = True, libr = lib2)_____no_output_____comms = {comm_pvals.motif.values.tolist()[k]:comm_pvals.pval.values.tolist()[k] for k in range(len(comm_pvals))}
comms_df = pd.DataFrame(comms, index = [0])
paths = {patho_pvals.motif.values.tolist()[k]:patho_pvals.pval.values.tolist()[k] for k in range(len(patho_pvals))}
paths_df = pd.DataFrame(paths, index = [0])
comb_df = pd.concat([comms_df, paths_df], axis = 0, ignore_index = True).T
comb_df.columns = ['commensal', 'pathobiont']
comb_df2 = comb_df2.apply(lambda x: -np.log10(x))_____no_output_____sns.scatterplot(x = 'commensal', y = 'pathobiont', data = comb_df2)
sns.despine(bottom = True, left = True)
plt.tight_layout()_____no_output_____


# Notebook from wconnell/metrx
Path: notebook/2019.12.15_rna-seq_pan_diseased_tissue-v-tissue.ipynb

import numpy as np
import pandas as pd

# viz
import matplotlib.pyplot as plt

# notebook settings
from IPython.core.interactiveshell import InteractiveShell
InteractiveShell.ast_node_interactivity = "all"

pd.set_option('display.max_columns', 1000)_____no_output_____

## Sample Prep_____no_output_____

samples = pd.read_csv('../data/TCGA/rna-seq_pan/meta/gdc_sample_sheet.2019-12-12.tsv', sep="\t")
# get file type
samples['data'] = [val[1] for i,val in samples['File Name'].str.split(".").items()]
samples['project'] = [val[1] for i,val in samples['Project ID'].str.split("-").items()]
samples['project'].value_counts()_____no_output_____samples['Sample Type'].value_counts()_____no_output_____samples.shape
cases = samples[samples['Sample Type']=='Primary Tumor'].sample(frac=1).copy()
cases.shape
proj = cases['project'].value_counts().head(10).index
cases = cases[cases['project'].isin(proj)]
cases['project'].value_counts()
cases.shape_____no_output_____

## Dataset Prep_____no_output_____

from sklearn.model_selection import train_test_split

target = 'project'
cases[target] = cases[target].astype('category')
train, test = train_test_split(cases)_____no_output_____import torch
import torch.nn as nn
from torch.optim import lr_scheduler
import torch.optim as optim
from torch.autograd import Variable

from trainer import fit
import visualization as vis
import numpy as np

if torch.cuda.is_available():
    cuda = torch.cuda.is_available()
    print("{} GPUs available".format(torch.cuda.device_count()))

colors = ['#1f77b4', '#ff7f0e', '#2ca02c', '#d62728',
              '#9467bd', '#8c564b', '#e377c2', '#7f7f7f',
              '#bcbd22', '#17becf']
classes = train[target].cat.categories.values
8 GPUs available
from tcga_datasets import TCGA, SiameseTCGA
root_dir = "../data/TCGA/rna-seq_pan/"
batch_size = 1

train_dataset = TCGA(root_dir, samples=train, train=True, target=target)
test_dataset = TCGA(root_dir, samples=test, train=False, target=target)
print('Loaded')
kwargs = {'num_workers': 1, 'pin_memory': True} if cuda else {}
train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True, **kwargs)
test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False, **kwargs)Loaded

## Siamese Network_____no_output_____

# Step 1 set up dataloader
siamese_train_dataset = SiameseTCGA(train_dataset) # Returns pairs of images and target same/different
siamese_test_dataset = SiameseTCGA(test_dataset)
batch_size = 64
kwargs = {'num_workers': 10, 'pin_memory': True} if cuda else {}
siamese_train_loader = torch.utils.data.DataLoader(siamese_train_dataset, batch_size=batch_size, shuffle=True, **kwargs)
siamese_test_loader = torch.utils.data.DataLoader(siamese_test_dataset, batch_size=batch_size, shuffle=False, **kwargs)

# Set up the network and training parameters
from tcga_networks import EmbeddingNet, SiameseNet
from losses import ContrastiveLoss
from metrics import AccumulatedAccuracyMetric

# Step 2
embedding_net = EmbeddingNet()
# Step 3
model = SiameseNet(embedding_net)
if cuda:
    model = nn.DataParallel(model)
    model.cuda()

# Step 4
margin = 1.
loss_fn = ContrastiveLoss(margin)
lr = 1e-3
optimizer = optim.Adam(model.parameters(), lr=lr)
scheduler = lr_scheduler.StepLR(optimizer, 8, gamma=0.1, last_epoch=-1)
n_epochs = 10
# print training metrics every log_interval * batch_size
log_interval = 4_____no_output_____train_loss, val_loss = fit(siamese_train_loader, siamese_test_loader, model, loss_fn, optimizer, scheduler,
    n_epochs, cuda, log_interval)Train: [0/4074 (0%)]	Loss: 0.262692
Train: [256/4074 (6%)]	Loss: 4.697056
Train: [512/4074 (12%)]	Loss: 0.960701
Train: [768/4074 (19%)]	Loss: 2.739669
Train: [1024/4074 (25%)]	Loss: 0.893237
Train: [1280/4074 (31%)]	Loss: 2.260341
Train: [1536/4074 (38%)]	Loss: 0.619737
Train: [1792/4074 (44%)]	Loss: 0.511726
Train: [2048/4074 (50%)]	Loss: 0.434056
Train: [2304/4074 (56%)]	Loss: 0.205737
Train: [2560/4074 (62%)]	Loss: 0.224834
Train: [2816/4074 (69%)]	Loss: 0.165997
Train: [3072/4074 (75%)]	Loss: 0.097918
Train: [3328/4074 (81%)]	Loss: 0.158752
Train: [3584/4074 (88%)]	Loss: 0.116707
Train: [3840/4074 (94%)]	Loss: 0.193258
Epoch: 1/10. Train set: Average loss: 0.9028
Epoch: 1/10. Validation set: Average loss: 0.1402
Train: [0/4074 (0%)]	Loss: 0.158404
Train: [256/4074 (6%)]	Loss: 0.107966
Train: [512/4074 (12%)]	Loss: 0.221601
Train: [768/4074 (19%)]	Loss: 0.238340
Train: [1024/4074 (25%)]	Loss: 0.183364
Train: [1280/4074 (31%)]	Loss: 0.114250
Train: [1536/4074 (38%)]	Loss: 0.114264
Train: [1792/4074 (44%)]	Loss: 0.103278
Train: [2048/4074 (50%)]	Loss: 0.118679
Train: [2304/4074 (56%)]	Loss: 0.085703
Train: [2560/4074 (62%)]	Loss: 0.113728
Train: [2816/4074 (69%)]	Loss: 0.089286
Train: [3072/4074 (75%)]	Loss: 0.070311
Train: [3328/4074 (81%)]	Loss: 0.055640
Train: [3584/4074 (88%)]	Loss: 0.055926
Train: [3840/4074 (94%)]	Loss: 0.068300
Epoch: 2/10. Train set: Average loss: 0.1144
Epoch: 2/10. Validation set: Average loss: 0.0554
Train: [0/4074 (0%)]	Loss: 0.055423
Train: [256/4074 (6%)]	Loss: 0.060450
Train: [512/4074 (12%)]	Loss: 0.054741
Train: [768/4074 (19%)]	Loss: 0.061552
Train: [1024/4074 (25%)]	Loss: 0.045619
Train: [1280/4074 (31%)]	Loss: 0.047550
Train: [1536/4074 (38%)]	Loss: 0.044481
Train: [1792/4074 (44%)]	Loss: 0.046999
Train: [2048/4074 (50%)]	Loss: 0.041552
Train: [2304/4074 (56%)]	Loss: 0.069584
Train: [2560/4074 (62%)]	Loss: 0.092018
Train: [2816/4074 (69%)]	Loss: 0.066347
Train: [3072/4074 (75%)]	Loss: 0.130705
Train: [3328/4074 (81%)]	Loss: 0.065747
Train: [3584/4074 (88%)]	Loss: 0.060220
Train: [3840/4074 (94%)]	Loss: 0.057487
Epoch: 3/10. Train set: Average loss: 0.0625
Epoch: 3/10. Validation set: Average loss: 0.0521
Train: [0/4074 (0%)]	Loss: 0.050497
Train: [256/4074 (6%)]	Loss: 0.093496
Train: [512/4074 (12%)]	Loss: 0.067881
Train: [768/4074 (19%)]	Loss: 0.041980
Train: [1024/4074 (25%)]	Loss: 0.053146
Train: [1280/4074 (31%)]	Loss: 0.037756
Train: [1536/4074 (38%)]	Loss: 0.038955
Train: [1792/4074 (44%)]	Loss: 0.049262
Train: [2048/4074 (50%)]	Loss: 0.039789
Train: [2304/4074 (56%)]	Loss: 0.041985
Train: [2560/4074 (62%)]	Loss: 0.038109
Train: [2816/4074 (69%)]	Loss: 0.042739
Train: [3072/4074 (75%)]	Loss: 0.027834
Train: [3328/4074 (81%)]	Loss: 0.039386
Train: [3584/4074 (88%)]	Loss: 0.030606
Train: [3840/4074 (94%)]	Loss: 0.037155
Epoch: 4/10. Train set: Average loss: 0.0465
Epoch: 4/10. Validation set: Average loss: 0.0412
Train: [0/4074 (0%)]	Loss: 0.057736
Train: [256/4074 (6%)]	Loss: 0.059791
Train: [512/4074 (12%)]	Loss: 0.033205
Train: [768/4074 (19%)]	Loss: 0.050647
Train: [1024/4074 (25%)]	Loss: 0.040725
Train: [1280/4074 (31%)]	Loss: 0.032147
Train: [1536/4074 (38%)]	Loss: 0.038437
Train: [1792/4074 (44%)]	Loss: 0.036398
Train: [2048/4074 (50%)]	Loss: 0.033560
Train: [2304/4074 (56%)]	Loss: 0.038458
Train: [2560/4074 (62%)]	Loss: 0.025303
Train: [2816/4074 (69%)]	Loss: 0.032039
Train: [3072/4074 (75%)]	Loss: 0.028003
Train: [3328/4074 (81%)]	Loss: 0.029301
Train: [3584/4074 (88%)]	Loss: 0.029260
Train: [3840/4074 (94%)]	Loss: 0.027436
Epoch: 5/10. Train set: Average loss: 0.0360
Epoch: 5/10. Validation set: Average loss: 0.0325
Train: [0/4074 (0%)]	Loss: 0.033827
Train: [256/4074 (6%)]	Loss: 0.026507
Train: [512/4074 (12%)]	Loss: 0.022447
Train: [768/4074 (19%)]	Loss: 0.110014
Train: [1024/4074 (25%)]	Loss: 0.366663
Train: [1280/4074 (31%)]	Loss: 0.086838
Train: [1536/4074 (38%)]	Loss: 0.052270
Train: [1792/4074 (44%)]	Loss: 0.063358
Train: [2048/4074 (50%)]	Loss: 0.094306
Train: [2304/4074 (56%)]	Loss: 0.098580
Train: [2560/4074 (62%)]	Loss: 0.095467
Train: [2816/4074 (69%)]	Loss: 0.106882
Train: [3072/4074 (75%)]	Loss: 0.066812
Train: [3328/4074 (81%)]	Loss: 0.041981
Train: [3584/4074 (88%)]	Loss: 0.046714
Train: [3840/4074 (94%)]	Loss: 0.042531
Epoch: 6/10. Train set: Average loss: 0.0887
Epoch: 6/10. Validation set: Average loss: 0.1909
Train: [0/4074 (0%)]	Loss: 0.128575
Train: [256/4074 (6%)]	Loss: 0.311432
Train: [512/4074 (12%)]	Loss: 0.211709
Train: [768/4074 (19%)]	Loss: 0.180004
Train: [1024/4074 (25%)]	Loss: 0.201150
Train: [1280/4074 (31%)]	Loss: 0.144121
Train: [1536/4074 (38%)]	Loss: 0.161725
Train: [1792/4074 (44%)]	Loss: 0.298485
Train: [2048/4074 (50%)]	Loss: 0.091914
Train: [2304/4074 (56%)]	Loss: 0.092737
Train: [2560/4074 (62%)]	Loss: 0.062440
Train: [2816/4074 (69%)]	Loss: 0.066464
Train: [3072/4074 (75%)]	Loss: 0.148429
Train: [3328/4074 (81%)]	Loss: 0.257082
Train: [3584/4074 (88%)]	Loss: 0.078517
Train: [3840/4074 (94%)]	Loss: 0.109666
Epoch: 7/10. Train set: Average loss: 0.1613
Epoch: 7/10. Validation set: Average loss: 0.1386
Train: [0/4074 (0%)]	Loss: 0.334296
Train: [256/4074 (6%)]	Loss: 0.156901
Train: [512/4074 (12%)]	Loss: 0.055085
Train: [768/4074 (19%)]	Loss: 0.059409
Train: [1024/4074 (25%)]	Loss: 0.060226
Train: [1280/4074 (31%)]	Loss: 0.060223
Train: [1536/4074 (38%)]	Loss: 0.053269
Train: [1792/4074 (44%)]	Loss: 0.047540
Train: [2048/4074 (50%)]	Loss: 0.059154
Train: [2304/4074 (56%)]	Loss: 0.052521
Train: [2560/4074 (62%)]	Loss: 0.042535
Train: [2816/4074 (69%)]	Loss: 0.039205
Train: [3072/4074 (75%)]	Loss: 0.042457
Train: [3328/4074 (81%)]	Loss: 0.043447
Train: [3584/4074 (88%)]	Loss: 0.110057
Train: [3840/4074 (94%)]	Loss: 0.112570
Epoch: 8/10. Train set: Average loss: 0.0692
Epoch: 8/10. Validation set: Average loss: 0.0528
Train: [0/4074 (0%)]	Loss: 16.195900
Train: [256/4074 (6%)]	Loss: 0.044408
Train: [512/4074 (12%)]	Loss: 0.115708
Train: [768/4074 (19%)]	Loss: 0.082974
Train: [1024/4074 (25%)]	Loss: 0.311366
Train: [1280/4074 (31%)]	Loss: 0.065507
Train: [1536/4074 (38%)]	Loss: 0.279541
Train: [1792/4074 (44%)]	Loss: 0.067656
Train: [2048/4074 (50%)]	Loss: 0.047125
Train: [2304/4074 (56%)]	Loss: 0.055082
Train: [2560/4074 (62%)]	Loss: 0.072999
Train: [2816/4074 (69%)]	Loss: 0.107642
Train: [3072/4074 (75%)]	Loss: 0.050504
Train: [3328/4074 (81%)]	Loss: 0.126473
Train: [3584/4074 (88%)]	Loss: 0.055208
Train: [3840/4074 (94%)]	Loss: 0.043276
Epoch: 9/10. Train set: Average loss: 0.3517
Epoch: 9/10. Validation set: Average loss: 0.0493
Train: [0/4074 (0%)]	Loss: 0.050822
Train: [256/4074 (6%)]	Loss: 0.082330
Train: [512/4074 (12%)]	Loss: 0.037701
Train: [768/4074 (19%)]	Loss: 0.038425
Train: [1024/4074 (25%)]	Loss: 0.036487
Train: [1280/4074 (31%)]	Loss: 0.053518
Train: [1536/4074 (38%)]	Loss: 0.031375
Train: [1792/4074 (44%)]	Loss: 0.035820
Train: [2048/4074 (50%)]	Loss: 0.034885
Train: [2304/4074 (56%)]	Loss: 0.029980
Train: [2560/4074 (62%)]	Loss: 0.035103
Train: [2816/4074 (69%)]	Loss: 0.029285
Train: [3072/4074 (75%)]	Loss: 0.030796
Train: [3328/4074 (81%)]	Loss: 0.031400
Train: [3584/4074 (88%)]	Loss: 0.047687
Train: [3840/4074 (94%)]	Loss: 0.028216
Epoch: 10/10. Train set: Average loss: 0.0393
Epoch: 10/10. Validation set: Average loss: 0.0372
plt.plot(range(0, n_epochs), train_loss, 'rx-')
plt.plot(range(0, n_epochs), val_loss, 'bx-')_____no_output_____train_embeddings_cl, train_labels_cl = vis.extract_embeddings(train_loader, model)
vis.plot_embeddings(train_embeddings_cl, train_labels_cl, classes)_____no_output_____train_embeddings_df = pd.DataFrame(train_embeddings_cl)
train_labels_tmp = train_labels_cl
train_embeddings_df[0].sort_values()
train_embeddings_df = train_embeddings_df.drop(3488)
train_embeddings_df[0].sort_values()
train_labels_tmp = np.delete(train_labels_tmp, 3488)
vis.plot_embeddings(train_embeddings_df.values, train_labels_tmp, classes)_____no_output_____val_embeddings_baseline, val_labels_baseline = vis.extract_embeddings(test_loader, model)
vis.plot_embeddings(val_embeddings_baseline, val_labels_baseline, classes)_____no_output_____train_dataset[3488]_____no_output_____classes_____no_output_____


# Notebook from robfalck/dymos
Path: docs/getting_started/collocation.ipynb

<code>
try:
    import openmdao.api as om
    import dymos as dm
except ImportError:
    !python -m pip install openmdao[notebooks]
    !python -m pip install dymos[docs]
    import openmdao.api as om
    import dymos as dm_____no_output_____
</code>

(getting_started:collocation:what_is_collocation)=
# What is collocation?

Dymos is focused on direct optimization methods based on an implicit integration technique, collocation.

The _state_ of a dynamic system defines its configuration at a point in time.
Common state variables for dynamic systems are position, velocity, and mass.
Given an initial state, a time duration, and ordinary differential equations that define the evolution of the states in time, the state of the system at the end of the time duration can be determined.
This is known as the _initial value problem_ (IVP) and there are numerous techniques and software packages that can be used to solve the IVP.
The initial value problem is typically the basis for the simulation of systems.
This time-marching approach is the basis for _shooting methods_ in optimization.
While Dymos supports shooting methods, its focus is on implicit collocation techniques, for performance reasons.

First, a few assumptions about the dynamic system:

1.  Over portions of the trajectory, the state is continuous (there are no instantaneous changes in the states)
2.  Over portions of the trajectory, the state is continuous in its first derivative w.r.t. time. (there are no instantaneous changes in the state rates)

If those two conditions are met, then it's reasonable to fit a polynomial to a plot of the state value over time.
Given the initial or final value of a state, and values of its rate at some number of points across the trajectory, then a polynomial can be fit whose initial or final value matches, and whose rates match the sampled rates.
The value of the state at any point throughout the trajectory can then be inferred by interpolating the collocated polynomial.

Solving this trajectory implicitly basically means _guessing_ values for the states along the trajectory.
While one could use a power series polynomial coefficients ($a t^2 + b t + c$), these coefficients can have vastly different values depending on the shape of the curve, making them difficult to scale numerically.
Instead, Lagrange interpolating polynomials are typically used, where the actual values to be interpolated at various points are the implicit variables.
This makes scaling the problem easier, and gives the implicit variables more intuitive meaning, making it easier to provide an initial guess.

## Example:  Simulating the fall of an object

As a "hello, world" problem for Dymos, consider the vertical fall of an object acting under gravity - like Galileo dropping cannonballs from the Tower of Pisa.
Ignoring aerodynamics and treating the ball as a point mass in a rectilinear gravity field, the motion is governed by:

\begin{align}
    \ddot{y} = -g
\end{align}

The techniques in Dymos are generally centered around first-order dynamics, so we convert the above equation into the following system of ODEs.

\begin{align}
    \dot{y} &= v_y \\
    \dot{v_y} &= -g
\end{align}

One common way to propagate this trajectory would be to use an "explicit time-marching" approach.
That is, we can use Euler's method or a Runge-Kutta approach to propagate the trajectory of the ball, one small slice of time at a time.
When performing trajectory optimization, the trajectory is typically evaluated at least dozens of time.
If the ODE for the system is computationally expensive, this time-marching approach can take a very long time.

Instead, we can use an implicit simulation technique to more efficiently model the trajectory.
First, let's assume that the value of the _states_ $y$ and $v_y$ smoothly vary in time (there's no teleportation or instantaneous changes in velocity).
Now we can fit a polynomial to the plot of each of these states with respect to time.
In this case, basic physics tells us that $v_y$ should be linear with time and $y$ should be quadratic with time.

Now for the simulation, lets solve a _boundary value problem_.
We know the initial and final height of the ball.
We know the initial velocity but not the terminal velocity.

_How long does it take the ball to reach the ground from an initial height of 100 m?_

We will explain how this problem is solved using the two implicit simulation techniques in Dymos:  high-order Gauss-Lobatto collocation, and the Radau Pseudospectral Method.

## Finding the trajectory using high-order Legendre-Gauss-Lobatto collocation

Using a 3rd-order polynomial segment to mimic the trajectory, we have two states and thus two polynomials.
Legendre-Gauss-Lobatto (LGL) collocation uses knowledge of the states and state-rates to form an interpolating polynomial.
A third-order polynomial requires four pieces of information to define it.
We'll take two points along the trajectory, and use those to guess the state values.
In Dymos, these points where state values are provided are called the _state discretization nodes_.
In LGL collocation, these points are chosen to be the LGL _nodes_ in the dimensionless time of the segment, which we call $\tau_s$.

Our initial guess for the trajectory might look something like this:

| Variables  | Description       | Value          | Notes                  |
|------------|-------------------|----------------|------------------------|
| $y_0$      | initial height    | 100 m          | fixed                  |
| $y_f$      | final height      | 0 m            | fixed                  |
| $v_{y0}$   | initial velocity  | 0 m/s          | fixed                  |
| $v_{yf}$   | final velocity    | -50 m/s        | free                   |
| $t_0$      | initial time      | 0              | fixed                  |
| $t_d$      | time duration     | 5              | free                   |
| $g$        | grav. acceleration| 9.80665 m/s^2  | assumed                |

To form 3rd-order polynomials for the states we need four pieces of information, but so far we only have the two endpoint values.
We'll use the state rates at the state discretization nodes to provide the other two values needed for interpolation.
To obtain these, we evaluate our ODE at the state discretization nodes.

\begin{align}
    \dot{y}_0 &= 0 \\
    \dot{y}_f &= -50 \\
    \dot{v}_{y0} &= -9.80665 \\
    \dot{v}_{yf} &= -9.80665
\end{align}

![Screenshot](scripts/lgl_animation_1.png)

But how do we know that our assumed trajectory is correct?
In LGL collocation, we check the interpolated slope of the polynomial at a 3rd point (the collocation node) and compare it to an evaluation of the ODE at the same point.
If the difference between the two is sufficiently close to zero, we can be somewhat confident that our polynomial is a reasonable match for the true time-history.
We call this difference in slope the collocation _defect_.

Having computed the ODE at the state discretization nodes, we can form interpolants for the values and the slope of the states at the collocation nodes.
Dymos uses Lagrange interpolants, such that the inputs to the system are the values of the states and controls, rather than less intuitive coefficients.
In this case, the state values at our single collocation node are:

\begin{align}
    y_m &= 50 \\
    v_{ym} &= -25
\end{align}

![Screenshot](scripts/lgl_animation_2.png)

The polynomial slopes at the collocation node are:

\begin{align}
    y'_m &= 50 \\
    v'_{ym} &= -25
\end{align}

Now, having the state and time values at the collocation nodes, we can again evaluate the ODE.

\begin{align}
    \dot{y}_m &= -17 \\
    \dot{v}_{ym} &= -9.80665
\end{align}

![Screenshot](scripts/lgl_animation_3.png)

In this case the known variables are the value of the polynomial at the left endpoint ($y_0$), and the value of the independent variable at the left and right endpoints of the interval to be fitted ($x_0$, $x_1$)
In this case, there are no _control_ inputs to the ODE.
However, if there were, the value of the control would also need to be provided at the collocation node.
Thus, in high-order Gauss-Lobatto collocation, the _control input nodes_ include both the state discretization nodes and the collocation nodes.

The free variables in the problem ($t_{duration}$ and $vy_f$) can be iterated until the difference between the interpolated state rates and the ODE-computed state rates is zero.

![Screenshot](scripts/lgl_solution_5.png)

## Finding the trajectory using the Radau Pseudospectral Method

Again, there are two states, and two corresponding 3rd-order polynomials representing the time history.
As with the Gauss-Lobatto method, a third-order polynomial requires four pieces of information to define it.
In the case of the Radau Pseudospectral Method, we'll use four state values, guessed at the Radau nodes in dimensionless time in addition to the right endpoint of the interval (the _state discretization nodes_).

A linear interpolation of the endpoint values will serve as the initial guess in this instance.
The three Radau nodes that don't include the right endpoint will serve as the _collocation nodes_ in this case.
Since these are a subset of the state discretization nodes, no interpolation of values is necessary...all of the state values have been provided.

![Screenshot](scripts/lgr_animation_0.png)

The derivative of the interpolating polynomial is computed using a differentiation matrix and the state discretization values.

![Screenshot](scripts/lgr_animation_2.png)

Now the ODE can be evaluated to calculate the state rates, and the results compared to the interpolated slopes.

![Screenshot](scripts/lgr_animation_3.png)

Again, the free variables are iterated until the defects are zero and any boundary conditions are met.

![Screenshot](scripts/lgr_solution_5.png)

## Multiple Segments

The simple examples above use one single 3rd-order polynomial to represent the time history of the states.
With more complex states, a single 3rd-order polynomial would not be capable of replicating the time history.
In practice, we can use more polynomial segments _and_ higher-order polynomials to better match the dynamics.

In the case of adding more segments, one can choose whether the segment boundaries are shared by the segments, or independent.
If they are shared, fewer design variables are necessary, but it may be more difficult for the optimizer to search for a solution.
This is called "compressed" transcription.
On the other hand, we can doubly-specify state values at segment boundaries as design variables, thus increasing the size of the problem.
In this "uncompressed" formulation, continuity constraints are imposed at the segment bounds to ensure that there are no discontinuities in the state time-history at segment bounds.

## LGL vs LGR Collocation

Gauss-Lobatto collocation results in smaller NLP sizes for the optimizer since fewer states are treated as design variables.
The cost of this reduction is the interpolation step in the Gauss-Lobatto algorithm.
While the Radau Pseudospectral method requires one evaluation of the ODE to assess the defects, the Gauss-Lobatto method requires two.
In addition, the interpolation step used by the Gauss-Lobatto method can result in interpolated state values falling well outside the user's expected range if the initial guess is not sufficiently accurate.
This can cause convergence issues if there are nonlinear solvers within the ODE that rely on a reasonable guess to achieve convergence.

## Satisfying Defects with a Nonlinear Solver

Typically, collocation problems are solved by posing the defects as constraints to an optimizer.
In Dymos, another option exists.
One can use an embedded Newton-solver satisfy the defect and continuity constraints.
This mode, which we call `solve_segments`, gives the Newton-Solver control over all but the first or final state value in the phase.
The optimizer only controls the initial or final value, and the solver is responsible for converging the trajectory time history.
This results in a shooting method, but one that is mathematically consistent with the collocation methods.
This `solve_segments` capability is useful if a user simply wants to propagate a dynamic system and not optimize it._____no_output_____


# Notebook from josh-wolpert/espin
Path: lessons/landlab/landlab-terrainbento/coupled_process_elements/model_basic_var_m_steady_solution_solution.ipynb

![terrainbento logo](../images/terrainbento_logo.png)

# terrainbento model Basic with variable $m$ steady-state solution_____no_output_____This model shows example usage of the Basic model from the TerrainBento package with a variable drainage-area exponent, $m$:

$\frac{\partial \eta}{\partial t} = - KQ^m S + D\nabla^2 \eta$

where $K$ and $D$ are constants, $Q$ is discharge, $S$ is local slope, $m$ is the drainage area exponent, and $\eta$ is the topography.

Note that the units of $K$ depend on $m$, so that the value of $K$ used in Basic cannot be meaningfully compared to other values of $K$ unless the valuess of $m$ are the same.

Refer to [Barnhart et al. (2019)](https://www.geosci-model-dev.net/12/1267/2019/) for further explaination. For detailed information about creating a Basic model, see [the detailed documentation](https://terrainbento.readthedocs.io/en/latest/source/terrainbento.derived_models.model_basic.html).

This notebook (a) shows the initialization and running of this model, (b) saves a NetCDF file of the topography, which we will use to make an oblique Paraview image of the landscape, and (c) creates a slope-area plot at steady state._____no_output_____

<code>
# import required modules
import os
import numpy as np

import matplotlib.pyplot as plt
import matplotlib
matplotlib.rcParams["font.size"] = 20
matplotlib.rcParams["pdf.fonttype"] = 42
%matplotlib inline

from landlab import imshow_grid
from landlab.io.netcdf import write_netcdf

from terrainbento import Basic

np.random.seed(42)_____no_output_____# create the parameter dictionary needed to instantiate the model

params = {
    # create the Clock.
    "clock": {
        "start": 0,
        "step": 10,
        "stop": 1e7
    },

    # Create the Grid
    "grid": {
        "RasterModelGrid": [
            (25, 40),
            {
                "xy_spacing": 40
            },
            {
                "fields": {
                    "node": {
                        "topographic__elevation": {
                            "random": [{
                                "where": "CORE_NODE"
                            }]
                        }
                    }
                }
            },
        ]
    },

    # Set up Boundary Handlers
    "boundary_handlers": {
        "NotCoreNodeBaselevelHandler": {
            "modify_core_nodes": True,
            "lowering_rate": -0.001
        }
    },
    # Parameters that control output.
    "output_interval": 1e4,
    "save_first_timestep": True,
    "output_prefix": "basicVm.",
    "fields": ["topographic__elevation"],

    # Parameters that control process and rates.
    "water_erodibility": 0.001,
    "m_sp": 0.25,
    "n_sp": 1.0,
    "regolith_transport_parameter": 0.01,
}_____no_output_____# the tolerance here is high, so that this can run on binder and for tests. (recommended value = 0.001 or lower).
tolerance = .001_____no_output_____# we can use an output writer to run until the model reaches steady state.
class run_to_steady(object):
    def __init__(self, model):
        self.model = model
        self.last_z = self.model.z.copy()
        self.tolerance = tolerance

    def run_one_step(self):
        if model.model_time > 0:
            diff = (self.model.z[model.grid.core_nodes] -
                    self.last_z[model.grid.core_nodes])
            if max(abs(diff)) <= self.tolerance:
                self.model.clock.stop = model._model_time
                print("Model reached steady state in " +
                      str(model._model_time) + " time units\n")
            else:
                self.last_z = self.model.z.copy()
                if model._model_time <= self.model.clock.stop - self.model.output_interval:
                    self.model.clock.stop += self.model.output_interval_____no_output_____# initialize the model using the Model.from_dict() constructor.
# We also pass the output writer here.
model = Basic.from_dict(params, output_writers={"class": [run_to_steady]})

# to run the model as specified, we execute the following line:
model.run()_____no_output_____#MAKE SLOPE-AREA PLOT

# plot nodes that are not on the boundary or adjacent to it
core_not_boundary = np.array(
    model.grid.node_has_boundary_neighbor(model.grid.core_nodes)) == False
plotting_nodes = model.grid.core_nodes[core_not_boundary]

# assign area_array and slope_array
area_array = model.grid.at_node["drainage_area"][plotting_nodes]
slope_array = model.grid.at_node["topographic__steepest_slope"][plotting_nodes]

# instantiate figure and plot
fig = plt.figure(figsize=(6, 3.75))
slope_area = plt.subplot()

slope_area.scatter(area_array,
                   slope_array,
                   marker="o",
                   c="k",
                   label="Model Basic (m=0.25)")

# make axes log and set limits
slope_area.set_xscale("log")
slope_area.set_yscale("log")

slope_area.set_xlim(9 * 10**1, 3 * 10**5)
slope_area.set_ylim(1e-2, 1e0)

# set x and y labels
slope_area.set_xlabel(r"Drainage area [m$^2$]")
slope_area.set_ylabel("Channel slope [-]")
slope_area.legend(scatterpoints=1, prop={"size": 12})
slope_area.tick_params(axis="x", which="major", pad=7)

plt.show()_____no_output_____# Save stack of all netcdfs for Paraview to use.
# model.save_to_xarray_dataset(filename="basicVm.nc",
#                              time_unit='years',
#                              reference_time='model start',
#                              space_unit='meters')

# remove temporary netcdfs
model.remove_output_netcdfs()_____no_output_____# make a plot of the final steady state topography
imshow_grid(model.grid, "topographic__elevation")_____no_output_____
</code>

## Next Steps

- We recommend you review the [terrainbento manuscript](https://www.geosci-model-dev.net/12/1267/2019/).

- There are three additional introductory tutorials:

    1) [Introduction terrainbento](../example_usage/Introduction_to_terrainbento.ipynb)

    2) [Introduction to boundary conditions in terrainbento](../example_usage/introduction_to_boundary_conditions.ipynb)

    3) [Introduction to output writers in terrainbento](../example_usage/introduction_to_output_writers.ipynb).

- Five examples of steady state behavior in coupled process models can be found in the following notebooks:

    1) [Basic](model_basic_steady_solution.ipynb) the simplest landscape evolution model in the terrainbento package.

    2) **This Notebook**: [BasicVm](model_basic_var_m_steady_solution.ipynb) which permits the drainage area exponent to change

    3) [BasicCh](model_basicCh_steady_solution.ipynb) which uses a non-linear hillslope erosion and transport law

    4) [BasicVs](model_basicVs_steady_solution.ipynb) which uses variable source area hydrology

    5) [BasisRt](model_basicRt_steady_solution.ipynb) which allows for two lithologies with different K values_____no_output_____


# Notebook from yourball/qubiter
Path: qubiter/jupyter_notebooks/Say_Hello_World_With_Qubiter.ipynb

## Say "Hello World"  With Qubiter
The purpose of this notebook is to illustrate how to use Qubiter to simulate ( i.e.,
predict the outcome of) a simple quantum circuit with a few basic gates

> Below, we won't always give the precise definition of each gate. You can find the
precise analytical/numerical definition of all gates implemented by Qubiter in the document entitled `qubiter_rosetta_stone.pdf`  included with the Qubiter distribution._____no_output_____$\newcommand{\bra}[1]{\left\langle{#1}\right|}$
$\newcommand{\ket}[1]{\left|{#1}\right\rangle}$
test: $\bra{\psi}M\ket{\phi}$_____no_output_____First change your working directory to the Qubiter directory in your computer, and add its path to the path environment variable._____no_output_____

import os
import sys
print(os.getcwd())
os.chdir('../../')
print(os.getcwd())
sys.path.insert(0,os.getcwd())C:\Users\rrtuc\Desktop\backed-up\python-projects\qubiter\qubiter\jupyter_notebooks
C:\Users\rrtuc\Desktop\backed-up\python-projects\qubiter

Suppose you are anywhere in your home ~ directory, and qubiter has been installed somewhere accessible via the path environmental variable. You can find where qubiter is installed like this, in case you want to cd there._____no_output_____

from qubiter.utilities_gen import find_path_to_qubiter
# this method returns the absolute path to the py file where the method is defined
path = find_path_to_qubiter()
print(path)C:\Users\rrtuc\Desktop\backed-up\python-projects\qubiter\qubiter\utilities_gen.py
from qubiter.SEO_writer import *
from qubiter.SEO_simulator import *
from qubiter.StateVec import *
from qubiter.SEO_MatrixProduct import *
import numpy as nploaded OneBitGates, WITHOUT autograd.numpy

Number of qubits is 4.
Note that we use "bit" for both qbits and cbits._____no_output_____

num_bits = 4_____no_output_____

Use a trivial circuit embedder that embeds 4 qubits into same 4 qubits._____no_output_____

emb = CktEmbedder(num_bits, num_bits)_____no_output_____

Open a writer, tell it where to write to.
We will use zero bit last (ZL) convention, which is the default for SEO_writer._____no_output_____

file_prefix = 'hello_world_test'
wr = SEO_writer(file_prefix, emb)_____no_output_____

Write Pauli matrices $\sigma_X, \sigma_Y,\sigma_Z$ at position 2._____no_output_____

wr.write_X(2)
wr.write_Y(2)
wr.write_Z(2)

# old way of doing it, still works
# wr.write_one_bit_gate(2, OneBitGates.sigx)
# wr.write_one_bit_gate(2, OneBitGates.sigy)
# wr.write_one_bit_gate(2, OneBitGates.sigz)_____no_output_____

Write 1 qubit Hadamard matrix at position 3._____no_output_____

wr.write_H(3)

# old way of doing it, still works
# wr.write_one_bit_gate(3, OneBitGates.had2)_____no_output_____

Rotate qubit 2 by $\pi$ along directions x, y, z successively.

> Note: We define $Ra(\theta) = exp(i\theta\sigma_a)$ for $a=X,Y,Z$. Others use
$Ra(\theta) = exp(-i\frac{\theta}{2}\sigma_a)$ instead.

> Note: $\theta$ in $Ra(\theta)$ is inserted in radians, but shows
up in the English File in degrees._____no_output_____

wr.write_Rx(2, np.pi)
wr.write_Ry(2, np.pi)
wr.write_Rz(2, np.pi)

# old way of doing it, still works dir=1,2,3
# wr.write_one_bit_gate(2, OneBitGates.rot_ax,[np.pi, dir])_____no_output_____

Rotate qubit 1 along a non-axis direction $\hat{n}$ characterized by a list of 3 angles.
$R(\theta_1, \theta_2, \theta_3) = \exp(i[\theta_1 \sigma_X +\theta_2\sigma_Y+\theta_3\sigma_Z])$_____no_output_____

wr.write_Rn(1, [np.pi, np.pi/2, np.pi/3])_____no_output_____

Definitions of S and T

$S = diag[1, i] = diag[1, e^{i\frac{\pi}{2}}]$

$T = \sqrt{S}= diag[1, e^{i\frac{\pi}{4}}]$
_____no_output_____Write  $S, S^\dagger, T, T^\dagger$ at position=2.

> These operations show up in the English File as `P1PH` and in the
Picture File as `@P`. That is because $P_1 = n =\ket{1}\bra{1} = diag(0, 1)$ and the operation
`P1PH` (i.e. $P_1$ Phase) by a phase angle $\theta$ equals the diagonal matrix $diag(1, e^{i\theta})$ _____no_output_____

wr.write_S(2)
wr.write_S(2, herm=True)
wr.write_T(2)
wr.write_T(2, herm=True)_____no_output_____

Write $CNOT = sigx(target\_pos)^{n(control\_pos)}$ with control_pos=3 and target_pos=1_____no_output_____

wr.write_cnot(3, 1)

# old way of doing it, still works
# control_pos = 3
# target_pos = 1
# trols = Controls.new_knob(num_bits, control_pos, kind=True)
# wr.write_controlled_one_bit_gate(
#     target_pos, trols, OneBitGates.sigx)_____no_output_____

At any point in the circuit, you can use a PRINT statement. This will print
on the console, immediately after you create
an object of the class SEO_simulator, a description of the state vector at that point in the circuit.
Various styles of description are pre-canned for your convenience, or
you can write your own. See use_PRINT() method of SEO_simulator class.
Let's use a PRINT statement now in the pre-canned style "ALL"._____no_output_____

wr.write_PRINT("ALL")_____no_output_____

Swap qubits 1 and 3_____no_output_____

wr.write_bit_swap(1, 3)_____no_output_____

 Recall that
$P_1 = n = \ket{1}\bra{1}=diag(0, 1)$ and a P1 phase (P1PH) by $\theta$ is $diag(1, e^{i\theta})$. Write a singly controlled P1PH with control=c=3, target=t=1 and rads = pi/3.
This gate equals $e^{i*rads*n(t) n(c)}$. _____no_output_____

wr.write_c_P1PH(3, 1, rads=np.pi/3)_____no_output_____

If rads=pi, c_P1PH equals  $(-1)^{n(t)n(c)} = \sigma_Z(t)^{n(c)}$,
which is commonly called a controlled Z and denoted by Cz. Write a Cz with c=3 and t=1._____no_output_____

wr.write_c_P1PH(3, 1)  # rads=np.pi is default_____no_output_____

Close English and Picture files._____no_output_____

wr.close_files()_____no_output_____

Look in files

* ../io_folder/hello_world_test_4_eng.txt
* ../io_folder/hello_world_test_4_ZLpic.txt

to see the quantum circuit that was generated._____no_output_____Once the English and Picture files are generated, you can ask the writer object wr to print them for you on screen_____no_output_____

wr.print_eng_file()SIGX	AT	2
SIGY	AT	2
SIGZ	AT	2
HAD2	AT	3
ROTX	180.0	AT	2
ROTY	180.0	AT	2
ROTZ	180.0	AT	2
ROTN	180.0	90.0	59.99999999999999	AT	1
P1PH	90.0	AT	2
P1PH	-90.0	AT	2
P1PH	45.0	AT	2
P1PH	-45.0	AT	2
SIGX	AT	1	IF	3T
PRINT	ALL
SWAP	3	1
P1PH	59.99999999999999	AT	1	IF	3T
P1PH	180.0	AT	1	IF	3T

wr.print_pic_file()|   X   |   |
|   Y   |   |
|   Z   |   |
H   |   |   |
|   Rx  |   |
|   Ry  |   |
|   Rz  |   |
|   |   R   |
|   @P  |   |
|   @P  |   |
|   @P  |   |
|   @P  |   |
@---+---X   |
PRINT	ALL
<---+--->   |
@---+---@P  |
@---+---@P  |

You can ask wr for the path to the English and Picture files_____no_output_____

print(wr.get_eng_file_path())C:\Users\rrtuc\Desktop\backed-up\python-projects\qubiter\qubiter/io_folder/hello_world_test_4_eng.txt
print(wr.get_pic_file_path())C:\Users\rrtuc\Desktop\backed-up\python-projects\qubiter\qubiter/io_folder/hello_world_test_4_ZLpic.txt

You can generate a log file with an inventory of the English file by creating
an object of the SEO_reader class with the flag `write_log` set to True_____no_output_____

rdr = SEO_reader(file_prefix, num_bits, write_log=True)_____no_output_____

The following file was just created

* ../io_folder/hello_world_test_4_log.txt_____no_output_____Once the log file is generated, you can ask the reader object rdr to print it for you on screen_____no_output_____

rdr.print_log_file()Number of lines in file = 17
Number of Elem. Ops = 16
Number of CNOTS (SIGX with single control) = 1
List of distinct variable numbers encountered (length=0)=
[]
List of distinct function names encountered (length=0)=
[]

You can ask rdr for the path to the log file_____no_output_____

print(rdr.get_log_file_path())C:\Users\rrtuc\Desktop\backed-up\python-projects\qubiter\qubiter/io_folder/hello_world_test_4_log.txt

Occasionally, especially for debugging purposes, you might want to display the
product of a SEO (sequence of elementary operations, sequence of gates) as a 2^num_bits dimensional
unitary matrix. This can be done with the class SEO_MatrixProduct. Simply
creating an object of this class multiplies the SEO and stores the result
in its attribute `self.prod_arr`. Next we print that array for our example_____no_output_____

mp = SEO_MatrixProduct(file_prefix, num_bits)
print('product array=')
print(np.array_str(mp.prod_arr,
                   precision=2, suppress_small=True))product array=
[[ 0.1 -0.61j  0.  +0.j    0.3 -0.15j  0.  +0.j   -0.  +0.j    0.  +0.j
  -0.  -0.j    0.  +0.j    0.1 -0.61j  0.  +0.j    0.3 -0.15j  0.  +0.j
  -0.  +0.j    0.  +0.j   -0.  -0.j    0.  +0.j  ]
 [ 0.  +0.j    0.1 -0.61j  0.  +0.j    0.3 -0.15j  0.  +0.j   -0.  +0.j
   0.  +0.j   -0.  -0.j    0.  +0.j    0.1 -0.61j  0.  +0.j    0.3 -0.15j
   0.  +0.j   -0.  +0.j    0.  +0.j   -0.  -0.j  ]
 [ 0.3 +0.15j  0.  +0.j   -0.1 -0.61j  0.  +0.j   -0.  -0.j    0.  +0.j
  -0.  +0.j    0.  +0.j   -0.3 -0.15j  0.  +0.j    0.1 +0.61j  0.  +0.j
   0.  +0.j    0.  +0.j    0.  -0.j    0.  +0.j  ]
 [ 0.  +0.j    0.3 +0.15j  0.  +0.j   -0.1 -0.61j  0.  +0.j   -0.  -0.j
   0.  +0.j   -0.  +0.j    0.  +0.j   -0.3 -0.15j  0.  +0.j    0.1 +0.61j
   0.  +0.j    0.  +0.j    0.  +0.j    0.  -0.j  ]
 [-0.  -0.j    0.  +0.j    0.  -0.j    0.  +0.j    0.1 -0.61j  0.  +0.j
   0.3 -0.15j  0.  +0.j   -0.  -0.j    0.  +0.j    0.  -0.j    0.  +0.j
   0.1 -0.61j  0.  +0.j    0.3 -0.15j  0.  +0.j  ]
 [ 0.  +0.j   -0.  -0.j    0.  +0.j    0.  -0.j    0.  +0.j    0.1 -0.61j
   0.  +0.j    0.3 -0.15j  0.  +0.j   -0.  -0.j    0.  +0.j    0.  -0.j
   0.  +0.j    0.1 -0.61j  0.  +0.j    0.3 -0.15j]
 [ 0.  -0.j    0.  +0.j   -0.  -0.j    0.  +0.j    0.3 +0.15j  0.  +0.j
  -0.1 -0.61j  0.  +0.j   -0.  +0.j    0.  +0.j    0.  +0.j    0.  +0.j
  -0.3 -0.15j  0.  +0.j    0.1 +0.61j  0.  +0.j  ]
 [ 0.  +0.j    0.  -0.j    0.  +0.j   -0.  -0.j    0.  +0.j    0.3 +0.15j
   0.  +0.j   -0.1 -0.61j  0.  +0.j   -0.  +0.j    0.  +0.j    0.  +0.j
   0.  +0.j   -0.3 -0.15j  0.  +0.j    0.1 +0.61j]
 [ 0.3 +0.15j  0.  +0.j   -0.1 -0.61j  0.  +0.j   -0.  -0.j    0.  +0.j
  -0.  +0.j    0.  +0.j    0.3 +0.15j  0.  +0.j   -0.1 -0.61j  0.  +0.j
  -0.  -0.j    0.  +0.j   -0.  +0.j    0.  +0.j  ]
 [ 0.  +0.j    0.3 +0.15j  0.  +0.j   -0.1 -0.61j  0.  +0.j   -0.  -0.j
   0.  +0.j   -0.  +0.j    0.  +0.j    0.3 +0.15j  0.  +0.j   -0.1 -0.61j
   0.  +0.j   -0.  -0.j    0.  +0.j   -0.  +0.j  ]
 [-0.58+0.22j  0.  +0.j   -0.28-0.19j  0.  +0.j    0.  +0.j    0.  +0.j
   0.  +0.j    0.  +0.j    0.58-0.22j  0.  +0.j    0.28+0.19j  0.  +0.j
  -0.  -0.j    0.  +0.j   -0.  -0.j    0.  +0.j  ]
 [ 0.  +0.j   -0.58+0.22j  0.  +0.j   -0.28-0.19j  0.  +0.j    0.  +0.j
   0.  +0.j    0.  +0.j    0.  +0.j    0.58-0.22j  0.  +0.j    0.28+0.19j
   0.  +0.j   -0.  -0.j    0.  +0.j   -0.  -0.j  ]
 [ 0.  -0.j    0.  +0.j   -0.  -0.j    0.  +0.j    0.3 +0.15j  0.  +0.j
  -0.1 -0.61j  0.  +0.j    0.  -0.j    0.  +0.j   -0.  -0.j    0.  +0.j
   0.3 +0.15j  0.  +0.j   -0.1 -0.61j  0.  +0.j  ]
 [ 0.  +0.j    0.  -0.j    0.  +0.j   -0.  -0.j    0.  +0.j    0.3 +0.15j
   0.  +0.j   -0.1 -0.61j  0.  +0.j    0.  -0.j    0.  +0.j   -0.  -0.j
   0.  +0.j    0.3 +0.15j  0.  +0.j   -0.1 -0.61j]
 [-0.  +0.j    0.  +0.j   -0.  +0.j    0.  +0.j   -0.58+0.22j  0.  +0.j
  -0.28-0.19j  0.  +0.j    0.  -0.j    0.  +0.j    0.  -0.j    0.  +0.j
   0.58-0.22j  0.  +0.j    0.28+0.19j  0.  +0.j  ]
 [ 0.  +0.j   -0.  +0.j    0.  +0.j   -0.  +0.j    0.  +0.j   -0.58+0.22j
   0.  +0.j   -0.28-0.19j  0.  +0.j    0.  -0.j    0.  +0.j    0.  -0.j
   0.  +0.j    0.58-0.22j  0.  +0.j    0.28+0.19j]]

Specify initial state vector for simulation. This example corresponds to $\ket{0}\ket{0}\ket{1}\ket{1}$. In ZL convention, last ket corresponds to bit 0._____no_output_____

init_st_vec = StateVec.get_standard_basis_st_vec([0, 0, 1, 1], ZL=True)_____no_output_____

Open a simulator. This automatically
calculates the final state vector for the quantum circuit in the English file subject to
the initial state vector that you give as input to the SEO_simulator constructor.
Note that the PRINT statement that we inserted into the English file, prints, as promised,
immediately after creating the SEO_simulator object._____no_output_____

sim = SEO_simulator(file_prefix, num_bits, init_st_vec)
*************************beginning PRINT output
PRINT line number=14
*********branch= pure
state vector:
ZL convention (Zero bit Last in state tuple)
(0001)ZL (0.30304576336566325-0.15152288168283165j) , prob= 0.11479591836734696
(1001)ZL (-0.10101525445522115-0.6123724356957945j) , prob= 0.3852040816326531
(0011)ZL (-0.10101525445522115-0.6123724356957945j) , prob= 0.3852040816326531
(1011)ZL (0.30304576336566325-0.15152288168283165j) , prob= 0.11479591836734696
total probability of state vector (=one if no measurements)= 1.0
dictionary with key=qubit, value=(Prob(0), Prob(1))
{0: (0.0, 1.0), 1: (0.5, 0.5), 2: (1.0, 0.0), 3: (0.5, 0.5)}
****************************ending PRINT output

Ask sim to print a description of final state vector_____no_output_____

sim.describe_st_vec_dict(print_st_vec=True, do_pp=True,
            omit_zero_amps=True, show_pp_probs=True)*********branch= pure
state vector:
ZL convention (Zero bit Last in state tuple)
(0001)ZL (0.30304576336566325-0.15152288168283165j) , prob= 0.11479591836734696
(1001)ZL (-0.10101525445522115-0.6123724356957945j) , prob= 0.3852040816326531
(0011)ZL (-0.10101525445522115-0.6123724356957945j) , prob= 0.3852040816326531
(1011)ZL (-0.28274554647478767-0.18668388874249606j) , prob= 0.11479591836734696
total probability of state vector (=one if no measurements)= 1.0
dictionary with key=qubit, value=(Prob(0), Prob(1))
{0: (0.0, 1.0), 1: (0.5, 0.5), 2: (1.0, 0.0), 3: (0.5, 0.5)}

The object sim of SEO_simulator, holds, at this point, the final state vector
for the evolution of the circuit subject to the initial state vector chosen.
You might want to sample the probability distribution defined
by that final state vector, and obtain counts of each observed multi-qubit state
for a given number of shots. This is the type
of output that a real qc device gives, albeit
our counts have no extrinsic noise. One can ask sim to simulate such counts as follows:_____no_output_____

counts = sim.get_counts(num_shots=100)
print(counts)OrderedDict([('0001ZL', 10), ('0011ZL', 36), ('1001ZL', 43), ('1011ZL', 11)])

And you can ask the Plotter class to plot those counts as follows:_____no_output_____

Plotter.plot_counts(counts)_____no_output_____