# Notebook from leylabmpi/animal_gut_metagenome_assembly
Path: HiSeqRuns_combined/04_assemblies/02_LLMGAG/04_LLMGAG_Ave.ipynb

Table of Contents

1  Goal

2  Var

3  Init

4  Selecting Aves samples

4.1  summary

5  LLMGAG

5.1  Run

6  Summary

6.1  Number of genes assembled & clustered

6.2  Taxonomy

6.2.1  Summary

6.3  Annotations

6.3.1  COG functional categories

6.3.1.1  Grouped by taxonomy

6.4  humann2 db genes

6.4.1  Summary

6.4.1.1  By Taxonomy

7  sessionInfo

_____no_output_____# Goal

* Metagenome assembly of Aves samples
  * assemblying genes by using `plass`_____no_output_____# Var_____no_output_____

work_dir = '/ebio/abt3_projects/databases_no-backup/animal_gut_metagenomes/wOutVertebrata/MG_assembly_ave/LLMGAG/'

samples_file = '/ebio/abt3_projects/Georg_animal_feces/data/metagenome/HiSeqRuns126-133-0138/wOutVertebrata/LLMGQC/samples_cov-gte0.3.tsv'
metadata_file = '/ebio/abt3_projects/Georg_animal_feces/data/mapping/unified_metadata_complete_190529.tsv'

pipeline_dir = '/ebio/abt3_projects/Georg_animal_feces/bin/llmgag/'
_____no_output_____

# Init_____no_output_____

library(dplyr)
library(tidyr)
library(ggplot2)
library(data.table)
source('/ebio/abt3_projects/Georg_animal_feces/code/misc_r_functions/init.R')_____no_output_____

# Selecting Aves samples_____no_output_____

meta = read.delim(metadata_file, sep='\t') %>%
    dplyr::select(SampleID, class, order, family, genus, scientific_name, diet, habitat)
meta %>% dfhead[1] 475   8
samps = read.delim(samples_file, sep='\t') %>%
    mutate(Sample = gsub('^XF', 'F', Sample))
samps %>% dfhead[1] 291   6
# checking overlap
setdiff(samps$Sample, meta$Sample)_____no_output_____# joining
samps = samps %>%
    inner_join(meta, c('Sample'='SampleID'))

samps %>% dfheadWarning message:
“Column `Sample`/`SampleID` joining character vector and factor, coercing into character vector”# all metadata
samps %>%
    group_by(class) %>%
    summarize(n = n()) %>%
    ungroup()_____no_output_____samps_f = samps %>%
    filter(class == 'Aves')

samps_f %>% dfhead[1] 63 13
outF = file.path(work_dir, 'samples_ave.tsv')
samps_f %>%
    arrange(class, order, family, genus) %>%
    write.table(outF, sep='\t', quote=FALSE, row.names=FALSE)
cat('File written:', outF, '\n')File written: /ebio/abt3_projects/databases_no-backup/animal_gut_metagenomes/wOutVertebrata/MG_assembly_ave/LLMGAG//samples_ave.tsv

## summary_____no_output_____

samps_f$Total.Sequences %>% summary_____no_output_____

# LLMGAG _____no_output_____

F = file.path(work_dir, 'config.yaml')
cat_file(F)#-- I/O --#
# table with sample --> read_file information
samples_file: /ebio/abt3_projects/databases_no-backup/animal_gut_metagenomes/wOutVertebrata/MG_assembly_ave/LLMGAG/samples_ave.tsv

# output location
output_dir: /ebio/abt3_projects/databases_no-backup/animal_gut_metagenomes/wOutVertebrata/MG_assembly_ave/LLMGAG/

#-- database --#
## eggnog mapper
eggnog_diamond_db: /ebio/abt3_projects/databases_no-backup/Eggnog/eggnog_proteins.dmnd
eggnog_db: /ebio/abt3_projects/databases_no-backup/Eggnog/eggnog.db
eggnog_og2lev_db: /ebio/abt3_projects/databases_no-backup/Eggnog/og2level.tsv.gz
## mmseqs taxonomy
mmseqs_tax_db: /ebio/abt3_projects/databases_no-backup/uniclust/uniclust50/uniclust50_2018_08_consensus
## checkM
checkM_data: /ebio/abt3_projects/databases_no-backup/checkM/
## sourmash
sourmash_db: /ebio/abt3_projects/databases_no-backup/sourmash/genbank-k31.sbt.json
sourmash_lca_db: /ebio/abt3_projects/databases_no-backup/sourmash/genbank-k31.lca.json.gz
# humann2 diamond
humann2_diamond_db: /ebio/abt3_projects/databases_no-backup/humann2/uniref50/uniref50_annotated.1.1.dmnd
# empanada
empanada_kegg_db: /ebio/abt3_projects/databases_no-backup/empanada/KOvsPATHWAY_BACTERIAL_KEGG_2013_07_15.tab
empanada_ptwy_db: /ebio/abt3_projects/databases_no-backup/empanada/map_kegg-pwy_name.txt.gz

#-- main params --#
## co-assemble all samples together or each sample individually?
all_sample_coassembly: False

#-- software parameters --#
# Notes:
## Use "Skip" to skip any step. If no params, just use ""
## Note: You must skip either metacompass_megahit or metacompass_metaspades (or both)
params:
  # reads
  subsample_reads: 10000000
  fastqc_on_raw: ""
  # assembly
  plass: --num-iterations 12 --min-seq-id 0.9
  # gene clustering
  linclust: --min-seq-id 0.90 -c 0.9 --kmer-per-seq 50
  # annotation
  eggnog_mapper_jobs: 20    # number of parallel batched jobs
  eggnog_mapper: ""
  eggnog_mapper_annot: ""
  # pathways
  empanada: --abundance_threshold 0.0
  # taxonomy
  mmseqs_tax_jobs: 40       # number of parallel batched jobs
  mmseqs_tax: -e 1e-5 --max-seqs 200 --lca-ranks "kingdom:phylum:class:order:family:genus:species"
  # mapping
  mmseqs_map: Skip #-s 2 -c 0.9 --cov-mode 2
  mmseqs_blast_filter: --qcov 0.95 --seq-id 0.95 --e-value 1e-20
  mmseqs_blast_to_bam: --mapq 30
  merge_tables: --no-dask    # dask is currently broken
  # formatting for humann2 db
  diamond: --evalue 1e-5 --sensitive
  diamond_db_to_mem: True

#-- snakemake pipeline --#
pipeline:
  snakemake_folder: ./
  script_folder: bin/scripts/
  temp_folder: /tmp/global/

## Run_____no_output_____```{bash}
(snakemake_dev) @ rick:/ebio/abt3_projects/vadinCA11/bin/llmgag
$ screen -L -S llmgag-ga-ave ./snakemake_sge.sh /ebio/abt3_projects/databases_no-backup/animal_gut_metagenomes/wOutVertebrata/MG_assembly_ave/LLMGAG/config.yaml cluster.json /ebio/abt3_projects/databases_no-backup/animal_gut_metagenomes/wOutVertebrata/MG_assembly_ave/LLMGAG/SGE_log 24
```_____no_output_____

pipelineInfo(pipeline_dir)LLMGAG
======

Ley Lab Metagenome Assembly of Genes (LLMGAG)

* Version: 0.1.3
* Authors:
  * Nick Youngblut <nyoungb2@gmail.com>
* Maintainers:
  * Nick Youngblut <nyoungb2@gmail.com>

--- conda envs ---
==> /ebio/abt3_projects/Georg_animal_feces/bin/llmgag//bin/envs/annotate.yaml <==
channels: !!python/tuple
- bioconda
dependencies:
- pigz
- bioconda::fasta-splitter
- bioconda::eggnog-mapper

==> /ebio/abt3_projects/Georg_animal_feces/bin/llmgag//bin/envs/checkm.yaml <==
channels: !!python/tuple
- bioconda
dependencies:
- python=2
- pigz
- bioconda::prodigal
- bioconda::pplacer
- bioconda::checkm-genome

==> /ebio/abt3_projects/Georg_animal_feces/bin/llmgag//bin/envs/das_tool.yaml <==
channels: !!python/tuple
- r
- bioconda
dependencies:
- pigz
- ruby
- r::r-base
- r::r-data.table
- r::r-domc
- r::r-ggplot2
- bioconda::pullseq
- bioconda::prodigal
- bioconda::blast
- bioconda::diamond
==> /ebio/abt3_projects/Georg_animal_feces/bin/llmgag//bin/envs/dask.yaml <==
channels: !!python/tuple
- conda-forge
dependencies:
- conda-forge::dask=0.18.1

==> /ebio/abt3_projects/Georg_animal_feces/bin/llmgag//bin/envs/empanda.yaml <==
channels: !!python/tuple
- bioconda
- conda-forge
dependencies:
- pigz
- conda-forge::numpy=1.15
- conda-forge::pandas
- conda-forge::r-argparse
- conda-forge::r-dplyr
- conda-forge::r-tidyr
- conda-forge::r-stringr
- conda-forge::r-data.table
- pip:
  - empanada

==> /ebio/abt3_projects/Georg_animal_feces/bin/llmgag//bin/envs/hadley.yaml <==
channels: !!python/tuple
- conda-forge
dependencies:
- conda-forge::r-ape
- conda-forge::r-dplyr
- conda-forge::r-tidyr
- conda-forge::r-ggplot2

==> /ebio/abt3_projects/Georg_animal_feces/bin/llmgag//bin/envs/humann2.yaml <==
channels: !!python/tuple
- bioconda
dependencies:
- pigz
- bioconda::prodigal
- bioconda::diamond=0.8.36

==> /ebio/abt3_projects/Georg_animal_feces/bin/llmgag//bin/envs/metabat2.yaml <==
channels: !!python/tuple
- BioBuilds
- bioconda
dependencies:
- pigz
- python=2.7
- BioBuilds::boost>=1.55.0
- scons>=2.1.0
- zlib>=1.2.4
- BioBuilds::binutils>=2.2.2

==> /ebio/abt3_projects/Georg_animal_feces/bin/llmgag//bin/envs/multiqc.yaml <==
channels: !!python/tuple
- bioconda
dependencies:
- bioconda::fastqc
- bioconda::multiqc

==> /ebio/abt3_projects/Georg_animal_feces/bin/llmgag//bin/envs/plass.yaml <==
channels: !!python/tuple
- bioconda
dependencies:
- pigz
- bioconda::seqtk
- bioconda::samtools
- bioconda::mmseqs2
- bioconda::plass

==> /ebio/abt3_projects/Georg_animal_feces/bin/llmgag//bin/envs/sourmash.yaml <==
channels: !!python/tuple
- bioconda
dependencies:
- bioconda::sourmash=2.0.0a9

==> /ebio/abt3_projects/Georg_animal_feces/bin/llmgag//bin/envs/vamb.yaml <==
channels: !!python/tuple
- bioconda
- conda-forge
dependencies:
- pigz
- pytorch
- numpy
- pysam

# Summary_____no_output_____## Number of genes assembled & clustered_____no_output_____

F = file.path(work_dir, 'assembly', 'plass', 'genes.faa')
cmd = glue::glue('grep -c ">" {fasta}', fasta=F)
n_raw_seqs = system(cmd, intern=TRUE)
cat('Number of assembled sequences:', n_raw_seqs, '\n')Number of assembled sequences: 53270560
F = file.path(work_dir, 'cluster', 'linclust', 'clusters_rep-seqs.faa')
cmd = glue::glue('grep -c ">" {fasta}', fasta=F)
n_rep_seqs = system(cmd, intern=TRUE)
cat('Number of cluster rep sequences:', n_rep_seqs, '\n')Number of cluster rep sequences: 6970073
F = file.path(work_dir, 'humann2_db', 'clusters_rep-seqs.faa.gz')
cmd = glue::glue('gunzip -c {fasta} | grep -c ">"', fasta=F)
n_h2_seqs = system(cmd, intern=TRUE)
cat('Number of humann2_db-formatted seqs:', n_h2_seqs, '\n')Number of humann2_db-formatted seqs: 1648787

## Taxonomy_____no_output_____

# reading in taxonomy table
## WARING: slow
F = file.path(work_dir, 'taxonomy', 'clusters_rep-seqs_tax_db.tsv.gz')
cmd = glue::glue('gunzip -c {file}', file=F)
coln = c('seqID', 'taxID', 'rank', 'spp', 'lineage')
levs = c('Domain', 'Phylum', 'Class', 'Order', 'Family', 'Genus', 'Species')
tax = fread(cmd, sep='\t', header=FALSE, col.names=coln, fill=TRUE) %>%
    separate(lineage, levs, sep=':')
tax %>% dfheadWarning message:
“Expected 7 pieces. Additional pieces discarded in 126610 rows [137, 148, 196, 225, 314, 360, 434, 444, 453, 462, 464, 473, 517, 638, 748, 975, 981, 1010, 1044, 1108, ...].”Warning message:
“Expected 7 pieces. Missing pieces filled with `NA` in 2450744 rows [1, 2, 3, 20, 21, 23, 24, 25, 26, 27, 28, 29, 43, 44, 46, 47, 48, 49, 61, 69, ...].”# number of sequences
tax$seqID %>% unique %>% length %>% print[1] 6970073
# which ranks found?
tax$rank %>% table %>% print.
           class           cohort           family            forma
            6368               33             6438              161
           genus       infraclass       infraorder          kingdom
            7070             1315               10               83
         no rank            order        parvorder           phylum
         3756947             6976                7             7918
         species    species group species subgroup         subclass
         3041632              447               73              100
       subfamily         subgenus       subkingdom         suborder
             101               13               40               42
       subphylum       subspecies         subtribe       superclass
              52           123084               74               23
     superfamily     superkingdom       superorder            tribe
              10             7947              106              128
        varietas
            2875
# number of classifications per seqID
tax %>%
    group_by(seqID) %>%
    summarize(n = n()) %>%
    ungroup() %>%
    .$n %>% summary_____no_output_____

### Summary_____no_output_____

# summarizing taxonomy
tax_s = tax %>%
    filter(Domain != '',
           Phylum != '') %>%
    group_by(Domain, Phylum) %>%
    summarize(n = seqID %>% unique %>% length) %>%
    ungroup()

tax_s %>% dfhead[1] 206   3
# plotting by phylum
p = tax_s %>%
    filter(n > 10) %>%
    mutate(Phylum = Phylum %>% reorder(n)) %>%
    ggplot(aes(Phylum, n, fill=Domain)) +
    geom_bar(stat='identity', position='dodge') +
    scale_y_log10() +
    labs(y = 'No. of genes') +
    coord_flip() +
    theme_bw() +
    theme(
        axis.text.y = element_text(size=7)
    )

dims(5,15)
plot(p)_____no_output_____# top phyla
tax_s %>%
    arrange(-n) %>%
    head(n=30)_____no_output_____# summarizing taxonomy
tax_s = tax %>%
    filter(Domain != '',
           Phylum != '',
           Class != '') %>%
    group_by(Domain, Phylum, Class) %>%
    summarize(n = seqID %>% unique %>% length) %>%
    ungroup()

tax_s %>% dfhead[1] 441   4
# top hits
tax_s %>%
    arrange(-n) %>%
    head(n=30)_____no_output_____

## Annotations_____no_output_____

F = file.path(work_dir, 'annotate', 'eggnog-mapper', 'clusters_rep-seqs.emapper.annotations.gz')
cmd = glue::glue('gunzip -c {file}', file=F, header=FALSE)
emap_annot = fread(cmd, sep='\t') %>%
  dplyr::select(-V6)
emap_annot %>% dfhead[1] 4428421      12
# adding taxonomy info
intersect(emap_annot$V1, tax$seqID) %>% length %>% print

emap_annot = emap_annot %>%
    left_join(tax, c('V1'='seqID'))

emap_annot %>% dfhead[1] 4428421
[1] 4428421      22
n_annot_seqs = emap_annot$V1 %>% unique %>% length
cat('Number of rep seqs with eggnog-mapper annotations:', n_annot_seqs, '\n')Number of rep seqs with eggnog-mapper annotations: 4428421

### COG functional categories

* [wiki on categories](https://ecoliwiki.org/colipedia/index.php/Clusters_of_Orthologous_Groups_%28COGs%29)_____no_output_____

# summarizing by functional group
emap_annot_s = emap_annot %>%
    dplyr::select(V1, V12) %>%
    separate(V12, LETTERS[1:6], sep=', ') %>%
    gather(X, COG_func_cat, -V1) %>%
    filter(!is.na(COG_func_cat),
           COG_func_cat != '') %>%
    dplyr::select(-X)

emap_annot_s %>% dfheadWarning message:
“Expected 6 pieces. Missing pieces filled with `NA` in 4428421 rows [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, ...].”# plotting summary
p = emap_annot_s %>%
    ggplot(aes(COG_func_cat)) +
    geom_bar() +
    labs(x='COG functional category', y='No. of genes') +
    theme_bw()

dims(9,3)
plot(p)_____no_output_____# plotting summary
p = emap_annot_s %>%
    group_by(COG_func_cat) %>%
    summarize(perc_abund = n() / n_annot_seqs * 100) %>%
    ungroup() %>%
    ggplot(aes(COG_func_cat, perc_abund)) +
    geom_bar(stat='identity') +
    labs(x='COG functional category', y='% of all genes') +
    theme_bw()

dims(9,3)
plot(p)_____no_output_____

#### Grouped by taxonomy_____no_output_____

emap_annot_s = emap_annot %>%
    dplyr::select(V1, V12) %>%
    separate(V12, LETTERS[1:6], sep=', ') %>%
    gather(X, COG_func_cat, -V1) %>%
    left_join(tax, c('V1'='seqID')) %>%
    filter(!is.na(COG_func_cat),
           COG_func_cat != '') %>%
    dplyr::select(-X)

emap_annot_s %>% dfheadWarning message:
“Expected 6 pieces. Missing pieces filled with `NA` in 4428421 rows [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, ...].”# plotting summary by domain
p = emap_annot_s %>%
    ggplot(aes(COG_func_cat)) +
    geom_bar() +
    facet_wrap(~ Domain, scales='free_y') +
    labs(x='COG functional category', y='No. of genes') +
    theme_bw()

dims(9,5)
plot(p)_____no_output_____# plotting summary by phylum
p = emap_annot_s %>%
    group_by(Phylum) %>%
    mutate(n = n()) %>%
    ungroup() %>%
    filter(n >= 1000) %>%
    mutate(Phylum = Phylum %>% reorder(-n)) %>%
    ggplot(aes(COG_func_cat, fill=Domain)) +
    geom_bar() +
    facet_wrap(~ Phylum, scales='free_y', ncol=3) +
    labs(x='COG functional category', y='No. of genes') +
    theme_bw()

dims(9,14)
plot(p)_____no_output_____

## humann2 db genes_____no_output_____

# gene IDs
F = file.path(work_dir, 'humann2_db', 'clusters_rep-seqs_annot-index.tsv')

hm2 = fread(F, sep='\t', header=TRUE) %>%
    separate(new_name, c('UniRefID', 'Gene_length', 'Taxonomy'), sep='\\|') %>%
    separate(Taxonomy, c('Genus', 'Species'), sep='\\.s__') %>%
    separate(Species, c('Species', 'TaxID'), sep='__taxID') %>%
    mutate(Genus = gsub('^g__', '', Genus))

hm2 %>% dfheadWarning message:
“Expected 2 pieces. Missing pieces filled with `NA` in 128016 rows [1, 5, 7, 14, 17, 20, 21, 38, 68, 118, 138, 167, 169, 327, 339, 347, 350, 359, 405, 476, ...].”# adding taxonomy
intersect(hm2$original_name, tax$seqID) %>% length %>% print

hm2 = hm2 %>%
    left_join(tax, c('original_name'='seqID'))

hm2 %>% dfhead[1] 1648787
[1] 1648787      16

### Summary_____no_output_____

# number of unique UniRef IDs
hm2$UniRefID %>% unique %>% length_____no_output_____# duplicate UniRef IDs
hm2 %>%
    group_by(UniRefID) %>%
    summarize(n = n()) %>%
    ungroup() %>%
    filter(n > 1) %>%
    arrange(-n) %>%
    head(n=30)_____no_output_____# number of genes with a taxID
hm2_f = hm2 %>%
    filter(!is.na(TaxID))

hm2_f %>% nrow _____no_output_____

#### By Taxonomy_____no_output_____

# number of UniRefIDs
hm2_f_s = hm2_f %>%
    group_by(Domain, Phylum) %>%
    summarize(n = UniRefID %>% unique %>% length) %>%
    ungroup()

p = hm2_f_s %>%
    filter(n >= 10) %>%
    mutate(Phylum = Phylum %>% reorder(n)) %>%
    ggplot(aes(Phylum, n, fill=Domain)) +
    geom_bar(stat='identity', position='dodge') +
    scale_y_log10() +
    coord_flip() +
    labs(y='No. of UniRef IDs') +
    theme_bw() +
    theme(
        axis.text.y = element_text(size=7)
    )

dims(5,10)
plot(p)_____no_output_____

# sessionInfo_____no_output_____

sessionInfo()_____no_output_____


# Notebook from hollylessthan/Book-Recommendation-From-Charles-Darwin
Path: notebook.ipynb

## 1. Darwin's bibliography

Charles Darwin is one of the few universal figures of science. His most renowned work is without a doubt his "On the Origin of Species" published in 1859 which introduced the concept of natural selection. But Darwin wrote many other books on a wide range of topics, including geology, plants or his personal life. In this notebook, we will automatically detect how closely related his books are to each other.

To this purpose, we will develop the bases of a content-based book recommendation system, which will determine which books are close to each other based on how similar the discussed topics are. The methods we will use are commonly used in text- or documents-heavy industries such as legal, tech or customer support to perform some common task such as text classification or handling search engine queries.

Let's take a look at the books we'll use in our recommendation system.
_____no_output_____

# Import library
import glob

# The books files are contained in this folder
folder = "datasets/"

# List all the .txt files and sort them alphabetically
files = [file for file in glob.glob(folder+"*.txt")]
# ... YOUR CODE FOR TASK 1 ...
files.sort()
print(files)['datasets/Autobiography.txt', 'datasets/CoralReefs.txt', 'datasets/DescentofMan.txt', 'datasets/DifferentFormsofFlowers.txt', 'datasets/EffectsCrossSelfFertilization.txt', 'datasets/ExpressionofEmotionManAnimals.txt', 'datasets/FormationVegetableMould.txt', 'datasets/FoundationsOriginofSpecies.txt', 'datasets/GeologicalObservationsSouthAmerica.txt', 'datasets/InsectivorousPlants.txt', 'datasets/LifeandLettersVol1.txt', 'datasets/LifeandLettersVol2.txt', 'datasets/MonographCirripedia.txt', 'datasets/MonographCirripediaVol2.txt', 'datasets/MovementClimbingPlants.txt', 'datasets/OriginofSpecies.txt', 'datasets/PowerMovementPlants.txt', 'datasets/VariationPlantsAnimalsDomestication.txt', 'datasets/VolcanicIslands.txt', 'datasets/VoyageBeagle.txt']

## 2. Load the contents of each book into Python

As a first step, we need to load the content of these books into Python and do some basic pre-processing to facilitate the downstream analyses. We call such a collection of texts a corpus. We will also store the titles for these books for future reference and print their respective length to get a gauge for their contents.
_____no_output_____

# Import libraries
import re, os

# Initialize the object that will contain the texts and titles
txts = []
titles = []

for n in files:
    # Open each file
    f = open(n, encoding='utf-8-sig')
    # Remove all non-alpha-numeric characters
    data = re.sub('[\W_]+', ' ', f.read())
    # ... YOUR CODE FOR TASK 2 ...
    # Store the texts and titles of the books in two separate lists
    txts.append(data)
    titles.append(os.path.basename(n).replace(".txt",""))

# Print the length, in characters, of each book
[len(t) for t in txts]_____no_output_____

## 3. Find "On the Origin of Species"

For the next parts of this analysis, we will often check the results returned by our method for a given book. For consistency, we will refer to Darwin's most famous book: "On the Origin of Species." Let's find to which index this book is associated.
_____no_output_____

# Browse the list containing all the titles
for i in range(len(titles)):
    # Store the index if the title is "OriginofSpecies"
    if titles[i]=='OriginofSpecies':
        ori = i
    # ... YOUR CODE FOR TASK 3 ...
print(ori)
# Print the stored index
# ... YOUR CODE FOR TASK 3 ...15

## 4. Tokenize the corpus

As a next step, we need to transform the corpus into a format that is easier to deal with for the downstream analyses. We will tokenize our corpus, i.e., transform each text into a list of the individual words (called tokens) it is made of. To check the output of our process, we will print the first 20 tokens of "On the Origin of Species".
_____no_output_____

# Define a list of stop words
stoplist = set('for a of the and to in to be which some is at that we i who whom show via may my our might as well'.split())

# Convert the text to lower case
txts_lower_case = [string.lower() for string in txts]

# Transform the text into tokens
txts_split = [string.split() for string in txts_lower_case]

# Remove tokens which are part of the list of stop words
texts = [[text for text in string if text not in stoplist ] for string in txts_split]

# Print the first 20 tokens for the "On the Origin of Species" book
# ... YOUR CODE FOR TASK 4 ..._____no_output_____

## 5. Stemming of the tokenized corpus

If you have read On the Origin of Species, you will have noticed that Charles Darwin can use different words to refer to a similar concept. For example, the concept of selection can be described by words such as selection, selective, select or selects. This will dilute the weight given to this concept in the book and potentially bias the results of the analysis.

To solve this issue, it is a common practice to use a stemming process, which will group together the inflected forms of a word so they can be analysed as a single item: the stem. In our On the Origin of Species example, the words related to the concept of selection would be gathered under the select stem.

As we are analysing 20 full books, the stemming algorithm can take several minutes to run and, in order to make the process faster, we will directly load the final results from a pickle file and review the method used to generate it.
_____no_output_____

import pickle

# Load the stemmed tokens list from the pregenerated pickle file
texts_stem = pickle.load(open('datasets/texts_stem.p', 'rb'))
texts_stem[:20]
# Print the 20 first stemmed tokens from the "On the Origin of Species" book
# ... YOUR CODE FOR TASK 5 ..._____no_output_____

## 6. Building a bag-of-words model

Now that we have transformed the texts into stemmed tokens, we need to build models that will be useable by downstream algorithms.

First, we need to will create a universe of all words contained in our corpus of Charles Darwin's books, which we call a dictionary. Then, using the stemmed tokens and the dictionary, we will create bag-of-words models (BoW) of each of our texts. The BoW models will represent our books as a list of all uniques tokens they contain associated with their respective number of occurrences.

To better understand the structure of such a model, we will print the five first elements of one of the "On the Origin of Species" BoW model.
_____no_output_____

# Load the functions allowing to create and use dictionaries
from gensim import corpora

# Create a dictionary from the stemmed tokens
dictionary = corpora.Dictionary(texts_stem)

# Create a bag-of-words model for each book, using the previously generated dictionary
bows = [dictionary.doc2bow(text) for text in texts_stem]

# Print the first five elements of the On the Origin of species' BoW model
# ... YOUR CODE FOR TASK 6 ...INFO:gensim.corpora.dictionary:adding document #0 to Dictionary(0 unique tokens: [])
INFO:gensim.corpora.dictionary:built Dictionary(28170 unique tokens: ['1', '11th', '1250', '12th', '13th']...) from 20 documents (total 2092316 corpus positions)

## 7. The most common words of a given book

The results returned by the bag-of-words model is certainly easy to use for a computer but hard to interpret for a human. It is not straightforward to understand which stemmed tokens are present in a given book from Charles Darwin, and how many occurrences we can find.

In order to better understand how the model has been generated and visualize its content, we will transform it into a DataFrame and display the 10 most common stems for the book "On the Origin of Species".
_____no_output_____

# Import pandas to create and manipulate DataFrames
import pandas as pd

# Convert the BoW model for "On the Origin of Species" into a DataFrame
df_bow_origin = pd.DataFrame(bows[ori])
print(df_bow_origin.head())

# Add the column names to the DataFrame
df_bow_origin.columns = ["index", "occurrences"]
# ... YOUR CODE FOR TASK 7 ...

# Add a column containing the token corresponding to the dictionary index
# ... YOUR CODE FOR TASK 7 ...
df_bow_origin["token"]=[dictionary[index] for index in df_bow_origin["index"]]
# Sort the DataFrame by descending number of occurrences and print the first 10 values
df_bow_origin.sort_values(by = "occurrences", ascending = False).head(10)
# ... YOUR CODE FOR TASK 7 ...    0   1
0   0  11
1   5  51
2   6   1
3   8   2
4  21   1

## 8. Build a tf-idf model

If it wasn't for the presence of the stem "speci", we would have a hard time to guess this BoW model comes from the On the Origin of Species book. The most recurring words are, apart from few exceptions, very common and unlikely to carry any information peculiar to the given book. We need to use an additional step in order to determine which tokens are the most specific to a book.

To do so, we will use a tf-idf model (term frequency–inverse document frequency). This model defines the importance of each word depending on how frequent it is in this text and how infrequent it is in all the other documents. As a result, a high tf-idf score for a word will indicate that this word is specific to this text.

After computing those scores, we will print the 10 words most specific to the "On the Origin of Species" book (i.e., the 10 words with the highest tf-idf score).
_____no_output_____

# Load the gensim functions that will allow us to generate tf-idf models
from gensim.models import TfidfModel

# Generate the tf-idf model
model = TfidfModel(bows)
model[bows[0]]

# Print the model for "On the Origin of Species"
# ... YOUR CODE FOR TASK 8 ...INFO:gensim.models.tfidfmodel:collecting document frequencies
INFO:gensim.models.tfidfmodel:PROGRESS: processing document #0
INFO:gensim.models.tfidfmodel:calculating IDF weights for 20 documents and 28169 features (106515 matrix non-zeros)

## 9. The results of the tf-idf model

Once again, the format of those results is hard to interpret for a human. Therefore, we will transform it into a more readable version and display the 10 most specific words for the "On the Origin of Species" book.
_____no_output_____

# Convert the tf-idf model for "On the Origin of Species" into a DataFrame
df_tfidf = pd.DataFrame(model[bows[ori]])

# Name the columns of the DataFrame id and score
df_tfidf.columns=['id','score']
# ... YOUR CODE FOR TASK 9 ...

# Add the tokens corresponding to the numerical indices for better readability
df_tfidf["token"]=[dictionary[i] for i in list(df_tfidf["id"])]
# Sort the DataFrame by descending tf-idf score and print the first 10 rows.
df_tfidf.sort_values(by = "score", ascending = False).head(10)_____no_output_____

## 10. Compute distance between texts

The results of the tf-idf algorithm now return stemmed tokens which are specific to each book. We can, for example, see that topics such as selection, breeding or domestication are defining "On the Origin of Species" (and yes, in this book, Charles Darwin talks quite a lot about pigeons too). Now that we have a model associating tokens to how specific they are to each book, we can measure how related to books are between each other.

To this purpose, we will use a measure of similarity called cosine similarity and we will visualize the results as a distance matrix, i.e., a matrix showing all pairwise distances between Darwin's books.
_____no_output_____

# Load the library allowing similarity computations
from gensim import similarities

# Compute the similarity matrix (pairwise distance between all texts)
sims = similarities.MatrixSimilarity(model[bows])

# Transform the resulting list into a dataframe
sim_df = pd.DataFrame(list(sims))

# Add the titles of the books as columns and index of the dataframe
sim_df.columns = titles
sim_df.index = titles

# Print the resulting matrix
sim_df
# ... YOUR CODE FOR TASK 10 ...WARNING:gensim.similarities.docsim:scanning corpus to determine the number of features (consider setting `num_features` explicitly)
INFO:gensim.similarities.docsim:creating matrix with 20 documents and 28170 features
DEBUG:gensim.similarities.docsim:PROGRESS: at document #0/20

## 11. The book most similar to "On the Origin of Species"

We now have a matrix containing all the similarity measures between any pair of books from Charles Darwin! We can now use this matrix to quickly extract the information we need, i.e., the distance between one book and one or several others.

As a first step, we will display which books are the most similar to "On the Origin of Species," more specifically we will produce a bar chart showing all books ranked by how similar they are to Darwin's landmark work.
_____no_output_____

# This is needed to display plots in a notebook
%matplotlib inline

# Import libraries
import matplotlib.pyplot as plt

# Select the column corresponding to "On the Origin of Species" and
v = sim_df["OriginofSpecies"]

# Sort by ascending scores
v_sorted = v.sort_values(ascending = True)

# Plot this data has a horizontal bar plot
v_sorted.plot.barh(x = 'lab', y = 'val', rot = 0).plot()

# ... YOUR CODE FOR TASK 11 ...

# Modify the axes labels and plot title for a better readability
plt.xlabel("Cosine distance")
plt.ylabel("")
plt.title("similarity between other books")
plt.show()
# ... YOUR CODE FOR TASK 11 ..._____no_output_____

## 12. Which books have similar content?

This turns out to be extremely useful if we want to determine a given book's most similar work. For example, we have just seen that if you enjoyed "On the Origin of Species," you can read books discussing similar concepts such as "The Variation of Animals and Plants under Domestication" or "The Descent of Man, and Selection in Relation to Sex." If you are familiar with Darwin's work, these suggestions will likely seem natural to you. Indeed, On the Origin of Species has a whole chapter about domestication and The Descent of Man, and Selection in Relation to Sex applies the theory of natural selection to human evolution. Hence, the results make sense.

However, we now want to have a better understanding of the big picture and see how Darwin's books are generally related to each other (in terms of topics discussed). To this purpose, we will represent the whole similarity matrix as a dendrogram, which is a standard tool to display such data. This last approach will display all the information about book similarities at once. For example, we can find a book's closest relative but, also, we can visualize which groups of books have similar topics (e.g., the cluster about Charles Darwin personal life with his autobiography and letters). If you are familiar with Darwin's bibliography, the results should not surprise you too much, which indicates the method gives good results. Otherwise, next time you read one of the author's book, you will know which other books to read next in order to learn more about the topics it addressed.
_____no_output_____

# Import libraries
from scipy.cluster import hierarchy

# Compute the clusters from the similarity matrix,
# using the Ward variance minimization algorithm
Z = hierarchy.linkage(V, 'ward')
# Display this result as a horizontal dendrogram
# ... YOUR CODE FOR TASK 12 ...
hierarchy.dendrogram(leaf_font_size=8, labels=sim_df.index, orientation="left")
_____no_output_____


# Notebook from nolanstr/bingo_nolan_fork
Path: examples/Tutorial_4_Symbolic_Regression.ipynb

# Bingo Tutorial 4: Symbolic Regression

## Goal: Given input data and output data, find the functional form of the equation_____no_output_____### Acyclic Graphs

Equations are represented as acyclic graphs in bingo.  The `AcyclicGraphChromosome` encapsulates a list of mathematical commands, performed in a set order, that define an equation.  These are called AGraphs in bingo._____no_output_____### Explicit Training Data
The way an AGraph's fitness is evaluated is whether or not it models some training data correctly. When both input and outputs are present in the training data, explicit regression is used.  This requires having some valid `ExplicitTrainingData`. `ExplicitTrainingData` requires x (input) and y (output) as numpy arrays._____no_output_____

import numpy as np

from bingo.symbolic_regression.explicit_regression import ExplicitTrainingData

x = np.linspace(-10, 10, 30).reshape([-1, 1])
y = x**2 + 3.5*x**3
training_data = ExplicitTrainingData(x, y)_____no_output_____import matplotlib.pyplot as plt
import matplotlib.animation as animation

plt.plot(training_data.x, training_data.y, 'ro')
plt.show()_____no_output_____

### AGraph Component Generator
AGraphs also require a component generator to generate elements of an acylic graph object. It plays a similar role as `get_random_float` in the ZeroMinExample jupyter notebook.

The dimension of the independent variable (x) is needed for the initialization of the component generator.  After initialization, mathematical operators can be added to the generator.  These operators constitute the building blocks from which the AGraphs will be made._____no_output_____

from bingo.symbolic_regression.agraph.component_generator import ComponentGenerator

component_generator = ComponentGenerator(input_x_dimension=x.shape[1])
component_generator.add_operator("+")
component_generator.add_operator("-")
component_generator.add_operator("*")_____no_output_____

### AGraph Generator
The AGraph generator will use the component generator to generate `AGraphChromosome` individuals. In addion to the component generator, the desired size of the AGraphs is needed in initialization.  This size corresponds to the number of maximum number of commands possible in the AGraph. In other words, a larger size allows for more complex equations but comes at the cost of longer evaluation times._____no_output_____

from bingo.symbolic_regression.agraph.generator import AGraphGenerator

agraph_generator = AGraphGenerator(agraph_size=10,
                                   component_generator=component_generator)_____no_output_____agraph = agraph_generator()
print("f(X_0) = ", agraph)_____no_output_____

Try rerunning the snippet above a few times to see some different equations produced by the generator.  Note that X_0 represents the first dimension of the independent variable.  Also note that "?" represent potential numerical constants, more on this later._____no_output_____### AGraph Variation
Set the the variation amongst the population per generation.
AGraphCrossover is single-point.  Mutation contains several possible mutation strategies, most of which are single-point.  See documentation for more in depth description of mutation types and tailoring the mutation function. Both the crossover and mutation require the component generator._____no_output_____

from bingo.symbolic_regression.agraph.crossover import AGraphCrossover
from bingo.symbolic_regression.agraph.mutation import AGraphMutation

crossover = AGraphCrossover(component_generator)
mutation = AGraphMutation(component_generator)_____no_output_____

### Evaluation: Explicit Regression

The type of regression that will be used is `ExplicitRegression` since the training data has both input and output data to train against. `ExplicitRegression` extends `FitnessFunction`; hence, may be passed to `ContinuousLocalOptimization` object as an argument. This is then passed to an `Evaluation` object which will run `ExplicitRegression` on all `AGraph` individuals.

The `ContinuousLocalOptimization` is responsible for finding the best numerical constants for the given explicit regression.  The numerical constants are represented by "?" before local optimization has been performed (as you may have seen in the AGraph Generator section). _____no_output_____

from bingo.symbolic_regression.explicit_regression import ExplicitRegression
from bingo.local_optimizers.continuous_local_opt import ContinuousLocalOptimization
from bingo.evaluation.evaluation import Evaluation

fitness = ExplicitRegression(training_data=training_data)
local_opt_fitness = ContinuousLocalOptimization(fitness, algorithm='lm')
evaluator = Evaluation(local_opt_fitness)_____no_output_____np.random.seed(16)
agraph = agraph_generator()
print("Before local optimization: f(X_0) = ", agraph)
_ = local_opt_fitness(agraph)
print("After local optimization: f(X_0) = ", agraph)_____no_output_____

The AGraphs can be easily evaluated at a given x, but note that **trying to evaluate an AGraph before the constants have been set will cause errors.**_____no_output_____

agraph_y = agraph.evaluate_equation_at(training_data.x)

plt.plot(training_data.x, training_data.y, 'ro')
plt.plot(training_data.x, agraph_y, 'b-')
plt.show()_____no_output_____

### Age Fitness Evolutionary Algorithm

The evoluaionary algorithm used in this example is `AgeFitnessEA`. This by default uses the `AgeFitnessSelection`. It also requires use of the `AGraphGenerator` in order to seed a random individuals.  The Age Fitness EA is used to combat premature convergence that can be seen in symbolic regression._____no_output_____

from bingo.evolutionary_algorithms.age_fitness import AgeFitnessEA

POPULATION_SIZE = 32
MUTATION_PROBABILITY = 0.4
CROSSOVER_PROBABILITY = 0.4

ea = AgeFitnessEA(evaluator, agraph_generator, crossover, mutation,
                  MUTATION_PROBABILITY, CROSSOVER_PROBABILITY, POPULATION_SIZE)_____no_output_____

### Pareto Front

A `ParetoFront` hall of fame object is useful in symbolic regression for tracking the best individuals, where fitness and equation complexity are both taken into consideration. The secondary key must be supplied to the `ParetoFront` object. The primary key can optionally be supplied instead of the default use of fitness. _____no_output_____

from bingo.stats.pareto_front import ParetoFront

def agraph_similarity(ag_1, ag_2):
    """a similarity metric between agraphs"""
    return ag_1.fitness == ag_2.fitness and ag_1.get_complexity() == ag_2.get_complexity()

pareto_front = ParetoFront(secondary_key=lambda ag: ag.get_complexity(),
                           similarity_function=agraph_similarity) _____no_output_____

### Evolution on an island
Once an `Island` is set up, evolution occurs in the same way as described in earlier examples._____no_output_____

from bingo.evolutionary_optimizers.island import Island
np.random.seed(2)

island = Island(ea, agraph_generator, POPULATION_SIZE, hall_of_fame=pareto_front)
print("Best individual\n f(X_0) =", island.get_best_individual())_____no_output_____

Run until convergence. Print the best result. We store each best individual in a list and use this to observe how the best solution evolves over time._____no_output_____

ERROR_TOLERANCE = 1e-6

best_indv_values = []
best_indv_values.append(island.get_best_individual())

while island.get_best_fitness() > ERROR_TOLERANCE:
    island.evolve(1)
    best_indv_values.append(island.get_best_individual())

print("Generation: ", island.generational_age)
print("Success!")
print("Best individual\n f(X_0) =", island.get_best_individual())_____no_output_____

We can look at the pareto then look at the Pareto front to see the tradeoff between fitness and complexity_____no_output_____

print(" FITNESS   COMPLEXITY    EQUATION")
for member in pareto_front:
    print("%.3e     " % member.fitness, member.get_complexity(),
          "     f(X_0) =", member)_____no_output_____

### Animation of evolution_____no_output_____

def animate_data(list_of_best_indv, training_data):

    fig, ax = plt.subplots()

    num_generations = len(list_of_best_indv)
    x = training_data.x
    y_actually = training_data.y
    y = list_of_best_indv
    plt.plot(training_data.x, training_data.y, 'ro')
    points, = ax.plot(x, y[0].evaluate_equation_at(x), 'b')
    points.set_label('Generation :' + str(0))
    legend = ax.legend(loc='upper right', shadow=True)

    def animate(i):
        ax.collections.clear()
        points.set_ydata(y[i].evaluate_equation_at(x))  # update the data
        points.set_label('Generation :' + str(i))
        legend = ax.legend(loc='upper right')
        return points, legend

    # Init only required for blitting to give a clean slate.
    def init():
        points.set_ydata(np.ma.array(x, mask=True))
        return points, points

    plt.xlabel('x', fontsize=15)
    plt.ylabel('y', fontsize=15)
    plt.title("Best Individual in Island", fontsize=12)
    ax.tick_params(axis='y', labelsize=12)
    ax.tick_params(axis='x', labelsize=12)
    plt.close()

    return animation.FuncAnimation(fig, animate, num_generations, init_func=init,
                                interval=250, blit=True)_____no_output_____from IPython.display import HTML
anim2 = animate_data(best_indv_values, training_data)
HTML(anim2.to_jshtml())_____no_output_____


# Notebook from uchiiii/SynapseML
Path: notebooks/features/vw/Vowpal Wabbit - Overview.ipynb

# VowalWabbit

[VowpalWabbit](https://github.com/VowpalWabbit/vowpal_wabbit) (VW) is a machine learning system which
pushes the frontier of machine learning with techniques such as online, hashing, allreduce,
reductions, learning2search, active, and interactive learning.
VowpalWabbit is a popular choice in ad-tech due to it's speed and cost efficacy.
Furthermore it includes many advances in the area of reinforcement learning (e.g. contextual bandits).

### Advantages of VowpalWabbit

-  **Composability**: VowpalWabbit models can be incorporated into existing
    SparkML Pipelines, and used for batch, streaming, and serving workloads.
-  **Small footprint**: VowpalWabbit memory consumption is rather small and can be controlled through '-b 18' or setNumBits method.
    This determines the size of the model (e.g. 2^18 * some_constant).
-  **Feature Interactions**: Feature interactions (e.g. quadratic, cubic,... terms) are created on-the-fly within the most inner
    learning loop in VW.
    Interactions can be specified by using the -q parameter and passing the first character of the namespaces that should be _interacted_.
    The VW namespace concept is mapped to Spark using columns. The column name is used as namespace name, thus one sparse or dense Spark ML vector corresponds to the features of a single namespace.
    To allow passing of multiple namespaces the VW estimator (classifier or regression) expose an additional property called _additionalFeatures_. Users can pass an array of column names.
-  **Simple deployment**: all native dependencies are packaged into a single jars (including boost and zlib).
-  **VowpalWabbit command line arguments**: users can pass VW command line arguments to control the learning process.
-  **VowpalWabbit binary models** Users can supply an inital VowpalWabbit model to start the training which can be produced outside of
    VW on Spark by invoking _setInitialModel_ and pass the model as a byte array. Similarly users can access the binary model by invoking
    _getModel_ on the trained model object.
-  **Java-based hashing** VWs version of murmur-hash was re-implemented in Java (praise to [JackDoe](https://github.com/jackdoe))
    providing a major performance improvement compared to passing input strings through JNI and hashing in C++.
-  **Cross language** VowpalWabbit on Spark is available on Spark, PySpark, and SparklyR.

### Limitations of VowpalWabbit on Spark

-  **Linux and CentOS only** The native binaries included with the published jar are built Linux and CentOS only.
    We're working on creating a more portable version by statically linking Boost and lib C++.
-  **Limited Parsing** Features implemented in the native VW parser (e.g. ngrams, skips, ...) are not yet implemented in
    VowpalWabbitFeaturizer.

### VowpalWabbit Usage:

-  VowpalWabbitClassifier: used to build classification models.
-  VowpalWabbitRegressor: used to build regression models.
-  VowpalWabbitFeaturizer: used for feature hashing and extraction. For details please visit [here](https://github.com/VowpalWabbit/vowpal_wabbit/wiki/Feature-Hashing-and-Extraction).
-  VowpalWabbitContextualBandit: used to solve contextual bandits problems. For algorithm details please visit [here](https://github.com/VowpalWabbit/vowpal_wabbit/wiki/Contextual-Bandit-algorithms)._____no_output_____## Heart Disease Detection with VowalWabbit Classifier

_____no_output_____#### Read dataset_____no_output_____

import os

if os.environ.get("AZURE_SERVICE", None) == "Microsoft.ProjectArcadia":
    from pyspark.sql import SparkSession
    spark = SparkSession.builder.getOrCreate()_____no_output_____df = spark.read.format("csv")\
  .option("header", True)\
  .option("inferSchema", True)\
  .load("wasbs://publicwasb@mmlspark.blob.core.windows.net/heart_disease_prediction_data.csv")
# print dataset basic info
print("records read: " + str(df.count()))
print("Schema: ")
df.printSchema()_____no_output_____display(df)_____no_output_____

#### Split the dataset into train and test_____no_output_____

train, test = df.randomSplit([0.85, 0.15], seed=1)_____no_output_____

#### Use VowalWabbitFeaturizer to convert data features into vector_____no_output_____

from synapse.ml.vw import VowpalWabbitFeaturizer
featurizer = VowpalWabbitFeaturizer(inputCols=df.columns[:-1], outputCol="features")
train_data = featurizer.transform(train)["target", "features"]
test_data = featurizer.transform(test)["target", "features"]_____no_output_____display(train_data.groupBy("target").count())_____no_output_____

#### Model Training_____no_output_____

from synapse.ml.vw import VowpalWabbitClassifier
model = VowpalWabbitClassifier(numPasses=20, labelCol="target", featuresCol="features").fit(train_data)_____no_output_____

#### Model Prediction_____no_output_____

predictions = model.transform(test_data)
display(predictions)_____no_output_____from synapse.ml.train import ComputeModelStatistics
metrics = ComputeModelStatistics(evaluationMetric='classification', labelCol='target', scoredLabelsCol='prediction').transform(predictions)
display(metrics)_____no_output_____

## Adult Census with VowpalWabbitClassifier

In this example, we predict incomes from the Adult Census dataset using Vowpal Wabbit (VW) Classifier in SynapseML._____no_output_____#### Read dataset and split them into train & test_____no_output_____

data = spark.read.parquet("wasbs://publicwasb@mmlspark.blob.core.windows.net/AdultCensusIncome.parquet")
data = data.select(["education", "marital-status", "hours-per-week", "income"])
train, test = data.randomSplit([0.75, 0.25], seed=123)
display(train)_____no_output_____

#### Model Training

We define a pipeline that includes feature engineering and training of a VW classifier. We use a featurizer provided by VW that hashes the feature names. Note that VW expects classification labels being -1 or 1. Thus, the income category is mapped to this space before feeding training data into the pipeline.

Note: VW supports distributed learning, and it's controlled by number of partitions of dataset._____no_output_____

from pyspark.sql.functions import when, col
from pyspark.ml import Pipeline
from synapse.ml.vw import VowpalWabbitFeaturizer, VowpalWabbitClassifier

# Define classification label
train = train.withColumn("label", when(col("income").contains("<"), 0.0).otherwise(1.0)).repartition(1)
print(train.count())

# Specify featurizer
vw_featurizer = VowpalWabbitFeaturizer(inputCols=["education", "marital-status", "hours-per-week"],
                                       outputCol="features")_____no_output_____

Note: "args" parameter lets you pass in any params not exposed through our API. Full command line argument docs can be found [here](https://github.com/VowpalWabbit/vowpal_wabbit/wiki/Command-Line-Arguments)._____no_output_____

# Define VW classification model
args = "--loss_function=logistic --quiet --holdout_off"
vw_model = VowpalWabbitClassifier(featuresCol="features",
                                  labelCol="label",
                                  args=args,
                                  numPasses=10)

# Create a pipeline
vw_pipeline = Pipeline(stages=[vw_featurizer, vw_model])_____no_output_____vw_trained = vw_pipeline.fit(train)_____no_output_____

#### Model Prediction

After the model is trained, we apply it to predict the income of each sample in the test set._____no_output_____

# Making predictions
test = test.withColumn("label", when(col("income").contains("<"), 0.0).otherwise(1.0))
prediction = vw_trained.transform(test)
display(prediction)_____no_output_____

Finally, we evaluate the model performance using ComputeModelStatistics function which will compute confusion matrix, accuracy, precision, recall, and AUC by default for classificaiton models._____no_output_____

from synapse.ml.train import ComputeModelStatistics
metrics = ComputeModelStatistics(evaluationMetric="classification",
                                 labelCol="label",
                                 scoredLabelsCol="prediction").transform(prediction)
display(metrics)_____no_output_____

## Boston house price prediction with VowpalWabbitRegressor - Quantile Regression

In this example, we show how to build regression model with VW using Boston's house price._____no_output_____#### Read dataset

We use [*Boston house price* dataset](https://scikit-learn.org/stable/modules/generated/sklearn.datasets.load_boston.html)
.
The data was collected in 1978 from Boston area and consists of 506 entries with 14 features including the value of homes.
We use `sklearn.datasets` module to download it easily, then split the set into training and testing by 75/25._____no_output_____

import math
from matplotlib.colors import ListedColormap, Normalize
from matplotlib.cm import get_cmap
import matplotlib.pyplot as plt
from synapse.ml.train import ComputeModelStatistics
from synapse.ml.vw import VowpalWabbitRegressor, VowpalWabbitFeaturizer
import numpy as np
import pandas as pd
from sklearn.datasets import load_boston_____no_output_____boston = load_boston()

feature_cols = ['f' + str(i) for i in range(boston.data.shape[1])]
header = ['target'] + feature_cols
df = spark.createDataFrame(
    pd.DataFrame(data=np.column_stack((boston.target, boston.data)), columns=header)
).repartition(1)
print("Dataframe has {} rows".format(df.count()))
display(df.limit(10))_____no_output_____train_data, test_data = df.randomSplit([0.75, 0.25], seed=42)_____no_output_____display(train_data.summary().toPandas())_____no_output_____train_data.show(10)_____no_output_____

Exploratory analysis: plot feature distributions over different target values._____no_output_____

features = train_data.columns[1:]
values = train_data.drop('target').toPandas()
ncols = 5
nrows = math.ceil(len(features) / ncols)

yy = [r['target'] for r in train_data.select('target').collect()]

f, axes = plt.subplots(nrows, ncols, sharey=True, figsize=(30,10))
f.tight_layout()

for irow in range(nrows):
    axes[irow][0].set_ylabel('target')
    for icol in range(ncols):
        try:
            feat = features[irow*ncols + icol]
            xx = values[feat]

            axes[irow][icol].scatter(xx, yy, s=10, alpha=0.25)
            axes[irow][icol].set_xlabel(feat)
            axes[irow][icol].get_yaxis().set_ticks([])
        except IndexError:
            f.delaxes(axes[irow][icol])_____no_output_____

#### VW-style feature hashing_____no_output_____

vw_featurizer = VowpalWabbitFeaturizer(
    inputCols=feature_cols,
    outputCol='features',
)
vw_train_data = vw_featurizer.transform(train_data)['target', 'features']
vw_test_data = vw_featurizer.transform(test_data)['target', 'features']
display(vw_train_data)_____no_output_____

#### Model training & Prediction

See [VW wiki](https://github.com/vowpalWabbit/vowpal_wabbit/wiki/Command-Line-Arguments) for command line arguments._____no_output_____

args = "--holdout_off --loss_function quantile -l 7 -q :: --power_t 0.7"
vwr = VowpalWabbitRegressor(
    labelCol='target',
    featuresCol='features',
    args=args,
    numPasses=200,
)

# To reduce number of partitions (which will effect performance), use `vw_train_data.repartition(1)`
vw_model = vwr.fit(vw_train_data.repartition(1))
vw_predictions = vw_model.transform(vw_test_data)

display(vw_predictions.limit(20).toPandas())_____no_output_____

#### Compute Statistics & Visualization_____no_output_____

metrics = ComputeModelStatistics(
    evaluationMetric='regression',
    labelCol='target',
    scoresCol='prediction'
).transform(vw_predictions)

vw_result = metrics.toPandas()
vw_result.insert(0, 'model', ['Vowpal Wabbit'])
display(vw_result)_____no_output_____cmap = get_cmap('YlOrRd')
target = np.array(test_data.select('target').collect()).flatten()
model_preds = [("Vowpal Wabbit", vw_predictions)]

f, axe = plt.subplots(figsize=(6, 6))
f.tight_layout()

preds = np.array(vw_predictions.select('prediction').collect()).flatten()
err = np.absolute(preds - target)
norm = Normalize()
clrs = cmap(np.asarray(norm(err)))[:, :-1]
plt.scatter(preds, target, s=60, c=clrs, edgecolors='#888888', alpha=0.75)
plt.plot((0, 60), (0, 60), linestyle='--', color='#888888')
axe.set_xlabel('Predicted values')
axe.set_ylabel('Actual values')
axe.set_title("Vowpal Wabbit")_____no_output_____

## Quantile Regression for Drug Discovery with VowpalWabbitRegressor

_____no_output_____#### Read dataset_____no_output_____

triazines = spark.read.format("libsvm")\
    .load("wasbs://publicwasb@mmlspark.blob.core.windows.net/triazines.scale.svmlight")_____no_output_____# print some basic info
print("records read: " + str(triazines.count()))
print("Schema: ")
triazines.printSchema()
display(triazines.limit(10))_____no_output_____

#### Split dataset into train and test_____no_output_____

train, test = triazines.randomSplit([0.85, 0.15], seed=1)_____no_output_____

#### Model Training_____no_output_____

from synapse.ml.vw import VowpalWabbitRegressor
model = (VowpalWabbitRegressor(numPasses=20, args="--holdout_off --loss_function quantile -q :: -l 0.1")
            .fit(train))_____no_output_____

#### Model Prediction_____no_output_____

scoredData = model.transform(test)
display(scoredData.limit(10))_____no_output_____from synapse.ml.train import ComputeModelStatistics
metrics = ComputeModelStatistics(evaluationMetric='regression',
                                 labelCol='label',
                                 scoresCol='prediction') \
            .transform(scoredData)
display(metrics)_____no_output_____

## VW Contextual Bandit_____no_output_____#### Read dataset_____no_output_____

data = spark.read.format("json").load("wasbs://publicwasb@mmlspark.blob.core.windows.net/vwcb_input.dsjson")_____no_output_____

Note: Actions are all five TAction_x_topic columns._____no_output_____

from pyspark.sql.functions import col
from pyspark.sql.types import IntegerType, DoubleType
data = data.withColumn('GUser_id', col('c.GUser.id'))\
  .withColumn('GUser_major', col('c.GUser.major'))\
  .withColumn('GUser_hobby', col('c.GUser.hobby'))\
  .withColumn('GUser_favorite_character', col('c.GUser.favorite_character'))\
  .withColumn('TAction_0_topic', col('c._multi.TAction.topic')[0])\
  .withColumn('TAction_1_topic', col('c._multi.TAction.topic')[1])\
  .withColumn('TAction_2_topic', col('c._multi.TAction.topic')[2])\
  .withColumn('TAction_3_topic', col('c._multi.TAction.topic')[3])\
  .withColumn('TAction_4_topic', col('c._multi.TAction.topic')[4])\
  .withColumn('chosenAction', col('_label_Action').cast(IntegerType()))\
  .withColumn('label', col('_labelIndex').cast(DoubleType()))\
  .withColumn('probability', col('_label_probability'))\
  .select('GUser_id', 'GUser_major', 'GUser_hobby', 'GUser_favorite_character', 'TAction_0_topic', 'TAction_1_topic', 'TAction_2_topic', 'TAction_3_topic', 'TAction_4_topic', 'chosenAction', 'label', 'probability')

print("Schema: ")
data.printSchema()_____no_output_____

Add pipeline to add featurizer, convert all feature columns into vector._____no_output_____

from synapse.ml.vw import VowpalWabbitFeaturizer, VowpalWabbitContextualBandit, VectorZipper
from pyspark.ml import Pipeline
pipeline = Pipeline(stages=[
  VowpalWabbitFeaturizer(inputCols=['GUser_id'], outputCol='GUser_id_feature'),
  VowpalWabbitFeaturizer(inputCols=['GUser_major'], outputCol='GUser_major_feature'),
  VowpalWabbitFeaturizer(inputCols=['GUser_hobby'], outputCol='GUser_hobby_feature'),
  VowpalWabbitFeaturizer(inputCols=['GUser_favorite_character'], outputCol='GUser_favorite_character_feature'),
  VowpalWabbitFeaturizer(inputCols=['TAction_0_topic'], outputCol='TAction_0_topic_feature'),
  VowpalWabbitFeaturizer(inputCols=['TAction_1_topic'], outputCol='TAction_1_topic_feature'),
  VowpalWabbitFeaturizer(inputCols=['TAction_2_topic'], outputCol='TAction_2_topic_feature'),
  VowpalWabbitFeaturizer(inputCols=['TAction_3_topic'], outputCol='TAction_3_topic_feature'),
  VowpalWabbitFeaturizer(inputCols=['TAction_4_topic'], outputCol='TAction_4_topic_feature'),
  VectorZipper(inputCols=['TAction_0_topic_feature', 'TAction_1_topic_feature', 'TAction_2_topic_feature', 'TAction_3_topic_feature','TAction_4_topic_feature'], outputCol='features')
])
tranformation_pipeline = pipeline.fit(data)
transformed_data = tranformation_pipeline.transform(data)

display(transformed_data)_____no_output_____

Build VowpalWabbit Contextual Bandit model and compute performance statistics._____no_output_____

estimator = VowpalWabbitContextualBandit() \
    .setArgs("--cb_explore_adf --epsilon 0.2 --quiet") \
    .setSharedCol('GUser_id_feature') \
    .setAdditionalSharedFeatures(["GUser_major_feature", "GUser_hobby_feature", "GUser_favorite_character_feature"]) \
    .setFeaturesCol('features') \
    .setUseBarrierExecutionMode(False)\
    .setChosenActionCol('chosenAction')\
    .setLabelCol('label')\
    .setProbabilityCol('probability')
model = estimator.fit(transformed_data)
display(model.getPerformanceStatistics())_____no_output_____


# Notebook from paulsbrookes/quimb
Path: docs/tensor-algorithms.ipynb

%matplotlib inline
from quimb.tensor import *_____no_output_____p = MPS_rand_state(n=20, bond_dim=50)
print("Site tags: '{}', site inds: '{}'".format(p.site_tag_id, p.site_ind_id))Site tags: 'I{}', site inds: 'k{}'
print(p)  # shows the full list of constituent tensorsMatrixProductState([
    Tensor(shape=(50, 2), inds=('_2ffccc0000000', 'k0'), tags={'I0'}),
    Tensor(shape=(50, 50, 2), inds=('_2ffccc0000000', '_2ffccc0000002', 'k1'), tags={'I1'}),
    Tensor(shape=(50, 50, 2), inds=('_2ffccc0000002', '_2ffccc0000004', 'k2'), tags={'I2'}),
    Tensor(shape=(50, 50, 2), inds=('_2ffccc0000004', '_2ffccc0000006', 'k3'), tags={'I3'}),
    Tensor(shape=(50, 50, 2), inds=('_2ffccc0000006', '_2ffccc0000008', 'k4'), tags={'I4'}),
    Tensor(shape=(50, 50, 2), inds=('_2ffccc0000008', '_2ffccc000000a', 'k5'), tags={'I5'}),
    Tensor(shape=(50, 50, 2), inds=('_2ffccc000000a', '_2ffccc000000c', 'k6'), tags={'I6'}),
    Tensor(shape=(50, 50, 2), inds=('_2ffccc000000c', '_2ffccc000000e', 'k7'), tags={'I7'}),
    Tensor(shape=(50, 50, 2), inds=('_2ffccc000000e', '_2ffccc000000A', 'k8'), tags={'I8'}),
    Tensor(shape=(50, 50, 2), inds=('_2ffccc000000A', '_2ffccc000000C', 'k9'), tags={'I9'}),
    Tensor(shape=(50, 50, 2), inds=('_2ffccc000000C', '_2ffccc000000E', 'k10'), tags={'I10'}),
    Tensor(shape=(50, 50, 2), inds=('_2ffccc000000E', '_2ffccc0000010', 'k11'), tags={'I11'}),
    Tensor(shape=(50, 50, 2), inds=('_2ffccc0000010', '_2ffccc0000012', 'k12'), tags={'I12'}),
    Tensor(shape=(50, 50, 2), inds=('_2ffccc0000012', '_2ffccc0000014', 'k13'), tags={'I13'}),
    Tensor(shape=(50, 50, 2), inds=('_2ffccc0000014', '_2ffccc0000016', 'k14'), tags={'I14'}),
    Tensor(shape=(50, 50, 2), inds=('_2ffccc0000016', '_2ffccc0000018', 'k15'), tags={'I15'}),
    Tensor(shape=(50, 50, 2), inds=('_2ffccc0000018', '_2ffccc000001a', 'k16'), tags={'I16'}),
    Tensor(shape=(50, 50, 2), inds=('_2ffccc000001a', '_2ffccc000001c', 'k17'), tags={'I17'}),
    Tensor(shape=(50, 50, 2), inds=('_2ffccc000001c', '_2ffccc000001e', 'k18'), tags={'I18'}),
    Tensor(shape=(50, 2), inds=('_2ffccc000001e', 'k19'), tags={'I19'}),
], structure='I{}', nsites=20)
p.show()  # 1D tensor networks also have a ascii ``show`` method 50 50 50 50 50 50 50 50 50 50 50 50 50 50 50 50 50 50 50
o--o--o--o--o--o--o--o--o--o--o--o--o--o--o--o--o--o--o--o
|  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |

We can then __canonicalize__ the MPS:_____no_output_____

p.left_canonize()
p.show() 2 4 8 16 32 50 50 50 50 50 50 50 50 50 50 50 50 50 50
>->->->-->-->-->-->-->-->-->-->-->-->-->-->-->-->-->--o
| | | |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |

And we can compute the inner product as:_____no_output_____

p.H @ p_____no_output_____

This relies on them sharing the same physical indices, ``site_ind_id``,
which the conjugated copy ``p.H`` naturally does.

Like any TN, we can __graph__ the overlap for example, and make use of the
site tags to color it:_____no_output_____

(p.H & p).graph(color=[f'I{i}' for i in range(30)])_____no_output_____

p2 = (p + p) / 2
p2.show() 4 8 16 32 64 100 100 100 100 100 100 100 100 100 100 100 100 100 100
o-o-o--o--o--o===o===o===o===o===o===o===o===o===o===o===o===o===o===o
| | |  |  |  |   |   |   |   |   |   |   |   |   |   |   |   |   |   |

Which doubles the bond dimension, as expected, but should still be normalized:_____no_output_____

p2.H @ p2_____no_output_____

Because the MPS is the addition of two identical states, it should also __compress__ right back down:_____no_output_____

p2.compress(form=10)
p2.show() 2 4 8 16 32 50 50 50 50 50 50 50 50 50 32 16 8 4 2
>->->->-->-->-->-->-->-->--o--<--<--<--<--<--<-<-<-<
| | | |  |  |  |  |  |  |  |  |  |  |  |  |  | | | |

Where we have also set the __orthogonality center__ at the site 10.

When tensor networks are imbued with a ``structure``, they
can be indexed with integers and slices, which automatically get
converted using ``TN.site_tag_id``:_____no_output_____

p2[10]  # get the tensor(s) with tag 'I10'._____no_output_____

Note the tensor has matching physical index ``'k10'``.

This tensor is the orthogonality center so:

       ->->-O-<-<-        +-O-+
    ... | | | | | ...  =  | | |
       ->->-O-<-<-        +-O-+
           i=10            i=10

should compute the normalization of the *whole* state:_____no_output_____

p2[10].H @ p2[10]  # all indices match -> inner product_____no_output_____

Or equivalently:_____no_output_____

p2[10].norm()_____no_output_____

If two tensor networks with the same ``structure`` are combined, it is propagated.
For example ``(p2.H & p2)`` can still be sliced.

Since the MPS is in canonical form, left and right pieces of the overlap
should form the identity. The following forms a TN of the inner product,
selects the *2* tensors corresponding to the last site (``-1``), contracts them,
then gets the underlying data:_____no_output_____

((p2.H & p2).select(-1) ^ all).data  # should be close to the identity_____no_output_____

A = MPO_rand_herm(20, bond_dim=7, tags=['HAM'])
pH = p.H

# This inplace modifies the indices of each to form overlap
p.align_(A, pH)

(pH & A & p).graph(color='HAM', iterations=1000)_____no_output_____

Compute the actual contraction (``...`` means contract everything, but use the structure if possible):_____no_output_____

(pH & A & p) ^ ..._____no_output_____

builder = SpinHam(S=1)
builder += 1/2, '+', '-'
builder += 1/2, '-', '+'
builder += 1, 'Z', 'Z'
H = builder.build_mpo(n=100)_____no_output_____

dmrg = DMRG2(H, bond_dims=[10, 20, 100, 100, 200], cutoffs=1e-10)_____no_output_____

The ``DMRG`` object will automatically detect OBC/PBC. Now we can solve to a certain absolute energy tolerance, showing progress and a schematic of the final state:_____no_output_____

dmrg.solve(tol=1e-6, verbosity=1)SWEEP-1, direction=R, max_bond=10, cutoff:1e-10
dmrg.state.show(max_width=80)     3 9 27 54 65 72 79 85 88 89 91 93 94 95 95 95 95 95 95 94 93 93 92 92
    >->->-->-->-->-->-->-->-->-->-->-->-->-->-->-->-->-->-->-->-->-->-->-- ...
    | | |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |
                                 ...
     92 91 91 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90
... >-->-->-->-->-->-->-->-->-->-->-->-->-->-->-->-->-->-->-->-->-->-->--> ...
    |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |
                                 ...
    90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 90 9
... -->-->-->-->-->-->-->-->-->-->-->-->-->-->-->-->-->-->-->-->-->-->-->- ...
      |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |
                                 ...
    0 90 90 90 90 90 90 90 91 92 94 97 98 98 98 97 96 94 93 90 87 83 78 73
... ->-->-->-->-->-->-->-->-->-->-->-->-->-->-->-->-->-->-->-->-->-->-->-- ...
     |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |  |
                                 ...
     64 53 27 9 3
... >-->-->-->->-o
    |  |  |  | | |

builder = SpinHam(S=1 / 2)
builder.add_term(1.0, 'Z', 'Z')
builder.add_term(0.9, 'Y', 'Y')
builder.add_term(0.8, 'X', 'X')
builder.add_term(0.6, 'Z')

H = NNI_ham_heis(20, bz=0.1)

# check the two site term
H()_____no_output_____

psi0 = MPS_neel_state(20)
tebd = TEBD(psi0, H)_____no_output_____

Now we are ready to evolve. By setting a ``tol``, the required timestep ``dt`` is computed for us:_____no_output_____

tebd.update_to(T=3, tol=1e-3)t=3, max-bond=34: 100%|██████████| 100/100 [00:01<00:00, 40.93%/s]

After the evolution we can see that entanglement has been generated throughout the chain:_____no_output_____

tebd.pt.show() 2 4 8 16 29 34 33 34 33 34 33 34 33 34 29 16 8 4 2
>->->->-->-->-->-->-->-->-->-->-->-->-->-->-->->->-o
| | | |  |  |  |  |  |  |  |  |  |  |  |  |  | | | |

import quimb as qu
Z = qu.pauli('Z')

# compute <psi0|Z_i|psi0> for neel state above
[
    psi0.gate(Z, i).H @ psi0
    for i in range(10)
]_____no_output_____

import quimb as qu

# some operators to apply
H = qu.hadamard()
CNOT = qu.controlled('not')

# setup an intitial register of qubits
n = 10
psi0 = MPS_computational_state('0' * n, tags='PSI0')

# apply hadamard to each site
for i in range(n):
    psi0.gate_(H, i, tags='H')

# apply CNOT to even pairs
for i in range(0, n, 2):
    psi0.gate_(CNOT, (i, i + 1), tags='CNOT')

# apply CNOT to odd pairs
for i in range(1, n - 1, 2):
    psi0.gate_(CNOT, (i, i + 1), tags='CNOT')_____no_output_____

Note we have used the inplace ``gate_`` (with a trailing
underscore) which modifies the original ``psi0`` object.
However ``psi0`` has its physical site indices mantained
such that it overall looks like the same object:_____no_output_____

sorted(psi0.outer_inds())_____no_output_____(psi0.H & psi0) ^ all_____no_output_____

But the network now contains the gates as additional tensors:_____no_output_____

psi0.graph(color=['PSI0', 'H', 'CNOT'], show_inds=True)_____no_output_____

With the swap and split method MPS form is always maintained, which
allows a canonical form and thus optimal trimming of singular values:_____no_output_____

n = 10
psi0 = MPS_computational_state('0' * n)

for i in range(n):
    # 'swap+split' will be ignore to one-site gates
    psi0.gate_(H, i, contract='swap+split')

# use Z-phase to create entanglement
Rz = qu.phase_gate(0.42)
for i in range(n):
    psi0.gate_(Rz, i, contract='swap+split')

for i in range(0, n, 2):
    psi0.gate_(CNOT, (i, i + 1), contract='swap+split')

for i in range(1, n - 1, 2):
    psi0.gate_(CNOT, (i, i + 1), contract='swap+split')

# act with one long-range CNOT
psi0.gate_(CNOT, (2, n - 2), contract='swap+split')_____no_output_____

We now still have an MPS, but with increased bond dimension:_____no_output_____

psi0.show() 2 2 4 4 4 4 4 2 2
>->->->->->->-o-o-<
| | | | | | | | | |

Finally, the eager (``contract=True``) method works fairly simply:_____no_output_____

psi0_CNOT = psi0.gate(CNOT, (1, n -2 ), contract=True)
psi0_CNOT.graph(color=[psi0.site_tag(i) for i in range(n)])_____no_output_____

Where we can see that the gate, site 1, and site 8 have been combined
into a new rank-6 tensor._____no_output_____


# Notebook from araman18/Summer-2020-Research
Path: Added Demos(ipynb)/demoPart1.ipynb

https://docs.google.com/document/d/1jo1R88kBe7U1PmCbhRvcneADTL0yc3EAcQABYuUEhfc/edit# Markup
De Novo Drug Design https://advances.sciencemag.org/content/4/7/eaap7885
# Reinforcement Learning For Structural Evolution.
This experiement ReLeaSe has two significant parts.
## PART 1 - Use a StackRNN to transform chemical formulas for compounds into a SMILE representation,
* SMILE is just a String ecnoding for this chemical formula that makes it easier to perform the second part of the experiment.

* A StackRNN which is a variant of an RNN is used to convert any new chemical formula into the SMILE version.

* An example can be seen in the 2 strings below converted from Carbon monoxide and carbon.

CO->CHEMBL14688

C->CHEMBL17564

* This conversion is especially useful when regarding formulas with double and triple bonds,like below.

CC(O)=O->CHEMBL539

CC(N)=S

* This property is especially useful in this use case since most of the formulas that will be modified are goingto be complex and thus a system like SMILE is used effectively.

## PART 2- Training a generative and predictive model together and use reinforcement learning to bias towards desired properties in the generated compounds.
* In this part of the experiment the valid SMILE strings are fed into a LSTM (Long short term memory network).
* There will be more elaboration about this in the future since there is a lot to say.
* The graphic below is a good description of the experiment flow taken from https://github.com/isayev/ReLeaSE
![image.png](attachment:image.png)_____no_output_____

!git clone https://github.com/isayev/ReLeaSE.git
%cd ReLeaSEfatal: destination path 'ReLeaSE' already exists and is not an empty directory.
/content/ReLeaSE
!wget -c https://repo.anaconda.com/miniconda/Miniconda3-4.5.4-Linux-x86_64.sh
!chmod +x Miniconda3-4.5.4-Linux-x86_64.sh
!bash ./Miniconda3-4.5.4-Linux-x86_64.sh -b -f -p /usr/local
!conda create -c rdkit -n my-rdkit-env rdkit python=3.6
!source activate my-rdkit-env
!conda install --yes --file conda_requirements.txt
!conda install -c rdkit rdkit nox cairo
!conda install pytorch
import sys
sys.path.append('/usr/local/lib/python3.6/site-packages')
!pip install  -r pip_requirements.txt--2020-06-02 23:02:50--  https://repo.anaconda.com/miniconda/Miniconda3-4.5.4-Linux-x86_64.sh
Resolving repo.anaconda.com (repo.anaconda.com)... 104.16.130.3, 104.16.131.3, 2606:4700::6810:8203, ...
Connecting to repo.anaconda.com (repo.anaconda.com)|104.16.130.3|:443... connected.
HTTP request sent, awaiting response... 416 Requested Range Not Satisfiable

    The file is already fully retrieved; nothing to do.

PREFIX=/usr/local
installing: python-3.6.5-hc3d631a_2 ...
Python 3.6.5 :: Anaconda, Inc.
installing: ca-certificates-2018.03.07-0 ...
installing: conda-env-2.6.0-h36134e3_1 ...
installing: libgcc-ng-7.2.0-hdf63c60_3 ...
installing: libstdcxx-ng-7.2.0-hdf63c60_3 ...
installing: libffi-3.2.1-hd88cf55_4 ...
installing: ncurses-6.1-hf484d3e_0 ...
installing: openssl-1.0.2o-h20670df_0 ...
installing: tk-8.6.7-hc745277_3 ...
installing: xz-5.2.4-h14c3975_4 ...
installing: yaml-0.1.7-had09818_2 ...
installing: zlib-1.2.11-ha838bed_2 ...
installing: libedit-3.1.20170329-h6b74fdf_2 ...
installing: readline-7.0-ha6073c6_4 ...
installing: sqlite-3.23.1-he433501_0 ...
installing: asn1crypto-0.24.0-py36_0 ...
installing: certifi-2018.4.16-py36_0 ...
installing: chardet-3.0.4-py36h0f667ec_1 ...
installing: idna-2.6-py36h82fb2a8_1 ...
installing: pycosat-0.6.3-py36h0a5515d_0 ...
installing: pycparser-2.18-py36hf9f622e_1 ...
installing: pysocks-1.6.8-py36_0 ...
installing: ruamel_yaml-0.15.37-py36h14c3975_2 ...
installing: six-1.11.0-py36h372c433_1 ...
installing: cffi-1.11.5-py36h9745a5d_0 ...
installing: setuptools-39.2.0-py36_0 ...
installing: cryptography-2.2.2-py36h14c3975_0 ...
installing: wheel-0.31.1-py36_0 ...
installing: pip-10.0.1-py36_0 ...
installing: pyopenssl-18.0.0-py36_0 ...
installing: urllib3-1.22-py36hbe7ace6_0 ...
installing: requests-2.18.4-py36he2e5f8d_1 ...
installing: conda-4.5.4-py36_0 ...
unlinking: ca-certificates-2020.1.1-0
unlinking: certifi-2020.4.5.1-py36_0
unlinking: conda-4.8.3-py36_0
unlinking: cryptography-2.9.2-py36h1ba5d50_0
unlinking: libgcc-ng-9.1.0-hdf63c60_0
unlinking: libstdcxx-ng-9.1.0-hdf63c60_0
unlinking: openssl-1.1.1g-h7b6447c_0
unlinking: python-3.6.8-h0371630_0
unlinking: setuptools-46.4.0-py36_0
unlinking: sqlite-3.26.0-h7b6447c_0
unlinking: tk-8.6.8-hbc83047_0
installation finished.
WARNING:
    You currently have a PYTHONPATH environment variable set. This may cause
    unexpected behavior when running the Python interpreter in Miniconda3.
    For best results, please verify that your PYTHONPATH only points to
    directories of packages that are compatible with the Python interpreter
    in Miniconda3: /usr/local

CondaValueError: prefix already exists: /usr/local/envs/my-rdkit-env

Solving environment: - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - done

## Package Plan ##

  environment location: /usr/local

  added / updated specs:
    - ipykernel
    - ipython
    - numpy
    - pytest
    - pytest-cov
    - scikit-learn==0.20.2
    - scipy
    - seaborn==0.9.0

The following packages will be downloaded:

    package                    |            build
    ---------------------------|-----------------
    cudatoolkit-8.0            |                3       322.4 MB
    cudnn-7.1.3                |        cuda8.0_0       229.6 MB
    ------------------------------------------------------------
                                           Total:       552.0 MB

The following packages will be UPDATED:

    ca-certificates: 2018.03.07-0         --> 2020.1.1-0
    certifi:         2018.4.16-py36_0     --> 2020.4.5.1-py36_0
    conda:           4.5.4-py36_0         --> 4.8.3-py36_0
    cryptography:    2.2.2-py36h14c3975_0 --> 2.9.2-py36h1ba5d50_0
    libgcc-ng:       7.2.0-hdf63c60_3     --> 9.1.0-hdf63c60_0
    libstdcxx-ng:    7.2.0-hdf63c60_3     --> 9.1.0-hdf63c60_0
    openssl:         1.0.2o-h20670df_0    --> 1.1.1g-h7b6447c_0
    python:          3.6.5-hc3d631a_2     --> 3.6.8-h0371630_0
    setuptools:      39.2.0-py36_0        --> 46.4.0-py36_0
    sqlite:          3.23.1-he433501_0    --> 3.26.0-h7b6447c_0
    tk:              8.6.7-hc745277_3     --> 8.6.8-hbc83047_0

The following packages will be DOWNGRADED:

    cudatoolkit:     10.1.243-h6bb024c_0  --> 8.0-3
    cudnn:           7.6.5-cuda10.1_0     --> 7.1.3-cuda8.0_0

Downloading and Extracting Packages
cudatoolkit-8.0      | 322.4 MB | : 100% 1.0/1 [00:36<00:00, 36.49s/it]
cudnn-7.1.3          | 229.6 MB | : 100% 1.0/1 [00:27<00:00, 27.64s/it]
Preparing transaction: | / done
Verifying transaction: \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / - \ | / done
Executing transaction: \ | / - done
Collecting package metadata (current_repodata.json): - \ | / - \ | / - \ | / - \ | / - \ | / done
Solving environment: \ | / - \ | / - \ | / - \ | / - \ | / - \ | /
The environment is inconsistent, please check the package plan carefully
The following packages are causing the inconsistency:

  - defaults/linux-64::pytorch==1.4.0=cuda101py36h02f0884_0
- \ | / - \ | / - \ | / - \ | / - \ done

## Package Plan ##

  environment location: /usr/local

  added / updated specs:
    - cairo
    - nox
    - rdkit

The following packages will be UPDATED:

  cudatoolkit                  pkgs/free::cudatoolkit-8.0-3 --> pkgs/main::cudatoolkit-10.1.243-h6bb024c_0
  cudnn                                     7.1.3-cuda8.0_0 --> 7.6.5-cuda10.1_0

Proceed ([y]/n)? y

Preparing transaction: / done
Verifying transaction: \ done
Executing transaction: / done
Collecting package metadata (current_repodata.json): - \ | / - \ done
Solving environment: / - \ | / - \ | / - \ | / - \ | / - \ | / - \ done

# All requested packages already installed.

Collecting mordred==1.1.2 (from -r pip_requirements.txt (line 1))
  Using cached https://files.pythonhosted.org/packages/80/8d/70f26c38ba4bd3bda46791bef71b67bcb7442d6671f66459ad8feaa8eb22/mordred-1.1.2.tar.gz
Collecting tensorboard==1.10.0 (from -r pip_requirements.txt (line 2))
  Using cached https://files.pythonhosted.org/packages/c6/17/ecd918a004f297955c30b4fffbea100b1606c225dbf0443264012773c3ff/tensorboard-1.10.0-py3-none-any.whl
Collecting tqdm==4.26.0 (from -r pip_requirements.txt (line 3))
  Using cached https://files.pythonhosted.org/packages/79/43/19c9fee28110cd47f73e6bc596394337fe9f3e5825b4de402bbf30b3beb5/tqdm-4.26.0-py2.py3-none-any.whl
Collecting xgboost==0.81 (from -r pip_requirements.txt (line 4))
  Using cached https://files.pythonhosted.org/packages/54/21/8b2ec99862903a6d3aed62ce156d21d114b8666e669c46d9e54041df9496/xgboost-0.81-py2.py3-none-manylinux1_x86_64.whl
Collecting tensorflow==1.13.0 (from -r pip_requirements.txt (line 5))
  Could not find a version that satisfies the requirement tensorflow==1.13.0 (from -r pip_requirements.txt (line 5)) (from versions: 0.12.1, 1.0.0, 1.0.1, 1.1.0rc0, 1.1.0rc1, 1.1.0rc2, 1.1.0, 1.2.0rc0, 1.2.0rc1, 1.2.0rc2, 1.2.0, 1.2.1, 1.3.0rc0, 1.3.0rc1, 1.3.0rc2, 1.3.0, 1.4.0rc0, 1.4.0rc1, 1.4.0, 1.4.1, 1.5.0rc0, 1.5.0rc1, 1.5.0, 1.5.1, 1.6.0rc0, 1.6.0rc1, 1.6.0, 1.7.0rc0, 1.7.0rc1, 1.7.0, 1.7.1, 1.8.0rc0, 1.8.0rc1, 1.8.0, 1.9.0rc0, 1.9.0rc1, 1.9.0rc2, 1.9.0, 1.10.0rc0, 1.10.0rc1, 1.10.0, 1.10.1, 1.11.0rc0, 1.11.0rc1, 1.11.0rc2, 1.11.0, 1.12.0rc0, 1.12.0rc1, 1.12.0rc2, 1.12.0, 1.12.2, 1.12.3, 1.13.0rc0, 1.13.0rc1, 1.13.0rc2, 1.13.1, 1.13.2, 1.14.0rc0, 1.14.0rc1, 1.14.0, 2.0.0a0, 2.0.0b0, 2.0.0b1)
No matching distribution found for tensorflow==1.13.0 (from -r pip_requirements.txt (line 5))
Traceback (most recent call last):
  File "/usr/local/lib/python3.6/runpy.py", line 183, in _run_module_as_main
    mod_name, mod_spec, code = _get_module_details(mod_name, _Error)
  File "/usr/local/lib/python3.6/runpy.py", line 142, in _get_module_details
    return _get_module_details(pkg_main_name, error)
  File "/usr/local/lib/python3.6/runpy.py", line 109, in _get_module_details
    __import__(pkg_name)
  File "/usr/local/lib/python3.6/site-packages/ipykernel/__init__.py", line 2, in <module>
    from .connect import *
  File "/usr/local/lib/python3.6/site-packages/ipykernel/connect.py", line 13, in <module>
    from IPython.core.profiledir import ProfileDir
  File "/usr/local/lib/python3.6/site-packages/IPython/__init__.py", line 55, in <module>
    from .core.application import Application
  File "/usr/local/lib/python3.6/site-packages/IPython/core/application.py", line 23, in <module>
    from traitlets.config.application import Application, catch_config_error
  File "/usr/local/lib/python3.6/site-packages/traitlets/config/__init__.py", line 6, in <module>
    from .application import *
  File "/usr/local/lib/python3.6/site-packages/traitlets/config/application.py", line 17, in <module>
    from decorator import decorator
ModuleNotFoundError: No module named 'decorator'
!python -m ipykernel install --user --name release --display-name ReLeaSE
import sys
sys.path.append('./release/')
Traceback (most recent call last):
  File "/usr/local/lib/python3.6/runpy.py", line 183, in _run_module_as_main
    mod_name, mod_spec, code = _get_module_details(mod_name, _Error)
  File "/usr/local/lib/python3.6/runpy.py", line 142, in _get_module_details
    return _get_module_details(pkg_main_name, error)
  File "/usr/local/lib/python3.6/runpy.py", line 109, in _get_module_details
    __import__(pkg_name)
  File "/usr/local/lib/python3.6/site-packages/ipykernel/__init__.py", line 2, in <module>
    from .connect import *
  File "/usr/local/lib/python3.6/site-packages/ipykernel/connect.py", line 13, in <module>
    from IPython.core.profiledir import ProfileDir
  File "/usr/local/lib/python3.6/site-packages/IPython/__init__.py", line 55, in <module>
    from .core.application import Application
  File "/usr/local/lib/python3.6/site-packages/IPython/core/application.py", line 23, in <module>
    from traitlets.config.application import Application, catch_config_error
  File "/usr/local/lib/python3.6/site-packages/traitlets/config/__init__.py", line 6, in <module>
    from .application import *
  File "/usr/local/lib/python3.6/site-packages/traitlets/config/application.py", line 17, in <module>
    from decorator import decorator
ModuleNotFoundError: No module named 'decorator'
import torch
import torch.nn as nn
from torch.optim.lr_scheduler import ExponentialLR, StepLR
import torch.nn.functional as F_____no_output_____use_cuda = torch.cuda.is_available()_____no_output_____import numpy as np
from tqdm import tqdm, trange
import pickle
from rdkit import Chem, DataStructs
from release.stackRNN import StackAugmentedRNN
from release.data import GeneratorData
from release.utils import canonical_smiles

import matplotlib.pyplot as plt
%matplotlib inline

import seaborn as sns/usr/local/lib/python3.6/dist-packages/statsmodels/tools/_testing.py:19: FutureWarning: pandas.util.testing is deprecated. Use the functions in the public API at pandas.testing instead.
  import pandas.util.testing as tm

# Explaining RNN's and the StackRNN used in this case
* An RNN is a type of neural network more akin to understanding and predicting sequential information such as stock market quotes.
* This is different from convolutional Neural networks which understand spatial input better.
* In this case the RNN is tasked with taking in past mappings of chemical formulas to SMILE representations and outputting SMILEs for any future inputs.
* Processing text data as input is a popular use for RNN which is what will be done in this case, break the input into sequences of characters
* Leverages the concept of sequential memory._____no_output_____

gen_data_path = './data/testingdata.smi'
tokens = ['<', '>', '#', '%', ')', '(', '+', '-', '/', '.', '1', '0', '3', '2', '5', '4', '7',
          '6', '9', '8', '=', 'A', '@', 'C', 'B', 'F', 'I', 'H', 'O', 'N', 'P', 'S', '[', ']',
          '\\', 'c', 'e', 'i', 'l', 'o', 'n', 'p', 's', 'r', '\n']
gen_data = GeneratorData(training_data_path=gen_data_path, delimiter='\t',
                         cols_to_read=[0], keep_header=True, tokens=tokens)_____no_output_____def plot_hist(prediction, n_to_generate):
    print("Mean value of predictions:", prediction.mean())
    print("Proportion of valid SMILES:", len(prediction)/n_to_generate)
    ax = sns.kdeplot(prediction, shade=True)
    ax.set(xlabel='Predicted pIC50',
           title='Distribution of predicted pIC50 for generated molecules')
    plt.show()_____no_output_____def estimate_and_update(generator, predictor, n_to_generate, **kwargs):
    generated = []
    pbar = tqdm(range(n_to_generate))
    for i in pbar:
        pbar.set_description("Generating molecules...")
        generated.append(generator.evaluate(gen_data, predict_len=120)[1:-1])

    sanitized = canonical_smiles(generated, sanitize=False, throw_warning=False)[:-1]
    unique_smiles = list(np.unique(sanitized))[1:]
    smiles, prediction, nan_smiles = predictor.predict(unique_smiles, get_features=get_fp)

    plot_hist(prediction, n_to_generate)

    return smiles, prediction_____no_output_____hidden_size = 1500
stack_width = 1500
stack_depth = 200
layer_type = 'GRU'
lr = 0.001
optimizer_instance = torch.optim.Adadelta

my_generator = StackAugmentedRNN(input_size=gen_data.n_characters, hidden_size=hidden_size,
                                 output_size=gen_data.n_characters, layer_type=layer_type,
                                 n_layers=1, is_bidirectional=False, has_stack=True,
                                 stack_width=stack_width, stack_depth=stack_depth,
                                 use_cuda=use_cuda,
                                 optimizer_instance=optimizer_instance, lr=lr)
print(my_generator)StackAugmentedRNN(
  (stack_controls_layer): Linear(in_features=1500, out_features=3, bias=True)
  (stack_input_layer): Linear(in_features=1500, out_features=1500, bias=True)
  (encoder): Embedding(45, 1500)
  (rnn): GRU(3000, 1500)
  (decoder): Linear(in_features=1500, out_features=45, bias=True)
  (log_softmax): LogSoftmax()
  (criterion): CrossEntropyLoss()
)
model_path = './checkpoints/generator/checkpoint_biggest_rnn'
my_generator.load_model(model_path)_____no_output__________no_output_____

## How a typical RNN works
* Image of a RNN below
![image.png](attachment:image.png)
* This unrolled network shows how an RNN uses previous information to guess values in the future.
* Activation value from previous timesteps is passed on to the next to help in the prediction, the only timestep that doesn't use previous info is timestep 1.
* A(n) or the activation passed on from the nth layer is calculated A(n) = g(waaa(n-1) + waxx(n-1) + ba)
* In this equation waa represents parameters passed between layers wax represents parameters of the input, ba represents a bias and g is the actiavtion function. tanh.Relu
* <hat>yhat</hat>n or the nework output for a specific layer is computed as $\hat{y}$n = g(wyaa(n)  + by). However this function uses a different activation function as the one which computes the actiavtion passed between layers - Sigmoid.
* It can be seen from these equations that RNN's use a dynamic programming approach to try to deal with sequential data calcualting y(N) from  y(n-1) y(n-2) etc, but the values that are closer in sequence to n tend to have more weight because of the vanishing gradient problem.
* Each time step is a layer.
* One problem here lies in the backpropogation method where in an RNN the earlier layers essentially learn less since the weights are changed less and less as the layers go further back in time.Vanishing gradient problem1
* RNN suffer from short term memory, mitigated by use of LSTM's and GRU's but they can learn long term dependancies using gates, knowing what to add to the hidden state both long term and short term._____no_output_____
## How the StackRNN works
* This experiement uses a Stack Augmented RNN in place of a normal RNN in order to convert formulas into smile strings.
* The goal is to be able to determine the pattern in forming sequences of letters that correspond to valid SMILE strings.
### What is different between a StackRNN and regular RNN's like GRU and LSTM.
* Normal RNN's like GRU's and LSTM's suffer from an inability to count, the most basic example of this is the Dyck language where a Dyck word must have balanced parentheses, but such words would not be able to be generated by a GRU or LSTM since they lack the memory to count parentheses and determine if it is balanced.
* Similarly SMILE strings must have properly matched parentheses with multiple types of brackers, so the RNN that is used for SMILES must have memory.
* Additionally, the rings of molecules muyst have correct valence for all atoms so we need stack memory to count that.
* Another issue of normal RNN's but not GRU's is the short term memory issue that is why in this experiment we use the GRU with a Stack which solves the problem of short term memory and counting.
* GRU's solve the short term memory problem by using gates, that regulate information flow, instead of just passing on the last few bits of information from the last few layers, the gates can learn what sequences of information are important whether they be from recently or a long time ago in the timestamps.
* The way this experiment integrates GRU's and a Stack is shown below.
![image.png](attachment:image.png)*
* The Stack RNN has a new neuron or cell strucutre on top of the GRU cell, and has 2 additional multiplicative gates which act as a memory stack, and this allows it to learn dependancies in the long term.
* The StackRNN also uses the SoftMax function as loss. This loss function provides a set of probabilities that adds up to one.
* This function is able to map the probability ditribution to the ouput varieties or in this case which values/weights to adjust.
* This is a good way of reducing the negative impact of the vanishing gradient problem since with this probabilistic approach to weights and value adjustment the lower levels or layers with smaller timestamps will not be neglected in modification._____no_output_____

!git clone --single-branch --branch develop https://github.com/Mariewelt/OpenChem.gitCloning into 'OpenChem'...
remote: Enumerating objects: 1561, done.
remote: Total 1561 (delta 0), reused 0 (delta 0), pack-reused 1561
Receiving objects: 100% (1561/1561), 108.66 MiB | 31.28 MiB/s, done.
Resolving deltas: 100% (789/789), done.
!pip install -r pip_requirements.txt
import sys
sys.path.append('/usr/local/lib/python3.6/site-packages')
from release.data import PredictorData
from release.utils import get_desc, get_fp
from mordred import Calculator, descriptors
Collecting mordred==1.1.2 (from -r pip_requirements.txt (line 1))
  Using cached https://files.pythonhosted.org/packages/80/8d/70f26c38ba4bd3bda46791bef71b67bcb7442d6671f66459ad8feaa8eb22/mordred-1.1.2.tar.gz
Collecting tensorboard==1.10.0 (from -r pip_requirements.txt (line 2))
  Using cached https://files.pythonhosted.org/packages/c6/17/ecd918a004f297955c30b4fffbea100b1606c225dbf0443264012773c3ff/tensorboard-1.10.0-py3-none-any.whl
Collecting tqdm==4.26.0 (from -r pip_requirements.txt (line 3))
  Using cached https://files.pythonhosted.org/packages/79/43/19c9fee28110cd47f73e6bc596394337fe9f3e5825b4de402bbf30b3beb5/tqdm-4.26.0-py2.py3-none-any.whl
Collecting xgboost==0.81 (from -r pip_requirements.txt (line 4))
  Using cached https://files.pythonhosted.org/packages/54/21/8b2ec99862903a6d3aed62ce156d21d114b8666e669c46d9e54041df9496/xgboost-0.81-py2.py3-none-manylinux1_x86_64.whl
Collecting tensorflow==1.13.0 (from -r pip_requirements.txt (line 5))
  Could not find a version that satisfies the requirement tensorflow==1.13.0 (from -r pip_requirements.txt (line 5)) (from versions: 0.12.1, 1.0.0, 1.0.1, 1.1.0rc0, 1.1.0rc1, 1.1.0rc2, 1.1.0, 1.2.0rc0, 1.2.0rc1, 1.2.0rc2, 1.2.0, 1.2.1, 1.3.0rc0, 1.3.0rc1, 1.3.0rc2, 1.3.0, 1.4.0rc0, 1.4.0rc1, 1.4.0, 1.4.1, 1.5.0rc0, 1.5.0rc1, 1.5.0, 1.5.1, 1.6.0rc0, 1.6.0rc1, 1.6.0, 1.7.0rc0, 1.7.0rc1, 1.7.0, 1.7.1, 1.8.0rc0, 1.8.0rc1, 1.8.0, 1.9.0rc0, 1.9.0rc1, 1.9.0rc2, 1.9.0, 1.10.0rc0, 1.10.0rc1, 1.10.0, 1.10.1, 1.11.0rc0, 1.11.0rc1, 1.11.0rc2, 1.11.0, 1.12.0rc0, 1.12.0rc1, 1.12.0rc2, 1.12.0, 1.12.2, 1.12.3, 1.13.0rc0, 1.13.0rc1, 1.13.0rc2, 1.13.1, 1.13.2, 1.14.0rc0, 1.14.0rc1, 1.14.0, 2.0.0a0, 2.0.0b0, 2.0.0b1)
No matching distribution found for tensorflow==1.13.0 (from -r pip_requirements.txt (line 5))