# Notebook from lafleur1/aparentGenomeTesting
Path: workingPredictions.ipynb

<code>
from __future__ import print_function
import keras
from keras.models import Sequential, Model, load_model
from keras import backend as K
import tensorflow as tf
import isolearn.keras as iso
import os
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.cm as cm
from keras.utils import plot_model

from aparent.predictor import *
##################################################
#import bioPython for working with FASTA files
from Bio import SeqIO
##################################################
Using TensorFlow backend.
/home/lafleur1/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
  _np_qint8 = np.dtype([("qint8", np.int8, 1)])
/home/lafleur1/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
  _np_quint8 = np.dtype([("quint8", np.uint8, 1)])
/home/lafleur1/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
  _np_qint16 = np.dtype([("qint16", np.int16, 1)])
/home/lafleur1/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:519: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
  _np_quint16 = np.dtype([("quint16", np.uint16, 1)])
/home/lafleur1/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:520: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
  _np_qint32 = np.dtype([("qint32", np.int32, 1)])
/home/lafleur1/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
  np_resource = np.dtype([("resource", np.ubyte, 1)])
/home/lafleur1/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
  _np_qint8 = np.dtype([("qint8", np.int8, 1)])
/home/lafleur1/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
  _np_quint8 = np.dtype([("quint8", np.uint8, 1)])
/home/lafleur1/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
  _np_qint16 = np.dtype([("qint16", np.int16, 1)])
/home/lafleur1/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
  _np_quint16 = np.dtype([("quint16", np.uint16, 1)])
/home/lafleur1/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
  _np_qint32 = np.dtype([("qint32", np.int32, 1)])
/home/lafleur1/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
  np_resource = np.dtype([("resource", np.ubyte, 1)])
#open the FASTA for GL000219.1 (https://www.ncbi.nlm.nih.gov/nuccore/GL000219.1?report=fasta) unplaced human genomic assembly

#bioPython instructions http://biopython.org/DIST/docs/tutorial/Tutorial.html
contigSeq = SeqIO.read("GL000219.1.fasta", "fasta")
seq = contigSeq.seq #actual genomic sequence from the file
print (len(seq))179198
#loading model
aparent_model = load_model('./saved_models/aparent_large_lessdropout_all_libs_no_sampleweights.h5')
plot_model(aparent_model, show_shapes = True, to_file='APARENTmodel.png')
aparent_encoder = get_aparent_encoder(lib_bias=4)
WARNING:tensorflow:From /home/lafleur1/.local/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py:4070: The name tf.nn.max_pool is deprecated. Please use tf.nn.max_pool2d instead.

#detecting peaks with signal smoothing for

chunk = seq
#predict where polyA will occur based on values of cut site predictions from APARENT model
x,y = find_polya_peaks_memoryFriendly(
    aparent_model,
    aparent_encoder,
    chunk,
    sequence_stride=1,
    conv_smoothing=False,
    peak_min_height=0.01,
    peak_min_distance=50,
    peak_prominence=(0.01, None),
)

"""
totalCut = np.zeros(len(chunk))
totalMask = np.zeros(len(chunk))
for i in range(0,numberfiles+1):

    totalCut = totalCut  + np.load("./numpyFiles/TESTINGPredSlices" + str(i) + ".npy")[:,:-1]
    totalMask = totalMask + np.load("./numpyFiles/TESTINGMaskSlices" + str(i) + ".npy")[:,:-1]

avgs = totalCut/totalMask
"""
print (max(y))
print (min(y))
np.save("GL000219_1_cutPreds", y)
x_data = np.arange(0,len(chunk))
plt.plot(x_data, y[0])
plt.title("PolyA cleavage profile")
plt.show()WARNING:tensorflow:From /home/lafleur1/.local/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py:422: The name tf.global_variables is deprecated. Please use tf.compat.v1.global_variables instead.

[8.20673961e-07 8.76481636e-07 7.72374885e-07 ... 1.93512354e-05
 2.17373354e-05 3.48301978e-06]
[8.20673961e-07 8.76481636e-07 7.72374885e-07 ... 1.93512354e-05
 2.17373354e-05 3.48301978e-06]
#detecting peaks
#playing with aparent.predict -> what is happening in there???
_____no_output_____
</code>


# Notebook from Ninomoriaty/iToolkit
Path: BashToolkits/ShellScript.ipynb

Guideline Level (for workflows)

Bash/Shell Script

Shell script writing, Cooperation with high-level language_____no_output_____# References
## Textbook
- TLCL 19.01 Part IV
- MSc BPSM Course Material in UoE
- Bioinformatics in SYSU

## Web Resource
-

# Lk
- `~/BashToolkits/LinuxShell.ipynb` in the same folder, which contains the command line and Linux system methods(TLCL 19.01 Part 123)
- `~/Practice/*` Shell scripts with integrative methods mentioned in this jupyter notebook
- `~/BioIToolkits/*` Some biological workflows implemented with shell scripts and other programming languages

# Pcp-Edit
- This notebook summarises the preliminary usage of bash/shell practice.

_____no_output_____# Mt-Notebook Environment
Lk: The same as the environment mentioned in `LinuxShell.ipynb`._____no_output_____

Part I Basic Concepts and Structure

Shell script writing, Cooperation with high-level language_____no_output_____---
---
# Cc-Basic

- **The Internal Field Separator (IFS): **The symbols used for word splitting; it has default values of space( ), tab(\t), and newline(\n).

# Pcp/Ab-Bash Shell Scripts

- Space sensitive
- String would be combine 'without' operator

_____no_output_____

# set the IFS
IFS="_"

# reset the IFS to the default value
unset IFS_____no_output_____

---
---
# Mt-Perform shell scirpts in terminal
_____no_output_____

# Execute a shell scripts in terminal
## Tips: Before execute the shell script, you should make sure that the script is executable with proper permission.
#$ script [options] [shell_script]
## Options
#(TODO) execute multiple scripts in one command line

script ./Practice/p1.sh

# Execute a script with specific interpreter
## Tips: It should be found in other programming language learning and the reason why it is here is that it is relative with shebang.
perl null.pl
python null.py

# _____no_output_____

---
---
# St-Shell script
## Cpn
- Shebang: "bang" line or "shebang" line
    - Def: The first line of the shell script to announce the executor/interpreter of the script
    - Pcp: Must be the first line in the script file.
    - Ab: Start with `#!`
    - F: Announce the particular executor/interpreter
    - Tips: Without the line above, you should specify shell terminal when you exec the .sh or python
- _____no_output_____

# Some example of shebang line
#!/bin/bash
#!/usr/bin/awk -f
#!/usr/bin/perl
#!/usr/bin/python_____no_output_____

Part II Practice

Shell script writing, Cooperation with high-level language_____no_output_____---
---
# Basic_____no_output_____## Arithmetic Operators
![Screenshot from 2020-10-01 02-33-30.png](https://i.loli.net/2020/10/01/QERfpuOT5Mh1Io3.png)_____no_output_____## Quoting
F: selectively suppress unwanted expansions.

### Single Quotes

F:  suppress all expansions

Tips: The F also means single quotes could only include string while double quotes could include other command and class.

### Double Quotes

F: All the special characters used by the shell lose their special meaning and are
treated as ordinary characters. The exceptions are $, \ (backslash), and ` (back-quote).(pathname expansion, arithmetic expansion, command substitution, escaping character)
_____no_output_____

# Single Quoting
# Differences between single quoting and double quoting
echo text ~/*.txt {a,b} $(echo foo) $((2+2)) $USER
text /home/me/ls-output.txt a b foo 4 me
echo "text ~/*.txt {a,b} $(echo foo) $((2+2)) $USER"
text ~/*.txt {a,b} foo 4 me
echo 'text ~/*.txt {a,b} $(echo foo) $((2+2)) $USER'
text ~/*.txt {a,b}
$(echo foo) $((2+2)) $USER

# Another exmaple
## single quotes could not include other class within it while double quotes can include.
# e.g. below
echo '$(echo "upg")'$ $(echo "upg") # return
echo "$(echo "upg")"$ upg # return

# Double Quoting
ls -l "two words.txt"
_____no_output_____

---
## Escaping Character: \ backslash

F:

1. suppress the expansion/special meaning even it is an exception in single/double quotes.
2. And it is a Cpn of control codes

![Screenshot from 2020-10-01 12-43-56.png](https://i.loli.net/2020/10/01/MYuyLvBlgGbkjRN.png)

_____no_output_____

echo "This apple\'s price is $2300"
## Outputs: This apple's price is
echo This apple\'s price is \$2300 \& it is wonderful\! \\\\
## Outputs: This apple's price is $2300 & it is wonderful! \\

# Sp \a
sleep 10; echo -e "Time's up\a" # -e would be interpret the escape sequence
sleep 10; echo "Time's up" $'\a' # '\a' is a variable and capable of exec

_____no_output_____

---
---
# Expansion, Grabbing and Regular Expressions
**Cc/Tips**
- Sequences
    - {a..z} # from a to z in alphbet
    - {00..99}
    - others could be seen in grabbing
- Grabbing
    -
_____no_output_____---
## Expansion
- Def: expanded into something else before the shell acts upon it.
_____no_output_____

#!/bin/bash

# Pathname Expansion
## Tips: default not reveal the hidden filesecho /usr/*/share
## show all the branch with share branch, good trick
echo [[:upper:]]*
## How to show the hidden files
echo .*  # however, current dir and last dir shown as . and .. would be included
echo .[!.]*  # only hidden files would be included.
echo .?*  # explanation for the last command why it would not include current dir .

# Tilde Expansion:  tilde character (~)
echo ~ # show the home dir of current user
echo ~[usr_name] # show the home dir of the user named [user_name]

# Arithmetic Expansion: exec calculation$((expression))
# only integers
$((2+(3**2)/3-2%2*3))
# e.g.

# Brace Expansion: create multiple text strings from a pattern containing braces
echo Front-{A,B,C}-Back
## Outputs: Front-A-Back Front-B-Back Front-C-Back
echo x_{1..5} # preamble 1 and postscript 5 limit the range of the list and .. means the other elements in the range
## Outputs: x_1 x_2 x_3 x_4 x_5
echo y_{01..015} # zero-padded would be ok and the result would like this:
## Outputs: $ y_001 y_002 y_003 y_004 y_005 y_006 y_007 y_008 y_009 y_010 y_011 y_012 y_013 y_014 y_015
echo {A..Z}
## Outputs: $ A B C D E F G H I J K L M N O P Q R S T U V W X Y Z
echo x_{A{1..5},B{2..8}}_y
## Outputs: $ x_A1_y x_A2_y x_A3_y x_A4_y x_A5_y x_B2_y x_B3_y x_B4_y x_B5_y x_B6_y x_B7_y x_B8_y

# Parameter Expansion: $[var]
echo $USER  # Parameter contains the USER name of this environment
echo $HOME  # show the home dir of your current user
printenv | less # show all the user name fo this environment

# Command Substitution: allows us to use the output of a command as an expansion.
$([command_or_pipelines])
ls -l $(which cp)
ls -l `which cp`
_____no_output_____

---
## Grabbing/Wildcards
- Wildcard: Specify a group of file name(e.g.: Ref: Table 4-3: Wildcard Examples)
- ![Screenshot from 2020-09-23 16-23-44.png](https://i.loli.net/2020/09/25/sjmtqpzwgRlVXUJ.png)
- **Tips**: In Nautilus/Files (the file manager for GNOME), you can select files using wildcards in the Search box
- **Further Reading**: https://linuxhint.com/bash_globbing_tutorial/_____no_output_____

# check the manual of Globbing
man 7 glob  # Wildcard Meaning
* # Matches any characters
? # Matches any single character
[characters] # Matches any character that is a member of the set characters
[!characters] # Matches any character that is not a member of the set characters
[[:class:]] # Matches any character that is a member of the specified class# check the manual of Globbing
man 7 glob  # Wildcard Meaning
* # Matches any characters
? # Matches any single character
[characters] # Matches any character that is a member of the set characters
[!characters] # Matches any character that is not a member of the set characters
[[:class:]] # Matches any character that is a member of the specified class
## Character Classes
[:alnum:] # Matches any alphanumeric character
[:alpha:] # Matches any alphabetic character
[:digit:] # Matches any numeral
[:lower:] # Matches any lowercase letter
[:upper:] # Matches any uppercase letter_____no_output_____

---
## Regular Expressions(Regex)
Att: Don't input spaces between commas(,) and objects in shell

Tips: Actually, the path name expansions is somehow similar with the regular expression, I think it may be because the path name expansion is naturally suitable for the regular expressions.

F: In fact, I think it could be considered as an universal syntax and it could be used in many situations, like expansions, grabbing. However, if the chart below is correct, they are quite different._____no_output_____

. matches any single character
? the preceding character matches 0 or 1 times
* the preceding character matches 0 or more times
+ the preceding character matches 1 or more times
x{n} the preceding character x matches exactly n times
x{n,m} the preceding character x matches at least n times and not more than m times
x{n,} the preceding character x matches at least n times
x{,m} the preceding character x matches not more than m times
[agd] the character is one of those included within the square brackets
[^agd] the character is NOT one of those included within the square brackets
[c-f] the dash within the square brackets operates as a range, in this case meaning any of the letters c, d, e, or f
(la) group several characters together, so (la){3} matches lalala.
(X|Y)  the logical OR operation
^x matches x (anchors) the beginning of the line
x$ matches x at the end of the line_____no_output_____

---
---
# System Control
_____no_output_____

# Wait for the specified number of seconds
sleep 10_____no_output_____


# Notebook from edeutsch/Community-Notebooks
Path: RegulomeExplorer/Association_GeneExpression_ClinicData.ipynb

# Regulome Explorer 2.X
Comparison of the average gene expression of two groups based on clinical data (catageorical)
_____no_output_____### Authenticate with Google  (IMPORTANT)
Our first step is to authenticate with Google -- you will need to be a member of a Google Cloud Platform (GCP) project, with authorization to run BigQuery jobs in order to run this notebook.  If you don't have access to a GCP project, please contact the ISB-CGC team for help (www.isb-cgc.org)_____no_output_____#### Import Python libraries_____no_output_____

from google.cloud import bigquery
import numpy as np
import pandas as pd
from scipy import stats_____no_output_____

#### Python convenience functions

- **`runQuery`**: a relatively generic BigQuery query-execution wrapper function which can be used to run a query in "dry-run"  mode or not:  the call to the `query()` function itself is inside a `try/except` block and if it fails we return `None`;  otherwise a "dry" will return an empty dataframe, and a "live" run will return the query results as a dataframe.  This function was modify from previous notebooks to handle user-defined parameteres necessary for the purpose of this notbeook.

_____no_output_____

def runQuery ( client, qString, ParameterList1, ParameterList2, ParameterList3, dryRun=False ):

  print ( "\n in runQuery ... " )
  if ( dryRun ):
    print ( "    dry-run only " )

  ## set up QueryJobConfig object
  job_config = bigquery.QueryJobConfig()

  query_params = [
        bigquery.ArrayQueryParameter('PARAMETERLIST1', 'STRING', ParameterList1 ),
        bigquery.ArrayQueryParameter('PARAMETERLIST2', 'STRING', ParameterList2 ),
        bigquery.ArrayQueryParameter('PARAMETERLIST3', 'STRING', ParameterList3 )
  ]
  job_config.query_parameters = query_params

  job_config.dry_run = dryRun
  job_config.use_query_cache = True
  job_config.use_legacy_sql = False

  ## run the query
  try:
    query_job = client.query ( qString, job_config=job_config )
    ## print ( "    query job state: ", query_job.state )
  except:
    print ( "  FATAL ERROR: query execution failed " )
    return ( None )

  ## return results as a dataframe (or an empty dataframe for a dry-run)
  if ( not dryRun ):
    try:
      df = query_job.to_dataframe()
      if ( query_job.total_bytes_processed==0 ):
        print ( "    the results for this query were previously cached " )
      else:
        print ( "    this query processed {} bytes ".format(query_job.total_bytes_processed) )
      if ( len(df) < 1 ):
        print ( "  WARNING: this query returned NO results ")
      return ( df )
    except:
      print ( "  FATAL ERROR: query execution failed " )
      return ( None )

  else:
    print ( "    if not cached, this query will process {} bytes ".format(query_job.total_bytes_processed) )
    ## return an empty dataframe
    return ( pd.DataFrame() )_____no_output_____

### SQL Building Functions

**`build_expdata`**. This function outputs a query string to create a table with three columns: patient ID, gene expresion, and gene name, for a user defined list of genes (PARAMETERLIST1) or all genes with gene expression data. Since a patient can have multiple samples, the gene expression value for each patient is averaged over the respective samples. _____no_output_____

def build_expdata ( study ):
  qString = """
  expdata AS (
       SELECT
           participantBarcode as patientid,
           Symbol,
           AVG( LOG10( normalized_count +1 )  ) as gene_exp
       FROM
          `pancancer-atlas.Filtered.EBpp_AdjustPANCAN_IlluminaHiSeq_RNASeqV2_genExp_filtered`
       WHERE
           #study = '__study__' AND Symbol IN UNNEST(@PARAMETERLIST1)
           study = '__study__' AND Symbol IS NOT NULL
       GROUP BY ParticipantBarcode, Symbol
  )""".replace('__study__',study)
  return qString
    _____no_output_____

**`build_statdata`**. This function outputs a query string to create a table with four columns: gene_name, category, n, sumx, and sumx2. n is the number of patients with the same clinical category, sumx (sumx2) is the sum of the gene expression (squared) data over all patients with the same category. _____no_output_____

def build_statdata( category ):
  qString ="""
  statdata AS (
       SELECT
           A.Symbol,
           B.__category__ as category,
           COUNT( A.patientid ) as n,
           SUM( A.gene_exp ) as sumx,
           SUM( A.gene_exp * A.gene_exp ) as sumx2
       FROM
           expdata A
       INNER JOIN `pancancer-atlas.Filtered.clinical_PANCAN_patient_with_followup_filtered`  B
           ON A.patientid = B.bcr_patient_barcode
       GROUP BY
           B.__category__, A.Symbol
  )""".replace('__category__', category)
  return qString  _____no_output_____

**`build_groupdate`**. Outputs a query string that summarize the statistical data for the desired catageory values  that are specified for group1 and group 2.  This catagory values are defined by a couple parameter lists, PARAMETERLIST2 and PARAMETERLIST3 for group 1 and group 2 respectively._____no_output_____

def build_groupdate():
  qString ="""
  group1 AS (
       SELECT Symbol, SUM( n ) as n, SUM( sumx ) as sumx, SUM( sumx2 ) as sumx2
       FROM statdata
       WHERE category IN UNNEST(@PARAMETERLIST2)
       GROUP BY Symbol
  ),
  group2 AS (
       SELECT Symbol, SUM( n ) as n, SUM( sumx ) as sumx, SUM( sumx2 ) as sumx2
       FROM statdata
       WHERE category IN UNNEST(@PARAMETERLIST3)
       GROUP BY Symbol
  )
  """
  return qString_____no_output_____

##  Start the analysis_____no_output_____

# Start by getting authorized.
bqclient = bigquery.Client()_____no_output_____

###  Specify Parameters_____no_output_____

study = 'ACC' # Select Tumor type
ClinicFeature1 = 'pathologic_stage'  # Clinical feature
#gene_list =['IGF2','ADAM6','GRWD1','PATE3','CASP8'] #gene nems, irrelevant if all genes are evaluated
group1 = [ 'Stage I', 'Stage II']   # Category values to define group 1
group2 = ['Stage III', 'Stage IV']  # Category values to define group 1
_____no_output_____

### Obtain data from Bigquery
We retreive data for t test evaluation. The ouput of the code is a dataframe with 7 columns corresponding to gene name (Symbol), the number of samples, average expression and variance of the two groups.
This computation can take several seconds._____no_output_____

# Build the sql code
sql = (
"""WITH
"""+
build_expdata( study ) + ',\n' +
build_statdata( ClinicFeature1 ) + ',\n' +
build_groupdate() + '\n' +
"""
SELECT
  A.Symbol,
  A.n as n1, (A.sumx/A.n) as avg1 , (A.sumx2 - A.sumx*A.sumx/A.n)/(A.n - 1) as var1 ,
  B.n as n2, (B.sumx/B.n) as avg2 , (B.sumx2 - B.sumx*B.sumx/B.n)/(B.n - 1) as var2
FROM
  group1 A
INNER JOIN
  group2 B
ON
  A.Symbol  = B.Symbol
"""
)

# bigquery
res0 = runQuery ( bqclient, sql, gene_list, group1, group2, dryRun=False )
 in runQuery ...
    the results for this query were previously cached

We can now take a look the computed statistics. We have to make sure that there is not NUL values which may indicate that a group has no elements._____no_output_____

res0[0:10]_____no_output_____

###  Analyze the results
We can now use python function to the data in the resulting dataframe (res0). For example we can compute t-scores and p-values to determine if the difference between the mean of the two groups is statistically significant_____no_output_____

# t statistics
f = lambda n1,n2,avg1,avg2,var1,var2  : (avg1 - avg2) / np.sqrt(var1/float(n1) + var2/float(n2) )
res0['tscore'] = res0.apply(lambda x: f(x.n1,x.n2,x.avg1,x.avg2,x.var1,x.var2), axis=1)

# p value
f = lambda  n1,n2,tscore :  (1.0 -  stats.t.cdf(abs(tscore), n1 + n2 - 2 )) * 2.0
res0['pvalue'] = res0.apply(lambda x: f(x.n1,x.n2,x.tscore), axis=1)

res0.sort_values( by= 'pvalue')[0:10]/Users/borisaguilar/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:2: RuntimeWarning: invalid value encountered in double_scalars

We can plot the data_____no_output_____

res0['tscore'].plot.kde()_____no_output_____


# Notebook from tobias-fyi/post_here_ds
Path: notebooks/post_here_2-small_full.ipynb

# Post Here: The Subreddit Suggester

## Small Model (limited dataset)

### Notebook by: _Tobias Reaper______no_output_____---

## Notebook outline

* [_Imports and Configuration_](#Imports-and-Configuration)
* [Introduction](#Introduction)
  * [The Problem](#The-Problem)
  * [The Solution (The App)](#The-Solution)
  * [My Role](#My-Role)
* [The Data](#The-Data)
  * [Wrangling](#Wrangling)
  * [Exploration](#Exploration)
* [Modeling](#Modeling)
  * [Challenges](#Challenges)
  * [Feature Selection](#Feature-Selection)
  * [Vectorization](#Vectorization)
  * [Baseline](#Baseline)
  * [Multinomial Naive Bayes](#Multinomial-Naive-Bayes)
* [Final Thoughts](#Final-Thoughts)_____no_output_____---

## Imports and Configuration_____no_output_____

# === General imports === #
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import os
import janitor_____no_output_____# === ML imports === #
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder
from sklearn.ensemble import RandomForestClassifier
from sklearn.naive_bayes import MultinomialNB
from sklearn.feature_selection import chi2, SelectKBest

# === NLP Imports === #
from sklearn.feature_extraction.text import TfidfVectorizer
# import spacy_____no_output_____# === Configure === #
# Load spacy language model
# nlp = spacy.load("en_core_web_md")

# Configure pandas display settings
pd.options.display.max_colwidth = 200

# Set random seed
seed = 92_____no_output_____

---

## Introduction_____no_output_____### The Problem

Reddit is an expansive site. Anyone who has spent any significant amount of time on it knows what I mean. There is a subreddit for seemingly every topic anyone could ever want to discuss or even think about (and many that most do not want think about).

Reddit is a powerful site; a tool for connecting and sharing information with like- or unlike-minded individuals around the world. When used well, it can be a very useful resource.

On the other hand, the deluge of information that's constantly piling into the pages of can be overwhelming and lead to wasted time. As with any tool, it can be used for good or for not-so-good.

A common problem that Redditors experience, particularly those who are relatively new to the site, is where to post content. Given that there are subreddits for just about everything, with wildly varying degrees of specificity it can be quite overwhelming trying to find the best place for each post.

Just to illustrate the point, some subreddits get _weirdly_ specific. I won't go into the _really_ weird or NSFW, but here are some good examples of what I mean by specific:

* [r/Borderporn](https://www.reddit.com/r/Borderporn/)
* [r/BreadStapledtoTrees](https://www.reddit.com/r/BreadStapledToTrees/)
* [r/birdswitharms](https://www.reddit.com/r/birdswitharms/)
* [r/totallynotrobots](https://old.reddit.com/r/totallynotrobots)

...need I go on? (If you're curious and/or want to be entertained indefinitely, here is a [thread](https://www.reddit.com/r/AskReddit/comments/dd49gw/what_are_some_really_really_weird_subreddits/) with these and much, much more.)

Most of the time when a post is deemed irrelevant to a particular subreddit, it will simply be removed by moderators or a bot. However, depending on the subreddit and how welcoming they are to newbies, sometimes it can lead to very unfriendly responses and/or bans.

So how does one go about deciding where to post or pose a question?

Post Here aims to take the guesswork out of this process._____no_output_____### The Solution

The goal with the Post Here app, as mentioned, is to provide a tool that makes it quick and easy to find the most appropriate subreddits for any given post. A user would simply provide the title and text of the their prospective post and the app would provide the user with a list of subreddit recommendations.

Recommendations are produced by a model attempts to predict which subreddit a given post would belong to. The model was built using Scikit-learn, and was trained on a large dataset of reddit posts. In order to serve the recommendations to the web app, an API was built using Flask and deployed to Heroku.

The live version of the app is linked below.

[Post Here: The Subreddit Suggester](https://github.com/tobias-fyi/post_here_ds)_____no_output_____### My Role

I worked on the Post Here app with a remote, interdisciplinary team of data scientists, machine learning engineers, and web developers. I was one of two machine learning engineers on the team, responsible for the entire process of building and training the machine learning models. The two data scientists on the team were primarily responsible for building and deploying the API.

The main challenge we ran into, which directed much of the iterative process, was scope management.

At this point in my machine learning journey, this was one of the larger datasets that I'd taken on. Uncompressed, the dataset we used was over 800mb of mostly natural language text. The dataset and the time constraint—we had less than four full days of work to finish the project—were the primary causes of the challenges we ended up facing.

With such a dataset, one important concept we had to keep in mind was the [curse of dimensionality](https://en.wikipedia.org/wiki/Curse_of_dimensionality), which is basically a title for the various problems and phenomena that occur when dealing with extremely highly dimensional datasets. When processed, a natural language dataset of this size would likely fall prey to this curse and may prove somewhat unwieldy without large amounts of processing power.

I ended up researching and applying various methods of addressing this problem in order to fit the processing/modeling pipeline on the free Heroku Dyno, with a memory limit of 500mb, while preserving adequate performance. Many of our deployments failed because the pipeline, when loaded into memory on the server, exceeded that limit.

One important tradeoff we had to wrangle with was how much, and in what ways we could limit the dataset—i.e. how many classes to try and predict, and how many observations per class to include when training. The original dataset contains data for 1,000 subreddits. It was not within the scope of a a four-day project to build a classification model of a caliber that could accurately classify 1,000 classes.

In the beginning, we did try to build a basic model trained on all 1,000 classes. But with the time and processing power I had, it proved to be untenable. In the end, we settled for a model that classified text into 305 subreddits with a test precision-at-k of .75, .88, and .92 for 'k' of 1, 3, and 5, respectively._____no_output_____---

## The Data_____no_output_____The dataset we ended up using to train the recommendation system is called the [Reddit Self-Post Classification Task dataset](https://www.kaggle.com/mswarbrickjones/reddit-selfposts), available on Kaggle thanks to Evolution AI. The full dataset clocks in at over 800mb, containing 1,013,000 rows: 1,000 posts each from 1,013 subreddits.

The data was posted to reddit between June 2016 and June 2018.

[more info from the article]

For more details on the dataset, including a nice interactive plot of all of the subreddits, refer to Evolution AI's [blog post](https://evolution.ai//blog/page/5/an-imagenet-like-text-classification-task-based-on-reddit-posts/)._____no_output_____### Wrangling and Exploration

As seems to be common with NLP projects, the process of wrangling the data was very much intertwined with the modeling process. Of course, this could be said about any machine learning project. However, I feel like it is particularly so in the case of NLP.

Therefore, this section—the one dedicated to data wrangling only—will be rather brief and basic. I go into much more detail in the Modeling section.

First, I needed to reduce the size of the dataset. I defined a subset of 12 categories which I thought were most relevant to the task at hand, and used that list to do the initial pruning. Those 12 categories left me with 305 unique subreddits and 305,000 rows. The list I used was as follows:

* health
* profession
* electronics
* hobby
* writing/stories
* advice/question
* social_group
* stem
* parenting
* books
* finance/money
* travel

Next, I took a random sample of those 305,000 rows. The result was a dataset with 91,500 rows, now consisting of between 250 and 340 rows per subreddit. If I tried to use all of the features (tokens, or words) that resulted from this corpus, even in its reduced state, it would still result in a serialized vocabulary and/or model too large for our free Heroku Dyno. However, the features used in the final model can be chosen based on how useful they are for the classification.

According to the dataset preview on Kaggle, there are quite a large number of missing values in each of the features—12%, 25%, and 39% of the subreddit, title, and selftext columns, respectively. However, I did not find any sign of those null values in the dataset nor mention of them in the dataset's companion blog post or article. I chocked it up to an error in the Kaggle preview.

Finally, I went about doing some basic preprocessing to get the data ready for vectorization. As described in the description page on Kaggle, newline and tab characters were replaced with their HTML equivalents, `<lb>` and `<tab>`. I removed those and other HTML entities using a simple regular expression. I also concatenated `title` and `selftext` into a single text feature in order to process them together._____no_output_____

# === Load the dataset === #
rspct = pd.read_csv("assets/data/rspct.tsv", sep="\t")
print(rspct.shape)
rspct.head(3)(1013000, 4)

#### Nulls

Kaggle says that 12%, 25%, and 39% of the subreddit, title, and selftext columns are null, respectively. If that is indeed the case, they did not get read into the dataframe correctly. However, it could be an error on Kaggle's part, seeing as there is no mention of these anywhere else in the description or blog post or article, nor sign of them during my explorations._____no_output_____

# === Null values === #
rspct.isnull().sum()_____no_output_____

#### Preprocess

To prune the list of subreddits, I'll load in the `subreddit_info.csv` file, join, then choose a certain number of categories (category_1) to filter on._____no_output_____

# === Load info === #
info = pd.read_csv("assets/data/subreddit_info.csv", usecols=["subreddit", "category_1", "category_2"])
print(info.shape)
info.head()(3394, 3)
# === Join the two dataframes === #
rspct = pd.merge(rspct, info, on="subreddit").drop(columns=["id"])
print(rspct.shape)
rspct.head()(1013000, 5)
# === Still no nulls === #
rspct.isnull().sum()  # That's a good sign_____no_output_____# === Look at categories === #
rspct["category_1"].value_counts()_____no_output_____# === Define list of categories to keep === #
keep_cats = [
    "health",
    "profession",
    "electronics",
    "hobby",
    "writing/stories",
    "advice/question",
    "social_group",
    "stem",
    "parenting",
    "books",
    "finance/money",
    "travel",
]

# === Prune dataset to above categories === #
# Overwriting to save memory
rspct = rspct[rspct["category_1"].isin(keep_cats)]
print(rspct.shape)
print("Unique subreddits:", len(rspct["subreddit"].unique()))
rspct.head(2)(305000, 5)
Unique subreddits: 305
# === Take a sample of that === #
rspct = rspct.sample(frac=.3, random_state=seed)
print(rspct.shape)
rspct.head()(91500, 5)
# === Clean up a bit === #
# Concatenate title and selftext
rspct["text"] = rspct["title"] + " " + rspct["selftext"]

# Drop categories
rspct = rspct.drop(columns=["category_1", "category_2", "title", "selftext"])

print(rspct.shape)
rspct.head(2)(91500, 2)
# === Remove <lb>, <tab>, and other HTML entities === #
# NOTE: takes a couple minutes to run
rspct["text"] = rspct["text"].str.replace("(<lb>)*|(<tab>)*|(&)*|(nbsp;)*", "")
rspct.head()_____no_output_____# === Reset the index === #
rspct = rspct.reset_index(drop=True)
rspct.head(2)_____no_output_____# === Save pruned dataset to file === #
rspct.to_csv("assets/data/rspct_small.csv")_____no_output_____# === List of subreddits === #
subreddits = rspct["subreddit"].unique()
print(len(subreddits))
subreddits[:50]305
rspct["subreddit"].value_counts()_____no_output_____

---

## Modeling_____no_output_____### Label Encoding_____no_output_____

# === Encode the target using LabelEncoder === #
# This process naively transforms each class of the target into a number
le = LabelEncoder() # Instantiate a new encoder instance
le.fit(y_train)  # Fit it on training label data

# Transform both using the trained instance
y_train = le.transform(y_train)
y_val = le.transform(y_val)
y_test  = le.transform(y_test)

y_train[:8]_____no_output_____

### Vectorization

A vectorizer is used to extract numerical features (information) from a corpus of natural language text. I used a [bag-of-words](https://en.wikipedia.org/wiki/Bag-of-words_model) method of vectorization, which for the most part, disregards grammar.

The output of this vectorizer is a document-term matrix, with the documents (observations, or rows) on one axes and the terms (words, bigrams) on the other. This matrix can be thought of as a sort of vocabulary, or text-number translator.

At first, the "vocabulary" derived from the corpus using the vectorizer was the largest object when serialized. Luckily, there are many options and parameters available to reduce its size, most of which are simply different methods for reducing the number of features (terms) it contains.

One option is to put a hard limit of 100,000 on the number of features in the vocabulary. This is a simple, naive limit on the generated features, and thus, the resulting vocabulary size.

I decided to remove stopwords before vectorization in hopes that this would reduce the size of the vector vocabulary. To my initial surprise, removing the stop words (using [NLTK](https://www.nltk.org/)'s list) actually increased the size of the serialized vocab from 59mb to 76mb.

After some consideration, I found this to be a reasonable result. I figured that many of the stop words are short ("I", "me", "my", etc.), and their removal caused the average length of words (and therefore bigrams as well) in the vocab to increase. While this may not account for the entirety of the difference, this provides some intuition as to why there is a difference.

Although the vocab without stop words was larger, I ended up using it anyways because it provided an extra ~0.01 in the precision-at-k score of the final model._____no_output_____

# === Look at lengths of stop words === #
lengths = []
three_or_below = []
for word in stop_words:
    lengths.append(len(word))
    if len(word) <= 4:
        three_or_below.append(len(word))

print(f"There are {len(stop_words)} stop words in the list.")
print(f"{len(three_or_below)} are 4 chars long or shorter.")
print(f"Average length is: {np.mean(lengths)}.")There are 179 stop words in the list.
109 are 4 chars long or shorter.
Average length is: 4.229050279329609.
# === Vectorize! === #
# Extract features from the text data using bag-of-words method
tfidf = TfidfVectorizer(
    max_features=100000,
    min_df=10,
    ngram_range=(1,2),
    stop_words=stop_words,  # Use nltk's stop words
)

# Fit the vectorizer on the feature column to create vocab (doc-term matrix)
vocab = tfidf.fit(X_train)

# Get sparse document-term matrices
X_train_sparse = vocab.transform(X_train)
X_val_sparse = vocab.transform(X_val)
X_test_sparse = vocab.transform(X_test)

X_train_sparse.shape, X_val_sparse.shape, X_test_sparse.shape_____no_output_____

### Feature Selection

As mentioned previously, the size of the corpus means the dimensionality of the featureset after vectorization will be very high. I passed in 100,000 as the maximum number of features to the vectorizer, limiting the initial size of the vocab. However, the features would have to be reduced more before training the model, as it is generally not good practice to have a larger number of features (100,000) than observations (91,500).

To reduce it down from that 100,000, I used a process called select k best, which does exactly what it sounds like: selects a certain number of the best features. The key aspect of this process is how to measure the value of the features; how to find which ones are the "best". The scoring function I used in this case is called [ch2](https://scikit-learn.org/stable/modules/generated/sklearn.feature_selection.chi2.html) (chi-squared).

This function calculates chi-squared statistics between each feature and the target, measuring the dependence, or correlation, between them. The intuition here is that features which are more correlated with the target are more likely to be useful to the model.

I played around with some different values for the maximum number of features to be selected. Ultimately, I was once again limited by the size of the free Heroku Dyno, and settled on 20,000. This allowed the deployment to go smoothly while retaining enough information for the model to have adequate performance._____no_output_____

# === Feature Selection === #
selector = SelectKBest(chi2, k=20000)

selector.fit(X_train_sparse, y_train)

X_train_select = selector.transform(X_train_sparse)
X_val_select = selector.transform(X_val_sparse)
X_test_select  = selector.transform(X_test_sparse)

X_train_select.shape, X_val_select.shape, X_test_select.shape_____no_output_____

### Model validation

In this case, the model has a target that it is attempting to predict—a supervised problem. Therefore, the performance can be measured on validation and test sets.

To test out the recommendations I copied some posts and put them through the prediction pipeline to see what kinds of subreddits were getting recommended. For the most part, the predictions were decent.

The cases where the recommendations were a little less than ideal happened when I pulled example posts from subreddits that were not in the training data. The model generally did a good job recommending similar subreddits._____no_output_____#### Baseline

For the baseline model, I decided to go with a basic random forest. This choice was somewhat arbitrary, though I was curious to see how a random forest would do with such a high target cardinality (number of classes/categories).

The baseline precision-at-k metrics for the random forest on the validation set were .54, .63, and .65, for 'k' of 1, 3, and 5, respectively._____no_output_____

# === Evaluate performance using precision-at-k === #
def precision_at_k(y_true, y_pred, k=5):
    y_true = np.array(y_true)
    y_pred = np.array(y_pred)
    y_pred = np.argsort(y_pred, axis=1)
    y_pred = y_pred[:, ::-1][:, :k]
    arr = [y in s for y, s in zip(y_true, y_pred)]
    return np.mean(arr)_____no_output_____# === Baseline RandomForest model === #
rfc = RandomForestClassifier(max_depth=32, n_jobs=-1, n_estimators=200)
rfc.fit(X_train_select, y_train)_____no_output_____# === Create predictions for validation set === #
y_pred_proba_rfc = rfc.predict_proba(X_val_select)

# === For each prediction, find the index with the highest probability === #
y_pred_rfc = np.argmax(y_pred_proba_rfc, axis=1)
y_pred_rfc[:10]_____no_output_____# === Evaluate precision at k for validation === #
print("Validation scores:")
print("  precision@1 =", np.mean(y_val == y_pred_rfc))
print("  precision@3 =", precision_at_k(y_val, y_pred_proba_rfc, 3))
print("  precision@5 =", precision_at_k(y_val, y_pred_proba_rfc, 5))Validation scores:
  precision@1 = 0.5368852459016393
  precision@3 = 0.6282786885245901
  precision@5 = 0.6502732240437158

#### Multinomial Naive Bayes

Multinomial naive Bayes is a probabilistic learning method for multinomially distributed data, and one of two classic naive Bayes algorithms used for text classification. I decided to iterate with this algorithm because it is meant for text classification tasks.

The precision-at-k metrics for the final Multinomial naive Bayes model on the validation set were .76, .88, and .9188, for 'k' of 1, 3, and 5, respectively. Performance on the test set was nearly identical: .75, .88, and .9159._____no_output_____

# === Naive Bayes model === #
nb = MultinomialNB(alpha=0.1)
nb.fit(X_train_select, y_train)_____no_output_____

#### Evaluate on validation set_____no_output_____

# === Create predictions for validation set === #
y_pred_proba_val = nb.predict_proba(X_val_select)

# For each prediction, find index with highest probability
y_pred_val = np.argmax(y_pred_proba_val, axis=1)
y_pred_val[:10]_____no_output_____# === Evaluate precision at k for validation === #
print("Validation scores:")
print("  precision@1 =", np.mean(y_val == y_pred_val))
print("  precision@3 =", precision_at_k(y_val, y_pred_proba_val, 3))
print("  precision@5 =", precision_at_k(y_val, y_pred_proba_val, 5))Validation scores:
  precision@1 = 0.7599726775956284
  precision@3 = 0.8834699453551913
  precision@5 = 0.9188524590163935

#### Evaluate on test set_____no_output_____

# === Create predictions for test set === #
y_pred_proba_test = nb.predict_proba(X_test_select)

# For each prediction, find index with highest probability
y_pred_test = np.argmax(y_pred_proba_test, axis=1)
y_pred_test[:10]_____no_output_____# === Evaluate precision at k for test === #
print("Test scores:")
print("  precision@1 =", np.mean(y_test == y_pred_test))
print("  precision@3 =", precision_at_k(y_test, y_pred_proba_test, 3))
print("  precision@5 =", precision_at_k(y_test, y_pred_proba_test, 5))Test scores:
  precision@1 = 0.7498360655737705
  precision@3 = 0.8834426229508197
  precision@5 = 0.9159562841530055

### Recommendations

The API should return a list of recommendations, not a single prediction. To accomplish this, I wrote a function that returns the top 5 most likely subreddits and their respective probabilities._____no_output_____

# === Function to serve predictions === #
# The main functionality of the predict API endpoint

def predict(title: str, submission_text: str, return_count: int = 5):
    """
    Serve subreddit predictions.

    Parameters
    ----------
    title : string
        Title of post.
    submission_text : string
        Selftext that needs a home.
    return_count    : integer
        The desired number of recommendations.

    Returns
    -------
    Python dictionary formatted as follows:
        [{'subreddit': 'PLC', 'proba': 0.014454},
         ...
         {'subreddit': 'Rowing', 'proba': 0.005206}]
    """
    # Concatenate title and post text
    fulltext = str(title) + str(submission_text)
    # Vectorize the post -> sparse doc-term matrix
    post_sparse = vocab.transform([fulltext])
    # Feature selection
    post_select = selector.transform(post_sparse)
    # Generate predicted probabilities from trained model
    proba = nb.predict_proba(post_select)
    # Wrangle into correct format
    proba_dict = (pd
                .DataFrame(proba, columns=[le.classes_])  # Classes as column names
                .T  # Transpose so column names become index
                .reset_index()  # Pull out index into a column
                .rename(columns={"level_0": "name", 0: "proba"})  # Rename for aesthetics
                .sort_values(by="proba", ascending=False)  # Sort by probability
                .iloc[:return_count]  # n-top predictions to serve
                .to_dict(orient="records")
               )
    proba_json = {"predictions": proba_dict}

    return proba_json_____no_output_____title_science = """Is there an evolutionary benefit to eating spicy food that lead to consumption across numerous cultures throughout history? Or do humans just like the sensation?"""

post_science = """I love spicy food and have done ever since I tried it. By spicy I mean HOT, like chilli peppers (we say spicy in England, I don't mean to state the obvious I'm just not sure if that's a global term and I've assumed too much before). I love a vast array of spicy foods from all around the world. I was just wondering if there was some evolutionary basis as to why spicy food managed to become some widely consumed historically. Though there seem to

It way well be that we just like a tingly mouth, the simple things in life."""

science_recs = predict(title_science, post_science)
science_recs_____no_output_____# === Test post from r/buildmeapc === #

title_pc = """Looking for help with a build"""

post_pc = """I posted my wants for my build about 2 months ago. Ordered them and when I went to build it I was soooooo lost. It took 3 days to put things together because I was afraid I would break something when I finally got the parts together it wouldn’t start, I was so defeated. With virtually replacing everything yesterday it finally booted and I couldn’t be more excited!"""

post_pc_recs = predict(title_pc, post_pc, 10)
post_pc_recs_____no_output_____# === Example post from 'r/learnprogramming' === #

post_title = """What to do about java vs javascript"""

post = """I am a new grad looking for a job and currently in the process with a company for a junior backend engineer role. I was under the impression that the position was Javascript but instead it is actually Java. My general programming and "leet code" skills are pretty good, but my understanding of Java is pretty shallow. How can I use the next three days to best improve my general Java knowledge? Most resources on the web seem to be targeting complete beginners. Maybe a book I can skim through in the next few days?

Edit:

A lot of people are saying "the company is a sinking ship don't even go to the interview". I just want to add that the position was always for a "junior backend engineer". This company uses multiple languages and the recruiter just told me the incorrect language for the specific team I'm interviewing for. I'm sure they're mainly interested in seeing my understanding of good backend principles and software design, it's not a senior lead Java position."""

# === Test out the function === #
post_pred = predict(post_title, post)  # Default is 5 results
post_pred_____no_output_____# === Test it out with another dummy post === #

title_book = "Looking for books with great plot twists"

# This one comes from r/suggestmeabook
post2 = """I've been dreaming about writing my own stort story for a while but I want to give it an unexpected ending. I've read lots of books, but none of them had the plot twist I want. I want to read books with the best plot twists, so that I can analyze what makes a good plot twist and write my own story based on that points. I don't like romance novels and I mostly enjoy sci-fi or historical books but anything beside romance novels would work for me, it doesn't have to be my type of novel. I'm open to experience after all. I need your help guys. Thanks in advance."""

# === This time with 10 results === #
post2_pred = predict(title_book, post2, 10)
post2_pred_____no_output_____

### Model deployment

As mentioned, the model, vocab, and feature selector were all serialized using Python's pickle module. In the Flask app, the pickled objects are loaded and ready for use, just like that.

I will go over the details of how the Flask app was set up in a separate blog post._____no_output_____

# === Create pickle func to make pickling (a little) easier === #

def picklizer(to_pickle, filename, path):
    """
    Creates a pickle file.

    Parameters
    ----------
    to_pickle : Python object
        The trained / fitted instance of the
        transformer or model to be pickled.
    filename : string
        The desired name of the output file,
        not including the '.pkl' extension.
    path : string or path-like object
        The path to the desired output directory.
    """
    import os
    import pickle

    # Create the path to save location
    picklepath = os.path.join(path, filename)

    # Use context manager to open file
    with open(picklepath, "wb") as p:
        pickle.dump(to_pickle, p)_____no_output_____# === Picklize! === #
filepath = "./assets/pickles"  # Change this accordingly

# Create directory if doesn't exist
os.makedirs(filepath, exist_ok=True)

# Export LabelEncoder as pickle
picklizer(le, "le.pkl", filepath)

# Export selector as pickle
picklizer(selector, "selector.pkl", filepath)

# Export vectorizer as pickle
picklizer(vocab, "vocab.pkl", filepath)

# Export naive bayes model as pickle
picklizer(nb, "nb.pkl", filepath)_____no_output_____

---

## Final Thoughts

For me, the most important and valuable aspects of this project were mainly surrounding the challenge of scope management. I constantly had to ask myself, "What is the best version of this I can create given our limitations?"

At first, I thought it would be feasible to predict all of the 1,000+ subreddits in the data, and wasted hours of valuable time attempting to do so. While I had tested various strategies of reducing the complexity of the model, the performance was rather terrible when it was trained on 100 or less examples of each of the complete list of subreddits.

The data scientist who I primarily worked with (we had one data scientist in addition to him and one other machine learning engineer, both of whom did not contribute significantly to the project) kept telling me that I should try reducing the number of classes first, allowing for more examples of each class and fewer classes for the model to predict.

Ultimately, this is the strategy that worked best, and I wasted valuable time by not listening to him the first few times he recommended that strategy. Good teamwork requires the members being humble and listening, something that I have taken to heart since the conclusion of this project.

### Scope Management, Revisited

As time was very short while building this initial recommendation API, there are many things that we wished we could have done but simply did not have the time. Here are a few of the more obvious improvements that could be made.

The first, and most obvious one, is to simply deploy to a more powerful server, such as one hosted on AWS Elastic Beanstalk or EC2. This way, we could use the entire dataset to train an optimal model without worrying (as much) about memory limits.

Second, I could use a Scikit-learn pipeline to validate and tune hyperparameters using cross-validation, instead of a separate validation set. Also, this pipeline could be serialized as a single large object, rather than as separate pieces (encoder, vectorizer, feature selector, and classifier). As a final note for this particular train of thought, [Joblib](https://joblib.readthedocs.io/en/latest/) could potentially provide more efficient serialization than the Pickle module, allowing a more complex pipeline to be deployed on the same server.

Third, a model could've been trained to classify the input post first into a broad category. Then, some sort of model could be used to to classify into a specific subreddit within that broad category. I'm not sure about the feasibility of the second part of this idea, but thought it could be an interesting one to explore.

Lastly, different classes and calibers of models could have been tested for use in the various steps in the pipeline. In this case, I'm referring primarily to using deep learning/neural networks. For example, word vectors could be generated with word embedding models such as Word2Vec. Or the process could be recreated with a library like PyTorch, and a framework like [FastText](https://github.com/facebookresearch/fastText/).

I plan to explore at least some of these in separate blog posts.

As always, thank you for reading! I'll see you in the next one._____no_output_____


# Notebook from kevingreenman/FLIP
Path: baselines/levenshtein_blosum62_baselines.ipynb

Test to see how well the Levenshtein distance can be used as a predictor for relevant datasets:_____no_output_____

# Import required modules
import os
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from Bio import SeqIO
from Bio.SubsMat.MatrixInfo import blosum62
from Levenshtein import distance as levenshtein_distance
from scipy.stats import spearmanr

# Create a new blosum matrix that has both directions.
SYMMETRIC_BLOSUM = {}
for (aa1, aa2), score in blosum62.items():
    SYMMETRIC_BLOSUM[(aa1, aa2)] = score
    SYMMETRIC_BLOSUM[(aa2, aa1)] = scoreC:\Users\t-bwittmann\Anaconda3\lib\site-packages\Bio\SubsMat\__init__.py:126: BiopythonDeprecationWarning: Bio.SubsMat has been deprecated, and we intend to remove it in a future release of Biopython. As an alternative, please consider using Bio.Align.substitution_matrices as a replacement, and contact the Biopython developers if you still need the Bio.SubsMat module.
  warnings.warn(

# General Functions_____no_output_____

def evaluate_task(task_filename, reference_seq):

    # Load the task file and filter to just the test data
    task_df = pd.read_csv(task_filename)
    task_df = task_df.loc[task_df.set == "test"].copy()

    # Create output arrays
    refseq_len = len(reference_seq)
    all_seqs = task_df.sequence.tolist()
    levenshteins = np.empty(len(all_seqs))
    blosum_scores = np.empty(len(all_seqs))

    # Calculate levenshtein distance between each sequence and the reference
    levenshteins = np.array([levenshtein_distance(reference_seq, new_seq)
                             for new_seq in task_df.sequence.values])

    # Calculate scores for each sequence
    calculate_blosum = True
    for i, new_seq in enumerate(all_seqs):

        # Score by levenshtein
        levenshteins[i] = levenshtein_distance(reference_seq, new_seq)

        # Continue to calculate blosum unless the data is not aligned
        if calculate_blosum:

            # Make sure the reference sequence and this sequence align
            seqs_aligned = len(new_seq) == refseq_len
            if not seqs_aligned:
                calculate_blosum = False
                blosum_scores = None
                continue

            # Calculate blosum scores
            blosum_scores[i] = sum(SYMMETRIC_BLOSUM[(aa1, aa2)] for
                                   aa1, aa2 in zip(reference_seq, new_seq))

    # Now get spearman rho and record. Negative levenshtein because we
    # expect a smaller distance to be correlated to larger fitness.
    l_rho, _ = spearmanr(-levenshteins, task_df.target.values)
    if blosum_scores is not None:
        b_rho, _ = spearmanr(blosum_scores, task_df.target.values)
    else:
        b_rho = None

    return l_rho, b_rho

def evaluate_tasks(refseq_fileloc, taskfolder, task_to_file_dict):

    # Get the reference sequence
    reference_seq = str(next(SeqIO.parse(refseq_fileloc, "fasta")).seq)

    # Loop over each task
    results = [["Task", "Levenshtein Rho", "BLOSUM62 Rho"]]
    for taskname, taskfile in task_to_file_dict.items():
        rhos = evaluate_task(os.path.join(taskfolder, taskfile),
                            reference_seq)
        results.append([taskname, *rhos])

    return results_____no_output_____

# AAV_____no_output_____

def levenshtein_to_fitness_aav():

    # Define the different aav inputs
    aav_refseq_file = "../tasks/aav/P03135.fasta"
    aav_taskfolder = "../tasks/aav/tasks"
    aav_task_to_file = {
        "design": "design_task_regression.csv",
        "design_reversed": "design_task_reversed_regression.csv",
        "natural1": "natural_task_1_regression.csv",
        "natural2": "natural_task_2_regression.csv"
    }

    return evaluate_tasks(aav_refseq_file,
                          aav_taskfolder,
                          aav_task_to_file)

levenshtein_to_fitness_aav()_____no_output_____

# Cas_____no_output_____

def levenshtein_to_fitness_cas():

    # Define the different cas inputs
    refseq_file = "../tasks/cas/cas9_sequence.fasta"
    taskfolder = "../tasks/cas/tasks/"
    task_to_file = {
        "neg": "pi_domain_log_negative_selection_regression.csv",
        "pos": "pi_domain_log_positive_selection_regression.csv"
    }

    return evaluate_tasks(refseq_file,
                          taskfolder,
                          task_to_file)

levenshtein_to_fitness_cas()_____no_output_____

# GB1_____no_output_____

def levenshtein_to_fitness_gb1():

    # Define the inputs
    refseq_file = "../tasks/gb1/5LDE_1.fasta"
    taskfolder = "../tasks/gb1/tasks/"
    task_to_file = {
        "task_1": "four_mutations_task_1.csv",
        "task_2": "four_mutations_task_2.csv",
        "task_3": "four_mutations_task_3.csv",
        "task_4": "four_mutations_task_4.csv"
    }

    return evaluate_tasks(refseq_file,
                          taskfolder,
                          task_to_file)

levenshtein_to_fitness_gb1()_____no_output_____