# Notebook from arischwartz/test
Path: ipynb/bac_genome/n1147_pt2/atomIncorp_sampDepth.ipynb

# Goal

Accuracy as a function of isotope incorporation & sampling depth

`acc ~ depth + incorp`

### Variable parameters:

* atom % isotope incorporation
  * 0, 15, 25, 50, 75, 100
* sampling depth
  * 1e2, 1e3, 1e4, 1e5
* n-reps (stocastic: taxon abundances & which incorporate)
  * 10 _____no_output_____# Setting paths_____no_output_____

# paths
import os

workDir = '/ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/'
buildDir = os.path.join(workDir, 'atomIncorp_sampDepth')
R_dir = '/ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/SIPSimR/scripts/'

fragFile = '/ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/ampFrags_kde.pkl'
genome_index = '/ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/genome_index.txt'_____no_output_____

# Init_____no_output_____

import glob
import itertools
import nestly_____no_output_____%load_ext pushmsgThe pushmsg extension is already loaded. To reload it, use:
  %reload_ext pushmsg
if not os.path.isdir(buildDir):
    os.makedirs(buildDir)
%cd $buildDir/ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth

## BD min/max_____no_output_____

## min G+C cutoff
min_GC = 13.5
## max G+C cutoff
max_GC = 80
## max G+C shift
max_13C_shift_in_BD = 0.036

min_BD = min_GC/100.0 * 0.098 + 1.66
max_BD = max_GC/100.0 * 0.098 + 1.66

max_BD = max_BD + max_13C_shift_in_BD

print('Min BD: {}'.format(min_BD))
print('Max BD: {}'.format(max_BD))Min BD: 1.67323
Max BD: 1.7744

# Nestly_____no_output_____

# making an experimental design file for qSIP
x = range(1,7)
y = ['control', 'treatment']

expDesignFile = os.path.join(buildDir, 'qSIP_exp_design.txt')
with open(expDesignFile, 'wb') as outFH:
    for i,z in itertools.izip(x,itertools.cycle(y)):
        line = '\t'.join([str(i),z])
        outFH.write(line + '\n')

!head $expDesignFile       1	control
2	treatment
3	control
4	treatment
5	control
6	treatment

## Nestly params_____no_output_____

# building tree structure
nest = nestly.Nest()

# varying params: TEST
#nest.add('percIncorp', [0, 100])
#nest.add('subsample_size', [100, 10000])
#nest.add('rep', [1])

# varying params
nest.add('percIncorp', [0, 15, 25, 50, 75, 100])
nest.add('subsample_size', [100, 1000, 10000, 100000])
nest.add('rep', range(1,11))

## set params
nest.add('percTaxa', [10], create_dir=False)
nest.add('abs', ['1e9'], create_dir=False)
#nest.add('abs', ['1e7'], create_dir=False)   # TESTING
nest.add('np', [4], create_dir=False)
nest.add('Monte_rep', [100000], create_dir=False)
nest.add('subsample_dist', ['uniform'], create_dir=False)
nest.add('min_BD', [min_BD], create_dir=False)
nest.add('max_BD', [max_BD], create_dir=False)
nest.add('DBL_scaling', [0.5], create_dir=False)
nest.add('bandwidth', [0.8], create_dir=False)
nest.add('heavy_BD_min', [1.71], create_dir=False)
nest.add('heavy_BD_max', [1.75], create_dir=False)
nest.add('topTaxaToPlot', [100], create_dir=False)
nest.add('padj', [0.1], create_dir=False)
nest.add('log2', [0.25], create_dir=False)

### input/output files
nest.add('buildDir', [buildDir], create_dir=False)
nest.add('R_dir', [R_dir], create_dir=False)
nest.add('genome_index', [genome_index], create_dir=False)
nest.add('fragFile', [fragFile], create_dir=False)
nest.add('exp_design', [expDesignFile], create_dir=False)

# building directory tree
nest.build(buildDir)

# bash file to run
bashFile = os.path.join(buildDir, 'SIPSimRun.sh')_____no_output_____

## Experimental design_____no_output_____

bashFileTmp = os.path.splitext(bashFile)[0] + '_expDesign.sh'
bashFileTmp_____no_output_____%%writefile $bashFileTmp
#!/bin/bash
source activate SIPSim
# OPENBLAS threads
export OMP_NUM_THREADS=1

echo '#-- Experimental design --#'

echo '# Making an isotope incorporation config file'
echo '## 3 replicate gradients for control & treatment'
SIPSim incorp_config_example \
  --percIncorpUnif {percIncorp} \
  --n_reps 3 \
  > incorp.config

echo '# Selecting incorporator taxa'
echo '## This is to make the gradient replicates consistent (qSIP finds mean among replicates)'
SIPSim KDE_select_taxa \
    -p {percTaxa} \
    {fragFile} \
    > incorporators.txt

echo '# Creating a community file (3 replicate control, 3 replicate treatment)'
SIPSim communities \
    --config incorp.config \
    {genome_index} \
    > comm.txt

echo '# simulating gradient fractions'
SIPSim gradient_fractions \
    --BD_min {min_BD} \
    --BD_max {max_BD} \
    comm.txt \
    > fracs.txt        Writing /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/SIPSimRun_expDesign.sh
!chmod 755 $bashFileTmp_____no_output_____%%bash -s $workDir $bashFileTmp $buildDir
# offset job start to prevent conda activate errors
sleep $[ ( $RANDOM % 10 )  + 1 ]s
source activate py2_ley0.4
# change to working dir
cd $1
# run job
nestrun --template-file $2 -d $3 --log-file exp_design.log -j 202017-07-30 15:16:48,135 * INFO * Template: ./SIPSimRun_expDesign.sh
2017-07-30 15:16:48,140 * INFO * [44847] Started ./SIPSimRun_expDesign.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100/1
2017-07-30 15:16:48,145 * INFO * [44850] Started ./SIPSimRun_expDesign.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100/2
2017-07-30 15:16:48,149 * INFO * [44858] Started ./SIPSimRun_expDesign.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100/3
2017-07-30 15:16:48,156 * INFO * [44866] Started ./SIPSimRun_expDesign.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100/4
2017-07-30 15:16:48,160 * INFO * [44874] Started ./SIPSimRun_expDesign.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100/5
2017-07-30 15:16:48,167 * INFO * [44885] Started ./SIPSimRun_expDesign.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100/6
2017-07-30 15:16:48,171 * INFO * [44890] Started ./SIPSimRun_expDesign.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100/7
2017-07-30 15:16:48,177 * INFO * [44900] Started ./SIPSimRun_expDesign.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100/8
2017-07-30 15:16:48,184 * INFO * [44908] Started ./SIPSimRun_expDesign.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100/9
2017-07-30 15:16:48,189 * INFO * [44916] Started ./SIPSimRun_expDesign.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100/10
2017-07-30 15:16:48,195 * INFO * [44924] Started ./SIPSimRun_expDesign.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/1000/1
2017-07-30 15:16:48,201 * INFO * [44931] Started ./SIPSimRun_expDesign.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/1000/2
2017-07-30 15:16:48,207 * INFO * [44940] Started ./SIPSimRun_expDesign.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/1000/3
2017-07-30 15:16:48,214 * INFO * [44946] Started ./SIPSimRun_expDesign.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/1000/4
2017-07-30 15:16:48,221 * INFO * [44958] Started ./SIPSimRun_expDesign.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/1000/5
2017-07-30 15:16:48,226 * INFO * [44964] Started ./SIPSimRun_expDesign.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/1000/6
2017-07-30 15:16:48,234 * INFO * [44972] Started ./SIPSimRun_expDesign.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/1000/7
2017-07-30 15:16:48,241 * INFO * [44980] Started ./SIPSimRun_expDesign.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/1000/8
2017-07-30 15:16:48,246 * INFO * [44986] Started ./SIPSimRun_expDesign.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/1000/9
2017-07-30 15:16:48,254 * INFO * [44994] Started ./SIPSimRun_expDesign.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/1000/10
2017-07-30 15:17:01,916 * INFO * [44916] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100/10 Finished with 0
2017-07-30 15:17:01,926 * INFO * [45628] Started ./SIPSimRun_expDesign.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/10000/1
2017-07-30 15:17:01,954 * INFO * [44964] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/1000/6 Finished with 0
2017-07-30 15:17:01,963 * INFO * [45639] Started ./SIPSimRun_expDesign.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/10000/2
2017-07-30 15:17:01,998 * INFO * [44866] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100/4 Finished with 0
2017-07-30 15:17:02,021 * INFO * [45647] Started ./SIPSimRun_expDesign.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/10000/3
2017-07-30 15:17:02,021 * INFO * [44890] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100/7 Finished with 0
2017-07-30 15:17:02,025 * INFO * [45650] Started ./SIPSimRun_expDesign.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/10000/4
2017-07-30 15:17:02,026 * INFO * [44900] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100/8 Finished with 0
2017-07-30 15:17:02,032 * INFO * [45660] Started ./SIPSimRun_expDesign.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/10000/5
2017-07-30 15:17:02,032 * INFO * [44885] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100/6 Finished with 0
2017-07-30 15:17:02,043 * INFO * [45671] Started ./SIPSimRun_expDesign.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/10000/6
2017-07-30 15:17:02,044 * INFO * [44931] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/1000/2 Finished with 0
2017-07-30 15:17:02,050 * INFO * [45677] Started ./SIPSimRun_expDesign.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/10000/7
2017-07-30 15:17:02,066 * INFO * [44858] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100/3 Finished with 0
2017-07-30 15:17:02,073 * INFO * [45687] Started ./SIPSimRun_expDesign.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/10000/8
2017-07-30 15:17:02,074 * INFO * [44994] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/1000/10 Finished with 0
2017-07-30 15:17:02,085 * INFO * [45691] Started ./SIPSimRun_expDesign.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/10000/9
2017-07-30 15:17:02,087 * INFO * [44850] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100/2 Finished with 0
2017-07-30 15:17:02,093 * INFO * [45698] Started ./SIPSimRun_expDesign.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/10000/10
2017-07-30 15:17:02,095 * INFO * [44924] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/1000/1 Finished with 0
2017-07-30 15:17:02,102 * INFO * [45710] Started ./SIPSimRun_expDesign.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100000/1
2017-07-30 15:17:02,137 * INFO * [44940] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/1000/3 Finished with 0
2017-07-30 15:17:02,144 * INFO * [45719] Started ./SIPSimRun_expDesign.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100000/2
2017-07-30 15:17:02,176 * INFO * [44908] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100/9 Finished with 0
2017-07-30 15:17:02,183 * INFO * [45727] Started ./SIPSimRun_expDesign.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100000/3
2017-07-30 15:17:02,183 * INFO * [44980] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/1000/8 Finished with 0
2017-07-30 15:17:02,189 * INFO * [45730] Started ./SIPSimRun_expDesign.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100000/4
2017-07-30 15:17:02,189 * INFO * [44946] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/1000/4 Finished with 0
2017-07-30 15:17:02,198 * INFO * [45738] Started ./SIPSimRun_expDesign.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100000/5
2017-07-30 15:17:02,198 * INFO * [44847] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100/1 Finished with 0
2017-07-30 15:17:02,203 * INFO * [45746] Started ./SIPSimRun_expDesign.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100000/6
2017-07-30 15:17:02,203 * INFO * [44958] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/1000/5 Finished with 0
2017-07-30 15:17:02,208 * INFO * [45755] Started ./SIPSimRun_expDesign.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100000/7
2017-07-30 15:17:02,212 * INFO * [44972] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/1000/7 Finished with 0
2017-07-30 15:17:02,219 * INFO * [45767] Started ./SIPSimRun_expDesign.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100000/8
2017-07-30 15:17:02,220 * INFO * [44874] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100/5 Finished with 0
2017-07-30 15:17:02,228 * INFO * [45774] Started ./SIPSimRun_expDesign.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100000/9
2017-07-30 15:17:02,243 * INFO * [44986] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/1000/9 Finished with 0
2017-07-30 15:17:02,251 * INFO * [45783] Started ./SIPSimRun_expDesign.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100000/10
2017-07-30 15:17:12,319 * INFO * [45639] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/10000/2 Finished with 0
2017-07-30 15:17:12,324 * INFO * [46417] Started ./SIPSimRun_expDesign.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/100/1
2017-07-30 15:17:12,396 * INFO * [45671] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/10000/6 Finished with 0
2017-07-30 15:17:12,416 * INFO * [46425] Started ./SIPSimRun_expDesign.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/100/2
2017-07-30 15:17:12,428 * INFO * [45628] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/10000/1 Finished with 0
2017-07-30 15:17:12,435 * INFO * [46433] Started ./SIPSimRun_expDesign.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/100/3
2017-07-30 15:17:12,465 * INFO * [45755] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100000/7 Finished with 0
2017-07-30 15:17:12,472 * INFO * [46441] Started ./SIPSimRun_expDesign.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/100/4
2017-07-30 15:17:12,472 * INFO * [45660] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/10000/5 Finished with 0
2017-07-30 15:17:12,478 * INFO * [46444] Started ./SIPSimRun_expDesign.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/100/5
2017-07-30 15:17:12,478 * INFO * [45698] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/10000/10 Finished with 0
2017-07-30 15:17:12,491 * INFO * [46453] Started ./SIPSimRun_expDesign.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/100/6
2017-07-30 15:17:12,491 * INFO * [45677] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/10000/7 Finished with 0
2017-07-30 15:17:12,498 * INFO * [46462] Started ./SIPSimRun_expDesign.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/100/7
2017-07-30 15:17:12,498 * INFO * [45687] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/10000/8 Finished with 0
2017-07-30 15:17:12,504 * INFO * [46468] Started ./SIPSimRun_expDesign.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/100/8
2017-07-30 15:17:12,505 * INFO * [45650] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/10000/4 Finished with 0
2017-07-30 15:17:12,510 * INFO * [46477] Started ./SIPSimRun_expDesign.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/100/9
2017-07-30 15:17:12,510 * INFO * [45719] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100000/2 Finished with 0
2017-07-30 15:17:12,516 * INFO * [46484] Started ./SIPSimRun_expDesign.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/100/10
2017-07-30 15:17:12,516 * INFO * [45691] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/10000/9 Finished with 0
2017-07-30 15:17:12,522 * INFO * [46494] Started ./SIPSimRun_expDesign.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/1000/1
2017-07-30 15:17:12,523 * INFO * [45647] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/10000/3 Finished with 0
2017-07-30 15:17:12,531 * INFO * [46504] Started ./SIPSimRun_expDesign.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/1000/2
2017-07-30 15:17:12,537 * INFO * [45738] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100000/5 Finished with 0
2017-07-30 15:17:12,543 * INFO * [46513] Started ./SIPSimRun_expDesign.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/1000/3
2017-07-30 15:17:12,574 * INFO * [45774] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100000/9 Finished with 0
2017-07-30 15:17:12,581 * INFO * [46521] Started ./SIPSimRun_expDesign.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/1000/4
2017-07-30 15:17:12,582 * INFO * [45710] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100000/1 Finished with 0
2017-07-30 15:17:12,587 * INFO * [46524] Started ./SIPSimRun_expDesign.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/1000/5
2017-07-30 15:17:12,629 * INFO * [45767] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100000/8 Finished with 0
2017-07-30 15:17:12,636 * INFO * [46537] Started ./SIPSimRun_expDesign.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/1000/6
2017-07-30 15:17:12,640 * INFO * [45783] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100000/10 Finished with 0
2017-07-30 15:17:12,650 * INFO * [46545] Started ./SIPSimRun_expDesign.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/1000/7
2017-07-30 15:17:12,650 * INFO * [45727] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100000/3 Finished with 0
2017-07-30 15:17:12,659 * INFO * [46552] Started ./SIPSimRun_expDesign.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/1000/8
2017-07-30 15:17:12,659 * INFO * [45730] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100000/4 Finished with 0
2017-07-30 15:17:12,664 * INFO * [46558] Started ./SIPSimRun_expDesign.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/1000/9
2017-07-30 15:17:12,664 * INFO * [45746] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100000/6 Finished with 0
2017-07-30 15:17:12,674 * INFO * [46569] Started ./SIPSimRun_expDesign.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/1000/10
2017-07-30 15:17:22,543 * INFO * [46433] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/100/3 Finished with 0
2017-07-30 15:17:22,549 * INFO * [47198] Started ./SIPSimRun_expDesign.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/10000/1
2017-07-30 15:17:22,709 * INFO * [46462] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/100/7 Finished with 0
2017-07-30 15:17:22,714 * INFO * [47206] Started ./SIPSimRun_expDesign.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/10000/2
2017-07-30 15:17:22,714 * INFO * [46444] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/100/5 Finished with 0
2017-07-30 15:17:22,722 * INFO * [47211] Started ./SIPSimRun_expDesign.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/10000/3
2017-07-30 15:17:22,723 * INFO * [46484] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/100/10 Finished with 0
2017-07-30 15:17:22,729 * INFO * [47219] Started ./SIPSimRun_expDesign.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/10000/4
2017-07-30 15:17:22,746 * INFO * [46468] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/100/8 Finished with 0
2017-07-30 15:17:22,764 * INFO * [47230] Started ./SIPSimRun_expDesign.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/10000/5
2017-07-30 15:17:22,765 * INFO * [46494] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/1000/1 Finished with 0
2017-07-30 15:17:22,769 * INFO * [47233] Started ./SIPSimRun_expDesign.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/10000/6
2017-07-30 15:17:22,777 * INFO * [46417] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/100/1 Finished with 0
2017-07-30 15:17:22,782 * INFO * [47246] Started ./SIPSimRun_expDesign.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/10000/7
2017-07-30 15:17:22,796 * INFO * [46477] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/100/9 Finished with 0
2017-07-30 15:17:22,809 * INFO * [47254] Started ./SIPSimRun_expDesign.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/10000/8
2017-07-30 15:17:22,809 * INFO * [46441] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/100/4 Finished with 0
2017-07-%pushmsg "exp_design complete: $buildDir"_____no_output_____

## SIPSim pipeline_____no_output_____

bashFileTmp = os.path.splitext(bashFile)[0] + '_SIPSim-pipeline.sh'
bashFileTmp_____no_output_____%%writefile $bashFileTmp
#!/bin/bash
# offset job start to prevent conda activate errors
sleep $[ ( $RANDOM % 10 )  + 1 ]s
source activate SIPSim
# OPENBLAS threads
export OMP_NUM_THREADS=1

echo '#-- SIPSim pipeline --#'

echo '# Adding diffusion'
SIPSim diffusion \
    -n {Monte_rep} \
    --bw {bandwidth} \
    --np {np} \
    {fragFile} \
    > ampFrags_KDE_dif.pkl

echo '# Adding DBL contamination; abundance-weighted smearing'
SIPSim DBL \
    -n {Monte_rep} \
    --comm comm.txt \
    --commx {DBL_scaling} \
    --np {np} \
    ampFrags_KDE_dif.pkl \
    > ampFrags_KDE_dif_DBL.pkl

echo '# Adding isotope incorporation to BD distribution'
SIPSim isotope_incorp \
    -n {Monte_rep} \
    --comm comm.txt \
    --taxa incorporators.txt \
    --np {np} \
    ampFrags_KDE_dif_DBL.pkl \
    incorp.config \
    > ampFrags_KDE_dif_DBL_inc.pkl

echo '# Simulating an OTU table'
SIPSim OTU_table \
    --abs {abs} \
    --np {np} \
    ampFrags_KDE_dif_DBL_inc.pkl \
    comm.txt \
    fracs.txt \
    > OTU_abs{abs}.txt

echo '# Simulating PCR'
SIPSim OTU_PCR \
    OTU_abs{abs}.txt \
    > OTU_abs{abs}_PCR.txt

echo '# Subsampling from the OTU table (simulating sequencing of the DNA pool)'
SIPSim OTU_subsample \
    --dist {subsample_dist} \
    --dist_params low:{subsample_size},high:{subsample_size} \
    OTU_abs{abs}_PCR.txt \
    > OTU_abs{abs}_PCR_sub.txt

echo '# Making a wide-formatted table'
SIPSim OTU_wide_long -w \
    OTU_abs{abs}_PCR_sub.txt \
    > OTU_abs{abs}_PCR_sub_w.txt

echo '# Making metadata (phyloseq: sample_data)'
SIPSim OTU_sample_data \
    OTU_abs{abs}_PCR_sub.txt \
    > OTU_abs{abs}_PCR_sub_meta.txt

#-- removing large intermediate files --#
rm -f ampFrags_KDE_dif.pkl
rm -f ampFrags_KDE_dif_DBL.pkl
rm -f ampFrags_KDE_dif_DBL_inc.pkl    Writing /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/SIPSimRun_SIPSim-pipeline.sh
!chmod 755 $bashFileTmp_____no_output_____%%bash -s $workDir $bashFileTmp $buildDir
source activate py2_ley0.4
cd $1

nestrun --template-file $2 -d $3 --log-file SIPSim_pipeline.log -j 6 --stop-on-error_____no_output_____%pushmsg "SIPSim pipeline complete: $buildDir"_____no_output_____

## Summary of simulated data_____no_output_____

bashFileTmp = os.path.splitext(bashFile)[0] + '_SIPSim-summary.sh'
bashFileTmp_____no_output_____%%writefile $bashFileTmp
#!/bin/bash
# offset job start to prevent conda activate errors
sleep $[ ( $RANDOM % 10 )  + 1 ]s
source activate SIPSim

echo "# Plotting taxon abundances"

# plotting 'raw' taxon abundances
Rscript {R_dir}OTU_taxonAbund.R \
    OTU_abs{abs}.txt \
    -r {topTaxaToPlot} \
    -o OTU_abs{abs}

# plotting 'sequenced' taxon abundances
Rscript {R_dir}OTU_taxonAbund.R \
    OTU_abs{abs}_PCR_sub.txt \
    -r {topTaxaToPlot} \
    -o OTU_abs{abs}_PCR_sub_____no_output_____!chmod 755 $bashFileTmp_____no_output_____%%bash -s $workDir $bashFileTmp $buildDir
source activate py2_ley0.4
cd $1

nestrun --template-file $2 -d $3 --log-file SIPSim_summary.log -j 20_____no_output_____

## HR-SIP_____no_output_____

bashFileTmp = os.path.splitext(bashFile)[0] + '_HRSIP.sh'
bashFileTmp_____no_output_____%%writefile $bashFileTmp
#!/bin/bash
# offset job start to prevent conda activate errors
sleep $[ ( $RANDOM % 10 )  + 1 ]s
source activate SIPSim

# phyloseq
## making phyloseq object from OTU table
Rscript {R_dir}phyloseq_make.R \
    OTU_abs{abs}_PCR_sub_w.txt \
    -s OTU_abs{abs}_PCR_sub_meta.txt \
    > OTU_abs{abs}_PCR_sub.physeq

## filtering phyloseq object to just 'heavy' fractions
Rscript {R_dir}phyloseq_edit.R \
    OTU_abs{abs}_PCR_sub.physeq \
    --BD_min {heavy_BD_min} \
    --BD_max {heavy_BD_max} \
    > OTU_abs{abs}_PCR_sub_filt.physeq

## making ordination
Rscript {R_dir}phyloseq_ordination.R \
    OTU_abs{abs}_PCR_sub_filt.physeq \
    OTU_abs{abs}_PCR_sub_filt_bray-NMDS.pdf

# DESeq2
Rscript {R_dir}phyloseq_DESeq2.R \
    --log2 {log2} \
    --hypo greater \
    --cont 1,3,5 \
    --treat 2,4,6 \
    OTU_abs{abs}_PCR_sub_filt.physeq \
    > OTU_abs{abs}_PCR_sub_filt_DESeq2_____no_output_____!chmod 755 $bashFileTmp_____no_output_____%%bash -s $workDir $bashFileTmp $buildDir
source activate py2_ley0.4
cd $1

nestrun --template-file $2 -d $3 --log-file HR-SIP.log -j 20_____no_output_____%pushmsg "HR-SIP complete: $buildDir"_____no_output_____

## MW-HR-SIP_____no_output_____

bashFileTmp = os.path.splitext(bashFile)[0] + '_MWHRSIP.sh'
bashFileTmp_____no_output_____%%writefile $bashFileTmp
#!/bin/bash
# offset job start to prevent conda activate errors
sleep $[ ( $RANDOM % 10 )  + 1 ]s
source activate SIPSim

## HR SIP pipeline
Rscript {R_dir}phyloseq_DESeq2.R \
    --log2 {log2} \
    --hypo greater \
    --cont 1,3,5 \
    --treat 2,4,6 \
    --occur_all 0.0,0.05,0.1,0.15,0.2,0.25,0.3,0.35,0.4,0.45,0.5 \
    -w 1.70-1.73,1.72-1.75,1.74-1.77 \
    --all OTU_abs1e9_PCR_sub_MW-all.txt \
    OTU_abs{abs}_PCR_sub.physeq \
    > OTU_abs{abs}_PCR_sub_filt_MW_DESeq2Overwriting /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/SIPSimRun_MWHRSIP.sh
!chmod 755 $bashFileTmp_____no_output_____%%bash -s $workDir $bashFileTmp $buildDir
source activate py2_ley0.4
cd $1

nestrun --template-file $2 -d $3 --log-file MW-HR-SIP.log -j 20_____no_output_____%pushmsg "MW-HR-SIP complete: $buildDir"_____no_output_____

## q-SIP_____no_output_____

bashFileTmp = os.path.splitext(bashFile)[0] + '_qSIP.sh'
bashFileTmp_____no_output_____%%writefile $bashFileTmp
#!/bin/bash
# offset job start to prevent conda activate errors
sleep $[ ( $RANDOM % 10 )  + 1 ]s
source activate SIPSim
# OPENBLAS threads
export OMP_NUM_THREADS=1

# qSIP
SIPSim qSIP \
    OTU_abs{abs}.txt \
    OTU_abs{abs}_PCR_sub.txt \
    > OTU_abs{abs}_PCR_sub_qSIP.txt

# qSIP: atom excess
SIPSim qSIP_atom_excess \
    --np {np} \
    OTU_abs{abs}_PCR_sub_qSIP.txt \
    {exp_design} \
    > OTU_abs{abs}_PCR_sub_qSIP_atom.txt  _____no_output_____!chmod 755 $bashFileTmp_____no_output_____%%bash -s $workDir $bashFileTmp $buildDir
source activate py2_ley0.4
cd $1

nestrun --template-file $2 -d $3 --log-file qSIP.log -j 6_____no_output_____%pushmsg "q-SIP complete: $buildDir"_____no_output_____

## delta-BD_____no_output_____

bashFileTmp = os.path.splitext(bashFile)[0] + '_dBD.sh'
bashFileTmp_____no_output_____%%writefile $bashFileTmp
#!/bin/bash
# offset job start to prevent conda activate errors
sleep $[ ( $RANDOM % 10 )  + 1 ]s
source activate SIPSim
# OPENBLAS threads
export OMP_NUM_THREADS=1

#deltaBD
SIPSim deltaBD \
    OTU_abs{abs}_PCR_sub.txt \
    {exp_design} \
    > OTU_abs{abs}_PCR_sub_dBD.txt  _____no_output_____!chmod 755 $bashFileTmp_____no_output_____%%bash -s $workDir $bashFileTmp $buildDir
source activate py2_ley0.4
cd $1

nestrun --template-file $2 -d $3 --log-file deltaBD.log -j 20_____no_output_____%pushmsg "deltaBD complete: $buildDir"_____no_output_____

## Making confusion matrices_____no_output_____

bashFileTmp = os.path.splitext(bashFile)[0] + '_cMtx.sh'
bashFileTmp_____no_output_____%%writefile $bashFileTmp
#!/bin/bash
# offset job start to prevent conda activate errors
sleep $[ ( $RANDOM % 10 )  + 1 ]s
source activate SIPSim

# HR-SIP
Rscript {R_dir}DESeq2_confuseMtx.R \
    --libs 2,4,6 \
    --padj {padj} \
    BD-shift_stats.txt \
    OTU_abs{abs}_PCR_sub_filt_DESeq2

# HR-SIP multiple 'heavy' BD windows
Rscript {R_dir}DESeq2_confuseMtx.R \
    --libs 2,4,6 \
    --padj {padj} \
    -o DESeq2_multi-cMtx \
    BD-shift_stats.txt \
    OTU_abs{abs}_PCR_sub_filt_MW_DESeq2

# qSIP
Rscript {R_dir}qSIP_confuseMtx.R \
    --libs 2,4,6 \
    BD-shift_stats.txt \
    OTU_abs{abs}_PCR_sub_qSIP_atom.txt

# heavy-SIP
Rscript {R_dir}heavy_confuseMtx.R \
    --treat 2,4,6 \
    --con 1,3,5 \
    --method 1 \
    BD-shift_stats.txt \
    OTU_abs{abs}_PCR_sub.txtOverwriting /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/SIPSimRun_cMtx.sh
!chmod 755 $bashFileTmp_____no_output_____%%bash -s $workDir $bashFileTmp $buildDir
source activate py2_ley0.4
cd $1

nestrun --template-file $2 -d $3 --log-file cMtx.log -j 202017-08-21 19:58:50,422 * INFO * Template: ./SIPSimRun_cMtx.sh
2017-08-21 19:58:50,427 * INFO * [42752] Started ./SIPSimRun_cMtx.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100/1
2017-08-21 19:58:50,432 * INFO * [42754] Started ./SIPSimRun_cMtx.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100/2
2017-08-21 19:58:50,437 * INFO * [42756] Started ./SIPSimRun_cMtx.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100/3
2017-08-21 19:58:50,443 * INFO * [42758] Started ./SIPSimRun_cMtx.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100/4
2017-08-21 19:58:50,448 * INFO * [42760] Started ./SIPSimRun_cMtx.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100/5
2017-08-21 19:58:50,455 * INFO * [42762] Started ./SIPSimRun_cMtx.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100/6
2017-08-21 19:58:50,461 * INFO * [42764] Started ./SIPSimRun_cMtx.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100/7
2017-08-21 19:58:50,468 * INFO * [42766] Started ./SIPSimRun_cMtx.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100/8
2017-08-21 19:58:50,475 * INFO * [42768] Started ./SIPSimRun_cMtx.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100/9
2017-08-21 19:58:50,481 * INFO * [42770] Started ./SIPSimRun_cMtx.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100/10
2017-08-21 19:58:50,488 * INFO * [42772] Started ./SIPSimRun_cMtx.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/1000/1
2017-08-21 19:58:50,493 * INFO * [42774] Started ./SIPSimRun_cMtx.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/1000/2
2017-08-21 19:58:50,497 * INFO * [42776] Started ./SIPSimRun_cMtx.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/1000/3
2017-08-21 19:58:50,503 * INFO * [42778] Started ./SIPSimRun_cMtx.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/1000/4
2017-08-21 19:58:50,508 * INFO * [42780] Started ./SIPSimRun_cMtx.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/1000/5
2017-08-21 19:58:50,516 * INFO * [42782] Started ./SIPSimRun_cMtx.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/1000/6
2017-08-21 19:58:50,522 * INFO * [42784] Started ./SIPSimRun_cMtx.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/1000/7
2017-08-21 19:58:50,528 * INFO * [42786] Started ./SIPSimRun_cMtx.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/1000/8
2017-08-21 19:58:50,536 * INFO * [42788] Started ./SIPSimRun_cMtx.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/1000/9
2017-08-21 19:58:50,542 * INFO * [42790] Started ./SIPSimRun_cMtx.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/1000/10
2017-08-21 19:59:14,749 * INFO * [42774] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/1000/2 Finished with 0
2017-08-21 19:59:14,756 * INFO * [43934] Started ./SIPSimRun_cMtx.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/10000/1
2017-08-21 19:59:15,476 * INFO * [42760] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100/5 Finished with 0
2017-08-21 19:59:15,483 * INFO * [43962] Started ./SIPSimRun_cMtx.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/10000/2
2017-08-21 19:59:16,320 * INFO * [42780] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/1000/5 Finished with 0
2017-08-21 19:59:16,330 * INFO * [43982] Started ./SIPSimRun_cMtx.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/10000/3
2017-08-21 19:59:17,399 * INFO * [42768] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100/9 Finished with 0
2017-08-21 19:59:17,405 * INFO * [44002] Started ./SIPSimRun_cMtx.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/10000/4
2017-08-21 19:59:18,208 * INFO * [42782] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/1000/6 Finished with 0
2017-08-21 19:59:18,212 * INFO * [44014] Started ./SIPSimRun_cMtx.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/10000/5
2017-08-21 19:59:18,501 * INFO * [42776] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/1000/3 Finished with 0
2017-08-21 19:59:18,509 * INFO * [44025] Started ./SIPSimRun_cMtx.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/10000/6
2017-08-21 19:59:18,536 * INFO * [42788] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/1000/9 Finished with 0
2017-08-21 19:59:18,545 * INFO * [44031] Started ./SIPSimRun_cMtx.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/10000/7
2017-08-21 19:59:18,546 * INFO * [42766] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100/8 Finished with 0
2017-08-21 19:59:18,552 * INFO * [44033] Started ./SIPSimRun_cMtx.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/10000/8
2017-08-21 19:59:19,425 * INFO * [42786] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/1000/8 Finished with 0
2017-08-21 19:59:19,430 * INFO * [44038] Started ./SIPSimRun_cMtx.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/10000/9
2017-08-21 19:59:19,677 * INFO * [42756] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100/3 Finished with 0
2017-08-21 19:59:19,684 * INFO * [44042] Started ./SIPSimRun_cMtx.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/10000/10
2017-08-21 19:59:19,933 * INFO * [42784] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/1000/7 Finished with 0
2017-08-21 19:59:19,940 * INFO * [44056] Started ./SIPSimRun_cMtx.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100000/1
2017-08-21 19:59:22,054 * INFO * [42752] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100/1 Finished with 0
2017-08-21 19:59:22,061 * INFO * [44086] Started ./SIPSimRun_cMtx.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100000/2
2017-08-21 19:59:22,928 * INFO * [42758] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100/4 Finished with 0
2017-08-21 19:59:22,934 * INFO * [44103] Started ./SIPSimRun_cMtx.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100000/3
2017-08-21 19:59:23,163 * INFO * [42770] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100/10 Finished with 0
2017-08-21 19:59:23,172 * INFO * [44112] Started ./SIPSimRun_cMtx.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100000/4
2017-08-21 19:59:23,247 * INFO * [42790] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/1000/10 Finished with 0
2017-08-21 19:59:23,256 * INFO * [44116] Started ./SIPSimRun_cMtx.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100000/5
2017-08-21 19:59:23,342 * INFO * [42764] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100/7 Finished with 0
2017-08-21 19:59:23,348 * INFO * [44120] Started ./SIPSimRun_cMtx.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100000/6
2017-08-21 19:59:23,969 * INFO * [42754] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100/2 Finished with 0
2017-08-21 19:59:23,976 * INFO * [44141] Started ./SIPSimRun_cMtx.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100000/7
2017-08-21 19:59:24,105 * INFO * [42778] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/1000/4 Finished with 0
2017-08-21 19:59:24,116 * INFO * [44145] Started ./SIPSimRun_cMtx.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100000/8
2017-08-21 19:59:24,642 * INFO * [42762] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100/6 Finished with 0
2017-08-21 19:59:24,649 * INFO * [44164] Started ./SIPSimRun_cMtx.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100000/9
2017-08-21 19:59:26,075 * INFO * [42772] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/1000/1 Finished with 0
2017-08-21 19:59:26,086 * INFO * [44237] Started ./SIPSimRun_cMtx.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100000/10
2017-08-21 19:59:44,003 * INFO * [43962] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/10000/2 Finished with 0
2017-08-21 19:59:44,014 * INFO * [45182] Started ./SIPSimRun_cMtx.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/100/1
2017-08-21 19:59:45,252 * INFO * [44025] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/10000/6 Finished with 0
2017-08-21 19:59:45,259 * INFO * [45226] Started ./SIPSimRun_cMtx.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/100/2
2017-08-21 19:59:47,877 * INFO * [44038] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/10000/9 Finished with 0
2017-08-21 19:59:47,884 * INFO * [45272] Started ./SIPSimRun_cMtx.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/100/3
2017-08-21 19:59:48,530 * INFO * [44056] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100000/1 Finished with 0
2017-08-21 19:59:48,536 * INFO * [45278] Started ./SIPSimRun_cMtx.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/100/4
2017-08-21 19:59:48,993 * INFO * [44141] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100000/7 Finished with 0
2017-08-21 19:59:49,000 * INFO * [45289] Started ./SIPSimRun_cMtx.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/100/5
2017-08-21 19:59:49,026 * INFO * [44031] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/10000/7 Finished with 0
2017-08-21 19:59:49,032 * INFO * [45293] Started ./SIPSimRun_cMtx.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/100/6
2017-08-21 19:59:49,299 * INFO * [43982] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/10000/3 Finished with 0
2017-08-21 19:59:49,307 * INFO * [45303] Started ./SIPSimRun_cMtx.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/100/7
2017-08-21 19:59:49,455 * INFO * [43934] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/10000/1 Finished with 0
2017-08-21 19:59:49,464 * INFO * [45315] Started ./SIPSimRun_cMtx.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/100/8
2017-08-21 19:59:50,568 * INFO * [44002] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/10000/4 Finished with 0
2017-08-21 19:59:50,576 * INFO * [45385] Started ./SIPSimRun_cMtx.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/100/9
2017-08-21 19:59:50,891 * INFO * [44116] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100000/5 Finished with 0
2017-08-21 19:59:50,904 * INFO * [45395] Started ./SIPSimRun_cMtx.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/100/10
2017-08-21 19:59:51,273 * INFO * [44014] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/10000/5 Finished with 0
2017-08-21 19:59:51,282 * INFO * [45411] Started ./SIPSimRun_cMtx.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/1000/1
2017-08-21 19:59:51,517 * INFO * [44120] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100000/6 Finished with 0
2017-08-21 19:59:51,530 * INFO * [45429] Started ./SIPSimRun_cMtx.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/1000/2
2017-08-21 19:59:51,760 * INFO * [44042] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/10000/10 Finished with 0
2017-08-21 19:59:51,798 * INFO * [45434] Started ./SIPSimRun_cMtx.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/1000/3
2017-08-21 19:59:52,341 * INFO * [44033] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/10000/8 Finished with 0
2017-08-21 19:59:52,349 * INFO * [45466] Started ./SIPSimRun_cMtx.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/1000/4
2017-08-21 19:59:53,160 * INFO * [44103] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100000/3 Finished with 0
2017-08-21 19:59:53,169 * INFO * [45502] Started ./SIPSimRun_cMtx.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/1000/5
2017-08-21 19:59:55,492 * INFO * [44164] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100000/9 Finished with 0
2017-08-21 19:59:55,502 * INFO * [45604] Started ./SIPSimRun_cMtx.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/1000/6
2017-08-21 19:59:56,569 * INFO * [44112] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100000/4 Finished with 0
2017-08-21 19:59:56,579 * INFO * [45664] Started ./SIPSimRun_cMtx.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/1000/7
2017-08-21 19:59:57,152 * INFO * [44086] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100000/2 Finished with 0
2017-08-21 19:59:57,161 * INFO * [45691] Started ./SIPSimRun_cMtx.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/1000/8
2017-08-21 19:59:57,646 * INFO * [44145] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100000/8 Finished with 0
2017-08-21 19:59:57,657 * INFO * [45726] Started ./SIPSimRun_cMtx.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/1000/9
2017-08-21 19:59:58,602 * INFO * [44237] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/0/100000/10 Finished with 0
2017-08-21 19:59:58,613 * INFO * [45820] Started ./SIPSimRun_cMtx.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/1000/10
2017-08-21 20:00:09,695 * INFO * [45226] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/100/2 Finished with 0
2017-08-21 20:00:09,708 * INFO * [46388] Started ./SIPSimRun_cMtx.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/10000/1
2017-08-21 20:00:13,174 * INFO * [45272] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/100/3 Finished with 0
2017-08-21 20:00:13,181 * INFO * [46456] Started ./SIPSimRun_cMtx.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/10000/2
2017-08-21 20:00:14,190 * INFO * [45278] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/100/4 Finished with 0
2017-08-21 20:00:14,199 * INFO * [46510] Started ./SIPSimRun_cMtx.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/10000/3
2017-08-21 20:00:14,205 * INFO * [45303] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/100/7 Finished with 0
2017-08-21 20:00:14,213 * INFO * [46516] Started ./SIPSimRun_cMtx.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/10000/4
2017-08-21 20:00:15,337 * INFO * [45289] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/100/5 Finished with 0
2017-08-21 20:00:15,348 * INFO * [46575] Started ./SIPSimRun_cMtx.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/10000/5
2017-08-21 20:00:17,451 * INFO * [45182] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/100/1 Finished with 0
2017-08-21 20:00:17,458 * INFO * [46632] Started ./SIPSimRun_cMtx.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/10000/6
2017-08-21 20:00:17,529 * INFO * [45411] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/1000/1 Finished with 0
2017-08-21 20:00:17,537 * INFO * [46636] Started ./SIPSimRun_cMtx.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/10000/7
2017-08-21 20:00:17,738 * INFO * [45293] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/100/6 Finished with 0
2017-08-21 20:00:17,743 * INFO * [46640] Started ./SIPSimRun_cMtx.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/10000/8
2017-08-21 20:00:17,923 * INFO * [45434] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/1000/3 Finished with 0
2017-08-21 20:00:17,928 * INFO * [46654] Started ./SIPSimRun_cMtx.sh in /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/10000/9
2017-08-21 20:00:20,805 * INFO * [45395] /ebio/abt3_projects/methanogen_host_evo/SIPSim_pt2/data/bac_genome1147/atomIncorp_sampDepth/15/100/10 Finished with 0
2017-08-21 20:00:20

### Aggregating confusion matrices_____no_output_____

def agg_cMtx(prefix):
    # all data
    x = prefix + '-cMtx_data.txt'
    !nestagg delim \
       -d $buildDir \
       -k percIncorp,subsample_size,rep \
       -o $x \
       --tab \
       $x

    # overall
    x = prefix + '-cMtx_overall.txt'
    !nestagg delim \
        -d $buildDir \
        -k percIncorp,subsample_size,rep \
        -o $x \
        --tab \
        $x

    # by class
    x = prefix + '-cMtx_byClass.txt'
    !nestagg delim \
        -d $buildDir \
        -k percIncorp,subsample_size,rep \
        -o $x \
        --tab \
        $x

agg_cMtx('DESeq2')
agg_cMtx('DESeq2_multi')
agg_cMtx('qSIP')
agg_cMtx('heavy') _____no_output_____%pushmsg "atomIncorp_sampleDepth complete!"_____no_output_____

# --End of simulation--_____no_output_____***_____no_output_____# Results_____no_output_____

F = os.path.join(buildDir, '*-cMtx_byClass.txt')
files = glob.glob(F)
files_____no_output_____# checking for errors
!find $buildDir -name "*log" | wc -l
!find $buildDir -name "*log" | xargs grep -i error1921


# Notebook from HelenaBelloff/Algorithms
Path: Algorithms Lab #2.ipynb

<code>
###################################
# Lab 2: Linked Lists and Sorting #
###################################

###
# Setup
###

# Enrollment link: http://rosalind.info/classes/enroll/083764b484/
# Course link: http://rosalind.info/classes/704/ (was 581 last year)

# BMI2005/
# labs/
#     lab1/
#         lab1.ipynb
#     rosalind/
#         rosalind_dna.txt
#         rosalind_gc.txt
_____no_output_____'''
Problem 1: Naive peak finding.
In class we discussed an algorithm to return a peak in a 1d array.
The code is shown below.
a) With the example array a, what are the peak(s)? 5, 7, and 8
b) Which peak is returned, and why? 7
c) What is the runtime of this algorithm? O(n)
'''
a = [6, 7, 4, 3, 8, 1, 4, 5]

def naive_peak_1d(a):
    n = len(a)
    for i in range(n):
        # check corner case at beginning
        if i == 0 and a[i] >= a[i+1]:
            return a[i]
        elif a[i] > a[i+1] and a[i] >= a[i-1]:
            return a[i]
        # check corner case at the end
        elif i == (n-1) and a[n-1] >= a[n-2]:
            return a[i]

naive_peak_1d(a)
_____no_output_____'''
Problem 2:
Rosalind: Insertion Sort

Translate the following pseudocode into python and verify
its performance on the downloaded file 'rosalind_ins.txt'

insertion_sort(A[1..n])
    for i <- 2 to n do
        k <- i
        while k > 1 and A[k] < A[k-1]
            SWAP(A[k-1], A[k])
            k <- k -1

Input: A string A of integers separated by spaces, e.g. A = '-323 2 32 ... 3'

Output: The number of times the swap operation was performed (hint: this will require
modifying the pseudocode a little bit) and (optionally) the sorted array.

'''

#temp = A[k-1]
#A[k-1] = A[k]
#A[k] = temp

# free parsing function
import numpy as np
with open('/Users/helenabelloff/Downloads/rosalind_ins (1).txt', 'r') as f:
    f.readline() # skips the line indicating number of entries
    A = np.array([int(x) for x in f.readline().strip().split(' ')])
    # f.readline() = '-323 2 32 ... 3\n'
    # f.readline().strip() = '-323 2 32 ... 3'
    # f.readline().strip().split(' ') = ['-323', '2', '32', ... '3']

# TO DO
def insertion_sort(A):
    count = 0
    for i in range(1,len(A)):
        k = i
        while k > 0 and A[k] < A[k-1]:
            A[k], A[k-1] = A[k-1], A[k]
            k = k - 1
            count += 1
    return count, A

count, sorted_A = insertion_sort(A)
print(count)
242764
'''
Problem 3. Comparing insertion and selection sorts
(a) Using the array A = E X A M P L E draw traces for each outer loop
iteration of insertion and selection sort.

(b) If the input array is already sorted, what are the runtimes of
selection and insertion sort? Which is faster? selection sort O(n^2), insertion sort O(n), insertion sort is faster

(c) Describe the worst-case input array. If the array is reversed order O(n^2)

(d) What is the expected runtime of insertion sort on an array with only 3 unique
values relative to the inputs in (b) and (c)?

Note that an analytical formula is not expected (e.g. O(n^3/2)) but rather empirical
demonstration is sufficient, i.e. conduct a doubling experiment (from n = 1 to n = 2048)
and empirically graph on a log-log plot the runtimes as a function of problem size.
- Create a function time_trial() that takes in an array as input and returns the time it
  takes for insertion sort to run on it.
- Create a function test_n() that returns the runtimes for a sorting algothrim on best, mid case
  (defined as the 3 unique values array)and worst case arrays as a function of n (their length).
  This function should generate the arrays in the process.
'''
# selection
def selection_sort(A):
    for i in range(len(A)):
        min_idx = i
        # inner loop compares the current value to all remaining
        # complexity and cost?
        for j in range(i + 1, len(A)):
            if A[min_idx] > A[j]:
                min_idx = j

        # work of exchange happens outside inner loop
        # complexity and cost?
        A[i], A[min_idx] = A[min_idx], A[i]
    return A

# insertion - take from Rosalind above
import time
import numpy as np

def time_trial(a, sort_method):
    start = time.process_time()
    sort_method(a)
    end = time.process_time() - start
    return end

def test_n(n, sort_method):
    # test best case (already sorted)
    best = np.array([x for x in range(n)])
    best_case_time = time_trial(best, sort_method)

    # test mid case (random series of 3 unique ints)
    mid = np.random.randint(low=0,high=3,size=n)
    mid_case_time = time_trial(mid, sort_method)

    # test worst case (reverse sorted)
    worst = np.copy(best)
    worst = list(worst[::-1])
    worst_case_time = time_trial(worst, sort_method)
    return [n, best_case_time, mid_case_time, worst_case_time]

n = 1
n_max = 2049
data = []
while n < n_max:
    data.append(test_n(n, insertion_sort))
    n = n * 2
data = np.log(np.array(data))

n_data = data[:,0]
best_data = data[:,1]
mid_data = data[:,2]
worst_data = data[:,3]

%matplotlib inline
import matplotlib.pyplot as plt
plt.figure(figsize=(6, 4))
plt.scatter(n_data, best_data)
plt.scatter(n_data, mid_data)
plt.scatter(n_data, worst_data)
_____no_output_____'''
Problem 3 (cont)
(e) What is the expected runtime of selection sort on an array with only 3 unique
values relative to the inputs in (b) and (c)? How does this differ from the
timings for insertion sort?

'''
n = 1
n_max = 2049
data = []
while n < n_max:
    data.append(test_n(n, selection_sort))
    n = n * 2
data = np.log(np.array(data))

n_data = data[:,0]
best_data = data[:,1]
mid_data = data[:,2]
worst_data = data[:,3]

%matplotlib inline
import matplotlib.pyplot as plt
plt.figure(figsize=(6, 4))
plt.scatter(n_data, best_data)
plt.scatter(n_data, mid_data)
plt.scatter(n_data, worst_data)

_____no_output_____'''
Problem 4: Linked lists and Pointers
https://xkcd.com/138/

Compare the following blocks of code. What are the values of a and b?
a)
a = [1,2,3]
b = a
b.append(4)

a = [1, 2, 3, 4]
b = [1, 2, 3, 4]
b is pointing to the same location in memory to a

b)
a = [1,2,3]
b = a
b = b.append(4)

print(a)
print(b)

a = [1,2,3]
b = none

c)
a = np.array([1,2,3])
b = a
b = np.append(b,4)

print(a)
print(b)

a = [1,2,3]
b = [1,2,3,4]

'''_____no_output_____'''
Example 5: Classes in Python
'''
class Student:
    # initialization / constructor
    def __init__(self, age, program):
        # attributes
        self.age = age
        self.program = program

    # method
    def getSummary(self):
        return str(self.age) + ', '  + str(self.program)

# create a new instance of Student
kevin = Student(27, 'MDPHD')

# modify age attribute
kevin.age = 28

# call example method
kevin.getSummary()_____no_output_____'''
Problem 4a: Linked List - the Node class
Online-formulation of LinkedList (moderate)
(a) Create a Node class with argument 'data' and has attributes 'initdata'
    (set to the argument initdata) and 'next' (set to None).
'''
# TO DO
class Node:
    def __init__(self, data):
        self.initdata = data
        self.next = None

'''
Problem 4b: Linked List - the LinkedList class
Assess the runtimes of the following LinkedList class methods
(a) add: add a new node to the front of the linked list
(b) size: return the number of linked list member nodes through a linear search of the list
(c) get_index: return the pointer to the node in index position n, where n is supplied to the function.
(d) remove_index: implement a method that removes an arbitrary linked list node from a supplied node index n.
(e) enumerate_items: list all the data values of nodes in order in the list
(f) remove_item: remove the first instance of particular data value (if it exists)
(g) search: returns True or False if query item exists in the linked list or not
'''
class LinkedList:
    def __init__(self):
        self.head = None

    def isEmpty(self):
        return self.head == None

    # a O(1)
    def add(self,item):
        newNode = Node(item) # new node with an item in it that points to None
        newNode.next = self.head # newNode.next points to None
        self.head = newNode # None becomes this new node

    # b O(n)
    def size(self):
        current = self.head
        count = 0
        while current != None:
            count = count + 1
            current = current.next

        return count

    # c O(n)
    def get_index(self, index):
        i = 0
        current = self.head
        while i < index:
            if current is None:
                print('index exceeds length')
                break
            else:
                current = current.next
                i += 1
        return current

    # d O(n)
    def remove_index(self,index):
        previous = None
        i = 0
        current = self.head
        while i < index:
            if current is None:
                print('index exceeds length')
                return current
            else:
                previous = current
                current = current.next
                i += 1
            print(current.data)
        if previous is None:
            self.head = current.next
        else:
            previous.next = current.next

    # e
    def enumerate_items(self):
        current = self.head
        while current is not None:
            print(current.data)
            current = current.next

    # f
    def remove_item(self,item):
        current = self.head
        previous = None
        found = False
        while not found:
            if current.data == item:
                found = True
            else:
                previous = current
                current = current.next

        if previous == None:
            self.head = current.next
        else:
            previous.next = current.next

    # g
    def search(self,item):
        current = self.head
        found = False
        while current != None and not found:
            if current.data == item:
                found = True
            else:
                current = current.next

        return found

# tests; construct a lists
mylist = LinkedList()
mylist.add('BMI2005') # 0
mylist.add('BMS') # 1
mylist.add('Biostatistics') # 2
mylist.add('RCR') # 3

# example operations/methods on instance
print('get size of list')
print(mylist.size())

print('\nget item in index 3')
print(mylist.get_index(3))
# BMS

print('\nget item in index 8 (DNE)')
mylist.get_index(8)

print('\nenumerate all items')
mylist.enumerate_items()

print('\nsearch for item RCR')
print(mylist.search('BMS'))

print('\nremove RCR')
mylist.remove_item('RCR')
print('item removed')

print('\nenumerate items again')
mylist.enumerate_items()

print('\nsearch for RCR; now gone')
print(mylist.search('RCR'))

print('\nremove index 1')
mylist.remove_index(1)

print('\nenumerate items')
mylist.enumerate_items()get size of list
4

get item in index 3
<__main__.Node object at 0x10fa36e10>

get item in index 8 (DNE)
index exceeds length

enumerate all items
'''
Document Distance, dictionary approach
Adapted from MIT OCW

Input: Two document strings D_1 and D_2

Output: Document distance d(D_1, D_2) defined as:

Define U as the sorted array of all words found in both D_1 and D_2
i.e. the union of D_1 and D_2. The 'word count vector' A_1 of length |U|
has positive integer entries in A_1[i] corresponding to the number of times word i
appears in document 1. E.g. if D_1 = 'the cat' and D_2 = 'the dog'
then U = ['cat', 'dog', 'the'] and A_1 = [1 0 1] and A_2 = [0 1 1]

Define the document distance as the product of counts of shared words, i.e.
d(D_1, D_2) = 1*0 + 0*1 + 1*1 = 1
Normalizing for word count (otherwise longer documents would have higher scores)
d'(D_1, D_2) = d(D_1, D_2) / ( |A_1| * |A_2| ) where |A_1| is the norm of vector A_1

d'(D_1, D_2) = dot product of A_1 and A_2

To make this more geometrically interpretable, we use np.arccos to take the inverse cosine
- a dot product of 1 evaluates to arccos of 0 meaning perfect similarity
- a dot product of 0 evaluates to arccos of 90 indicating orthogonality, or complete difference
- note that arccos returns radians

What is the runtime of your program in terms of D_1 and D_2?
'''

doc1 = 'Lorem ipsum dolor'.lower()
doc2 = 'ipsum dolor sit amet'.lower()

# 1. Get union of the lists of words (parse the docs)
# 2. Sort that union
# 3. convert from list of words to a list of counts
# 4. get the dot product / distance

import numpy as np
# parse the docs
# [x.lower() for x in ...]
doc1_list = doc1.split(' ')
doc2_list = doc2.split(' ')

# union
union = list(set(doc1_list).union(set(doc2_list)))
union.sort()

# list of counts
doc1_counts = np.zeros(len(union))
doc2_counts = np.zeros(len(union))

for i in range(len(union)):
    doc1_counts[i] = doc1.count(union[i])
    doc2_counts[i] = doc2.count(union[i])

print(doc2_counts)

#Don't worry about this dot product part
from numpy import linalg as LA
doc1_norm = LA.norm(np.array(list(doc1_counts.values())))
doc2_norm = LA.norm(np.array(list(doc2_counts.values())))
dp = total / (doc1_norm * doc2_norm)
print(np.across(dp) * 180)
[1. 1. 1. 0. 1.]

</code>


# Notebook from IBMDecisionOptimization/DO-Samples
Path: jupyter/Cloud Pak for Data v4.0.x/House Building with worker skills.ipynb

# House Building with worker skills

This tutorial includes everything you need to set up decision optimization engines, build constraint programming models.

Table of contents:

-  [Describe the business problem](#Describe-the-business-problem)
*  [How decision optimization (prescriptive analytics) can help](#How--decision-optimization-can-help)
*  [Use decision optimization](#Use-decision-optimization)
    *  [Step 1: Set up the engines](#Step-1:-Set-up-the-prescriptive-engine)
    -  [Step 2: Model the Data](#Step-2:-Model-the-data)
    -  [Step 3: Set up the prescriptive model](#Step-3:-Set-up-the-prescriptive-model)
        * [Define the decision variables](#Define-the-decision-variables)
        * [Express the business constraints](#Express-the-business-constraints)
        * [Express the objective](#Express-the-objective)
        * [Solve with Decision Optimization solve service](#Solve-with-Decision-Optimization-solve-service)
    *  [Step 4: Investigate the solution and run an example analysis](#Step-4:-Investigate-the-solution-and-then-run-an-example-analysis)
*  [Summary](#Summary)
****_____no_output_____### Describe the business problem

* This is a problem of building five houses in different locations; the masonry, roofing, painting, etc. must be scheduled. Some tasks must necessarily take place before others and these requirements are expressed through precedence constraints.

* There are three workers, and each worker has a given skill level for each task. Each task requires one worker; the worker assigned must have a non-null skill level for the task. A worker can be assigned to only one task at a time.

* Each house has a deadline.

* The objective is to maximize the skill levels of the workers assigned to the tasks._____no_output_____*****
## How  decision optimization can help
* Prescriptive analytics technology recommends actions based on desired outcomes, taking into account specific scenarios, resources, and knowledge of past and current events. This insight can help your organization make better decisions and have greater control of business outcomes.

* Prescriptive analytics is the next step on the path to insight-based actions. It creates value through synergy with predictive analytics, which analyzes data to predict future outcomes.

* Prescriptive analytics takes that insight to the next level by suggesting the optimal way to handle that future situation. Organizations that can act fast in dynamic conditions and make superior decisions in uncertain environments gain a strong competitive advantage.

+ For example:
    + Automate complex decisions and trade-offs to better manage limited resources.
    + Take advantage of a future opportunity or mitigate a future risk.
    + Proactively update recommendations based on changing events.
    + Meet operational goals, increase customer loyalty, prevent threats and fraud, and optimize business processes.
_____no_output_____
About Detailed Scheduling concepts

 Scheduling consists of assigning start and completion times to a set of activities while satisfying different types of constraints (resource availability, precedence relationships, … ) and optimizing some criteria (minimizing tardiness, …)

 Time is considered as a continuous dimension: domain of possible start/completion times for an activity is potentially very large

Besides start and completion times of activities, other types of decision variables are often involved in real industrial scheduling problems (resource allocation, optional activities …)

_____no_output_____## Use decision optimization_____no_output_____### Step 1: Set up the prescriptive engine_____no_output_____For display of the solution, ensure that last version of matplotlib is available:_____no_output_____

try:
   import matplotlib
   if matplotlib.__version__ < "1.4.3":
        !pip install --upgrade matplotlib
except:
   !pip install --user matplotlib_____no_output_____

Now, import all required modeling functions that are provided by the Constraint Programming docplex.cp package. _____no_output_____

from docplex.cp.model import CpoModel
from sys import stdout
from collections import namedtuple_____no_output_____

### Step 2: Model the data_____no_output_____Planning contains the number of houses and the max amount of periods (days) for our schedule._____no_output_____

NB_HOUSES = 5
MAX_AMOUNT_OF_PERIODS = 318
HOUSES = range(1, NB_HOUSES + 1)_____no_output_____

All tasks must start and end between 0 and the max amount of periods._____no_output_____

period_domain = (0, MAX_AMOUNT_OF_PERIODS)_____no_output_____

For each task type in the house building project, the following table shows the duration of the task in days along with the tasks that must be finished before the task can start. A worker can only work on one task at a time; each task, once started, may not be interrupted.

| *Task* | *Duration* | *Preceding tasks* |
|---|---|---|
| masonry 	| 35 |	|
| carpentry | 15 | masonry |
| plumbing 	| 40 | masonry |
| ceiling 	| 15 | masonry |
| roofing 	| 5  | carpentry |
| painting 	| 10 | ceiling |
| windows 	| 5  | roofing |
| facade 	| 10 | roofing, plumbing |
| garden 	| 5  | roofing, plumbing |
| moving 	| 5  | windows, facade, garden, painting | _____no_output_____##### Tasks' durations_____no_output_____

Task = (namedtuple("Task", ["name", "duration"]))
TASKS = {Task("masonry",   35),
         Task("carpentry", 15),
         Task("plumbing",  40),
         Task("ceiling",   15),
         Task("roofing",    5),
         Task("painting",  10),
         Task("windows",    5),
         Task("facade",    10),
         Task("garden",     5),
         Task("moving",     5),
        }_____no_output_____

##### Task precedences_____no_output_____

TaskPrecedence = (namedtuple("TaskPrecedence", ["beforeTask", "afterTask"]))
TASK_PRECEDENCES = {TaskPrecedence("masonry",   "carpentry"),
                    TaskPrecedence("masonry",   "plumbing"),
                    TaskPrecedence("masonry",   "ceiling"),
                    TaskPrecedence("carpentry", "roofing"),
                    TaskPrecedence("ceiling",   "painting"),
                    TaskPrecedence("roofing",   "windows"),
                    TaskPrecedence("roofing",   "facade"),
                    TaskPrecedence("plumbing",  "facade"),
                    TaskPrecedence("roofing",   "garden"),
                    TaskPrecedence("plumbing",  "garden"),
                    TaskPrecedence("windows",   "moving"),
                    TaskPrecedence("facade",    "moving"),
                    TaskPrecedence("garden",    "moving"),
                    TaskPrecedence("painting",  "moving"),
                   }_____no_output_____

There are three workers with varying skill levels in regard to the ten tasks. If a worker has a skill level of zero for a task, he may not be assigned to the task.

| *Task* | *Joe* | *Jack* | *Jim* |
|---|---|---|---|
|masonry   |9 |	5 |	0|
|carpentry |7 |	0 |	5|
|plumbing  |0 |	7 |	0|
|ceiling   |5 |	8 |	0|
|roofing   |6 |	7 |	0|
|painting  |0 |	9 |	6|
|windows   |8 |	0 |	5|
|façade    |5 |	5 |	0|
|garden    |5 |	5 |	9|
|moving    |6 |	0 |	8|_____no_output_____##### Workers Names_____no_output_____

WORKERS = {"Joe", "Jack", "Jim"}_____no_output_____

##### Workers Name and level for each of there skill_____no_output_____

Skill = (namedtuple("Skill", ["worker", "task", "level"]))
SKILLS = {Skill("Joe",  "masonry",   9),
          Skill("Joe",  "carpentry", 7),
          Skill("Joe",  "ceiling",   5),
          Skill("Joe",  "roofing",   6),
          Skill("Joe",  "windows",   8),
          Skill("Joe",  "facade",    5),
          Skill("Joe",  "garden",    5),
          Skill("Joe",  "moving",    6),
          Skill("Jack", "masonry",   5),
          Skill("Jack", "plumbing",  7),
          Skill("Jack", "ceiling",   8),
          Skill("Jack", "roofing",   7),
          Skill("Jack", "painting",  9),
          Skill("Jack", "facade",    5),
          Skill("Jack", "garden",    5),
          Skill("Jim",  "carpentry", 5),
          Skill("Jim",  "painting",  6),
          Skill("Jim",  "windows",   5),
          Skill("Jim",  "garden",    9),
          Skill("Jim",  "moving",    8)
          }_____no_output_____

##### Utility functions_____no_output_____find_tasks: returns the task it refers to in the TASKS vector_____no_output_____

def find_tasks(name):
    return next(t for t in TASKS if t.name == name)_____no_output_____

find_skills: returns the skill it refers to in the SKILLS vector_____no_output_____

def find_skills(worker, task):
    return next(s for s in SKILLS if (s.worker == worker) and (s.task == task))_____no_output_____

find_max_level_skill: returns the tuple "skill" where the level is themaximum for a given task_____no_output_____

def find_max_level_skill(task):
    st = [s for s in SKILLS if s.task == task]
    return next(sk for sk in st if sk.level == max([s.level for s in st]))_____no_output_____

### Step 3: Set up the prescriptive model_____no_output_____
Create the model container

The model is represented by a Python object that is filled with the different model elements (variables, constraints, objective function, etc). The first thing to do is then to create such an object:_____no_output_____

mdl = CpoModel(name="HouseBuilding")_____no_output_____

#### Define the decision variables_____no_output_____
Concept: interval variable

 What for?

 Modeling an interval of time during which a particular property holds

(an activity executes, a resource is idle, a tank must be non-empty, …)

 Example:

interval_var(start=(0,1000), end=(0,1000), size=(10,20))

Properties:

The **value** of an interval variable is an integer interval [start,end)

**Domain**  of possible values: [0,10), [1,11), [2,12),...[990,1000), [0,11),[1,12),...

Domain of interval variables is represented **compactly** in CP Optimizer (a few bounds: smin, smax, emin, emax, szmin, szmax)

_____no_output_____For each house, an interval variable is created for each task.

This interval must start and end inside the period_domain and its duration is set as the value stated in TASKS definition._____no_output_____

tasks = {}   # dict of interval variable for each house and task
for house in HOUSES:
    for task in TASKS:
        tasks[(house, task)] = mdl.interval_var(start=period_domain,
                                                end=period_domain,
                                                size=task.duration,
                                                name="house {} task {}".format(house, task))_____no_output_____

Concept: optional interval variable

Interval variables can be defined as being **optional** that is, it is part of the decisions of the problem to decide whether the interval will be **present** or **absent** in the solution

 What for?

 Modeling optional activities, alternative execution modes for activities, and … most of the discrete decisions in a schedule

 Example:

interval_var(optional=True, start=(0,1000), end=(0,1000), size=(10,20))

Properties:

An optional interval variable has an additional possible value in its domain (absence value)

**Optionality** is a powerful property that you must learn to leverage in your models

_____no_output_____For each house, an __optional__ interval variable is created for each skill.

Skill being a tuple (worker, task, level), this means that for each house, an __optional__ interval variable is created for each couple worker-task such that the skill level of this worker for this task is > 0.

The "**set_optional()**" specifier allows a choice between different variables, thus between different couples house-skill.
This means that the engine decides if the interval will be present or absent in the solution._____no_output_____

wtasks = {}  # dict of interval variable for each house and skill
for house in HOUSES:
    for skill in SKILLS:
        iv = mdl.interval_var(name='H' + str(house) + '-' + skill.task + '(' + skill.worker + ')')
        iv.set_optional()
        wtasks[(house, skill)] = iv_____no_output_____

#### Express the business constraints_____no_output_____
Temporal constraints
_____no_output_____
Concept: precedence constraint

 What for?

Modeling temporal constraints between interval variables

Modeling constant or variable minimal delays

Properties

Semantic of the constraints handles optionality (as for all constraints in CP Optimizer).

Example of endBeforeStart:

end_before_start(a,b,z)

present(a) AND present(b)  ⇒  end(a)+z ⩽ start(b)

_____no_output_____The tasks in the model have precedence constraints that are added to the model._____no_output_____

for h in HOUSES:
    for p in TASK_PRECEDENCES:
        mdl.add(mdl.end_before_start(tasks[(h, find_tasks(p.beforeTask))], tasks[(h, find_tasks(p.afterTask))]))_____no_output_____

Alternative workers
_____no_output_____
Concept: alternative constraint

 What for?

Modeling alternative resource/modes/recipes

In general modeling a discrete selection in the schedule

 Example:

alternative(a,[b1,...,bn])

Remark: Master interval variable **a** can of course be optional

_____no_output_____To constrain the solution so that exactly one of the interval variables wtasks associated with a given task of a given house is to be present in the solution, an "**alternative**" constraint is used._____no_output_____

for h in HOUSES:
    for t in TASKS:
        mdl.add(mdl.alternative(tasks[(h, t)], [wtasks[(h, s)] for s in SKILLS if (s.task == t.name)], 1))_____no_output_____

No overlap constraint
_____no_output_____
Concept: No-overlap constraint

 Constraint noOverlap schedules a group of interval variables in such a way that they do not overlap in time.

 Absent interval variables are ignored.

It is possible to constrain minimum delays between intervals using transition matrix.

It is possible to constraint the first, last in the sequence or next or preceding interval

_____no_output_____To add the constraints that a given worker can be assigned only one task at a given moment in time, a  **noOverlap** constraint is used._____no_output_____

for w in WORKERS:
    mdl.add(mdl.no_overlap([wtasks[(h, s)] for h in HOUSES for s in SKILLS if s.worker == w]))_____no_output_____

#### Express the objective_____no_output_____The presence of an interval variable in tasks in the solution must be accounted for in the objective. Thus for each of these possible tasks, the cost is incremented by the product of the skill level and the expression representing the presence of the interval variable in the solution.

The objective of this problem is to maximize the skill level used for all the tasks._____no_output_____

obj = mdl.sum([s.level * mdl.presence_of(wtasks[(h, s)]) for s in SKILLS for h in HOUSES])
mdl.add(mdl.maximize(obj))_____no_output_____

#### Solve the model
The model is now completely defined. It is time to solve it !_____no_output_____

# Solve the model
print("\nSolving model....")
msol = mdl.solve(TimeLimit=10)_____no_output_____

### Step 4: Investigate the solution and then run an example analysis_____no_output_____

print("Solve status: " + msol.get_solve_status())
if msol.is_solution():
    stdout.write("Solve time: " + str(msol.get_solve_time()) + "\n")
    # Sort tasks in increasing begin order
    ltasks = []
    for hs in HOUSES:
        for tsk in TASKS:
            (beg, end, dur) = msol[tasks[(hs, tsk)]]
            ltasks.append((hs, tsk, beg, end, dur))
    ltasks = sorted(ltasks, key = lambda x : x[2])
    # Print solution
    print("\nList of tasks in increasing start order:")
    for tsk in ltasks:
        print("From " + str(tsk[2]) + " to " + str(tsk[3]) + ", " + tsk[1].name + " in house " + str(tsk[0]))
else:
    stdout.write("No solution found\n")_____no_output_____

#### Import graphical tools_____no_output_____*You can set __POP\_UP\_GRAPHIC=True__ if you prefer a pop up graphic window instead of an inline one.*_____no_output_____

POP_UP_GRAPHIC=False_____no_output_____import docplex.cp.utils_visu as visu
import matplotlib.pyplot as plt
if not POP_UP_GRAPHIC:
    %matplotlib inline
#Change the plot size
from pylab import rcParams
rcParams['figure.figsize'] = 15, 3_____no_output_____

#### Draw solution_____no_output_____#### Useful functions _____no_output_____To facilitate the display of tasks names, keep only the first n characters._____no_output_____

def compact_name(name,n): return name[:n]_____no_output_____if msol and visu.is_visu_enabled():
    workers_colors = {}
    workers_colors["Joe"] = 'lightblue'
    workers_colors["Jack"] = 'violet'
    workers_colors["Jim"] = 'lightgreen'
    visu.timeline('Solution per houses', 0, MAX_AMOUNT_OF_PERIODS)
    for h in HOUSES:
        visu.sequence(name="house " + str(h))
        for s in SKILLS:
            wt = msol.get_var_solution(wtasks[(h,s)])
            if wt.is_present():
                color = workers_colors[s.worker]
                wtname = compact_name(s.task,2)
                visu.interval(wt, color, wtname)
    visu.show()_____no_output_____

The purpose of this function is to compact the names of the different tasks to make the graphical display more readable.

For example "H3-garden" becomes "G3"_____no_output_____

def compact_house_task(name):
    loc, task = name[1:].split('-', 1)
    return task[0].upper() + loc_____no_output_____

A green color is used in the display when the task is using the most skilled worker. A red color is used in the display when the task does not use the most skilled worker._____no_output_____

if msol and visu.is_visu_enabled():
    visu.timeline('Solution per workers', 0, MAX_AMOUNT_OF_PERIODS)
    for w in WORKERS:
        visu.sequence(name=w)
        for h in HOUSES:
            for s in SKILLS:
                if s.worker == w:
                    wt = msol.get_var_solution(wtasks[(h,s)])
                    if wt.is_present():
                        ml = find_max_level_skill(s.task).level
                        if s.level == ml:
                            color = 'lightgreen'
                        else:
                            color = 'salmon'
                        wtname = compact_house_task(wt.get_name())
                        visu.interval(wt, color, wtname)
    visu.show()_____no_output_____

Going further with Constraint Programming

The last available installable package is available on Pypi here: https://pypi.python.org/pypi/docplex

A complete set of modeling examples can be downloaded here: https://github.com/IBMDecisionOptimization/docplex-examples  _____no_output_____## Summary

You have learned how to set up and use the IBM Decision Optimization CPLEX Modeling for Python to build and solve a Constraint Programming model._____no_output_____#### References
* Decision Optimization CPLEX Modeling for Python documentation
* Watson Studio documentation_____no_output_____

Copyright © 2017-2021. This notebook and its source code are released under the terms of the MIT License._____no_output_____

Love this notebook?
Don't have an account yet?

Share it with your colleagues and help them discover the power of Watson Studio!
Sign Up

_____no_output_____