# Notebook from sriramsrikant/pheromoneFinder
Path: notebooks/pheromoneFinder_py3-testbed_210718.ipynb

## Test-bed notebook to find taxonomic groupings for pheromoneFinder
### Authored by Sri Srikant, Murray and Gaudet Labs, Harvard
Modified to run on my sager since my xps15 died, installed the following packages with potential for version issues.
1. biopython 1.78
2. dendropy 4.5.1

[210718] Triumphant return to my xps15, but I am going to modify all the data folders to point to the Google Drive folder in my Windows installation._____no_output_____

##This is to test code snippets for the pheromoneFinder codebase
import sys
import os
from os import listdir
from os.path import isfile, join

import traceback
import re

from Bio import SeqIO
from Bio.Seq import Seq
##from Bio.Alphabet import IUPAC ##deprecated in biopython 1.78
from collections import defaultdict
from Bio.SeqUtils.ProtParam import ProteinAnalysis
from Bio.Blast.Applications import NcbiblastxCommandline
from Bio.Blast.Applications import NcbiblastnCommandline
from Bio.Blast import NCBIXML
import xml.etree.ElementTree as ET

import dendropy
import numpy as np
import seaborn as sns
from pylab import *
import matplotlib.pyplot as plt
##Set pdf ouput of matplotlib to have embedded fonts that are editable in Illustrator
plt.rcParams['pdf.fonttype'] = 42

import csv
import math
import operator
from operator import attrgetter
from pprint import pprint

data_dir = os.path.join("/home", "ssrikant", "WindowsFiles", "Fungi_pheromoneFinder", "Cluster")
figure_dir = os.path.join(data_dir, "figures")

#Setting matplotlib to be inline
%matplotlib inline_____no_output_____tree_dir = os.path.join(data_dir, "tree")_____no_output_____#Printing file list
fileList = [f for f in os.listdir(tree_dir) if os.path.isfile(os.path.join(tree_dir, f))]
print("List of files:")
print("\n".join(fileList))List of files:
332taxa_index.txt
332_2408OGs_timetree_mcmctree.nwk
332_2408OGs_timetree_reltime.nwk
332genomes_Shen2017.xlsx
332genomes_Shen2017.csv
taxonGroups.csv
taxonGroups_240max-dist_210718.csv
taxonGroups_180max-dist_mating-regulation.csv
taxonGroups_180max-dist_mating-regulation_210721.csv
taxonGroups_240max-dist.csv
taxonGroups_240max-dist_mcmc.csv
taxonGroups_240max-dist_reltime.csv
##Importing Newick phylogenetic trees
phyloTree = dendropy.Tree.get(file=open(os.path.join(tree_dir, "332_2408OGs_timetree_reltime.nwk"), 'r'), schema="newick")
phyloTree_mcmc = dendropy.Tree.get(file=open(os.path.join(tree_dir, "332_2408OGs_timetree_mcmctree.nwk"), 'r'), schema="newick")_____no_output_____

### Code blocks below are to calculate and plot the pair-wise distances of species
These are using the dendropy.Tree.pdc(), to read in the Newick reltime tree from the Shen XX, Cell (2018) paper to identify clades of species that have diverged within a time horizon.
> Written originally in Windows and moved to WSL Ubuntu 18.04 for further work and testing to move to Odyssey._____no_output_____

distMatrix = np.zeros((len(phyloTree.taxon_namespace), len(phyloTree.taxon_namespace)))
pdc = phyloTree.phylogenetic_distance_matrix()
for i, t1 in enumerate(phyloTree.taxon_namespace):
    for j, t2 in enumerate(phyloTree.taxon_namespace[:i]):
        #print("Distance between '%s' and '%s': %s" % (t1.label, t2.label, pdc(t1, t2)))
        distMatrix[i,j] = pdc(t1, t2);
#print(distMatrix)
print(phyloTree.taxon_namespace[0].label, "\t", phyloTree.taxon_namespace[-1].label, "\n",
      pdc.distance(phyloTree.taxon_namespace[0], phyloTree.taxon_namespace[-1]))
print(len(distMatrix.flatten()))
mask = np.zeros_like(distMatrix, dtype=bool) #np.bool deprecated with identical implementation as bool
mask[np.triu_indices_from(mask)] = True
#print(pdc.distance(phyloTree.taxon_namespace[:], phyloTree.taxon_namespace[:]))
fig1 = figure(figsize=(16,6))
ax1 = subplot(121)
#ax = sns.heatmap(distMatrix[0:99, 0:99], yticklabels=phyloTree.taxon_namespace[0:99])
ax1 = sns.heatmap(distMatrix, mask=mask, cmap="Greys_r")

ax2 = subplot(122)
plot = list(filter(lambda a: a != 0, distMatrix.flatten()))
ax2 = sns.histplot(plot, kde=True)
##ax2 = sns.distplot(plot, kde=True, rug=True) ##distplot is being deprecated and replacing with axis-level histplot
pairwise_dist_title = " ".join([str(len(plot)), "pairwise distances in tree"])
plt.title(pairwise_dist_title);
#savefig(os.path.join(figure_dir, "Shen_332yeast_pairwise-dist_reltime.pdf"), dpi=300, transparent=True)yHMPu5000041678 Debaryomyces prosopidis 	 yHMPu5000034742 Lipomyces suomiensis
 904.3625675999999
110224
distMatrix = np.zeros((len(phyloTree_mcmc.taxon_namespace), len(phyloTree_mcmc.taxon_namespace)))
pdc = phyloTree_mcmc.phylogenetic_distance_matrix()
for i, t1 in enumerate(phyloTree_mcmc.taxon_namespace):
    for j, t2 in enumerate(phyloTree_mcmc.taxon_namespace[:i]):
        #print("Distance between '%s' and '%s': %s" % (t1.label, t2.label, pdc(t1, t2)))
        distMatrix[i,j] = pdc(t1, t2);
#print(distMatrix)
print(phyloTree_mcmc.taxon_namespace[0].label, "\t", phyloTree_mcmc.taxon_namespace[-1].label, "\n",
      pdc.distance(phyloTree_mcmc.taxon_namespace[0], phyloTree_mcmc.taxon_namespace[-1]))
print(len(distMatrix.flatten()))
mask = np.zeros_like(distMatrix, dtype=bool) #np.bool deprecated with identical implementation as bool
mask[np.triu_indices_from(mask)] = True
#print(pdc.distance(phyloTree.taxon_namespace[:], phyloTree.taxon_namespace[:]))
fig1 = figure(figsize=(16,6))
ax1 = subplot(121)
#ax = sns.heatmap(distMatrix[0:99, 0:99], yticklabels=phyloTree.taxon_namespace[0:99])
ax1 = sns.heatmap(distMatrix, mask=mask, cmap="Greys_r")

ax2 = subplot(122)
plot = list(filter(lambda a: a != 0, distMatrix.flatten()))
ax2 = sns.histplot(plot, kde=True)
##ax2 = sns.distplot(plot, kde=True, rug=True) ##distplot is being deprecated and replacing with axis-level histplot
pairwise_dist_title = " ".join([str(len(plot)), "pairwise distances in tree"])
plt.title(pairwise_dist_title);
#savefig(os.path.join(figure_dir, "Shen_332yeast_pairwise-dist_reltime.pdf"), dpi=300, transparent=True)Babjeviella inositovora 	 yHMPu5000034742 Lipomyces suomiensis
 8.084041
110224

#### Code-base to identify the time-horizon for the conservation of a-factor-like (lipidated) fungal pheromones
I am going to assume that the "conservation" here corresponds to pheromones that can be recognized from one species to another. The best examples I have are S. cerevisiae and K. lactis, within the Saccharomycetaceae and the 5 species in the Yarrowia clade._____no_output_____

##Identifying the evolutionary time horizon for the divergence of all species in the Yarrowia clade.
yarrowiaPair = []
yarrowiaDist = []
for i, t1 in enumerate(phyloTree.taxon_namespace):
    for j, t2 in enumerate(phyloTree.taxon_namespace[:i]):
        if ((("Yarrowia" in t1.label) and ("Yarrowia" in t2.label)) or (("Yarrowia" in t2.label) and ("Yarrowia" in t1.label))):
            print("Distance between '%s' and '%s': %s" % (t1.label, t2.label, pdc(t1, t2)))
            yarrowiaPair.append([t1.label, t2.label])
            yarrowiaDist.append(pdc(t1, t2))
index, value = max(enumerate(yarrowiaDist), key=operator.itemgetter(1))
print("Max. divergence time of Yarrowia is between: ", yarrowiaPair[index], "\t of time: ", yarrowiaDist[index])
print(max(yarrowiaDist))Distance between 'Yarrowia keelungensis' and 'yHMPu5000035643 Yarrowia bubula': 46.3610224
Distance between 'Yarrowia lipolytica' and 'yHMPu5000035643 Yarrowia bubula': 46.3610223
Distance between 'Yarrowia lipolytica' and 'Yarrowia keelungensis': 26.3931419
Distance between 'Yarrowia deformans' and 'yHMPu5000035643 Yarrowia bubula': 46.3610224
Distance between 'Yarrowia deformans' and 'Yarrowia keelungensis': 26.393141999999997
Distance between 'Yarrowia deformans' and 'Yarrowia lipolytica': 20.5187099
Distance between 'yHMPu5000035645 Yarrowia divulgata' and 'yHMPu5000035643 Yarrowia bubula': 46.3610224
Distance between 'yHMPu5000035645 Yarrowia divulgata' and 'Yarrowia keelungensis': 26.393141999999997
Distance between 'yHMPu5000035645 Yarrowia divulgata' and 'Yarrowia lipolytica': 20.5187099
Distance between 'yHMPu5000035645 Yarrowia divulgata' and 'Yarrowia deformans': 6.2020266
Max. divergence time of Yarrowia is between:  ['Yarrowia keelungensis', 'yHMPu5000035643 Yarrowia bubula'] 	 of time:  46.3610224
46.3610224
##Identifying the evolutionary divergence time between the Saccharomycetaceae (K. lactis) and Saccharomycodaceae (Hanseniaspora)
speciesPair = []
speciesDist = []
for i, t1 in enumerate(phyloTree.taxon_namespace):
    for j, t2 in enumerate(phyloTree.taxon_namespace[:i]):
        if ((("lactis" in t1.label) and ("Hanseniaspora" in t2.label)) or (("lactis" in t2.label) and ("Hanseniaspora" in t1.label))):
            print("Distance between '%s' and '%s': %s" % (t1.label, t2.label, pdc(t1, t2)))
            speciesPair.append([t1.label, t2.label])
            speciesDist.append(pdc(t1, t2))
index, value = max(enumerate(speciesDist), key=operator.itemgetter(1))
print("Max. divergence time of Saccharomyces MFA1/2-like is between: ", speciesPair[index], "\t of time: ", speciesDist[index])
print(max(speciesDist))Distance between 'Kluyveromyces lactis' and 'Hanseniaspora vinae': 295.48907810000003
Distance between 'Kluyveromyces lactis' and 'yHMPu5000034957 Hanseniaspora osmophila': 295.48907810000003
Distance between 'Kluyveromyces lactis' and 'yHMPu5000035695 Hanseniaspora pseudoguilliermondii': 295.489078
Distance between 'Kluyveromyces lactis' and 'yHMPu5000034963 Hanseniaspora clermontiae': 295.489078
Distance between 'Kluyveromyces lactis' and 'Hanseniaspora uvarum': 295.489078
Distance between 'Kluyveromyces lactis' and 'Hanseniaspora valbyensis': 295.48907810000003
Distance between 'Kluyveromyces lactis' and 'yHMPu5000035696 Hanseniaspora singularis': 295.48907810000003
Max. divergence time of Saccharomyces MFA1/2-like is between:  ['Kluyveromyces lactis', 'Hanseniaspora vinae'] 	 of time:  295.48907810000003
295.48907810000003
##Identifying the evolutionary time horizon for the divergence of all species in the Saccharomycetaceae clade (tested as between S. cerevisiae and K. lactis).
speciesPair = []
speciesDist = []
for i, t1 in enumerate(phyloTree.taxon_namespace):
    for j, t2 in enumerate(phyloTree.taxon_namespace[:i]):
        if ((("cere" in t1.label) and ("lactis" in t2.label)) or (("cere" in t2.label) and ("lactis" in t1.label))):
            print("Distance between '%s' and '%s': %s" % (t1.label, t2.label, pdc(t1, t2)))
            speciesPair.append([t1.label, t2.label])
            speciesDist.append(pdc(t1, t2))
index, value = max(enumerate(speciesDist), key=operator.itemgetter(1))
print("Max. divergence time of Saccharomyces MFA1/2-like is between: ", speciesPair[index], "\t of time: ", speciesDist[index])
print(max(speciesDist))Distance between 'Kluyveromyces lactis' and 'Saccharomyces cerevisiae': 237.6215988
Max. divergence time of Saccharomyces MFA1/2-like is between:  ['Kluyveromyces lactis', 'Saccharomyces cerevisiae'] 	 of time:  237.6215988
237.6215988
##Repeating in the mcmctree
##Identifying the evolutionary time horizon for the divergence of all species in the Saccharomycetaceae clade (tested as between S. cerevisiae and K. lactis).
speciesPair = []
speciesDist = []
for i, t1 in enumerate(phyloTree_mcmc.taxon_namespace):
    for j, t2 in enumerate(phyloTree_mcmc.taxon_namespace[:i]):
        if ((("cere" in t1.label) and ("lactis" in t2.label)) or (("cere" in t2.label) and ("lactis" in t1.label))):
            print("Distance between '%s' and '%s': %s" % (t1.label, t2.label, pdc(t1, t2)))
            speciesPair.append([t1.label, t2.label])
            speciesDist.append(pdc(t1, t2))
index, value = max(enumerate(speciesDist), key=operator.itemgetter(1))
print("Max. divergence time of Saccharomyces MFA1/2-like is between: ", speciesPair[index], "\t of time: ", speciesDist[index])
print(max(speciesDist))Distance between 'Kluyveromyces lactis' and 'Saccharomyces cerevisiae': 2.27999
Max. divergence time of Saccharomyces MFA1/2-like is between:  ['Kluyveromyces lactis', 'Saccharomyces cerevisiae'] 	 of time:  2.27999
2.27999
##Identifying the distance of closest neighbor to the Yarrowia lineage. As seen it is already more than the value expected to share pheromones; though the divergence time
##is not a hard limit that arises naturally.
speciesPair = []
speciesDist = []
for i, t1 in enumerate(phyloTree.taxon_namespace):
    for j, t2 in enumerate(phyloTree.taxon_namespace[:i]):
        if ((("Nadsonia" in t1.label) and ("Yarrowia" in t2.label)) or (("Nadsonia" in t2.label) and ("Yarrowia" in t1.label))):
            print("Distance between '%s' and '%s': %s" % (t1.label, t2.label, pdc(t1, t2)))
            speciesPair.append([t1.label, t2.label])
            speciesDist.append(pdc(t1, t2))
index, value = max(enumerate(speciesDist), key=operator.itemgetter(1))
print("Max. divergence time of: ", speciesPair[index], "\t of time: ", speciesDist[index])
print(max(speciesDist))Distance between 'yHMPu5000035643 Yarrowia bubula' and 'yHMPu5000034635 Nadsonia fulvescens': 611.5004867
Distance between 'yHMPu5000035643 Yarrowia bubula' and 'Nadsonia fulvescens var elongata': 611.5004867
Distance between 'Yarrowia keelungensis' and 'yHMPu5000034635 Nadsonia fulvescens': 611.5004867
Distance between 'Yarrowia keelungensis' and 'Nadsonia fulvescens var elongata': 611.5004867
Distance between 'Yarrowia lipolytica' and 'yHMPu5000034635 Nadsonia fulvescens': 611.5004865999999
Distance between 'Yarrowia lipolytica' and 'Nadsonia fulvescens var elongata': 611.5004865999999
Distance between 'Yarrowia deformans' and 'yHMPu5000034635 Nadsonia fulvescens': 611.5004867
Distance between 'Yarrowia deformans' and 'Nadsonia fulvescens var elongata': 611.5004867
Distance between 'yHMPu5000035645 Yarrowia divulgata' and 'yHMPu5000034635 Nadsonia fulvescens': 611.5004867
Distance between 'yHMPu5000035645 Yarrowia divulgata' and 'Nadsonia fulvescens var elongata': 611.5004867
Max. divergence time of:  ['yHMPu5000035643 Yarrowia bubula', 'yHMPu5000034635 Nadsonia fulvescens'] 	 of time:  611.5004867
611.5004867

#### Code-block to identify the time-horizon for the conservation of the regulation of mating genes.
This is to identify the time-horizon of evolutionary conservation of the regulation of mating genes. The Sorrells TR, Nature (2015) [doi: 10.1038/nature14613] paper inspired the basis of looking for an evolutionary relationship to the motifs for mating regulatory TFs, specifically STE12 in the Saccharomyces clade._____no_output_____

##Identifying the evolutionary time horizon for the divergence of all species in the Saccharomycetaceae clade (tested as between S. cerevisiae and K. lactis).
speciesPair = []
speciesDist = []
for i, t1 in enumerate(phyloTree.taxon_namespace):
    for j, t2 in enumerate(phyloTree.taxon_namespace[:i]):
        if ((("cerevisiae" in t1.label) and ("polyspora" in t2.label)) or (("cerevisiae" in t2.label) and ("polyspora" in t1.label))):
            print("Distance between '%s' and '%s': %s" % (t1.label, t2.label, pdc(t1, t2)))
            speciesPair.append([t1.label, t2.label])
            speciesDist.append(pdc(t1, t2))
index, value = max(enumerate(speciesDist), key=operator.itemgetter(1))
print("Max. divergence time of first clade (Saccharomyces) described in Sorrells et al is between: ", speciesPair[index], "\t of time: ", speciesDist[index])
print(max(speciesDist))Distance between 'Saccharomyces cerevisiae' and 'Vanderwaltozyma polyspora': 181.2832848
Max. divergence time of first clade (Saccharomyces) described in Sorrells et al is between:  ['Saccharomyces cerevisiae', 'Vanderwaltozyma polyspora'] 	 of time:  181.2832848
181.2832848
##Identifying the evolutionary time horizon for the divergence of all species in the Saccharomycetaceae clade (tested as between S. cerevisiae and K. lactis).
speciesPair = []
speciesDist = []
for i, t1 in enumerate(phyloTree.taxon_namespace):
    for j, t2 in enumerate(phyloTree.taxon_namespace[:i]):
        if ((("kluyveri" in t1.label) and ("lactis" in t2.label)) or (("kluyveri" in t2.label) and ("lactis" in t1.label))):
            print("Distance between '%s' and '%s': %s" % (t1.label, t2.label, pdc(t1, t2)))
            speciesPair.append([t1.label, t2.label])
            speciesDist.append(pdc(t1, t2))
index, value = max(enumerate(speciesDist), key=operator.itemgetter(1))
print("Max. divergence time of second clade (Kluyveromyces) described in Sorrells et al is between: ", speciesPair[index], "\t of time: ", speciesDist[index])
print(max(speciesDist))Distance between 'Kluyveromyces lactis' and 'Lachancea kluyveri': 214.5338346
Max. divergence time of second clade (Kluyveromyces) described in Sorrells et al is between:  ['Kluyveromyces lactis', 'Lachancea kluyveri'] 	 of time:  214.5338346
214.5338346
##Identifying the evolutionary time horizon for the divergence of all species in the Saccharomycetaceae clade (tested as between S. cerevisiae and K. lactis).
speciesPair = []
speciesDist = []
for i, t1 in enumerate(phyloTree.taxon_namespace):
    for j, t2 in enumerate(phyloTree.taxon_namespace[:i]):
        if ((("bicuspidata" in t1.label) and ("guilliermondii" in t2.label)) or (("bicuspidata" in t2.label) and ("guilliermondii" in t1.label))):
            print("Distance between '%s' and '%s': %s" % (t1.label, t2.label, pdc(t1, t2)))
            speciesPair.append([t1.label, t2.label])
            speciesDist.append(pdc(t1, t2))
index, value = max(enumerate(speciesDist), key=operator.itemgetter(1))
print("Max. divergence time of third clade (Candida) described in Sorrells et al is between: ", speciesPair[index], "\t of time: ", speciesDist[index])
print(max(speciesDist))Distance between 'Meyerozyma guilliermondii' and 'Metschnikowia bicuspidata': 213.7973007
Distance between 'yHMPu5000035695 Hanseniaspora pseudoguilliermondii' and 'Metschnikowia bicuspidata': 486.48659929999997
Max. divergence time of third clade (Candida) described in Sorrells et al is between:  ['yHMPu5000035695 Hanseniaspora pseudoguilliermondii', 'Metschnikowia bicuspidata'] 	 of time:  486.48659929999997
486.48659929999997
##Identifying the evolutionary time horizon for the divergence of all species in the Saccharomycetaceae clade (tested as between S. cerevisiae and K. lactis).
###This doesn't seem like the right pair. I am going to ignore this for now.
speciesPair = []
speciesDist = []
for i, t1 in enumerate(phyloTree.taxon_namespace):
    for j, t2 in enumerate(phyloTree.taxon_namespace[:i]):
        if ((("polymorpha" in t1.label) and ("tannophilus" in t2.label)) or (("polymorpha" in t2.label) and ("tannophilus" in t1.label))):
            print("Distance between '%s' and '%s': %s" % (t1.label, t2.label, pdc(t1, t2)))
            speciesPair.append([t1.label, t2.label])
            speciesDist.append(pdc(t1, t2))
index, value = max(enumerate(speciesDist), key=operator.itemgetter(1))
print("Max. divergence time of fourth clade (Pichia) described in Sorrells et al is between: ", speciesPair[index], "\t of time: ", speciesDist[index])
print(max(speciesDist))Distance between 'Ogataea parapolymorpha' and 'Pachysolen tannophilus': 386.6871759
Distance between 'Ogataea polymorpha' and 'Pachysolen tannophilus': 386.6871759
Max. divergence time of fourth clade (Pichia) described in Sorrells et al is between:  ['Ogataea parapolymorpha', 'Pachysolen tannophilus'] 	 of time:  386.6871759
386.6871759

----
Inferences from the runs above:
1. From the 4 clades, it seems like a good time horizon of conservation of mating gene motifs are 180, since I am hoping for a conservative estimate, i.e. the horizon that gives me maximum confidence of shared motifs.
2. This is also consistent with the fact that the only time-horizon of STE12 motif conserved in a-specific gene promoters is for the clade of Saccharomyces.
3. Another possible thought is to restrict this search within the major clades that I have. But would this work?
----
----_____no_output_____### First-pass taxonomic grouping based on the divergence times
Let us assume that the list of taxons are ordered such that neighbors in the list are the closest and we can therefore limit are check of relatedness to the immediate neighbor and only move on once we classify as within the group or out of the group.
> I'm not sure if this is true for an arbitrary tree. I am going to think about how to generalize based on just slicing lists based on the distance measures. I can move down a list of taxon_names and then remove members that are within a particular divergence time.
>
> {191026/1628hrs} I've managed to code a more generalized classifier 3 cells down.

Also it is worth considering that the divergenceLimit that is appropriate is that which separates the Saccharomyces cerevisiae and Kluyveromyces lactis species on the tree since the pheromones can be found by sequence homology. Need to confirm this by tBLASTN._____no_output_____

##Trying to build groupings of strains that are within a known value of divergence time.
##I am going to round up from the value estimated in the yarrowia group.
divergenceLimit = 50.0;
taxoGroups = [[phyloTree.taxon_namespace[0].label]]
for i, t1 in enumerate(phyloTree.taxon_namespace[:-2]):
    #print(i)
    if pdc(t1, phyloTree.taxon_namespace[i+1]) <= divergenceLimit:
        taxoGroups[-1].append(phyloTree.taxon_namespace[i+1].label)
    else:
        taxoGroups.append([phyloTree.taxon_namespace[i+1].label])
print(len(taxoGroups))

##Hist of group sizes
groupSize = []
for taxoGroup in taxoGroups:
    groupSize.append(len(taxoGroup))
binList = list(range(max(groupSize)))
print(max(groupSize))
ax1 = sns.histplot(groupSize, kde=False, bins=binList, stat='count', element='step', fill=False);
##ax1 = sns.distplot(groupSize, kde=False, bins=binList, hist_kws={"histtype": "step", "linewidth": 3, "alpha": 1});
#plt.ylim(0, 50)
#ax2 = sns.swarmplot(groupSize);

# with open(os.path.join(tree_dir, "taxonGroups.csv"), 'w') as writeFile:
#     writer = csv.writer(writeFile)
#     writer.writerows(taxoGroups)212
23
##Trying to build groupings of strains that are within a known value of divergence time.
##I am going to be as loose about including more species by doubling the divergence time from the yarrowia group (above).
divergenceLimit = 100.0;
taxoGroups = [[phyloTree.taxon_namespace[0].label]]
for i, t1 in enumerate(phyloTree.taxon_namespace[:-2]):
    #print(i)
    if pdc(t1, phyloTree.taxon_namespace[i+1]) <= divergenceLimit:
        taxoGroups[-1].append(phyloTree.taxon_namespace[i+1].label)
    else:
        taxoGroups.append([phyloTree.taxon_namespace[i+1].label])
print(len(taxoGroups))

##Hist of group sizes
groupSize = []
for taxoGroup in taxoGroups:
    groupSize.append(len(taxoGroup))
binList = list(range(max(groupSize)))
#print(binList)
ax1 = sns.histplot(groupSize, kde=False, bins=binList, stat='count', element='step', fill=False);143
## Building a loop through leaf names that can sort taxonomic groups. This is a generalized search that should identify species groupings for pheromone conservation.
divergenceLimit = 240.0;

taxoGroups = []
taxonList = []
#print(taxoGroups, taxonList)
for taxon in phyloTree.taxon_namespace:
    taxonList.append(taxon.label)
#treeClone = phyloTree.clone(depth=1)
for taxon in phyloTree.taxon_namespace:
    while taxon.label in taxonList:
        taxoGroups.append([taxon.label])
        for taxonCompare in phyloTree.taxon_namespace:
            if (pdc(taxon, taxonCompare) <= divergenceLimit) and (taxonCompare != taxon):
                taxoGroups[-1].append(taxonCompare.label)
                taxonList.remove(taxonCompare.label)
        taxonList.remove(taxon.label)

print("Total number of monophyletic groups:\t", len(taxoGroups))

##Hist of group sizes
groupSize = []
for taxoGroup in taxoGroups:
    groupSize.append(len(taxoGroup))
binList = list(range(int(math.ceil(1.1*max(groupSize)))))
print("Number of species in largest monophyletic group:\t", max(groupSize))
print(groupSize)
fig1 = figure(figsize=(8,4))
ax1 = sns.histplot(groupSize, kde=False, bins=binList, stat='count', element='step', fill=False, color='black');
#ax1 = sns.distplot(groupSize, kde=False, bins=binList, hist_kws={"histtype": "step", "linewidth": 3, "alpha": 1});
#plt.ylim(0, 50)
#ax2 = sns.swarmplot(groupSize);
#savefig(os.path.join(figure_dir, 'pheromone-phyloGroups_species-grouping-density_210913.pdf'), dpi=300, transparent=True, bbox_inches='tight')

with open(os.path.join(tree_dir, "taxonGroups_240max-dist_reltime.csv"), 'w') as writeFile:
    writer = csv.writer(writeFile)
    writer.writerows(taxoGroups)_____no_output_____###Phylogenetic groups that correspond to infered time-horizon of lipidated pheromone conservation.
print(*[group for group in taxoGroups if len(group)>=1], sep = "\n")['yHMPu5000041678 Debaryomyces prosopidis', 'yHMPu5000041829 Debaryomyces fabryi', 'yHMPu5000041824 Debaryomyces subglobosus', 'Debaryomyces hansenii', 'yHMPu5000041693 Debaryomyces nepalensis', 'yHMPu5000041713 Debaryomyces maramus', 'Millerozyma acaciae', 'yHMPu5000035296 Priceomyces carsonii', 'yHMPu5000035297 Priceomyces castillae', 'Priceomyces haplophilus', 'yHMPu5000034606 Priceomyces medius', 'Scheffersomyces lignosus', 'Scheffersomyces stipitis', 'Spathaspora passalidarum', 'Spathaspora girioi', 'Spathaspora arborariae', 'Spathaspora gorwiae', 'Spathaspora hagerdaliae', 'Candida sojae', 'Candida tropicalis', 'Candida albicans', 'Candida dubliniensis', 'yHMPu5000034990 Candida corydali', 'Lodderomyces elongisporus', 'Candida orthopsilosis', 'Candida parapsilosis', 'yHMPu5000034654 Aciculoconidium aculeatum', 'yHMPu5000034665 Kodamaea laetipori', 'yHMPu5000034648 Candida restingae', 'yHMPu5000035677 Kodamaea ohmeri', 'yHMPu5000035022 Candida emberorum', 'yHMPu5000035018 Candida canberraensis', 'Candida tanzawaensis', 'yHMPu5000035011 Candida pyralidae', 'Wickerhamia fluorescens', 'yHMPu5000035031 Candida kruisii', 'yHMPu5000035033 Candida cretensis', 'yHMPu5000035032 Candida gatunensis', 'Candida auris', 'yHMPu5000035707 Candida heveicola', 'yHMPu5000034986 Candida oregonensis', 'yHMPu5000034988 Candida fructus', 'Clavispora lusitaniae', 'yHMPu5000035335 Candida blattae', 'Candida intermedia', 'yHMPu5000041840 Candida wancherniae', 'yHMPu5000041862 Candida golubevii', 'Metschnikowia bicuspidata', 'yHMPu5000041743 Candida hawaiiana', 'Metschnikowia kipukae', 'Metschnikowia hibisci', 'Metschnikowia aberdeeniae', 'Metschnikowia shivogae', 'Metschnikowia drakensbergensis', 'Metschnikowia proteae', 'Metschnikowia borealis', 'Metschnikowia ipomoeae', 'Metschnikowia continentalis', 'Metschnikowia cerradonensis', 'Metschnikowia santaceciliae', 'Metschnikowia lockheadii', 'Metschnikowia matae maris', 'Metschnikowia matae', 'Metschnikowia kamakouana', 'Metschnikowia hawaiiensis', 'Metschnikowia mauinuiana', 'Metschnikowia hamakuensis', 'Metschnikowia arizonensis', 'Metschnikowia bowlesiae', 'Metschnikowia dekortum', 'Metschnikowia similis', 'yHMPu5000034973 Danielozyma ontarioensis', 'yHMPu5000035681 Candida gotoi', 'yHMPu5000035679 Candida rhagii', 'yHMPu5000035318 Hyphopichia heimii', 'Hyphopichia burtonii', 'Candida homilentoma', 'yHMPu5000035641 Yamadazyma scolyti', 'Candida tenuis', 'yHMPu5000035640 Yamadazyma philogaea', 'yHMPu5000035252 Yamadazyma nakazawae', 'yHMPu5000041833 Candida tammaniensis', 'yHMPu5000041863 Candida gorgasii', 'yHMPu5000041855 Candida ascalaphidarum', 'yHMPu5000034632 Candida athensensis', 'Meyerozyma guilliermondii', 'yHMPu5000035662 Meyerozyma caribbica', 'Candida carpophila', 'yHMPu5000035667 Kurtzmaniella cleridarum', 'yHMPu5000035302 Candida fragi', 'yHMPu5000034643 Candida schatavii']
['yHMPu5000034999 Cephaloascus fragrans', 'yHMPu5000034998 Cephaloascus albidus']
['Babjeviella inositovora']
['Pachysolen tannophilus']
['yHMPu5000034918 Nakazawaea holstii', 'Nakazawaea peltata', 'yHMPu5000034884 Peterozyma toletana', 'yHMPu5000034883 Peterozyma xylosa']
['Komagataella pastoris', 'yHMPu5000034939 Komagataella pseudopastoris', 'yHMPu5000026274 Komagataella populi']
['yHMPu5000026142 Citeromyces matritensis', 'yHMPu5000034950 Citeromyces hawaiiensis', 'yHMPu5000034952 Citeromyces siamensis']
['yHMPu5000034932 Kuraishia ogatae']
['yHMPu5000034933 Kuraishia molischiana', 'Kuraishia capsulata']
['Candida boidinii JCM9604', 'yHMPu5000034901 Ogataea methylivora', 'yHMPu5000034890 Ogataea ramenticola', 'yHMPu5000034903 Ogataea naganishii', 'Candida succiphila', 'yHMPu5000026197 Brettanomyces custersianus', 'Dekkera bruxellensis', 'Brettanomyces anomalus', 'yHMPu5000035674 Kregervanrija delftensis', 'yHMPu5000035675 Kregervanrija fluxuum', 'yHMPu5000034631 Martiniozyma abiesophila', 'yHMPu5000034614 Saturnispora silvae', 'yHMPu5000034611 Saturnispora mendoncae', 'yHMPu5000034610 Saturnispora hagleri', 'yHMPu5000034613 Saturnispora serradocipensis', 'yHMPu5000035659 Saturnispora dispora', 'yHMPu5000034615 Saturnispora zaruensis', 'yHMPu5000034612 Saturnispora saitoi', 'yHMPu5000035301 Pichia terricola', 'Candida sorboxylosa', 'Pichia membranifaciens', 'yHMPu5000034629 Pichia exigua', 'yHMPu5000034622 Pichia occidentalis', 'yHMPu5000034625 Pichia kudriavzevii', 'yHMPu5000034623 Pichia norvegensis', 'yHMPu5000034624 Pichia nakasei', 'yHMPu5000034627 Pichia heedii', 'yHMPu5000035040 Ambrosiozyma maleeae', 'yHMPu5000026145 Ambrosiozyma vanderkliftii', 'yHMPu5000035041 Ambrosiozyma pseudovanderkliftii', 'Ambrosiozyma kashinagacola', 'Ambrosiozyma monospora', 'yHMPu5000034946 Ambrosiozyma oregonensis', 'yHMPu5000026137 Ambrosiozyma ambrosiae', 'yHMPu5000034947 Ambrosiozyma philentoma', 'Candida arabinofermentans', 'Ogataea methanolica', 'yHMPu5000034886 Ogataea trehalophila', 'yHMPu5000034892 Ogataea pilisensis', 'yHMPu5000034636 Ogataea nitratoaversa', 'yHMPu5000034637 Ogataea populiabae', 'yHMPu5000034887 Ogataea trehaloabstinens', 'yHMPu5000034885 Ogataea zsoltii', 'yHMPu5000034897 Ogataea glucozyma', 'yHMPu5000034891 Ogataea pini', 'yHMPu5000026124 Ogataea henricii', 'yHMPu5000034904 Ogataea nonfermentans', 'yHMPu5000034902 Ogataea minuta', 'yHMPu5000034893 Ogataea philodendra', 'Ogataea parapolymorpha', 'Ogataea polymorpha', 'yHMPu5000034899 Ogataea kodamae']
['Saccharomycopsis malanga', 'yHMPu5000034607 Saccharomycopsis capsularis']
['Ascoidea rubescens', 'Ascoidea asiatica']
['Hanseniaspora vinae', 'yHMPu5000034957 Hanseniaspora osmophila', 'yHMPu5000035695 Hanseniaspora pseudoguilliermondii', 'yHMPu5000034963 Hanseniaspora clermontiae', 'Hanseniaspora uvarum', 'yHMPu5000035684 Kloeckera hatyaiensis', 'Hanseniaspora valbyensis', 'yHMPu5000035696 Hanseniaspora singularis']
['yHMPu5000035629 Yueomyces sinensis', 'Tetrapisispora blattae', 'Vanderwaltozyma polyspora', 'yHMPu5000034876 Tetrapisispora iriomotensis', 'Tetrapisispora phaffii', 'yHMPu5000034875 Tetrapisispora fleetii', 'yHMPu5000034877 Tetrapisispora namnaonensis', 'Nakaseomyces castellii', 'Nakaseomyces bacillisporus', 'Candida glabrata', 'Nakaseomyces bracarensis', 'Nakaseomyces delphensis', 'Nakaseomyces nivariensis', 'Saccharomyces eubayanus', 'Saccharomyces uvarum', 'Saccharomyces arboricola', 'Saccharomyces kudriavzevii', 'Saccharomyces mikatae', 'Saccharomyces cerevisiae', 'Saccharomyces paradoxus', 'yHAB153 Kazachstania rosinii', 'yHAB155 Kazachstania spencerorum', 'yHAB160 Kazachstania kunashirensis', 'yHAB134 Kazachstania turicensis', 'yHAB158 Kazachstania viticola', 'Kazachstania africana', 'yHAB132 Kazachstania martiniae', 'yHAB148 Kazachstania intestinalis', 'yHAB136 Kazachstania bromeliacearum', 'Kazachstania naganishii', 'yHAB147 Kazachstania taianensis', 'yHAB166 Kazachstania yakushimaensis', 'yHAB154 Kazachstania transvaalensis', 'yHAB164 Kazachstania aerobia', 'yHAB159 Kazachstania solicola', 'yHAB133 Kazachstania unispora', 'yHAB143 Kazachstania siamensis', 'Naumovozyma castellii', 'Naumovozyma dairenensis', 'Zygosaccharomyces bailii', 'yHMPu5000034866 Zygosaccharomyces bisporus', 'yHMPu5000034865 Zygosaccharomyces kombuchaensis', 'Zygosaccharomyces rouxii', 'yHMPu5000026256 Zygotorulaspora mrakii', 'yHMPu5000034862 Zygotorulaspora florentina', 'yHMPu5000034881 Torulaspora pretoriensis', 'yHMPu5000026152 Torulaspora franciscae', 'Torulaspora delbrueckii', 'yHMPu5000035652 Torulaspora maleeae', 'yHMPu5000035651 Torulaspora microellipsoides', 'Lachancea kluyveri', 'Lachancea cidri', 'Lachancea fermentati', 'Lachancea mirantina', 'Lachancea waltii', 'Lachancea thermotolerans', 'Lachancea quebecensis', 'Lachancea dasiensis', 'Lachancea lanzarotensis', 'Lachancea fantastica', 'Lachancea meyersii', 'Lachancea nothofagi', 'yHMPu5000034709 Kluyveromyces aestuarii', 'Kluyveromyces marxianus', 'yHMPu5000034710 Kluyveromyces dobzhanskii', 'Kluyveromyces lactis', 'Ashbya aceri', 'Eremothecium gossypii', 'Eremothecium cymbalariae', 'Eremothecium sinecaudum', 'Eremothecium coryli']
['yHMPu5000035261 Candida ponderosae']
['yHMPu5000034594 Starmera quercuum', 'yHMPu5000034597 Candida stellimalicola', 'yHMPu5000035658 Starmera amethionina']
['Wickerhamomyces anomalus', 'Wickerhamomyces ciferrii', 'yHMPu5000035271 Wickerhamomyces bovis', 'yHMPu5000035045 Barnettozyma hawaiiensis', 'yHMPu5000035046 Barnettozyma populi', 'yHMPu5000035044 Barnettozyma californica', 'yHMPu5000035037 Candida montana', 'yHMPu5000035671 Phaffomyces antillensis', 'yHMPu5000035670 Phaffomyces opuntiae', 'yHMPu5000035673 Candida orba', 'yHMPu5000035672 Phaffomyces thermotolerans', 'yHMPu5000035048 Barnettozyma salicaria', 'yHMPu5000035047 Barnettozyma pratensis', 'yHMPu5000035274 Wickerhamomyces alni', 'yHMPu5000035286 Candida azyma', 'yHMPu5000035639 Wickerhamomyces canadensis', 'yHMPu5000035268 Wickerhamomyces hampshirensis', 'yHMPu5000034967 Candida freyschussii', 'yHMPu5000035687 Cyberlindnera suaveolens', 'yHMPu5000035686 Cyberlindnera saturnus', 'yHMPu5000034978 Cyberlindnera mrakii', 'yHMPu5000034979 Cyberlindnera misumaiensis', 'yHMPu5000035699 Cyberlindnera maclurae', 'yHMPu5000035690 Candida vartiovaarae', 'Cyberlindnera jadinii', 'yHMPu5000035688 Cyberlindnera xylosilytica', 'Cyberlindnera fabianii JCM3601', 'yHMPu5000035325 Cyberlindnera petersonii', 'yHMPu5000035689 Candida mycetangii', 'yHMPu5000035703 Cyberlindnera americana']
['Sporopachydermia quercuum', 'yHMPu5000034604 Sporopachydermia lactativora']
['Alloascoidea hylecoeti']
['yHMPu5000034635 Nadsonia fulvescens', 'Nadsonia fulvescens var elongata']
['yHMPu5000035633 Candida hispaniensis']
['yHMPu5000035643 Yarrowia bubula', 'Yarrowia keelungensis', 'Yarrowia lipolytica', 'Yarrowia deformans', 'yHMPu5000035645 Yarrowia divulgata']
['yHMPu5000035665 Middelhovenomyces tepae']
['yHMPu5000034661 Dipodascus albidus', 'yHMPu5000041822 Dipodascus geniculatus']
['Geotrichum candidum']
['yHMPu5000041818 Magnusiomyces tetrasperma']
['Saprochaete clavata']
['yHMPu5000035244 Candida incommunis', 'yHMPu5000034974 Deakozyma indianensis']
['Candida versatilis']
['Starmerella bombicola JCM9596', 'Candida apicola']
['Wickerhamiella domercqiae']
['yHMPu5000034646 Wickerhamiella cacticola']
['Candida infanticola']
['yHMPu5000034674 Blastobotrys muscicola']
['yHMPu5000034667 Blastobotrys serpentis']
['yHMPu5000034681 Blastobotrys americana', 'yHMPu5000034671 Blastobotrys peoriensis', 'Arxula adeninivorans', 'yHMPu5000034669 Blastobotrys raffinofermentans']
['yHMPu5000034675 Blastobotrys mokoenaii']
['yHMPu5000034673 Blastobotrys nivea', 'yHMPu5000034670 Blastobotrys proliferans']
['yHMPu5000035691 Candida salmanticensis']
['yHMPu5000035242 Zygoascus ofunaensis']
['yHMPu5000035243 Zygoascus meyerae']
['Sugiyamaella lignohabitans']
['yHMPu5000034605 Spencermartinsiella europaea']
['yHMPu5000034660 Diddensiella caesifluorescens']
['yHMPu5000035650 Trigonopsis variabilis']
['yHMPu5000035282 Trigonopsis vinaria']
['yHMPu5000035279 Tortispora starmeri']
['yHMPu5000035654 Tortispora ganteri', 'Tortispora caseinolytica']
['yHMPu5000034655 Botryozyma nematodophila']
['yHMPu5000034761 Lipomyces lipofer']
['yHMPu5000034758 Lipomyces japonicus']
['yHMPu5000034757 Lipomyces doorenjongii']
['yHMPu5000034760 Lipomyces kononenkoae', 'Lipomyces starkeyi', 'yHMPu5000034754 Lipomyces arxii', 'yHMPu5000034749 Lipomyces mesembrius']
['yHMPu5000034748 Lipomyces oligophaga']
['yHMPu5000034742 Lipomyces suomiensis']
#print(phyloTree.taxon_namespace)
test = [];
taxonList = phyloTree.taxon_namespace;
for taxon in phyloTree.taxon_namespace:
    if taxon in taxonList:
        #print(taxon.label)
        for taxonCompare in phyloTree.taxon_namespace:
            test.append(pdc(taxon, taxonCompare))
print(len(test))110224

#### Code to print out Phylo-groups within a certain evolutionary distance threshold where the pheromones are expected to be conserved
Work started on 20200204 at 1545hrs. I am just going to publish groups that have 2 or more species within the group._____no_output_____

##Making sure the species names have underscores instead of white space analogous to what is listed in the dictionary "332taxa_index.txt" (and "332genomes_Shen2017.csv").
iterCount = 0;
for iterCount in range(len(taxoGroups)):
    #print(len(taxoGroups[iterCount]))
    iterCount1 = 0;
    for iterCount1 in range(len(taxoGroups[iterCount])):
        taxoGroups[iterCount][iterCount1] = taxoGroups[iterCount][iterCount1].replace(" ", "_")
    #print(taxoGroups[iterCount])
#print("Done modifying species names in ", len(taxoGroups), "taxon groups. Good Luck!")_____no_output_____##I need to import the csv from 332taxa_index into a dictionary with the columns {"old_speceis_names" : "original_genome_id"}. I think there is a direct way to do this.
with open(os.path.join(tree_dir, "332genomes_Shen2017.csv"), mode='r') as infile:
    reader = csv.reader(infile)
    fileDict = {rows[1]:rows[0] for rows in reader}

#print(len(fileDict))
#print(taxoGroups[0][0], fileDict[taxoGroups[0][0]])

##I am going to print out the species lits in the Phylo groups of Fungi and I hope to pool these species together to judge the copy number in these groups.
if not(os.path.isdir(os.path.join(tree_dir, "PhyloGroups_pheromones"))):
    os.mkdir(os.path.join(tree_dir, "PhyloGroups_pheromones"))

groupCount = 0
for groupCount in range(len(taxoGroups)):
    if len(taxoGroups[groupCount]) >= 2:
        outputHandle = open(os.path.join(tree_dir, "PhyloGroups_pheromones", "phyloGroup_"+str(groupCount+1)+".txt"), "w")
        for species in taxoGroups[groupCount]:
            #print(fileDict[species])
            outputHandle.write(fileDict[species]+".fas.caax.asn"+"\n")
        outputHandle.close()_____no_output_____

### Version that uses the mcmc tree.
[211107] Started the generation of the phylogrouping and lists._____no_output_____

##Using the mcmctree.
## Building a loop through leaf names that can sort taxonomic groups. This is a generalized search that should identify species groupings for pheromone conservation.
divergenceLimit = 2.40;

taxoGroups = []
taxonList = []
for taxon in phyloTree_mcmc.taxon_namespace:
    taxonList.append(taxon.label)
#treeClone = phyloTree.clone(depth=1)
for taxon in phyloTree_mcmc.taxon_namespace:
    while taxon.label in taxonList:
        taxoGroups.append([taxon.label])
        for taxonCompare in phyloTree_mcmc.taxon_namespace:
            if (pdc(taxon, taxonCompare) <= divergenceLimit) and (taxonCompare != taxon):
                taxoGroups[-1].append(taxonCompare.label)
                taxonList.remove(taxonCompare.label)
        taxonList.remove(taxon.label)

print("Total number of monophyletic groups:\t", len(taxoGroups))

##Hist of group sizes
groupSize = []
for taxoGroup in taxoGroups:
    groupSize.append(len(taxoGroup))
binList = list(range(int(math.ceil(1.1*max(groupSize)))))
print("Number of species in largest monophyletic group:\t", max(groupSize))
print(groupSize)
fig1 = figure(figsize=(8,4))
ax1 = sns.histplot(groupSize, kde=False, bins=binList, stat='count', element='step', fill=False, color='black');
#ax1 = sns.distplot(groupSize, kde=False, bins=binList, hist_kws={"histtype": "step", "linewidth": 3, "alpha": 1});
#plt.ylim(0, 50)
#ax2 = sns.swarmplot(groupSize);
#savefig(os.path.join(figure_dir, 'pheromone-phyloGroups_species-grouping-density_210913.pdf'), dpi=300, transparent=True, bbox_inches='tight')

with open(os.path.join(tree_dir, "taxonGroups_240max-dist_mcmc.csv"), 'w') as writeFile:
    writer = csv.writer(writeFile)
    writer.writerows(taxoGroups)Total number of monophyletic groups:	 42
Number of species in largest monophyletic group:	 71
[1, 7, 4, 27, 33, 6, 1, 6, 4, 1, 2, 2, 1, 2, 2, 3, 3, 1, 2, 1, 4, 47, 2, 2, 8, 71, 1, 3, 2, 28, 1, 2, 2, 6, 1, 5, 8, 15, 2, 1, 3, 9]

#### Code-chunk to identify time-horizon of mating regulatory motifs in the yeast lineage_____no_output_____

## Building a loop through leaf names that can sort taxonomic groups. This is a generalized search that should identify species groupings for
divergenceLimit = 180.0;

taxonGroups_regulation = []
taxonList_regulation = []
for taxon in phyloTree.taxon_namespace:
    taxonList_regulation.append(taxon.label)
#treeClone = phyloTree.clone(depth=1)
for taxon in phyloTree.taxon_namespace:
    while taxon.label in taxonList_regulation:
        taxonGroups_regulation.append([taxon.label])
        for taxonCompare in phyloTree.taxon_namespace:
            if (pdc(taxon, taxonCompare) <= divergenceLimit) and (taxonCompare != taxon):
                taxonGroups_regulation[-1].append(taxonCompare.label)
                taxonList_regulation.remove(taxonCompare.label)
        taxonList_regulation.remove(taxon.label)

print("Total number of monophyletic groups:\t", len(taxonGroups_regulation))

##Hist of group sizes
groupSize = []
for taxonGroup_regulation in taxonGroups_regulation:
    groupSize.append(len(taxonGroup_regulation))
binList = list(range(int(math.ceil(1.1*max(groupSize)))))
print("Number of species in largest monophyletic group:\t", max(groupSize))
print(groupSize)
fig1 = figure(figsize=(8,4))
ax1 = sns.histplot(groupSize, kde=False, bins=binList, stat='count', element='step', color='black', fill=False);
#plt.ylim(0, 50)
#ax2 = sns.swarmplot(groupSize);
#savefig(os.path.join(figure_dir, 'mating-regulation-phyloGroups_species-grouping-density_210913.pdf'), dpi=300, transparent=True, bbox_inches='tight')

#print(taxonGroups_regulation[0])
with open(os.path.join(tree_dir, "taxonGroups_180max-dist_mating-regulation.csv"), 'w') as writeFile:
    writer = csv.writer(writeFile)
    writer.writerows(taxonGroups_regulation)Total number of monophyletic groups:	 79
Number of species in largest monophyletic group:	 47
[7, 31, 33, 6, 6, 1, 4, 3, 2, 1, 1, 2, 2, 3, 3, 1, 2, 1, 4, 47, 2, 1, 1, 2, 6, 7, 32, 11, 1, 11, 4, 5, 1, 1, 2, 2, 14, 14, 1, 1, 1, 2, 1, 5, 1, 2, 1, 1, 1, 2, 1, 2, 1, 1, 1, 1, 1, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 4, 1, 1]
#print(phyloTree.taxon_namespace)
test = [];
taxonList_regulation = phyloTree.taxon_namespace;
for taxon in phyloTree.taxon_namespace:
    if taxon in taxonList_regulation:
        #print(taxon.label)
        for taxonCompare in phyloTree.taxon_namespace:
            test.append(pdc(taxon, taxonCompare))
print(len(test))110224
print(*[group for group in taxonGroups_regulation if len(group)>=1], sep = "\n")['yHMPu5000041678 Debaryomyces prosopidis', 'yHMPu5000041829 Debaryomyces fabryi', 'yHMPu5000041824 Debaryomyces subglobosus', 'Debaryomyces hansenii', 'yHMPu5000041693 Debaryomyces nepalensis', 'yHMPu5000041713 Debaryomyces maramus', 'Millerozyma acaciae']
['yHMPu5000035296 Priceomyces carsonii', 'yHMPu5000035297 Priceomyces castillae', 'Priceomyces haplophilus', 'yHMPu5000034606 Priceomyces medius', 'Scheffersomyces lignosus', 'Scheffersomyces stipitis', 'Spathaspora passalidarum', 'Spathaspora girioi', 'Spathaspora arborariae', 'Spathaspora gorwiae', 'Spathaspora hagerdaliae', 'Candida sojae', 'Candida tropicalis', 'Candida albicans', 'Candida dubliniensis', 'yHMPu5000034990 Candida corydali', 'Lodderomyces elongisporus', 'Candida orthopsilosis', 'Candida parapsilosis', 'yHMPu5000034654 Aciculoconidium aculeatum', 'yHMPu5000034665 Kodamaea laetipori', 'yHMPu5000034648 Candida restingae', 'yHMPu5000035677 Kodamaea ohmeri', 'yHMPu5000035022 Candida emberorum', 'yHMPu5000035018 Candida canberraensis', 'Candida tanzawaensis', 'yHMPu5000035011 Candida pyralidae', 'Wickerhamia fluorescens', 'yHMPu5000035031 Candida kruisii', 'yHMPu5000035033 Candida cretensis', 'yHMPu5000035032 Candida gatunensis']
['Candida auris', 'yHMPu5000035707 Candida heveicola', 'yHMPu5000034986 Candida oregonensis', 'yHMPu5000034988 Candida fructus', 'Clavispora lusitaniae', 'yHMPu5000035335 Candida blattae', 'Candida intermedia', 'yHMPu5000041840 Candida wancherniae', 'yHMPu5000041862 Candida golubevii', 'Metschnikowia bicuspidata', 'yHMPu5000041743 Candida hawaiiana', 'Metschnikowia kipukae', 'Metschnikowia hibisci', 'Metschnikowia aberdeeniae', 'Metschnikowia shivogae', 'Metschnikowia drakensbergensis', 'Metschnikowia proteae', 'Metschnikowia borealis', 'Metschnikowia ipomoeae', 'Metschnikowia continentalis', 'Metschnikowia cerradonensis', 'Metschnikowia santaceciliae', 'Metschnikowia lockheadii', 'Metschnikowia matae maris', 'Metschnikowia matae', 'Metschnikowia kamakouana', 'Metschnikowia hawaiiensis', 'Metschnikowia mauinuiana', 'Metschnikowia hamakuensis', 'Metschnikowia arizonensis', 'Metschnikowia bowlesiae', 'Metschnikowia dekortum', 'Metschnikowia similis']
['yHMPu5000034973 Danielozyma ontarioensis', 'yHMPu5000035681 Candida gotoi', 'yHMPu5000035679 Candida rhagii', 'yHMPu5000035318 Hyphopichia heimii', 'Hyphopichia burtonii', 'Candida homilentoma']
['yHMPu5000035641 Yamadazyma scolyti', 'Candida tenuis', 'yHMPu5000035640 Yamadazyma philogaea', 'yHMPu5000035252 Yamadazyma nakazawae', 'yHMPu5000041833 Candida tammaniensis', 'yHMPu5000041863 Candida gorgasii']
['yHMPu5000041855 Candida ascalaphidarum']
['yHMPu5000034632 Candida athensensis', 'Meyerozyma guilliermondii', 'yHMPu5000035662 Meyerozyma caribbica', 'Candida carpophila']
['yHMPu5000035667 Kurtzmaniella cleridarum', 'yHMPu5000035302 Candida fragi', 'yHMPu5000034643 Candida schatavii']
['yHMPu5000034999 Cephaloascus fragrans', 'yHMPu5000034998 Cephaloascus albidus']
['Babjeviella inositovora']
['Pachysolen tannophilus']
['yHMPu5000034918 Nakazawaea holstii', 'Nakazawaea peltata']
['yHMPu5000034884 Peterozyma toletana', 'yHMPu5000034883 Peterozyma xylosa']
['Komagataella pastoris', 'yHMPu5000034939 Komagataella pseudopastoris', 'yHMPu5000026274 Komagataella populi']
['yHMPu5000026142 Citeromyces matritensis', 'yHMPu5000034950 Citeromyces hawaiiensis', 'yHMPu5000034952 Citeromyces siamensis']
['yHMPu5000034932 Kuraishia ogatae']
['yHMPu5000034933 Kuraishia molischiana', 'Kuraishia capsulata']
['Candida boidinii JCM9604']
['yHMPu5000034901 Ogataea methylivora', 'yHMPu5000034890 Ogataea ramenticola', 'yHMPu5000034903 Ogataea naganishii', 'Candida succiphila']
['yHMPu5000026197 Brettanomyces custersianus', 'Dekkera bruxellensis', 'Brettanomyces anomalus', 'yHMPu5000035674 Kregervanrija delftensis', 'yHMPu5000035675 Kregervanrija fluxuum', 'yHMPu5000034631 Martiniozyma abiesophila', 'yHMPu5000034614 Saturnispora silvae', 'yHMPu5000034611 Saturnispora mendoncae', 'yHMPu5000034610 Saturnispora hagleri', 'yHMPu5000034613 Saturnispora serradocipensis', 'yHMPu5000035659 Saturnispora dispora', 'yHMPu5000034615 Saturnispora zaruensis', 'yHMPu5000034612 Saturnispora saitoi', 'yHMPu5000035301 Pichia terricola', 'Candida sorboxylosa', 'Pichia membranifaciens', 'yHMPu5000034629 Pichia exigua', 'yHMPu5000034622 Pichia occidentalis', 'yHMPu5000034625 Pichia kudriavzevii', 'yHMPu5000034623 Pichia norvegensis', 'yHMPu5000034624 Pichia nakasei', 'yHMPu5000034627 Pichia heedii', 'yHMPu5000035040 Ambrosiozyma maleeae', 'yHMPu5000026145 Ambrosiozyma vanderkliftii', 'yHMPu5000035041 Ambrosiozyma pseudovanderkliftii', 'Ambrosiozyma kashinagacola', 'Ambrosiozyma monospora', 'yHMPu5000034946 Ambrosiozyma oregonensis', 'yHMPu5000026137 Ambrosiozyma ambrosiae', 'yHMPu5000034947 Ambrosiozyma philentoma', 'Candida arabinofermentans', 'Ogataea methanolica', 'yHMPu5000034886 Ogataea trehalophila', 'yHMPu5000034892 Ogataea pilisensis', 'yHMPu5000034636 Ogataea nitratoaversa', 'yHMPu5000034637 Ogataea populiabae', 'yHMPu5000034887 Ogataea trehaloabstinens', 'yHMPu5000034885 Ogataea zsoltii', 'yHMPu5000034897 Ogataea glucozyma', 'yHMPu5000034891 Ogataea pini', 'yHMPu5000026124 Ogataea henricii', 'yHMPu5000034904 Ogataea nonfermentans', 'yHMPu5000034902 Ogataea minuta', 'yHMPu5000034893 Ogataea philodendra', 'Ogataea parapolymorpha', 'Ogataea polymorpha', 'yHMPu5000034899 Ogataea kodamae']
['Saccharomycopsis malanga', 'yHMPu5000034607 Saccharomycopsis capsularis']
['Ascoidea rubescens']
['Ascoidea asiatica']
['Hanseniaspora vinae', 'yHMPu5000034957 Hanseniaspora osmophila']
['yHMPu5000035695 Hanseniaspora pseudoguilliermondii', 'yHMPu5000034963 Hanseniaspora clermontiae', 'Hanseniaspora uvarum', 'yHMPu5000035684 Kloeckera hatyaiensis', 'Hanseniaspora valbyensis', 'yHMPu5000035696 Hanseniaspora singularis']
['yHMPu5000035629 Yueomyces sinensis', 'Tetrapisispora blattae', 'Vanderwaltozyma polyspora', 'yHMPu5000034876 Tetrapisispora iriomotensis', 'Tetrapisispora phaffii', 'yHMPu5000034875 Tetrapisispora fleetii', 'yHMPu5000034877 Tetrapisispora namnaonensis']
['Nakaseomyces castellii', 'Nakaseomyces bacillisporus', 'Candida glabrata', 'Nakaseomyces bracarensis', 'Nakaseomyces delphensis', 'Nakaseomyces nivariensis', 'Saccharomyces eubayanus', 'Saccharomyces uvarum', 'Saccharomyces arboricola', 'Saccharomyces kudriavzevii', 'Saccharomyces mikatae', 'Saccharomyces cerevisiae', 'Saccharomyces paradoxus', 'yHAB153 Kazachstania rosinii', 'yHAB155 Kazachstania spencerorum', 'yHAB160 Kazachstania kunashirensis', 'yHAB134 Kazachstania turicensis', 'yHAB158 Kazachstania viticola', 'Kazachstania africana', 'yHAB132 Kazachstania martiniae', 'yHAB148 Kazachstania intestinalis', 'yHAB136 Kazachstania bromeliacearum', 'Kazachstania naganishii', 'yHAB147 Kazachstania taianensis', 'yHAB166 Kazachstania yakushimaensis', 'yHAB154 Kazachstania transvaalensis', 'yHAB164 Kazachstania aerobia', 'yHAB159 Kazachstania solicola', 'yHAB133 Kazachstania unispora', 'yHAB143 Kazachstania siamensis', 'Naumovozyma castellii', 'Naumovozyma dairenensis']
['Zygosaccharomyces bailii', 'yHMPu5000034866 Zygosaccharomyces bisporus', 'yHMPu5000034865 Zygosaccharomyces kombuchaensis', 'Zygosaccharomyces rouxii', 'yHMPu5000026256 Zygotorulaspora mrakii', 'yHMPu5000034862 Zygotorulaspora florentina', 'yHMPu5000034881 Torulaspora pretoriensis', 'yHMPu5000026152 Torulaspora franciscae', 'Torulaspora delbrueckii', 'yHMPu5000035652 Torulaspora maleeae', 'yHMPu5000035651 Torulaspora microellipsoides']
['Lachancea kluyveri']
['Lachancea cidri', 'Lachancea fermentati', 'Lachancea mirantina', 'Lachancea waltii', 'Lachancea thermotolerans', 'Lachancea quebecensis', 'Lachancea dasiensis', 'Lachancea lanzarotensis', 'Lachancea fantastica', 'Lachancea meyersii', 'Lachancea nothofagi']
['yHMPu5000034709 Kluyveromyces aestuarii', 'Kluyveromyces marxianus', 'yHMPu5000034710 Kluyveromyces dobzhanskii', 'Kluyveromyces lactis']
['Ashbya aceri', 'Eremothecium gossypii', 'Eremothecium cymbalariae', 'Eremothecium sinecaudum', 'Eremothecium coryli']
['yHMPu5000035261 Candida ponderosae']
['yHMPu5000034594 Starmera quercuum']
['yHMPu5000034597 Candida stellimalicola', 'yHMPu5000035658 Starmera amethionina']
['Wickerhamomyces anomalus', 'Wickerhamomyces ciferrii']
['yHMPu5000035271 Wickerhamomyces bovis', 'yHMPu5000035045 Barnettozyma hawaiiensis', 'yHMPu5000035046 Barnettozyma populi', 'yHMPu5000035044 Barnettozyma californica', 'yHMPu5000035037 Candida montana', 'yHMPu5000035671 Phaffomyces antillensis', 'yHMPu5000035670 Phaffomyces opuntiae', 'yHMPu5000035673 Candida orba', 'yHMPu5000035672 Phaffomyces thermotolerans', 'yHMPu5000035048 Barnettozyma salicaria', 'yHMPu5000035047 Barnettozyma pratensis', 'yHMPu5000035274 Wickerhamomyces alni', 'yHMPu5000035286 Candida azyma', 'yHMPu5000035639 Wickerhamomyces canadensis']
['yHMPu5000035268 Wickerhamomyces hampshirensis', 'yHMPu5000034967 Candida freyschussii', 'yHMPu5000035687 Cyberlindnera suaveolens', 'yHMPu5000035686 Cyberlindnera saturnus', 'yHMPu5000034978 Cyberlindnera mrakii', 'yHMPu5000034979 Cyberlindnera misumaiensis', 'yHMPu5000035699 Cyberlindnera maclurae', 'yHMPu5000035690 Candida vartiovaarae', 'Cyberlindnera jadinii', 'yHMPu5000035688 Cyberlindnera xylosilytica', 'Cyberlindnera fabianii JCM3601', 'yHMPu5000035325 Cyberlindnera petersonii', 'yHMPu5000035689 Candida mycetangii', 'yHMPu5000035703 Cyberlindnera americana']
['Sporopachydermia quercuum']
['yHMPu5000034604 Sporopachydermia lactativora']
['Alloascoidea hylecoeti']
['yHMPu5000034635 Nadsonia fulvescens', 'Nadsonia fulvescens var elongata']
['yHMPu5000035633 Candida hispaniensis']
['yHMPu5000035643 Yarrowia bubula', 'Yarrowia keelungensis', 'Yarrowia lipolytica', 'Yarrowia deformans', 'yHMPu5000035645 Yarrowia divulgata']
['yHMPu5000035665 Middelhovenomyces tepae']
['yHMPu5000034661 Dipodascus albidus', 'yHMPu5000041822 Dipodascus geniculatus']
['Geotrichum candidum']
['yHMPu5000041818 Magnusiomyces tetrasperma']
['Saprochaete clavata']
['yHMPu5000035244 Candida incommunis', 'yHMPu5000034974 Deakozyma indianensis']
['Candida versatilis']
['Starmerella bombicola JCM9596', 'Candida apicola']
['Wickerhamiella domercqiae']
['yHMPu5000034646 Wickerhamiella cacticola']
['Candida infanticola']
['yHMPu5000034674 Blastobotrys muscicola']
['yHMPu5000034667 Blastobotrys serpentis']
['yHMPu5000034681 Blastobotrys americana', 'yHMPu5000034671 Blastobotrys peoriensis']
['Arxula adeninivorans', 'yHMPu5000034669 Blastobotrys raffinofermentans']
['yHMPu5000034675 Blastobotrys mokoenaii']
['yHMPu5000034673 Blastobotrys nivea']
['yHMPu5000034670 Blastobotrys proliferans']
['yHMPu5000035691 Candida salmanticensis']
['yHMPu5000035242 Zygoascus ofunaensis']
['yHMPu5000035243 Zygoascus meyerae']
['Sugiyamaella lignohabitans']
['yHMPu5000034605 Spencermartinsiella europaea']
['yHMPu5000034660 Diddensiella caesifluorescens']
['yHMPu5000035650 Trigonopsis variabilis']
['yHMPu5000035282 Trigonopsis vinaria']
['yHMPu5000035279 Tortispora starmeri']
['yHMPu5000035654 Tortispora ganteri', 'Tortispora caseinolytica']
['yHMPu5000034655 Botryozyma nematodophila']
['yHMPu5000034761 Lipomyces lipofer']
['yHMPu5000034758 Lipomyces japonicus']
['yHMPu5000034757 Lipomyces doorenjongii']
['yHMPu5000034760 Lipomyces kononenkoae', 'Lipomyces starkeyi', 'yHMPu5000034754 Lipomyces arxii', 'yHMPu5000034749 Lipomyces mesembrius']
['yHMPu5000034748 Lipomyces oligophaga']
['yHMPu5000034742 Lipomyces suomiensis']
##Making sure the species names have underscores instead of white space analogous to what is listed in the dictionary "332taxa_index.txt" (and "332genomes_Shen2017.csv").
iterCount = 0;
for iterCount in range(len(taxonGroups_regulation)):
    #print(len(taxoGroups_regulation[iterCount]))
    iterCount1 = 0;
    for iterCount1 in range(len(taxonGroups_regulation[iterCount])):
        taxonGroups_regulation[iterCount][iterCount1] = taxonGroups_regulation[iterCount][iterCount1].replace(" ", "_")
    #print(taxonGroups_regulation[iterCount])
print("Done modifying species names in ", len(taxonGroups_regulation), "taxon groups. Good Luck!")Done modifying species names in  79 taxon groups. Good Luck!
##I need to import the csv from 332taxa_index into a dictionary with the columns {"old_speceis_names" : "original_genome_id"}. I think there is a direct way to do this.
with open(os.path.join(tree_dir, "332genomes_Shen2017.csv"), mode='r') as infile:
    reader = csv.reader(infile)
    fileDict = {rows[1]:rows[0] for rows in reader}

#print(len(fileDict))
#print(taxonGroups_regulation[0][0])
#print(taxonGroups_regulation[0][0], fileDict[taxonGroups_regulation[0][0]])

##I am going to print out the species lits in the Phylo groups of Fungi and I hope to pool these species together to judge the copy number in these groups.
if not(os.path.isdir(os.path.join(tree_dir, "PhyloGroups_regulation"))):
    os.mkdir(os.path.join(tree_dir, "PhyloGroups_regulation"))

groupCount = 0
for groupCount in range(len(taxonGroups_regulation)):
    if len(taxonGroups_regulation[groupCount]) >= 2:
        outputHandle = open(os.path.join(tree_dir, "PhyloGroups_regulation", "phyloGroup_"+str(groupCount+1)+".txt"), "w")
        for species in taxonGroups_regulation[groupCount]:
            #print(fileDict[species])
            outputHandle.write("promoters_0_"+fileDict[species]+".fasta"+"\n")
        outputHandle.close()_____no_output_____##I need to import the csv from 332taxa_index into a dictionary with the columns {"old_speceis_names" : "original_genome_id"}. I think there is a direct way to do this.
with open(os.path.join(tree_dir, "332genomes_Shen2017.csv"), mode='r') as infile:
    reader = csv.reader(infile)
    fileDict = {rows[1]:rows[0] for rows in reader}

#print(len(fileDict))
#print(taxonGroups_regulation[0][0])
#print(taxonGroups_regulation[0][0], fileDict[taxonGroups_regulation[0][0]])

##I am going to print out the species lits in the Phylo groups of Fungi and I hope to pool these species together to judge the copy number in these groups.
if not(os.path.isdir(os.path.join(tree_dir, "PhyloGroups_candidate_regulation"))):
    os.mkdir(os.path.join(tree_dir, "PhyloGroups_candidate_regulation"))

groupCount = 0
for groupCount in range(len(taxonGroups_regulation)):
    if len(taxonGroups_regulation[groupCount]) >= 2:
        outputHandle = open(os.path.join(tree_dir, "PhyloGroups_candidate_regulation", "phyloGroup_"+str(groupCount+1)+".txt"), "w")
        for species in taxonGroups_regulation[groupCount]:
            #print(fileDict[species])
            outputHandle.write(fileDict[species]+".fas.caax.asn.promoters"+"\n")
        outputHandle.close()_____no_output_____

## Code to sort all CAAX-Asn candidates for unique loci
>Date started 191211, 1500 IST

This is a small code block to accept a fasta list of all the candidates I identify from a genome and then try to get a unique list for finding multiple copies of a candidate at different genome loci (obviously)._____no_output_____

##Importing the FASTA file, this won't be used when I add this to the python codebase for functionality. Remember to run the first two blocks for all library imports.
inputFile = "Yarli1_AssemblyScaffolds-put-v2p3_180508.fasta"
fileHandle = open(data_dir+inputFile, "r")
candidates = list(SeqIO.parse(fileHandle, "fasta"))
fileHandle.close()

print(candidates[0].id)
#print(candidates[0].id.split("||")[1].split("[")[3].split("-")[:])
print("There are %i candidates" % len(candidates))Yali0A_1||[forward][1][45693-45756-45768][Translate:MSNLTSHRVGVRSTPRRAISSCSVS*]
There are 399 candidates

#### Algorthm for stack search
- The simplest approach is to brute-force the search and basically compare the current candidate to with all others to see if there is something that shares the chromID, direction and CAAX-STOP positions while having a higher candidate number.
- Another option is to just invert the stack and pull candidates one after another and since the first candidate from the back would be the largest ORF at any given CAAX-STOP, then all I would have to do is compare the current ID to the last in the new list I am making, and if it is the same keep moving, else append to the list I am making._____no_output_____

##Function to parse candidates and eliminate those that share a CAAX tag (retain the largest ORF among those that share CAAX-STOP).
##Trying the cheat version with a reversed iterator.
uniqueCandidates = []
skipCount = 0
for candidate in reversed(candidates):
    #print(candidate.id)
    if (uniqueCandidates != []):
        if (candidate.id.split("||")[0] == uniqueCandidates[-1].id.split("||")[0]) and (candidate.id.split("||")[1].split("[")[1] == uniqueCandidates[-1].id.split("||")[1].split("[")[1]):
            if candidate.id.split("||")[1].split("[")[3].split("-")[2] == uniqueCandidates[-1].id.split("||")[1].split("[")[3].split("-")[2]:
                skipCount += 1
            else:
                uniqueCandidates.append(candidate)
        else:
            uniqueCandidates.append(candidate)
    else:
        uniqueCandidates.append(candidate)

print("Skipped %i candidates" % skipCount)
uniqueCandidates.reverse()
print(len(uniqueCandidates))

##printing output.
outputHandle = open(data_dir+inputFile.split(".f")[0]+"_unique.fasta", "w")
for candidate in uniqueCandidates:
    outputHandle.write(">"+str(candidate.id)+"\n"+str(candidate.seq)+"\n")
outputHandle.close()Skipped 147 candidates
252
for candidate in uniqueCandidates:
    print(candidate.id)_____no_output_____


# Notebook from leylabmpi/animal_gut_metagenome_assembly
Path: multi-study/05i_LLMGA_PRJNA476660.ipynb

Table of Contents

1  Goal

2  Var

3  Init

4  LLMGA

4.1  Setup

4.2  config

4.3  Run

5  Summary

5.1  No. of genomes

5.2  CheckM

5.3  Taxonomy

5.3.1  Taxonomic novelty

5.3.2  Quality ~ taxonomy

6  sessionInfo

_____no_output_____# Goal

* running `LLMGA` on metagenome datasets
* studyID = PRJNA476660
  * host = pangolin_____no_output_____# Var_____no_output_____

work_dir = '/ebio/abt3_projects/Georg_animal_feces/data/metagenome/multi-study/BioProjects/'
tmp_out_dir = '/ebio/abt3_projects/databases_no-backup/animal_gut_metagenomes/multi-study_MG-asmbl/'
pipeline_dir = '/ebio/abt3_projects/methanogen_host_evo/bin/llmga-find-refs/'

studyID = 'PRJNA476660'
threads = 24_____no_output_____

# Init_____no_output_____

library(dplyr)
library(tidyr)
library(ggplot2)
library(data.table)
set.seed(8304)

source('/ebio/abt3_projects/Georg_animal_feces/code/misc_r_functions/init.R')_____no_output_____

# LLMGA_____no_output_____## Setup_____no_output_____

out_dir = file.path(tmp_out_dir, studyID)
make_dir(out_dir)
out_dir = file.path(out_dir, 'LLMGA')
make_dir(out_dir)Created directory: /ebio/abt3_projects/databases_no-backup/animal_gut_metagenomes/multi-study_MG-asmbl//PRJNA476660
Created directory: /ebio/abt3_projects/databases_no-backup/animal_gut_metagenomes/multi-study_MG-asmbl//PRJNA476660/LLMGA
ref_genomes = file.path(work_dir, studyID, 'LLMGA-find-refs/references/ref_genomes.fna')
cat(ref_genomes)/ebio/abt3_projects/Georg_animal_feces/data/metagenome/multi-study/BioProjects//PRJNA476660/LLMGA-find-refs/references/ref_genomes.fna

## config_____no_output_____

cat_file(file.path(out_dir, 'config.yaml'))#-- I/O --#
# table with sample --> read_file information
samples_file: /ebio/abt3_projects/Georg_animal_feces/data/metagenome/multi-study/BioProjects/PRJNA476660/LLMGQC/final/samples.txt

# output location
output_dir: /ebio/abt3_projects/databases_no-backup/animal_gut_metagenomes/multi-study_MG-asmbl/PRJNA476660/LLMGA/

#-- reference genome(s) for metacompass --#
metacompass_ref: /ebio/abt3_projects/Georg_animal_feces/data/metagenome/multi-study/BioProjects/PRJNA476660/LLMGA-find-refs/references/ref_genomes.fna

#-- database --#
kraken2_db: /ebio/abt3_projects/databases_no-backup/kraken2/nt_db/hash.k2d
krakenuniq_db: /ebio/abt3_projects/databases_no-backup/krakenuniq/taxonomy/nodes.dmp
checkM_data: /ebio/abt3_projects/databases_no-backup/checkM/
sourmash_db: /ebio/abt3_projects/databases_no-backup/sourmash/genbank-k31.sbt.json
sourmash_lca_db: /ebio/abt3_projects/databases_no-backup/sourmash/v2/genbank-k31.lca.json.gz
gtdbtk_db: /ebio/abt3_projects/databases_no-backup/GTDB/release89/db_info.md

#-- re-running --#
# use this to prevent re-running the assembly steps if you just need to rerun post-assembly steps
skip_assembly: False

#-- software parameters --#
# Notes:
## Use "Skip" to skip any step. If no params, just use ""
## Note: You must skip either metacompass_megahit or metacompass_metaspades (or both)
## Note: for the *_batches params, the number must be <= the number of MAGs
## Note: using >15mil paired-end reads may not scale well!
## Note: subsampling just applies to samples with > the number of reads
## Note: for diff. cov. binning, you can select a certain number of samples to use (the origin sample is always used)
params:
  subsample_reads: 10000000
  fastqc_on_raw: ""

  # metacompass
  ## ref-based assembly; use "Skip" to skip the ref-based assembly
  metacompass: ""
  metacompass_buildcontig: --pickref breadth --mincov 3 -l 500 -n T -b F -u F
  ## de-novo assembly; use "Skip" to skip the de-novo assembly
  metacompass_metaspades: -k auto --only-assembler
  metacompass_megahit: Skip       # eg., (--min-count 3 --min-contig-len 500 --presets meta-sensitive)
  ## contig length cutoff
  metacompass_min_contig_len: 1500            # min contig length retained
  metacompass_derep_contigs: minidentity=100 minscaf=500 minoverlappercent=95

  # co-assembly
  ## normalization/subsampling/dereplication
  bbnorm_metacompass_unmapped: Skip            # eg., (target=100 k=31 minkmers=15 prefilter=t passes=1)
  subsample_combined: Skip                     # max number of read pairs to use for co-assembly (eg., 10000000)
  bbnorm_metacompass_unmapped_combined: Skip   # eg., (target=100 k=31 minkmers=15 prefilter=t passes=1)
  ## co-assembly (default => Skipped)
  coassemble_metaspades_hybrid: Skip          # eg., (-k auto --only-assembler)

  # combined, final contigs
  combine_all_contigs: Skip                 # 'Skip' will cause all samples to be binned seperately; must not be skipped if using co-assembly
  metaquast: --max-ref-number 0             # job run on rick to use internet if `--max-ref-number` > 0
  contig_rename: minscaf=1500               # minscaf = scaffold length cutoff
  cut_up_fasta: Skip                        # cutting long contigs for possibly better binning (eg., -c 20000 -o 0)

  # contig binning
  ## mapping
  num_map_samples: all                 # how many samples to use for differential cov. binning ('all' = all samples)
  ### bowtie2 (more sensitive than kraken)
  samtools: -q 0                              # -q = MAPQ cutoff
  bam_to_depth: --percentIdentity 97
  ### kraken (faster than bowtie2, but less sensitive)
  kraken2: Skip #--memory-mapping
  krakenuniq_build: --kmer-len 31 --minimizer-len 15
  krakenuniq: --hll-precision 12
  krakenuniq_kmer_cutoff: 1000
  ## binning
  ### maxbin2 (2 different binning parameters used)
  maxbin2_low_prob: -min_contig_length 1500 -markerset 40  -prob_threshold 0.6
  maxbin2_high_prob: -min_contig_length 1500 -markerset 40  -prob_threshold 0.8
  ### metabat (2 different binning parameters used)
  metabat2_low_PE: --minContig 1500 --minCV 0.5 --minCVSum 0.5  --maxP 92 --maxEdges 150 --seed 8394
  metabat2_high_PE: --minContig 1500 --minCV 0.5 --minCVSum 0.5 --maxP 97 --maxEdges 500 --seed 8394
  ### vamb
  vamb: Skip #-m 1500

  # bin refinement/assessment
  ## selecting the 'best' bins from all binning methods
  das_tool: --search_engine diamond
  ## bin assessment
  bin_batches: 4                               # process MAGs in batches
  checkm:  --tab_table
  sourmash_compute: --scaled 10000 -k 31
  sourmash_gather: -k 31 --dna
  fastani_batches: 4                           # process MAGs in batches
  fastani: --fragLen 1000 --minFrag 50 -k 16
  gtdbtk_classify_wf: --min_perc_aa 10
  drep: -comp 50 -con 5 -sa 99
  ### anivo; use "Skip" on anvio_gen_contigs_db to skip all of anvio
  anvio_gen_contigs_db: Skip                 # eg., (--skip-mindful-splitting --kmer-size 4)
  anvio_run_ncbi_cogs: --cog-data-dir /ebio/abt3_projects/databases_no-backup/anvio_v4/
  anvio_centrifuge: -x /ebio/abt3_projects/databases_no-backup/centrifuge/p+h+v
  anvio_profile: --min-mean-coverage 0 --min-contig-length 2000 --cluster-contigs
  anvio_merge: -S coassembly_contigs --skip-concoct-binning
  anvio_interactive_script: -P 8080 -C DAS_Tool

#-- snakemake pipeline --#
# your username will be added automatically to the `temp_folder` path
pipeline:
  bwlimit: 100m                            # rsync bwlimit
  snakemake_folder: ./
  script_folder: bin/scripts/
  temp_folder: /tmp/global/
  random_number_seed: 83421

## Run_____no_output_____```
(snakemake_dev) @ rick:/ebio/abt3_projects/vadinCA11/bin/llmga
$ screen -L -S llmga-PRJNA476660 ./snakemake_sge.sh /ebio/abt3_projects/databases_no-backup/animal_gut_metagenomes/multi-study_MG-asmbl/PRJNA476660/LLMGA/config.yaml cluster.json /ebio/abt3_projects/databases_no-backup/animal_gut_metagenomes/multi-study_MG-asmbl/PRJNA476660/LLMGA/SGE_log 24
```_____no_output_____

pipelineInfo('/ebio/abt3_projects/vadinCA11/bin/llmga')LLMGA
=====

Ley Lab Metagenome Aseembly (LLMGA)

* Version: 0.6.2
* Authors:
  * Nick Youngblut <nyoungb2@gmail.com>
* Maintainers:
  * Nick Youngblut <nyoungb2@gmail.com>

--- conda envs ---
==> /ebio/abt3_projects/vadinCA11/bin/llmga/bin/envs/anvio.yaml <==
channels:
- conda-forge
- bioconda
dependencies:
- pigz
- bioconda::centrifuge
- bioconda::anvio>=5.1.0
==> /ebio/abt3_projects/vadinCA11/bin/llmga/bin/envs/bbmap.yaml <==
channels:
- conda-forge
- bioconda
dependencies:
- pigz
- bioconda::bbmap=37.90
==> /ebio/abt3_projects/vadinCA11/bin/llmga/bin/envs/bowtie2.yaml <==
channels:
- conda-forge
- bioconda
dependencies:
- pigz
- bioconda::samtools
- bioconda::bowtie2
- bioconda::bedtools
==> /ebio/abt3_projects/vadinCA11/bin/llmga/bin/envs/checkm.yaml <==
channels:
- conda-forge
- bioconda
dependencies:
- python=2
- pigz
- bioconda::prodigal
- bioconda::pplacer
- bioconda::checkm-genome

==> /ebio/abt3_projects/vadinCA11/bin/llmga/bin/envs/concoct.yaml <==
channels:
- conda-forge
- bioconda
dependencies:
- pigz
- bioconda::pysam=0.14.1
- bioconda::concoct=0.4.0
==> /ebio/abt3_projects/vadinCA11/bin/llmga/bin/envs/coverage.yaml <==
channels:
- conda-forge
- bioconda
dependencies:
- pigz
- conda-forge::biopython
- bioconda::samtools
- bioconda::bbmap
==> /ebio/abt3_projects/vadinCA11/bin/llmga/bin/envs/das_tool.yaml <==
channels:
- conda-forge
- bioconda
- r
dependencies:
- pigz
- bioconda::das_tool
==> /ebio/abt3_projects/vadinCA11/bin/llmga/bin/envs/drep.yaml <==
channels:
- conda-forge
- bioconda
dependencies:
- pigz
- bioconda::mash
- bioconda::checkm-genome
- bioconda::drep
==> /ebio/abt3_projects/vadinCA11/bin/llmga/bin/envs/fastani.yaml <==
channels:
- conda-forge
- bioconda
dependencies:
- pigz
- bioconda::fastani
==> /ebio/abt3_projects/vadinCA11/bin/llmga/bin/envs/fastqc.yaml <==
channels:
- conda-forge
- bioconda
dependencies:
- bioconda::fastqc

==> /ebio/abt3_projects/vadinCA11/bin/llmga/bin/envs/gtdbtk.yaml <==
channels:
- conda-forge
- bioconda
dependencies:
- pigz
- bioconda::gtdbtk

==> /ebio/abt3_projects/vadinCA11/bin/llmga/bin/envs/hadley.yaml <==
channels:
- conda-forge
dependencies:
- conda-forge::r-ape
- conda-forge::r-dplyr
- conda-forge::r-tidyr
- conda-forge::r-ggplot2

==> /ebio/abt3_projects/vadinCA11/bin/llmga/bin/envs/maxbin2.yaml <==
channels:
- conda-forge
- bioconda
dependencies:
- pigz
- bioconda::perl-lwp-simple
- bioconda::maxbin2=2.2.4

==> /ebio/abt3_projects/vadinCA11/bin/llmga/bin/envs/megahit.yaml <==
channels:
- conda-forge
- bioconda
dependencies:
- pigz
- bioconda::megahit=1.1.3
==> /ebio/abt3_projects/vadinCA11/bin/llmga/bin/envs/metabat2.yaml <==
channels:
- BioBuilds
- conda-forge
- bioconda
dependencies:
- pigz
- python=2.7
- BioBuilds::boost>=1.55.0
- scons>=2.1.0
- zlib>=1.2.4
- BioBuilds::binutils>=2.2.2

==> /ebio/abt3_projects/vadinCA11/bin/llmga/bin/envs/metacompass.yaml <==
channels:
- conda-forge
- bioconda
dependencies:
- pigz=2.3.4
- bioconda::seqtk
- bioconda::bioawk
- bioconda::blast=2.7.1
- bioconda::meryl=2013
- bioconda::samtools=1.8
- bioconda::megahit=1.1.3
- bioconda::spades=3.12.0
- bioconda::pilon=1.22=1
- bioconda::bowtie2=2.3.4.1
==> /ebio/abt3_projects/vadinCA11/bin/llmga/bin/envs/multiqc.yaml <==
channels:
- conda-forge
- bioconda
dependencies:
- bioconda::multiqc

==> /ebio/abt3_projects/vadinCA11/bin/llmga/bin/envs/quast.yaml <==
channels:
- conda-forge
- bioconda
dependencies:
- python=2.7
- bioconda::quast

==> /ebio/abt3_projects/vadinCA11/bin/llmga/bin/envs/seqtk.yaml <==
channels:
- conda-forge
- bioconda
dependencies:
- pigz
- bioconda::seqtk

==> /ebio/abt3_projects/vadinCA11/bin/llmga/bin/envs/sourmash.yaml <==
channels:
- conda-forge
- bioconda
dependencies:
- bioconda::sourmash

==> /ebio/abt3_projects/vadinCA11/bin/llmga/bin/envs/spades.yaml <==
channels:
- conda-forge
- bioconda
dependencies:
- pigz
- bioconda::spades=3.12.0
==> /ebio/abt3_projects/vadinCA11/bin/llmga/bin/envs/vamb.yaml <==
channels:
- conda-forge
- bioconda
- pytorch
dependencies:
- pip
- pigz
- python=3
- cudatoolkit=9.0
- pytorch::torchvision
- pytorch::pytorch
- pip:
  - https://github.com/jakobnissen/vamb/archive/v2.0.1.zip

# Summary_____no_output_____

asmbl_dir = out_dir = file.path(tmp_out_dir, studyID, 'LLMGA')

checkm_markers_file = file.path(asmbl_dir, 'checkm', 'markers_qa_summary.tsv')
gtdbtk_bac_sum_file = file.path(asmbl_dir, 'gtdbtk', 'gtdbtk_bac_summary.tsv')
gtdbtk_arc_sum_file = file.path(asmbl_dir, 'gtdbtk', 'gtdbtk_ar_summary.tsv')

bin_dir = file.path(asmbl_dir, 'bin')
das_tool_dir = file.path(asmbl_dir, 'bin_refine', 'DAS_Tool')
drep_dir = file.path(asmbl_dir, 'drep', 'drep')_____no_output_____# bin genomes
## maxbin2
bin_files = list.files(bin_dir, '*.fasta$', full.names=TRUE, recursive=TRUE)
bin = data.frame(binID = gsub('\\.fasta$', '', basename(bin_files)),
                 fasta = bin_files,
                 binner = bin_files %>% dirname %>% basename,
                 sample = bin_files %>% dirname %>% dirname %>% basename)

## metabat2
bin_files = list.files(bin_dir, '*.fa$', full.names=TRUE, recursive=TRUE)
X = data.frame(binID = gsub('\\.fa$', '', basename(bin_files)),
               fasta = bin_files,
               binner = bin_files %>% dirname %>% basename,
               sample = bin_files %>% dirname %>% dirname %>% basename)

## combine
bin = rbind(bin, X)
X = NULL
bin %>% dfhead[1] 249   4
# DAS-tool genomes
dastool_files = list.files(das_tool_dir, '*.fa$', full.names=TRUE, recursive=TRUE)

dastool = data.frame(binID = gsub('\\.fa$', '', basename(dastool_files)),
                    fasta = dastool_files)
dastool %>% dfhead[1] 29  2
# drep genome files
P = file.path(drep_dir, 'dereplicated_genomes')
drep_files = list.files(P, '*.fa$', full.names=TRUE)

drep = data.frame(binID = gsub('\\.fa$', '', basename(drep_files)),
                  fasta = drep_files)
drep %>% dfhead[1] 26  2
# checkm info
markers_sum = read.delim(checkm_markers_file, sep='\t')
markers_sum %>% nrow %>% print

drep_j = drep %>%
    inner_join(markers_sum, c('binID'='Bin.Id'))

drep_j %>% dfhead[1] 29
# gtdb
## bacteria
X = read.delim(gtdbtk_bac_sum_file, sep='\t') %>%
    dplyr::select(-other_related_references.genome_id.species_name.radius.ANI.AF.) %>%
    separate(classification, c('Domain', 'Phylum', 'Class', 'Order', 'Family', 'Genus', 'Species'), sep=';')
X %>% nrow %>% print
## archaea
# Y = read.delim(gtdbtk_arc_sum_file, sep='\t') %>%
#     dplyr::select(-other_related_references.genome_id.species_name.radius.ANI.AF.) %>%
#     separate(classification, c('Domain', 'Phylum', 'Class', 'Order', 'Family', 'Genus', 'Species'), sep=';')
# Y %>% nrow %>% print

## combined
drep_j = drep_j %>%
    left_join(X, c('binID'='user_genome'))

## status
X = Y = NULL
drep_j %>% dfhead[1] 29

## No. of genomes_____no_output_____

cat('Number of binned genomes:', bin$fasta %>% unique %>% length)Number of binned genomes: 249cat('Number of DAS-Tool passed genomes:', dastool$binID %>% unique %>% length)Number of DAS-Tool passed genomes: 29cat('Number of 99% ANI de-rep genomes:', drep_j$binID %>% unique %>% length)Number of 99% ANI de-rep genomes: 26

## CheckM_____no_output_____

# checkm stats
p = drep_j %>%
    dplyr::select(binID, Completeness, Contamination) %>%
    gather(Metric, Value, -binID) %>%
    ggplot(aes(Value)) +
    geom_histogram(bins=30) +
    labs(y='No. of MAGs\n(>=99% ANI derep.)') +
    facet_grid(Metric ~ ., scales='free_y') +
    theme_bw()

dims(4,3)
plot(p)_____no_output_____

## Taxonomy_____no_output_____

# summarizing by taxonomy
p = drep_j %>%
    unite(Taxonomy, Phylum, Class, sep=';', remove=FALSE) %>%
    group_by(Taxonomy, Phylum) %>%
    summarize(n = n()) %>%
    ungroup() %>%
    ggplot(aes(Taxonomy, n, fill=Phylum)) +
    geom_bar(stat='identity') +
    coord_flip() +
    labs(y='No. of MAGs\n(>=99% ANI derep.)') +
    theme_bw()

dims(7,4)
plot(p)_____no_output_____# summarizing by taxonomy
p = drep_j %>%
    unite(Taxonomy, Phylum, Class, Family, sep=';', remove=FALSE) %>%
    group_by(Taxonomy, Phylum) %>%
    summarize(n = n()) %>%
    ungroup() %>%
    ggplot(aes(Taxonomy, n, fill=Phylum)) +
    geom_bar(stat='identity') +
    coord_flip() +
    labs(y='No. of MAGs\n(>=99% ANI derep.)') +
    theme_bw()

dims(7,4)
plot(p)_____no_output_____

### Taxonomic novelty_____no_output_____

# no close ANI matches
p = drep_j %>%
    unite(Taxonomy, Phylum, Class, sep=';', remove=FALSE) %>%
    mutate(closest_placement_ani = closest_placement_ani %>% as.character,
           closest_placement_ani = ifelse(closest_placement_ani == 'N/A',
                                          0, closest_placement_ani),
           closest_placement_ani = ifelse(is.na(closest_placement_ani),
                                          0, closest_placement_ani),
           closest_placement_ani = closest_placement_ani %>% as.Num) %>%
    mutate(has_species_placement = ifelse(closest_placement_ani >= 95,
                                          'ANI >= 95%', 'No match')) %>%
    ggplot(aes(Taxonomy, fill=Phylum)) +
    geom_bar() +
    facet_grid(. ~ has_species_placement) +
    coord_flip() +
    labs(y='Closest placement ANI') +
    theme_bw()

dims(7,4)
plot(p)_____no_output_____p = drep_j %>%
    filter(Genus == 'g__') %>%
    unite(Taxonomy, Phylum, Class, Order, Family, sep='; ', remove=FALSE) %>%
    mutate(Taxonomy = stringr::str_wrap(Taxonomy, 45),
           Taxonomy = gsub(' ', '', Taxonomy)) %>%
    group_by(Taxonomy, Phylum) %>%
    summarize(n = n()) %>%
    ungroup() %>%
    ggplot(aes(Taxonomy, n, fill=Phylum)) +
    geom_bar(stat='identity') +
    coord_flip() +
    labs(y='No. of MAGs lacking a\ngenus-level classification') +
    theme_bw()

dims(6.5,3)
plot(p)_____no_output_____

### Quality ~ taxonomy_____no_output_____

p = drep_j %>%
    unite(Taxonomy, Phylum, Class, sep='; ', remove=FALSE) %>%
    dplyr::select(Taxonomy, Phylum, Completeness, Contamination) %>%
    gather(Metric, Value, -Taxonomy, -Phylum) %>%
    ggplot(aes(Taxonomy, Value, color=Phylum)) +
    geom_boxplot() +
    facet_grid(. ~ Metric, scales='free_x') +
    coord_flip() +
    labs(y='CheckM quality') +
    theme_bw()

dims(7,3.5)
plot(p)_____no_output_____# just unclassified at genus/species
p = drep_j %>%
    filter(Genus == 'g__' | Species == 's__') %>%
    unite(Taxonomy, Phylum, Class, sep='; ', remove=FALSE) %>%
    dplyr::select(Taxonomy, Phylum, Completeness, Contamination) %>%
    gather(Metric, Value, -Taxonomy, -Phylum) %>%
    ggplot(aes(Taxonomy, Value, color=Phylum)) +
    geom_boxplot() +
    facet_grid(. ~ Metric, scales='free_x') +
    coord_flip() +
    labs(y='CheckM quality') +
    theme_bw()

dims(7,3.5)
plot(p)_____no_output_____

# sessionInfo_____no_output_____

sessionInfo()_____no_output_____


# Notebook from therobotacademy/kaggle-anomaly-detection
Path: data/NASAbearingDataset-pyodKNN_using_PCA.ipynb

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
from scipy import stats

import matplotlib.pyplot as plt
from matplotlib.pyplot import figure

import os
for dirname, _, filenames in os.walk('./input/NASA-bearing-dataset'):
    for filename in filenames:
        print(os.path.join(dirname, filename))./input/NASA-bearing-dataset/merged_dataset_BearingTest_2.csv

# Basic EDA_____no_output_____

# Read the CSV file and set first column as the dataframe index
dataset = pd.read_csv("./input/NASA-bearing-dataset/merged_dataset_BearingTest_2.csv", index_col=0)
dataset.describe()_____no_output_____dataset.head()_____no_output_____# The last rows shows the bearing failure (acceleration -> 0)
dataset.tail()_____no_output_____

## Extract each bearing data and plot together the 4 signals_____no_output_____

bearing1 = dataset['Bearing 1']
bearing1_index = bearing1.index.values
bearing2 = dataset['Bearing 2']
bearing2_index = bearing1.index.values
bearing3 = dataset['Bearing 3']
bearing3_index = bearing1.index.values
bearing4 = dataset['Bearing 4']
bearing4_index = bearing1.index.values_____no_output_____figure(figsize=(15, 4), dpi=80)

bearing1.plot(color='green', label='Bearing 1')
bearing2.plot(color='yellow', label='Bearing 2')
bearing3.plot(color='orange', label='Bearing 3')
bearing4.plot(color='blue', label='Bearing 4')
plt.xlabel('Timestamp')
plt.ylabel('Acceleration')
plt.legend(loc="upper left")
plt.title('Time series for all 4 accelerometers', fontweight ="bold")
plt.show()_____no_output_____

# Normalize the dataset_____no_output_____

from sklearn import preprocessing

# Dataset is scaled so that maximum for every column is 1
scaler = preprocessing.MinMaxScaler()
dataset_scaled = pd.DataFrame(scaler.fit_transform(dataset),
                              columns=dataset.columns,
                              index=dataset.index)
dataset_scaled.describe()_____no_output_____# Let's plot the dataset again
bearing1 = dataset_scaled['Bearing 1']
bearing1_index = bearing1.index.values
bearing2 = dataset_scaled['Bearing 2']
bearing2_index = bearing1.index.values
bearing3 = dataset_scaled['Bearing 3']
bearing3_index = bearing1.index.values
bearing4 = dataset_scaled['Bearing 4']
bearing4_index = bearing1.index.values

figure(figsize=(15, 4), dpi=80)

bearing1.plot(color='green', label='Bearing 1')
bearing2.plot(color='yellow', label='Bearing 2')
bearing3.plot(color='orange', label='Bearing 3')
bearing4.plot(color='blue', label='Bearing 4')
plt.xlabel('Timestamp')
plt.ylabel('Acceleration')
plt.legend(loc="upper left")
plt.title('Time series for all 4 accelerometers (normalized)', fontweight ="bold")
plt.show()_____no_output_____

# PCA model: Principal Components analysis
Apply dimensionality reduction to deal with only 2 signals_____no_output_____

from sklearn.decomposition import PCA
pca = PCA(n_components=2, svd_solver= 'full')
dataset_PCA = pca.fit_transform(dataset_scaled)
dataset_PCA = pd.DataFrame(dataset_PCA)
dataset_PCA.index = dataset_scaled.index
dataset_PCA.columns = ['Dimension 1', 'Dimension 2']
dataset_PCA.head()_____no_output_____# Normalize PCA data
dataset_PCA = pd.DataFrame(scaler.fit_transform(dataset_PCA),
                              columns=dataset_PCA.columns,
                              index=dataset_PCA.index)
dataset_PCA.describe()_____no_output_____# Let's plot the Principal Components
dimension1 = dataset_PCA['Dimension 1']
dimension1_index = dimension1.index.values
dimension2 = dataset_PCA['Dimension 2']
dimension2_index = dimension2.index.values

figure(figsize=(15, 8), dpi=80)

dimension1.plot(color='orangered', label='Dimension 1')
dimension2.plot(color='salmon', label='Dimension 2')
bearing1.plot(color='dimgray', label='Bearing 1')
bearing2.plot(color='darkgray', label='Bearing 2')
bearing3.plot(color='lightsteelblue', label='Bearing 3')
bearing4.plot(color='lavender', label='Bearing 4')
plt.xlabel('Timestamp')
plt.ylabel('Acceleration')
plt.legend(loc="upper left")
plt.title('Time series for Principal Components', fontweight ="bold")
plt.show()_____no_output_____# VISUALIZE THE DATA IN THE PCA DIMENSIONAL SPACE
# Transform dataframe columns to np arrays
sample_size = 800
#sample_size = dimension1.shape[0] # =984
dim1_arr = np.array(dimension1)[:sample_size]
dim2_arr = np.array(dimension2)[:sample_size]
# Create a meshgrid
xx, yy = np.meshgrid(np.linspace(0, 1, 200),
                     np.linspace(0, 1, 200))
# scatter plot
figure(figsize=(7, 6), dpi=80)
plt.scatter(dim1_arr, dim2_arr, marker='x')
plt.xlim((0,1))
plt.ylim((0,1))
plt.xlabel('Dimension 1')
plt.ylabel('Dimension 2')
plt.title('Scatter plot: first 800 points', fontweight ="bold")
plt.show()_____no_output_____# Look at how the points are spreading in the index range 800-900
sample_size = 900
dim1_arr = np.array(dimension1)[:sample_size]
dim2_arr = np.array(dimension2)[:sample_size]
# Create a meshgrid
xx, yy = np.meshgrid(np.linspace(0, 1, 200),
                     np.linspace(0, 1, 200))
# scatter plot
figure(figsize=(7, 6), dpi=80)
plt.scatter(dim1_arr, dim2_arr, marker='x')
plt.xlim((0,1))
plt.ylim((0,1))
plt.xlabel('Dimension 1')
plt.ylabel('Dimension 2')
plt.title('Scatter plot: first 900 points', fontweight ="bold")
plt.show()_____no_output_____# Whole data (984 points)
# You can see how the dots are spreaded when approaching the bearing failure
dim1_arr = np.array(dimension1)
dim2_arr = np.array(dimension2)
# Create a meshgrid
xx, yy = np.meshgrid(np.linspace(0, 1, 200),
                     np.linspace(0, 1, 200))
# scatter plot
figure(figsize=(7, 6), dpi=80)
plt.scatter(dim1_arr, dim2_arr, marker='x')
plt.xlabel('Dimension 1')
plt.ylabel('Dimension 2')
plt.title('Scatter plot: whole dataset (984 points)', fontweight ="bold")
plt.show()_____no_output_____

# KNN Analysis_____no_output_____

# Extract PCA dimension arrays
dim1_arr = np.array(dataset_PCA['Dimension 1'])
dim2_arr = np.array(dataset_PCA['Dimension 2'])_____no_output_____dataset_PCA_____no_output_____from pyod.models.knn import KNN
from pyod.utils.data import get_outliers_inliers
import matplotlib.font_manager

# Setting the percentage of outliers
outlier_fraction = 0.15 # To catch all (formerly, =0.20)

# Training the classifier
clf = KNN()
#clf = KNN(contamination = outlier_fraction)
clf.fit(dataset_PCA)_____no_output_____

## Compute scores and threshold for labeling outliers_____no_output_____

scores_pred = clf.decision_function(dataset_PCA)*-1
print ("Scores' predictions range from", "{:.4f}".format(min(scores_pred)), "to {:.4f}".format(max(scores_pred)) )Scores' predictions range from -0.6132 to -0.0002
y_pred = clf.predict(dataset_PCA) # Dataset is not labeled, so this prediction cannot be used to compute any error
# !!! We're applying unsupervised learning, just look at from which index points are labeled as outliers
y_pred[-100:]_____no_output_____## We'll use y_pred to identify outliers
# Storing the outliers and inliners in different numpy arrays
X_outliers, X_inliers = get_outliers_inliers(np.array(dataset_PCA), y_pred)
n_inliers = len(X_inliers)
n_outliers = len(X_outliers)
print("There are", n_inliers, "inliers and", n_outliers, "outliers")There are 892 inliers and 92 outliers
# threshold value to consider a
# datapoint inlier or outlier
threshold = stats.scoreatpercentile(scores_pred, 100 * outlier_fraction)
print ("Threshold value to label outliers is" , "{:.4f}".format(threshold) )Threshold value to label outliers is -0.0072

## Scatter plot of outliers_____no_output_____

# decision function calculates the raw
# anomaly score for every point
Z = clf.decision_function(np.c_[xx.ravel(), yy.ravel()]) * -1
Z = Z.reshape(xx.shape)_____no_output_____# fill blue colormap from minimum anomaly
# score to threshold value
subplot = plt.subplot(1, 1, 1)
subplot.contourf(xx, yy, Z, levels = np.linspace(Z.min(),
                  threshold, 10), cmap = plt.cm.Blues_r)

# draw red contour line where anomaly
# score is equal to threshold
a = subplot.contour(xx, yy, Z, levels =[threshold],
                     linewidths = 2, colors ='red')

# fill orange contour lines where range of anomaly
# score is from threshold to maximum anomaly score
subplot.contourf(xx, yy, Z, levels =[threshold, Z.max()], colors ='orange')

# scatter plot of inliers with white dots
#b = subplot.scatter(dim1_arr[:-n_outliers], dim2_arr[:-n_outliers],
#                                    c ='white', s = 20, edgecolor ='k')

# scatter plot of outliers with black dots
c = subplot.scatter(dim1_arr[-n_outliers:], dim2_arr[-n_outliers:],
                                    c ='black', s = 20, edgecolor ='k')
subplot.axis('tight')

#   [a.collections[0], b, c],
#   ['learned decision function', 'true inliers', 'true outliers'],
subplot.legend(
    [a.collections[0], c],
    ['learned decision function', 'true outliers'],
    prop = matplotlib.font_manager.FontProperties(size = 10),
    loc ='lower right')

subplot.set_title('K-Nearest Neighbours (true inliers not plotted for clarity)')
subplot.set_xlim((0.3, 1))
subplot.set_ylim((0, 1))
plt.show() _____no_output_____

## Time series plot highlighting outliers_____no_output_____

dataset_PCA['outlier'] = y_pred
dataset_PCA_outliers = dataset_PCA[ dataset_PCA['outlier']==1 ]

dimension1_outliers = dataset_PCA_outliers['Dimension 1']
dimension2_outliers = dataset_PCA_outliers['Dimension 2']_____no_output_____figure(figsize=(15, 4), dpi=80)

tail_size    = 900
x_ticks_span = 150

dim1_arr = dimension1[-tail_size:]
dim1_arr_index = dataset_PCA.index[-tail_size:]

plt.plot(dim1_arr_index, dim1_arr, color='green', label='acceleration dim1')
# https://www.tutorialspoint.com/numpy/numpy_matplotlib.htm
plt.plot(dimension1_outliers.index, dimension1_outliers, 'v', color='red', label='outliers')

plt.xlabel('Timestamp')
plt.xlim(0,tail_size)
plt.xticks(np.arange(0, tail_size+1, x_ticks_span), fontsize=10, rotation = 45)
plt.ylabel('Acceleration')

plt.legend(loc="upper left")
plt.title('Time series for PCA dim1 showing outliers', fontweight ="bold")
plt.show()_____no_output_____figure(figsize=(15, 4), dpi=80)

tail_size    = 900
x_ticks_span = 150

dim2_arr = dimension2[-tail_size:]
dim2_arr_index = dataset_PCA.index[-tail_size:]

plt.plot(dim2_arr_index, dim2_arr, color='green', label='acceleration dim2')
# https://www.tutorialspoint.com/numpy/numpy_matplotlib.htm
plt.plot(dimension2_outliers.index, dimension2_outliers, 'v', color='red', label='outliers')

plt.xlabel('Timestamp')
plt.xlim(0,tail_size)
plt.xticks(np.arange(0, tail_size+1, x_ticks_span), fontsize=10, rotation = 45)
plt.ylabel('Acceleration')

plt.legend(loc="upper left")
plt.title('Time series for PCA dim2 showing outliers', fontweight ="bold")
plt.show()_____no_output_____

#### We had to manually select outlier_fraction=0.15 to detects outliers in dim2
Failure is predicted to happen 2 days after detecting (non-isolated) outliers_____no_output_____