# Notebook from dr-guangtou/caterpillar
Path: notebook/photocal/ps1_refcat_test.ipynb

%matplotlib inline
%load_ext autoreload
%autoreload 2

import os
import copy

import numpy as np

from astropy.io import fits
from astropy.wcs import WCS
from astropy.table import Table, Column, vstack

import matplotlib.pyplot as pltThe autoreload extension is already loaded. To reload it, use:
  %reload_ext autoreload
import healpy as hp_____no_output_____from caterpillar import catalog_____no_output_____

#### PS-1 Reference catalog_____no_output_____

ps1_dir = '/Users/song/work/data/chunks-qz-star-v3/'

fits_pattern = os.path.join(ps1_dir, 'ps1-{:05d}.fits')

nside = 32
indexing = 'ring'_____no_output_____

#### Merian COSMOS stack_____no_output_____

cosmos_dir = '/Users/song/work/merian/cosmos/cosmos_stack/'

cosmos_deep = fits.open(os.path.join(cosmos_dir, 'c4d_210306_011257_osj_N708_deep.fits.fz'))_____no_output_____cosmos_wcs = WCS(cosmos_deep[1].header)_____no_output_____ra, dec = 150.117, 2.38802 _____no_output_____cosmos_deep[1].data.shape_____no_output_____cosmos_wcs_____no_output_____

### LegacyPipe version_____no_output_____

from astrometry.util.util import radecdegtohealpix, healpix_xy_to_ring, healpix_xy_to_nested_____no_output_____healpix_xy_to_ring(radecdegtohealpix(ra, dec, nside), nside)_____no_output_____

### Healpy + Astropy version_____no_output_____

hp.pixelfunc.ang2pix(nside, ra, dec, lonlat=True, nest=False)_____no_output_____PS1_DIR = '/Users/song/work/data/chunks-qz-star-v3/'

PS1_PATTERN = 'ps1-{:05d}.fits'_____no_output_____dec_size, ra_size = cosmos_wcs.array_shape

step, margin = 100, 10

xx_arr, yy_arr = np.meshgrid(
    np.linspace(1 - margin, ra_size + margin, 2 + int((ra_size + 2 * margin) / step)),
    np.linspace(1 - margin, dec_size + margin, 2 + int((dec_size + 2 * margin) / step)))

ra_arr, dec_arr = cosmos_wcs.all_pix2world(xx_arr.ravel(), yy_arr.ravel(), 1)

hp_pixels = set()
for rr, dd in zip(ra_arr, dec_arr):
    hp_pixels.add(hp.pixelfunc.ang2pix(nside, rr, dd, lonlat=True, nest=False))

ref_cats = []
for pix in hp_pixels:
    ref_cats.append(Table.read(fits_pattern.format(pix)))_____no_output_____ra_min, ra_max = ra_arr.min(), ra_arr.max()
dec_min, dec_max = dec_arr.min(), dec_arr.max()
ra_sep = (ra_max - ra_min) / 20
dec_sep = (dec_max - dec_min) / 20

cosmos_ref = vstack(ref_cats)

cosmos_cut = cosmos_ref[
    (cosmos_ref['RA'] > ra_min - ra_sep) &
    (cosmos_ref['RA'] < ra_max + ra_sep) &
    (cosmos_ref['DEC'] > dec_min - dec_sep) &
    (cosmos_ref['DEC'] < dec_max + dec_sep)
]

len(cosmos_cut)_____no_output_____plt.scatter(cos_cat['RA'], cos_cat['DEC'], s=1, alpha=0.1)

ra_0, dec_0 = cosmos_wcs.all_pix2world(0, 0, 1)
ra_1, dec_1 = cosmos_wcs.all_pix2world(ra_size, dec_size, 1)
plt.scatter(ra_0, dec_0, s=10, c='r')
plt.scatter(ra_1, dec_1, s=10, c='r')_____no_output_____cosmos_cut.write('/Users/song/work/merian/cosmos/cosmos_ps1_refcat.fits', overwrite=True)_____no_output_____np.min(cosmos_cut['MEDIAN'], axis=0)

_ = plt.hist(cosmos_cut['MEDIAN'][:, 1])_____no_output_____cosmos_cut['NMAG_OK'][0]_____no_output_____ps1_cat = catalog.ReferenceCatalog(PS1_DIR, catalog.PS1_PATTERN)

cos_cat = ps1_cat.get_stars_hdu(cosmos_deep[1])_____no_output_____len(cos_cat)_____no_output_____


# Notebook from czbiohub/scRFE
Path: scripts/practiceScripts/scRFE_jun19_FROMANGELA.ipynb

# scRFEjun19_____no_output_____

<code>
# AnnDatasubset
# Angela sent to Madeline on June 22_____no_output_____# Imports
import numpy as np
import pandas as pd
import scanpy as sc
import random
from anndata import read_h5ad
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
from sklearn.feature_selection import SelectFromModel
from sklearn.metrics import accuracy_score
from sklearn.feature_selection import RFE
from sklearn.feature_selection import RFECV
import seaborn as sns
import matplotlib.pyplot as plt
import scanpy.external as sce
import logging as logg_____no_output_____mouse_tfs = pd.read_csv("/home/angela/maca/GO_term_summary_20171110_222852.csv")
mouse_tfs.head()_____no_output_____adata = read_h5ad(
    "/home/angela/maca/tabula-muris-senis/0_data_ingest/01_figure_1/tabula-muris-senis-droplet-processed-official-annotations.h5ad")
adata_____no_output_____adata = adata[:,adata.var_names[adata.var_names.isin(mouse_tfs['Symbol'])]]
adata_____no_output_____# adataLiver = read_h5ad('/Users/madelinepark/Downloads/Liver_droplet.h5ad')

adataLiver = adata[adata.obs['tissue']=='Liver'].copy()
adataLiver_____no_output_____def remove_cats(adata):

    cat_columns = adata.obs.select_dtypes(['category']).columns
    adata.obs[cat_columns] = adata.obs[cat_columns].astype(str)

    return adata_____no_output_____def filterNormalize (dataMatrix, classOfInterest):
    np.random.seed(644685)
#     sc.logging.print_versions()
#     sc.settings.verbosity = 3
#     sc.logging.print_versions()
#     dataMatrix.obs['n_counts'] = dataMatrix.X.sum(axis=1).A1
    sc.pp.filter_cells(dataMatrix, min_genes=0)
    sc.pp.filter_genes(dataMatrix, min_cells=0)
#     dataMatrix = dataMatrix[dataMatrix.obs['n_counts'] > 1500, :]
#     sc.pp.normalize_per_cell(dataMatrix, counts_per_cell_after=1e5)
#     sc.pp.log1p(dataMatrix)
#     dataMatrix.raw = dataMatrix
    dataMatrix = dataMatrix[dataMatrix.obs[classOfInterest]!='nan']
    dataMatrix = dataMatrix[~dataMatrix.obs[classOfInterest].isna()]

    return dataMatrix_____no_output_____def labelSplit (dataMatrix, classOfInterest, labelOfInterest):
    dataMatrix = filterNormalize (dataMatrix, classOfInterest)
    dataMatrix.obs['classification_group'] = 'B'
    dataMatrix.obs.loc[dataMatrix.obs[dataMatrix.obs[classOfInterest]==labelOfInterest]
                   .index,'classification_group'] = 'A'
    return dataMatrix_____no_output_____def downsampleToSmallestCategory(dataMatrix,
        classOfInterest = 'classification_group',
        random_state = None,
        min_cells = 15,
        keep_small_categories = True
) -> sc.AnnData:
    """
    returns an annData object in which all categories in 'classOfInterest' have
    the same size
    classOfInterest
        column with the categories to downsample
    min_cells
        Minimum number of cells to downsample.
        Categories having less than `min_cells` are discarded unless
        keep_small_categories is True
    keep_small_categories
        Be default categories with less than min_cells are discarded.
        Set to true to keep them
    """

    counts = dataMatrix.obs[classOfInterest].value_counts(sort=False)
    if len(counts[counts < min_cells]) > 0 and keep_small_categories is False:
        logg.warning(
            "The following categories have less than {} cells and will be "
            "ignored: {}".format(min_cells, dict(counts[counts < min_cells]))
        )
    min_size = min(counts[counts >= min_cells])
    sample_selection = None
    for sample, num_cells in counts.items():
        if num_cells <= min_cells:
            if keep_small_categories:
                sel = dataMatrix.obs.index.isin(
                    dataMatrix.obs[dataMatrix.obs[classOfInterest] == sample].index)
            else:
                continue
        else:
            sel = dataMatrix.obs.index.isin(
                dataMatrix.obs[dataMatrix.obs[classOfInterest] == sample]
                .sample(min_size, random_state=random_state)
                .index
            )
        if sample_selection is None:
            sample_selection = sel
        else:
            sample_selection |= sel
    logg.info(
        "The cells in category {!r} had been down-sampled to have each {} cells. "
        "The original counts where {}".format(classOfInterest, min_size, dict(counts))
    )
    return dataMatrix[sample_selection].copy()_____no_output_____def makeOneForest (dataMatrix, classOfInterest, labelOfInterest, nEstimators = 5000,
                   randomState = 0,  nJobs = -1, oobScore = True, Step = 0.2, Cv = 5):
    """
    Builds and runs a random forest for one label in a class of interest

    Parameters
    ----------
    dataMatrix : anndata object
        The data file of interest
    classOfInterest : str
        The class you will split the data by in the set of dataMatrix.obs
    labelOfInterest : str
        The specific label within the class that the random forezt will run a
        "one vs all" classification on
    nEstimators : int
        The number of trees in the forest
    randomState : int
        Controls random number being used
    nJobs : int
        The number of jobs to run in parallel
    oobScore : bool
        Whether to use out-of-bag samples to estimate the generalization accuracy
    Step : float
        Corresponds to percentage of features to remove at each iteration
    Cv : int
        Determines the cross-validation splitting strategy

    Returns
    -------
    feature_selected : list
        list of top features from random forest
    selector.estimator_.feature_importances_ : list
        list of top ginis corresponding to to features

    """
    splitDataMatrix = labelSplit (dataMatrix, classOfInterest, labelOfInterest)
    downsampledMatrix = downsampleToSmallestCategory (dataMatrix = splitDataMatrix,
        classOfInterest = 'classification_group',
        random_state = None, min_cells = 15, keep_small_categories = False)
    print('downsampled Matrix')
#     print("labelOfInterest")

    print(pd.DataFrame(downsampledMatrix.obs.groupby(['classification_group',classOfInterest])[classOfInterest].count()))
#     display (downsampledMatrix.obs['classification_group'])
#     print(set(downsampledMatrix.obs['classification_group']))

    feat_labels = downsampledMatrix.var_names
    X = downsampledMatrix.X
    y = downsampledMatrix.obs['classification_group']

    clf = RandomForestClassifier(n_estimators = nEstimators, random_state = randomState,
                                 n_jobs = nJobs, oob_score = oobScore)
    selector = RFECV(clf, step = Step, cv = Cv)

    clf.fit(X, y)
    selector.fit(X, y)
    feature_selected = feat_labels[selector.support_]
#     display(downsampledMatrix.obs)

    dataMatrix.obs['classification_group'] = 'B'
#     print('corresponding')
#     display(downsampledMatrix.obs)

    return feature_selected, selector.estimator_.feature_importances______no_output_____def resultWrite (classOfInterest, results_df, labelOfInterest,
                feature_selected, feature_importance):

    column_headings = []
    column_headings.append(labelOfInterest)
    column_headings.append(labelOfInterest + '_gini')
    resaux = pd.DataFrame(columns = column_headings)
    resaux[labelOfInterest] = feature_selected
    resaux[labelOfInterest + '_gini'] = feature_importance
    resaux = resaux.sort_values(by = [labelOfInterest + '_gini'], ascending = False)
    resaux.reset_index(drop = True, inplace = True)

    results_df = pd.concat([results_df, resaux], axis=1)
    return results_df _____no_output_____def scRFE(adata, classOfInterest, nEstimators = 5000, randomState = 0,
                  nJobs = -1, oobScore = True, Step = 0.2, Cv = 5):
    """
    Builds and runs a random forest with one vs all classification for each label
    for one class of interest

    Parameters
    ----------
    dataMatrix : anndata object
        The data file of interest
    classOfInterest : str
        The class you will split the data by in the set of dataMatrix.obs
    labelOfInterest : str
        The specific label within the class that the random forezt will run a
        "one vs all" classification on
    nEstimators : int
        The number of trees in the forest
    randomState : int
        Controls random number being used
    nJobs : int
        The number of jobs to run in parallel
    oobScore : bool
        Whether to use out-of-bag samples to estimate the generalization accuracy
    Step : float
        Corresponds to percentage of features to remove at each iteration
    Cv : int
        Determines the cross-validation splitting strategy

    Returns
    -------
    results_df : pd.DataFrame
        Dataframe with results for each label in the class, formatted as
        "label" for one column, then "label + gini" for the corresponding column

    """
    dataMatrix = adata.copy()
    dataMatrix = remove_cats(dataMatrix)

    print("Original dataset ",dataMatrix.shape)
    dataMatrix = filterNormalize (dataMatrix, classOfInterest)
    print("Filtered dataset ",dataMatrix.shape)
    print(pd.DataFrame(dataMatrix.obs.groupby([classOfInterest])[classOfInterest].count()))

    results_df = pd.DataFrame()
    for labelOfInterest in np.unique(dataMatrix.obs[classOfInterest]):
        print(labelOfInterest)
        dataMatrix_labelOfInterest = dataMatrix.copy()
        feature_selected, feature_importance = makeOneForest(dataMatrix_labelOfInterest,
                                                             classOfInterest,
                          labelOfInterest = labelOfInterest)

        results_df = resultWrite (classOfInterest, results_df,
                            labelOfInterest = labelOfInterest,
                    feature_selected = feature_selected,
                    feature_importance = feature_importance)

#     finaldf = makeOneForest (dataMatrix, classOfInterest, labelOfInterest = labelOfInterest)[2]

    return results_df_____no_output_____senis_facs_tfs_age = scRFE (adata, classOfInterest = 'age',
                       nEstimators = 10, randomState = 0,
                  nJobs = -1, oobScore = True, Step = 0.2, Cv = 3)Original dataset  (245389, 1044)
Filtered dataset  (245389, 1044)
       age
age
18m  44645
1m   25980
21m  35828
24m  37660
30m  55674
3m   45602
18m
adata_____no_output_____senis_facs_tfs_age_____no_output_____age = '24m'
sum(senis_droplet_tfs_age[age].isin(senis_facs_tfs_age[age]))/sum(~senis_droplet_tfs_age[age].isna())*100_____no_output_____sum(senis_facs_tfs_age['3m'].isin(senis_droplet_tfs_age['3m']))/sum(~senis_facs_tfs_age['3m'].isna())*100_____no_output_____sum(~senis_facs_tfs_age['3m'].isna()),sum(~senis_droplet_tfs_age['3m'].isna())_____no_output_____sum(senis_facs_tfs_age['3m'].isin(senis_droplet_tfs_age['3m']))_____no_output_____senis_droplet_tfs_age.to_csv("senis_droplet_tfs_age.csv")_____no_output_____senis_facs_tfs_age.to_csv("senis_facs_tfs_age.csv")_____no_output_____liverAgeSmallnorm = scRFE (dataMatrix = adataLiver, classOfInterest = 'age',
                       nEstimators = 10, randomState = 0,
                  nJobs = -1, oobScore = True, Step = 0.2, Cv = 3)_____no_output_____liverAgeSmall.to_csv("scRFE_Liver_droplet_age.csv")_____no_output_____liverAgeSmallnonorm_____no_output_____liverAgeSmallnorm.to_csv("scRFE_Liver_facs_age.csv")_____no_output_____
</code>


# Notebook from Roli-Wilhelm/Buckley_Lab_SIP_project_protocols
Path: sequence_analysis_walkthrough/QIIME2_Processing_Pipeline.ipynb

> ### Pipeling to Process Raw Sequences into Phyloseq Object with DADA2 ###
* Prep for Import to QIIME2  (Combine two index files)
* Import to QIIME2
* Demultiplex
* Denoise and Merge
* Prepare OTU Tables and Rep Sequences  *(Note: sample names starting with a digit will break this step)*
* Classify Seqs

*100% Appropriated from the "Atacama Desert Tutorial" for QIIME2*

### Pipeline can handle both 16S rRNA gene and ITS sequences (in theory)####
* Tested on 515f and 806r
* Tested on ITS1

### Commands to Install Dependencies ####
##### || QIIME2 ||
** Note: QIIME2 is still actively in development, and I've noticed frequent new releases. Check for the most up-to-date conda install file <https://docs.qiime2.org/2017.11/install/native/#install-qiime-2-within-a-conda-environment>

* wget https://data.qiime2.org/distro/core/qiime2-2018.2-py35-linux-conda.yml
* conda env create -n qiime2-2018.2 --file qiime2-2018.2-py35-linux-conda.yml
* source activate qiime2-pipeline

##### || Copyrighter rrn Database ||
* The script will automatically install the curated GreenGenes rrn attribute database
* https://github.com/fangly/AmpliCopyrighter

##### || rpy2 (don't use conda version) ||
* pip install rpy2

##### || phyloseq ||
* conda install -c r r-igraph
* Rscript -e "source('http://bioconductor.org/biocLite.R');biocLite('phyloseq')"

##### || R packages ||
* ape   (natively installed in conda environment)

### Citations ###
* Caporaso, J. G., Kuczynski, J., Stombaugh, J., Bittinger, K., Bushman, F. D., Costello, E. K., *et al.* (2010). QIIME allows analysis of high-throughput community sequencing data. Nature methods, 7(5), 335-336.

* McMurdie and Holmes (2013) phyloseq: An R Package for Reproducible Interactive Analysis and Graphics of Microbiome Census Data. PLoS ONE. 8(4):e61217

* Paradis E., Claude J. & Strimmer K. 2004. APE: analyses of phylogenetics and evolution in R language. Bioinformatics 20: 289-290.

* Angly, F. E., Dennis, P. G., Skarshewski, A., Vanwonterghem, I., Hugenholtz, P., & Tyson, G. W. (2014). CopyRighter: a rapid tool for improving the accuracy of microbial community profiles through lineage-specific gene copy number correction. Microbiome, 2(1), 11.

###### Last Modified by R. Wilhelm on October 12th, 2017 ######
_____no_output_____# Step 1: User Input_____no_output_____

import os, re, numpy as np

# Provide the directory for your index and read files (you can do multiple independently in one go)
bioblitz = '/home/roli/BioBlitz.2017/SV_based/'

# Prepare an object with the name of the library, the name of the directory object (created above), and the metadatafile name
#datasets = [['name',directory1,'metadata1','domain of life'],['name',directory2,'metadata2','domain of life']]
datasets = [['bioblitz',bioblitz,'metadata.tsv','bacteria']]

# Ensure your reads files are named accordingly (or modify to suit your needs)
readFile1 = 'read1.fq.gz'
readFile2 = 'read2.fq.gz'
indexFile1 = 'index_read1.fq.gz'
indexFile2 = 'index_read2.fq.gz'

# Set # of Processors to Use
processors = "20"

# Classification Database to Use
# options: "Silva" [default] | "GreenGenes"
db = "Silva"

## Enter Minimum Support for Keeping QIIME Classification
# Note: Classifications that do not meet this criteria will simply be retained, but labeled 'putative'
min_support = 0.8_____no_output_____

# Step 2: Concatenate Barcodes for QIIME2 Pipeline_____no_output_____

## Note: QIIME takes a single barcode file. The command 'extract_barcodes.py' concatenates the forward and reverse read barcode and attributes it to a single read.

# See http://qiime.org/tutorials/processing_illumina_data.html

for dataset in datasets:
    directory = dataset[1]
    index1 = directory+indexFile1
    index2 = directory+indexFile2

    # Run extract_barcodes to merge the two index files
    !python2 /opt/anaconda2/bin/extract_barcodes.py --input_type barcode_paired_end -f $index1 -r $index2 --bc1_len 8 --bc2_len 8 -o $directory/output

    # QIIME2 import requires a directory containing files names: forward.fastq.gz, reverse.fastq.gz and barcodes.fastq.gz
    !ln -s $directory$readFile1 $directory/output/forward.fastq.gz
    !ln -s $directory$readFile2 $directory/output/reverse.fastq.gz

    # Gzip the barcodes files (apparently necessary)
    !pigz -p 5 $directory/output/barcodes.fastq

    # Removed orphaned reads files (not needed)
    !rm $directory/output/reads?.fastq
_____no_output_____

# Step 3: Import into QIIME2_____no_output_____

for dataset in datasets:
    name = dataset[0]
    directory = dataset[1]

    os.system(' '.join([
        "qiime tools import",
        "--type EMPPairedEndSequences",
        "--input-path "+directory+"output/",
        "--output-path "+directory+"output/"+name+".qza"
    ]))

    # This more direct command is broken by the fact QIIME uses multiple dashes in their arguments (is my theory)
    #!qiime tools import --type EMPPairedEndSequences --input-path $directory/output --output-path $directory/output/$name.qza
     _____no_output_____

# Step 4: Demultiplex_____no_output_____

########
## Note: The barcode you supply to QIIME is now a concatenation of your forward and reverse barcode.
# Your 'forward' barcode is actually the reverse complement of your reverse barcode and the 'reverse' is your forward barcode. The file 'primers.complete.csv' provides this information corresponding to the Buckley Lab 'primer number'
# This quirk could be corrected in how different sequencing facilities pre-process the output from the sequencer

##
## SLOW STEP (~ 2 - 4 hrs)
##

for dataset in datasets:
    name = dataset[0]
    directory = dataset[1]
    metadata = dataset[2]

    os.system(' '.join([
        "qiime demux emp-paired",
        "--m-barcodes-file "+directory+metadata,
        "--m-barcodes-column BarcodeSequence",
        "--i-seqs "+directory+"output/"+name+".qza",
        "--o-per-sample-sequences "+directory+"output/"+name+".demux"
    ]))

    # This more direct command is broken by the fact QIIME uses multiple dashes in their arguments (is my theory)
    #!qiime demux emp-paired --m-barcodes-file $directory/$metadata --m-barcodes-category BarcodeSequence --i-seqs $directory/output/$name.qza --o-per-sample-sequences $directory/output/$name.demux
    _____no_output_____

# Step 5: Visualize Quality Scores and Determine Trimming Parameters_____no_output_____

## Based on the Graph Produced using the Following Command enter the trim and truncate values. Trim refers to the start of a sequence and truncate the total length (i.e. number of bases to remove from end)

# The example in the Atacam Desert Tutorial trims 13 bp from the start of each read and does not remove any bases from the end of the 150 bp reads:
#  --p-trim-left-f 13 \
#  --p-trim-left-r 13 \
#  --p-trunc-len-f 150 \
#  --p-trunc-len-r 150

for dataset in datasets:
    name = dataset[0]
    directory = dataset[1]

    os.system(' '.join([
        "qiime demux summarize",
        "--i-data "+directory+"/output/"+name+".demux.qza",
        "--o-visualization "+directory+"/output/"+name+".demux.QC.summary.qzv"
    ]))

    ## Take the output from this command and drop it into:
    #https://view.qiime2.org

wait_for_user = input("The script will now wait for you to input trimming parameters in the next cell. You will need to take the .qzv files for each library and visualize them at <https://view.qiime2.org>. This is hopefully temporary, while QIIME2 developers improve on q2view.\n\n[ENTER ANYTHING. THIS IS ONLY MEANT TO PAUSE THE PIPELINE]")
print("\nThe script is now proceeding. Stay tuned to make sure trimming works.")_____no_output_____

# Step 6: Trimming Parameters | USER INPUT REQUIRED_____no_output_____

## User Input Required
trim_dict = {}

## Input your trimming parameters into a python dictionary for all libraries
#trim_dict["LibraryName1"] = [trim_forward, truncate_forward, trim_reverse, truncate_reverse]
#trim_dict["LibraryName2"] = [trim_forward, truncate_forward, trim_reverse, truncate_reverse]

## Example
trim_dict["bioblitz"] = [1, 240, 1, 190]_____no_output_____

# Step 7: Trim, Denoise and Join (aka 'Merge') Reads Using DADA2_____no_output_____

##
## SLOW STEP (~ 6 - 8 hrs, IF multithreading is used)
##

for dataset in datasets:
    name = dataset[0]
    directory = dataset[1]

    os.system(' '.join([
        "qiime dada2 denoise-paired",
        "--i-demultiplexed-seqs "+directory+"/output/"+name+".demux.qza",
        "--o-table "+directory+"/output/"+name+".table",
        "--o-representative-sequences "+directory+"/output/"+name+".rep.seqs.final",
        "--p-trim-left-f "+str(trim_dict[name][0]),
        "--p-trim-left-r "+str(trim_dict[name][2]),
        "--p-trunc-len-f "+str(trim_dict[name][1]),
        "--p-trunc-len-r "+str(trim_dict[name][3]),
        "--p-n-threads",
        processors
    ]))_____no_output_____

# Step 8: Create Summary of OTUs_____no_output_____

for dataset in datasets:
    name = dataset[0]
    directory = dataset[1]
    metadata = dataset[2]

    os.system(' '.join([
        "qiime feature-table summarize",
        "--i-table "+directory+"/output/"+name+".table.qza",
        "--o-visualization "+directory+"/output/"+name+".table.qzv",
        "--m-sample-metadata-file "+directory+metadata
    ]))

    os.system(' '.join([
        "qiime feature-table tabulate-seqs",
        "--i-data "+directory+"/output/"+name+".rep.seqs.final.qza",
        "--o-visualization "+directory+"/output/"+name+".rep.seqs.final.qzv"
    ])) _____no_output_____

# Step 9: Make Phylogenetic Tree_____no_output_____

for dataset in datasets:
    name = dataset[0]
    directory = dataset[1]
    metadata = dataset[2]
    domain = dataset[3]

    if domain != "fungi":
        # Generate Alignment with MAFFT
        os.system(' '.join([
            "qiime alignment mafft",
            "--i-sequences "+directory+"/output/"+name+".rep.seqs.final.qza",
            "--o-alignment "+directory+"/output/"+name+".rep.seqs.aligned.qza",
            "--p-n-threads",
            processors
        ]))

        # Mask Hypervariable parts of Alignment
        os.system(' '.join([
            "qiime alignment mask",
            "--i-alignment "+directory+"/output/"+name+".rep.seqs.aligned.qza",
            "--o-masked-alignment "+directory+"/output/"+name+".rep.seqs.aligned.masked.qza"
        ]))

        # Generate Tree with FastTree
        os.system(' '.join([
            "qiime phylogeny fasttree",
            "--i-alignment "+directory+"/output/"+name+".rep.seqs.aligned.masked.qza",
            "--o-tree "+directory+"/output/"+name+".rep.seqs.tree.unrooted.qza",
            "--p-n-threads",
            processors
        ]))

        # Root Tree
        os.system(' '.join([
            "qiime phylogeny midpoint-root",
            "--i-tree "+directory+"/output/"+name+".rep.seqs.tree.unrooted.qza",
            "--o-rooted-tree "+directory+"/output/"+name+".rep.seqs.tree.final.qza"
        ]))
_____no_output_____

# Step 10: Classify Seqs_____no_output_____### Example of creating Silva Classifier DB  (very slow: use a pre-built one if possible)

#### For Silva, remove all the alignment information (unsure of the impact of keeping it) using the following python code:
```python
import re
from Bio import SeqIO

output = open("silva.nr_v128.fasta", "w")

for record in SeqIO.parse(open("silva.nr_v128.align", "rU"), "fasta") :
    seq = str(record.seq)
    seq = re.sub("\.|-","",seq)  # Remove "." and "-"

    output.write(">"+record.id+"\n"+seq+"\n")```

#### Import fasta sequence file and taxonomy file as .qza
```bash
qiime tools import
  --type 'FeatureData[Sequence]'
  --input-path silva.nr_v128.fasta
  --output-path silva.nr_v128.qza

qiime tools import
  --type 'FeatureData[Taxonomy]'
  --source-format HeaderlessTSVTaxonomyFormat
  --input-path silva.nr_v128.tax
  --output-path silva.nr_v128.taxonomy.qza```

#### Run QIIME2 'fit-classifier-naive-bayes'
```bash
qiime feature-classifier fit-classifier-naive-bayes
  --i-reference-reads silva.nr_v128.qza
  --i-reference-taxonomy silva.nr_v128.taxonomy.qza
  --o-classifier silva.nr_v128.nb.classifier.qza```_____no_output_____

## Note: Different QIIME2 versions can conflict with previously donwloaded databases. This section might have to be updated.
try:
    if db == "GreenGenes":
        classification_db = "/home/db/GreenGenes/qiime2_13.8.99_515.806_nb.classifier.qza"
    else:
        classification_db = "/home/db/Silva/silva.nr_v128.nb.classifier.qza"

except:
        classification_db = "/home/db/Silva/silva.nr_v128.nb.classifier.qza"

for dataset in datasets:
    name = dataset[0]
    directory = dataset[1]
    metadata = dataset[2]
    domain = dataset[3]

    # Classify
    if domain == 'bacteria':
        os.system(' '.join([
            "qiime feature-classifier classify-sklearn",
            "--i-classifier",
            classification_db,
            "--i-reads "+directory+"/output/"+name+".rep.seqs.final.qza",
            "--o-classification "+directory+"/output/"+name+".taxonomy.final.qza",
            "--p-n-jobs",
            processors
        ]))

    if domain == 'fungi':
        os.system(' '.join([
            "qiime feature-classifier classify-sklearn",
            "--i-classifier /home/db/UNITE/qiime2_unite_ver7.99_20.11.2016_classifier.qza",
            "--i-reads "+directory+"/output/"+name+".rep.seqs.final.qza",
            "--o-classification "+directory+"/output/"+name+".taxonomy.final.qza",
            "--p-n-jobs",
            processors
        ]))

    # Output Summary
    os.system(' '.join([
        "qiime metadata tabulate",
        "--m-input-file "+directory+"/output/"+name+".taxonomy.final.qza",
        "--o-visualization "+directory+"/output/"+name+".taxonomy.final.summary.qzv"
    ])) _____no_output_____

# Step 11: Prepare Data for Import to Phyloseq_____no_output_____

## Make Function to Re-Format Taxonomy File to Contain Full Column Information
# and factor in the certain of the taxonomic assignment

def format_taxonomy(tax_file, classification_db, min_support):
    output = open(re.sub(".tsv",".fixed.tsv",tax_file), "w")

    # Silva db lacks species classifications
    if classification_db == "GreenGenes":
        full_rank_length = 7
        output.write("\t".join(["OTU","Domain","Phylum","Class","Order","Family","Genus","Species"])+"\n")
    else:
        full_rank_length = 6
        output.write("\t".join(["OTU","Domain","Phylum","Class","Order","Family","Genus"])+"\n")

    with open(tax_file, "r") as f:
        next(f) #skip header

        for line in f:
            line = line.strip()
            line = line.split("\t")

            read_id = line[0]
            tax_string = line[1]

            ## Remove All Underscore Garbage (I need aesthetics)
            if classification_db == "GreenGenes":
                tax_string = re.sub("k__|p__|c__|o__|f__|g__|s__","",tax_string)
            else:
                tax_string = re.sub("_cl|_or|_fa|_ge","",tax_string)

            # Split full rank into ranks
            full_rank = tax_string.split(";")

            # Getting trailing empty tab in Silva
            if full_rank[len(full_rank)-1] == "":
                    full_rank = full_rank[:-1]

            ## Identify the Lowest Classified Taxonomic Rank
            # Account for cases when a taxonomic rank contains an empty space (common in GreenGenes output)
            last_classified = full_rank[len(full_rank)-1]

            count = 1
            while last_classified == " ":
                last_classified = full_rank[len(full_rank)-count]
                count = count + 1

            # Annotate the last classified as 'putative' if it does not meet the minimum support criteria
            # Older versions of this script contain code to designate all taxonomic ranks as 'putative' in this case, but
            # this seems conservative
            if float(line[2]) < float(min_support):
                    full_rank[full_rank.index(last_classified)] = "putative "+last_classified
                    last_classified = "putative "+last_classified

            # Add in columns containing unclassified taxonomic information
            try: # In Silva, many classifications are a single entry (which breaks from the reliance on lists for full_rank.index)
                for n in range(full_rank.index(last_classified)+1, full_rank_length, 1):
                    try:
                        full_rank[n] = "unclassified "+last_classified
                    except:
                        full_rank.append("unclassified "+last_classified)
            except:
                for n in range(0, full_rank_length, 1):
                    try:
                        full_rank[n] = "unclassified "+last_classified
                    except:
                        full_rank.append("unclassified "+last_classified)

            # Clean-up the trailing whitespace introduced in Silva classification
            if classification_db == "Silva":
                full_rank = [x.strip(' ') for x in full_rank]

            # Write Taxonomy to File
            output.write(read_id+"\t"+'\t'.join(full_rank)+"\n")

    return()_____no_output_____#####################
## Export from QIIME2

for dataset in datasets:
    name = dataset[0]
    directory = dataset[1]
    metadata = dataset[2]
    domain = dataset[3]

    ## Final Output Names
    fasta_file = directory+"/output/"+name+".rep.seqs.final.fasta"
    tree_file = directory+"/output/"+name+".tree.final.nwk"
    tax_file = directory+"/output/"+name+".taxonomy.final.tsv"
    count_table = directory+"/output/"+name+".counts.final.biom"

    # Export Classifications
    os.system(' '.join([
        "qiime tools export",
        directory+"/output/"+name+".taxonomy.final.qza",
        "--output-dir "+directory+"/output/"
    ]))

    # Reformat Classifications to meet phyloseq format
    format_taxonomy(directory+"/output/taxonomy.tsv", db, min_support)

    # Export SV Table
    os.system(' '.join([
        "qiime tools export",
        directory+"/output/"+name+".table.qza",
        "--output-dir "+directory+"/output/"
    ]))

    # Export SV Sequences
    os.system(' '.join([
        "qiime tools export",
        directory+"/output/"+name+".rep.seqs.final.qza",
        "--output-dir "+directory+"/output/"
    ]))

    # Export Tree
    os.system(' '.join([
        "qiime tools export",
        directory+"/output/"+name+".rep.seqs.tree.final.qza",
        "--output-dir "+directory+"/output/"
    ]))

    # Rename Exported Files
    %mv $directory/output/dna-sequences.fasta $fasta_file
    %mv $directory/output/feature-table.biom $count_table
    %mv $directory/output/taxonomy.fixed.tsv $tax_file

    if domain == "bacteria":
        %mv $directory/output/tree.nwk $tree_file_____no_output_____

# Step 13: Get 16S rRNA Gene Copy Number (rrn)_____no_output_____

## This step is based on the database contructed for the software 'copyrighter'
## The software itself lacked information about datastructure (and, the import of a biom from QIIME2 failed, likely because there are multiple versions of the biom format)
downloaded = "N"

if db != "Silva":
    for dataset in datasets:
        name = dataset[0]
        directory = dataset[1]
        domain = dataset[3]

        if domain == 'bacteria':
            if downloaded == "N":
                ## Download copyrighter database
                !git clone https://github.com/fangly/AmpliCopyrighter $directory/temp/

                ## There are multiple GreenGenes ID numbers for a given taxonomic string.
                ## However, the copyrighter database uses the same average rrn copy number.
                ## We will therefore just use the taxonomic strings, since QIIME2 does not output the ID numbers

                !sed -e '1,1075178d; 1078115d' $directory/temp/data/201210/ssu_img40_gg201210.txt > $directory/output/copyrighter.tax.strings.tsv

                ## Create Dictionary of rrnDB
                rrnDB = {}

                with open(directory+"/output/copyrighter.tax.strings.tsv", "r") as f:
                    for line in f:
                        line = line.strip()
                        line = line.split("\t")

                        try:
                            rrnDB[line[0]] = line[1]

                        except:
                            pass

                downloaded = "Y"

            ## Attribute rrn to readID from taxonomy.tsv
            output = open(directory+"/output/"+name+".seqID.to.rrn.final.tsv","w")
            output.write("Feature ID\trrn\n")

            with open(directory+"/output/taxonomy.tsv", "r") as f:
                missing = 0
                total = 0
                next(f)  # Skip Header

                for line in f:
                    line = line.strip()
                    line = line.split("\t")

                    seqID = line[0]

                    try:
                        rrn = rrnDB[line[1]]

                    except:
                        rrn = "NA"
                        missing = missing + 1

                    total = total + 1
                    output.write(seqID+"\t"+rrn+"\n")

            print("\nPercent of OTUs Missing {:.1%}".format(float(missing)/total))
            print("Don't Panic! The majority of missing OTUs could be low abundance.")_____no_output_____

# Step 14: Import into Phyloseq_____no_output_____

## Setup R-Magic for Jupyter Notebooks
import rpy2
%load_ext rpy2.ipython

def fix_biom_conversion(file):
    with open(file, 'r') as fin:
        data = fin.read().splitlines(True)
    with open(file, 'w') as fout:
        fout.writelines(data[1:])_____no_output_____import pandas as pd
%R library(phyloseq)
%R library(ape)

for dataset in datasets:
    name = dataset[0]
    directory = dataset[1]
    metadata = dataset[2]
    domain = dataset[3]

    #### IMPORT DATA to R
    ## For '.tsv' files, use Pandas to create a dataframe and then pipe that to R
    ## For '.biom' files, first convert using 'biom convert' on the command-line
    ## Had problems importing the count table with pandas, opted for using read.table in R

    # Import Taxonomy File
    tax_file = pd.read_csv(directory+"/output/"+name+".taxonomy.final.tsv", sep="\t")
    %R -i tax_file
    %R rownames(tax_file) = tax_file$OTU
    %R tax_file$OTU <- NULL
    %R tax_file <- tax_file[sort(row.names(tax_file)),] #read names must match the count_table

    # Import Sample Data
    sample_file = pd.read_table(directory+metadata, keep_default_na=False)
    %R -i sample_file
    %R rownames(sample_file) = sample_file$X.SampleID
    %R sample_file$X.SampleID <- NULL
    %R sample_file$LinkerPrimerSequence <- NULL  ## Clean-up some other stuff

    # Import Count Data
    os.system(' '.join([
        "biom convert",
        "-i",
        directory+"/output/"+name+".counts.final.biom",
        "-o",
        directory+"/output/"+name+".counts.final.tsv",
        "--to-tsv"
    ]))

    # The biom converter adds a stupid line that messes with the table formatting
    fix_biom_conversion(directory+"/output/"+name+".counts.final.tsv")

    # Finally import
    count_table = pd.read_csv(directory+"/output/"+name+".counts.final.tsv", sep="\t")
    %R -i count_table
    %R rownames(count_table) = count_table$X.OTU.ID
    %R count_table$X.OTU.ID <- NULL
    %R count_table <- count_table[sort(row.names(count_table)),] #read names must match the tax_table

    # If you've used numbers in your sample names, R prepends them with an 'X'. Normally, you can tell R 'check.names=F' to avoid this behaviour, but I couldn't find this functionality with the pandas import function
    %R colnames(count_table)[grep("^X", colnames(count_table))] <- gsub("X","",colnames(count_table)[grep("^X", colnames(count_table))])

    # Convert to Phyloseq Objects
    %R p_counts = otu_table(count_table, taxa_are_rows = TRUE)
    %R p_samples = sample_data(sample_file)
    %R p_tax = tax_table(tax_file)
    %R taxa_names(p_tax) <- rownames(tax_file) # phyloseq throws out rownames
    %R colnames(p_tax) <- colnames(tax_file) # phyloseq throws out colnames

    # Merge Phyloseq Objects
    %R p = phyloseq(p_counts, p_tax)

    # Import Phylogenetic Tree
    if domain == "bacteria":
        tree_file = directory+"/output/"+name+".tree.final.nwk"
        %R -i tree_file
        %R p_tree <- read.tree(tree_file)

        # Combine All Objects into One Phyloseq
        %R p_final <- merge_phyloseq(p, p_samples, p_tree)

    else:
        # Combine All Objects into One Phyloseq
        %R p_final <- merge_phyloseq(p, p_samples)

    # Save Phyloseq Object as '.rds'
    output = directory+"/output/p_"+name+".final.rds"
    %R -i output
    %R saveRDS(p_final, file = output)

    # Confirm Output
    %R print(p_final)_____no_output_____

# Step 15: Clean-up Intermediate Files and Final Outputs_____no_output_____

for dataset in datasets:
    directory = dataset[1]
    metadata = dataset[2]

    # Remove Files
    if domain == "bacteria":
        %rm -r $directory/output/*tree.unrooted.qza
        %rm -r $directory/output/*aligned.masked.qza

    %rm $directory/output/*.biom
    %rm $directory/output/*barcodes.fastq.gz
    %rm $directory/output/taxonomy.tsv
    %rm $directory/output/forward.fastq.gz # Just the symlink
    %rm $directory/output/reverse.fastq.gz # Just the symlink

    ## 16S rRNA gene copy number info
    if db != "Silva":
        %rm $directory/output/copyrighter.tax.strings.tsv
        %mv $directory/output/*.seqID.to.rrn.final.tsv $directory/final/

    # Separate Final Files
    %mkdir $directory/final/
    %mv $directory/output/*.final.rds $directory/final/
    %mv $directory/output/*.taxonomy.final.tsv $directory/final/
    %mv $directory/output/*.counts.final.tsv $directory/final/
    %mv $directory/output/*.final.fasta $directory/final/
    %cp $directory$metadata $directory/final/
    %mv $directory/output/*.nwk $directory/final/

    # Rename Intermediate File Directory
    %mv $directory/output/ $directory/intermediate_files

print("Your sequences have been successfully saved to 'final' and 'intermediate_files'")_____no_output_____

QIIME2 Bioconductor phyloseq microbiome bioinformatics evolution phylogenetics


# Notebook from mrtommyb/spotty
Path: notebooks/Tissot_mapping_K18-test-RANDOM_VISITS-062420.ipynb

<code>
import matplotlib.pyplot as plt
import cartopy.crs as ccrs
import pandas as pd
import numpy as np
from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas
import matplotlib.animation as animation
from scipy import stats
from datetime import date

import warnings
warnings.filterwarnings("ignore")

%matplotlib inline_____no_output_____k = 0.0172020989 # Gaussian Grav Constant
grav = k**2 # work in units of AU, DAYS and solar masses
rsun_to_AU = 0.00465047
gm = 2.959122083e-4
rEarth = 6370.# km
rEarth_to_rSun = 0.0091577

mstar = 0.359#MSun
Rstar = 0.4445
Rplanet = 2.6*rEarth_to_rSun
#Rp_Rs = np.sqrt(np.pi/4)*0.054
Rp_Rs = 0.054
tdepth_ = Rp_Rs**2
Pplanet = 32.93963#days
b = 0.#0.774
Prot = 38.6*24.#hours

aplanet = (Pplanet**(2./3.))*((grav*mstar)**(1./3.))/((2.*np.pi)**(2./3.))

tdur_days = (Pplanet/np.pi)*np.arcsin(np.sqrt((Rstar**2 + Rplanet**2) + b**2)/(aplanet/rsun_to_AU))
tdur_hrs = tdur_days*24.
tdur_hrs = 2.7 # Kruse et al. 2019

print('tdur [calc] =', "%0.2f" % tdur_hrs, 'hours')
print('tdur [Kruse 2019] = 2.7 hours')
print('Prot(star) = ', Prot, '[hours]')

#star rotation of 1 degree in (1./360.)*Prot hours
Prot_1hr = (1./360.)*Prot
print('1 degree stellar rotation =', Prot_1hr, 'hours')
print('1 degree stellar rotation =', Prot_1hr/tdur_hrs, 'transit duration')
#
HST_orbit_ = 5./6.#0.001*Pplanet*24# In hours
HST_orbits_per_degree_stellar_rotation = Prot_1hr/HST_orbit_
Degrees_stellar_rotation_per_HST_orbit_ = 1./HST_orbits_per_degree_stellar_rotation
print()
print('1 HST orbit =', HST_orbit_, 'hours (FIXED!!)')
print('1 transit duration =', tdur_hrs/HST_orbit_, 'HST orbits')
print('1 degree stellar rotation =', HST_orbits_per_degree_stellar_rotation, 'HST orbits')
print('1 HST orbit =', Degrees_stellar_rotation_per_HST_orbit_, 'degrees stellar rotation')
#print('6 HST orbit = ', 6*obs_duration_, 'hours')
star_size = 750 # pixels, FIXED!
print('1 HST orbit =', star_size/(tdur_hrs/HST_orbit_), 'pixels traveled across star')
pixels_per_HST_visit_round_of_10 = int((star_size/(tdur_hrs/HST_orbit_) // 10)*10)
print('1 HST orbit =', pixels_per_HST_visit_round_of_10, 'pixels traveled across star (FIXED!!!)')tdur [calc] = 2.70 hours
tdur [Kruse 2019] = 2.7 hours
Prot(star) =  926.4000000000001 [hours]
1 degree stellar rotation = 2.5733333333333337 hours
1 degree stellar rotation = 0.9530864197530865 transit duration

1 HST orbit = 0.8333333333333334 hours (FIXED!!)
1 transit duration = 3.24 HST orbits
1 degree stellar rotation = 3.088 HST orbits
1 HST orbit = 0.3238341968911917 degrees stellar rotation
1 HST orbit = 231.48148148148147 pixels traveled across star
1 HST orbit = 230 pixels traveled across star (FIXED!!!)
#aa = ((star_size/(tdur_hrs/HST_orbit_)) // 10)*10
#print(aa)_____no_output_____amplitude_mag = 10e-3# magnitude
amplitude_flux = 10**(-0.4*amplitude_mag)

amplitude_flux_____no_output_____Teff_ = 3500
Tspots = 3000
#
UV_min, UV_max, IR_min, IR_max = 0.42, 0.88, 0.9, 1.8 # in micron
spectral_resolution = 500
#
area_covered, number_of_spots, aspect_ratio = 0.05, 20, 1.
#
number_of_rotation_phases = 1440#360
#
HST_filter = [1.12,1.63]
#HST_filter_=np.array([1.135,1.165,1.195,1.225,1.255,1.285,1.315,1.345,1.375,1.405,1.435,1.465,1.495,\
#                    1.525,1.555,1.585,1.615])
IR_min, IR_max = min(HST_filter), max(HST_filter)_____no_output_____#url = 'https://phoenix.ens-lyon.fr/Grids/BT-Settl/CIFIST2011_2015/SPECTRA/'
#url = 'https://phoenix.ens-lyon.fr/Grids/AMES-Dusty/SPECTRA/'
#fn = 'lte0' + str(int(Teff_/100)) + '.0-5.0-0.0a+0.0.BT-Settl.spec.7.xz'
#fn = 'lte' + str(int(Teff_/100)) + '-5.0-0.0.AMES-dusty.7.gz'
#tt = 'https://phoenix.ens-lyon.fr/Grids/AMES-Dusty/SPECTRA/lte35-5.0-0.0.AMES-dusty.7.gz'
#print(url+fn)
#print(tt)
#df = pd.read_csv(url+fn, na_filter= False).values_____no_output_____fn_model_Teff = f"T{Teff_}_g5.0_solar.txt"
fn_model_spots = f"T{Tspots}_g5.0_solar.txt"
#
stellar_spec_Teff = pd.read_csv(fn_model_Teff, header = 7, sep = '\t')
stellar_spec_Teff = stellar_spec_Teff.values
wavelength_model_Teff = stellar_spec_Teff[:,0]
stellar_flux_model_Teff = stellar_spec_Teff[:,1]
#
stellar_spec_spots = pd.read_csv(fn_model_spots, header = 7, sep = '\t')
stellar_spec_spots = stellar_spec_spots.values
wavelength_model_spots = stellar_spec_spots[:,0]
stellar_flux_model_spots = stellar_spec_spots[:,1]_____no_output_____interp_model_spectra_ = 'n'

if interp_model_spectra_ == 'y':
    print('Model Spectra Interpolated to Spectral Resolution of ' + str(spectral_resolution))
    filter_UV = np.linspace(UV_min, UV_max, spectral_resolution)
    filter_IR = np.linspace(IR_min, IR_max, spectral_resolution)
#
    stellar_flux_model_interp_Teff_IR = np.interp(filter_IR, wavelength_model_Teff/1e4, stellar_flux_model_Teff)
    stellar_flux_model_interp_spots_IR = np.interp(filter_IR, wavelength_model_spots/1e4, stellar_flux_model_spots)
#
    stellar_flux_model_interp_Teff_UV = np.interp(filter_UV, wavelength_model_Teff/1e4, stellar_flux_model_Teff)
    stellar_flux_model_interp_spots_UV = np.interp(filter_UV, wavelength_model_spots/1e4, stellar_flux_model_spots)
else:
    idx_UV_, = np.where((wavelength_model_Teff/1e4 >= UV_min) & (wavelength_model_Teff/1e4 <= UV_max))
    idx_IR_, = np.where((wavelength_model_Teff/1e4 >= IR_min) & (wavelength_model_Teff/1e4 <= IR_max))
#
    filter_UV = wavelength_model_Teff[idx_UV_]/1e4
    filter_IR = wavelength_model_Teff[idx_IR_]/1e4
#
    stellar_flux_model_interp_Teff_IR = stellar_flux_model_Teff[idx_IR_]
    stellar_flux_model_interp_spots_IR = stellar_flux_model_spots[idx_IR_]
#
    stellar_flux_model_interp_Teff_UV = stellar_flux_model_Teff[idx_UV_]
    stellar_flux_model_interp_spots_UV = stellar_flux_model_spots[idx_UV_]_____no_output_____#Sep 2019 Benneke submission: depths_Benneke, depth_err_ = \
#np.array([2937, 2886, 2874, 2912, 2913, 2897, 2901, 2970, 2955, \
#2952, 2978, 2914,2929, 2893, 2862, 2897, 2884]), 30.+np.zeros((17))

# Dec 2019 Benneke accepted:
depths_Benneke, depth_err_ = np.array([2916, 2900, 2873, 2910, 2909, 2898, 2897, 2970, 2955, 2953, 2971, 2907, \
                                       2934, 2891, 2869, 2896, 2878]), 25.+np.zeros((17))

depths_Tsiaras, depth_err_ = np.array([2905, 2939, 2903, 2922, 2891, 2897, 2919, 2965, 2955, 2976, 2990, 2895, \
                                       2930, 2921, 2875, 2927, 2925]), 30.+np.zeros((17))
idx_HST, = np.where((filter_IR >= min(HST_filter)) & (filter_IR <= max(HST_filter)))

HST_depths_ = depths_Benneke
#HST_depths_ = depths_Tsiaras

normalized_HST_depths_ = HST_depths_# - np.nanmean(HST_depths_)_____no_output_____fig = plt.figure(figsize=(20,5))

#plt.plot(filter_UV, stellar_flux_model_interp_Teff_UV, 'b')
plt.plot(filter_IR, stellar_flux_model_interp_Teff_IR, 'r')
plt.plot(filter_IR, stellar_flux_model_interp_spots_IR, 'b')
plt.xlabel('Wavelength [micron]', fontsize = 16)
plt.ylabel('NexGen flux density', fontsize = 16)
#plt.plot(filter_IR, stellar_flux_model_interp_Teff_IR - stellar_flux_model_interp_spots_IR)
plt.xlim(HST_filter[0], HST_filter[1])
#plt.ylim(0.,2e3)

plt.show()_____no_output_____#### MAKE FIGURES ####

def fig_rotated_star_map_(phase_,\
                          star_cube_orbit_1,star_cube_orbit_2,star_cube_orbit_3,star_cube_orbit_4,\
                          nn,ii,observations_per_orbit_,save_fig):

    for kk in range(observations_per_orbit_):
        fig = plt.figure(figsize=(20,10))
        grid = plt.GridSpec(observations_per_orbit_, 4, wspace=0., hspace=0.)

        plt.subplot(grid[kk,0])
        plt.imshow(star_cube_orbit_1[:,:,kk], cmap='hot', vmin=-1., vmax=1.)
        plt.axis('off')
        if kk == 0:
            plt.title('Visit ' + str(ii+1) + '; Orbit 1')

        plt.subplot(grid[kk,1])
        plt.imshow(star_cube_orbit_2[:,:,kk], cmap='hot', vmin=-1., vmax=1.)
        plt.axis('off')
        if kk == 0:
            plt.title('Visit ' + str(ii+1) + '; Orbit 2')

        plt.subplot(grid[kk,2])
        plt.imshow(star_cube_orbit_3[:,:,kk], cmap='hot', vmin=-1., vmax=1.)
        plt.axis('off')
        if kk == 0:
            plt.title('Visit ' + str(ii+1) + '; Orbit 3')

        plt.subplot(grid[kk,3])
        plt.imshow(star_cube_orbit_4[:,:,kk], cmap='hot', vmin=-1., vmax=1.)
        plt.axis('off')
        if kk == 0:
            plt.title('Visit ' + str(ii+1) + '; Orbit 4')

#            plt.title('Rot. Phase = ' + str(round(phase_[jj],2)))

#            plt.show()
    if save_fig == 'y':
        plt.savefig(str(nn) + '_star_visit_' + str(ii+1) + '.png')
#    else:
#        plt.show()

    return

def fig_full_lc_(phase_full_rotation, lc_full, phase_, lc_transit, nn, ii, save_fig):
    fig = plt.figure(figsize=(20,3))
    plt.plot(phase_full_rotation/360., lc_full, 'ko-')#, ms = 12)
    plt.plot(phase_/360., lc_transit, 'r-')#, ms = 10)
#                plt.xlim(-0.001, 0.004)
#                plt.ylim(0.995, 1.005)
    plt.title('Norm. Lightcurve, Visit ' + str(ii+1))
    plt.xlabel('Phase')
#            else:

    left, bottom, width, height = [0.65, 0.65, 0.15, 0.2]
    ax2 = fig.add_axes([left, bottom, width, height])
#                ax2.plot(phase_full_rotation/360., lc_full, 'k.')
    ax2.plot(phase_/360., lc_transit, 'ro-')#, ms = 10)
    plt.xlim(0.995*min(phase_/360.), 1.005*max(phase_/360.))
    plt.ylim(0.995*min(lc_transit), 1.005*max(lc_transit))
#    plt.xlabel('Phase')

#            plt.imshow(star_map_copy_, cmap='hot', vmin=-1., vmax=1., origin = 'bottom')
#            plt.xlabel('Lon [deg]', fontsize = 12)
#            plt.ylabel('Lat [deg]', fontsize = 12)
#            plt.show()
    if save_fig == 'y':
        path_ = '/Users/vkostov/Desktop/Ideas_etc/Mapping/for_Overleaf/'
        plt.savefig(path_ + 'RandomMap_' + str(int(area_covered*100)) + 'percent_SpotCoverage_' \
                    + str(nn) + '_LcVisit_' + str(ii+1) + '.png')
    else:
        plt.show()

    return

def fig_plot_individual_visits(number_of_visits_, filter_IR_HST, mean_subtracted_diff_flux_arr_, \
                              bin_centers_, mean_subtracted_binned_spec_per_visit, nn, save_fig):
    fig = plt.figure(figsize=(20,15))

    for jj in range(number_of_visits_):

        plt.subplot(number_of_visits_, 1, jj+1)
        plt.plot(filter_IR[idx_HST], mean_subtracted_diff_flux_arr_[jj,:], 'k', alpha = 0.25)
        plt.plot(bin_centers_, mean_subtracted_binned_spec_per_visit[jj,:],'ro-',ms=10)
        plt.title('Diff spectrum, Visit ' + str(jj+1))

#        plt.show()
    if save_fig == 'y':
        path_ = '/Users/vkostov/Desktop/Ideas_etc/Mapping/for_Overleaf/'
        plt.savefig(path_ + 'Individual_visits_iter_' + str(nn) + '.png')
    else:
        plt.show()

    return

def fig_plot_averaged_spec_all_visits_(bin_centers_, averaged_spec_all_visits_binned_, normalized_HST_depths_,\
                                       nn, save_fig):
    fig = plt.figure(figsize=(20,3))
    plt.plot(bin_centers_, averaged_spec_all_visits_binned_, 'rs-', ms = 7, lw = 15, alpha = 0.5)
    plt.errorbar(bin_centers_, normalized_HST_depths_, yerr = 1.*depth_err_, fmt = 'bs-', ms = 7, \
                    label = 'K2-18 HST', lw = 15, alpha = 0.5)

    plt.title('Averaged Difference Spectrum, (OOT-ITR)/ITR)')
#        plt.show()
    if save_fig == 'y':
        path_ = '/Users/vkostov/Desktop/Ideas_etc/Mapping/for_Overleaf/'
        plt.savefig(path_ + 'Averaged_spec_all_visits_iter_' + str(nn) + '.png')
    else:
        plt.show()

    return_____no_output_____def spot_sizes(area_covered, ndc, aspect_ratio):

    Atot = np.round(4.*np.pi*rEarth**2)
    rad = np.random.random(ndc)
    ntot = np.sum((np.pi*(rad**2))/aspect_ratio)
    normalization = Atot*area_covered/ntot

    rad_real = np.around(rad*(normalization**0.5))

    return rad_real_____no_output_____def create_photosphere_map(spot_sizes_, faculae_size, lon, lat, photosphere_id, spots_id):

    fig = plt.figure(figsize=[5.,2.5], dpi = 72)# This makes a 360x180 map in Jupyter. Brute force, but it works...
    proj = ccrs.PlateCarree()
    ax = plt.axes(projection = proj, fc = 'r')

    canvas = FigureCanvas(fig)
    plt.gca().set_position([0, 0, 1, 1])

    ax.set_global()
    ax.outline_patch.set_linewidth(0.)
    ax.set_extent([-180, 180, -90, 90])

    for ii in range(len(spot_sizes_)):

#        add_faculae=ax.tissot(rad_km=faculae_size[ii], lons=lon[ii], lats=lat[ii], n_samples=500, fc='b', alpha=0.5)
        add_spots = ax.tissot(rad_km=spot_sizes_[ii], lons=lon[ii], lats=lat[ii], n_samples=500, fc='r', alpha=0.1)
#        XXXXXX
#        THIS IS WRONG!!!!

    canvas.draw()       # draw the canvas, cache the renderer
    s, (width, height) = canvas.print_to_buffer()
#    img_ = np.fromstring(s, np.uint8).reshape((height, width, 4))
    img_ = np.frombuffer(s, np.uint8).reshape((height, width, 4))

#    photosphere_id = 0.25
#    spots_id = 0.04
    photosphere = 1. + np.zeros((img_.shape[0],img_.shape[1]))
    photosphere[img_[:,:,3] != 0] = photosphere_id#0.05
    photosphere[img_[:,:,2] != 255] = spots_id#0.45

    plt.close()
#    plt.show()
    return photosphere_____no_output_____#photosphere_id = 0.25
#spots_id = 0.04
#planet_id = -1.
#spot_lat_ = -60. + 120*np.random.random(number_of_spots)
#spot_lon_ = -180 + 360.*np.random.random(number_of_spots)
#spot_sizes_ = spot_sizes(area_covered, number_of_spots, aspect_ratio)
#faculae_size = 1.5*spot_sizes_
#star_map = create_photosphere_map(spot_sizes_, faculae_size, spot_lon_, spot_lat_, photosphere_id, spots_id)
#print(star_map.shape)
#plt.imshow(star_map)_____no_output_____def rotate_star(star_map, lon, do_plot_, add_planet_, photosphere_id, spots_id):

    n_rot_phases = len(lon)
#    lon = np.linspace(0.,360, n_rot_phases)

    proj = ccrs.Orthographic(central_longitude=0., central_latitude=0.)

    tmp_arr = np.zeros((750,750,n_rot_phases))

    for ii in range(n_rot_phases):
        if ii != 180.:
            lon_min_, lon_max_ = -180+lon[ii], 180+lon[ii]
        else:
            lon_min_, lon_max_ = -180+lon[ii]+1e-1, 180+lon[ii]+1e-1

        plt.figure(figsize=(5,5))
        ax = plt.axes(projection = proj, fc = 'r')
        ax.outline_patch.set_linewidth(0.)

        aa = ax.imshow(star_map, origin='upper', transform=ccrs.PlateCarree(), extent=[lon_min_, lon_max_, -90, 90], \
                    cmap='hot', vmin=-1., vmax=1.)
        img_ = aa.get_array()
        photosphere = 1. + np.zeros((img_.shape[0],img_.shape[1]))
#        photosphere_id = 0.25
#        spots_id = 0.04
        photosphere[img_ == photosphere_id] = photosphere_id
        photosphere[img_ == spots_id] = spots_id

        tmp_arr[:,:,ii] = photosphere

#        rr_ = (Rp_Rs*star_cube.shape[0])
#        center_ = int(0.5*star_cube.shape[0])
#        print(rr_, center_)
#        star_cube[center_ - rr_:center_ + rr_, 300:360, ii] = planet_id

        if do_plot_ == 'y':
            plt.show()
        else:
            plt.close()

    return tmp_arr_____no_output_____def fraction_covered_(star_cube, number_of_rotation_phases, photosphere_id, spots_id, planet_id):

    photosphere = photosphere_id
    spots = spots_id
    planet = planet_id
    frac_photosphere = np.zeros(number_of_rotation_phases)
    frac_spots = np.zeros(number_of_rotation_phases)
    frac_planet = np.zeros(number_of_rotation_phases)

    for ii in range(number_of_rotation_phases):
        area_visible_disk = star_cube[:,:,ii].copy()
        area_visible_disk = star_cube[:,:,ii][star_cube[:,:,ii] != 1.].size

        frac_photosphere[ii] = float(star_cube[:,:,ii][star_cube[:,:,ii] == photosphere].size)
        frac_photosphere[ii] = frac_photosphere[ii]/area_visible_disk

        frac_spots[ii] = float(star_cube[:,:,ii][star_cube[:,:,ii] == spots].size)
        frac_spots[ii] = frac_spots[ii]/area_visible_disk

        frac_planet[ii] = float(star_cube[:,:,ii][star_cube[:,:,ii] == planet].size)
        frac_planet[ii] = frac_planet[ii]/area_visible_disk

#    print(frac_photosphere)
#    print(frac_spots)
#    print(frac_planet)
#    print(frac_photosphere + frac_spots + frac_planet)

    return frac_photosphere, frac_spots, frac_planet_____no_output_____def make_flux_cube(star_cube, number_of_rotation_phases, photosphere_id, spots_id, planet_id):
    frac_photosphere, frac_spots, frac_planet = \
    fraction_covered_(star_cube, number_of_rotation_phases, photosphere_id, spots_id, planet_id)
    frac_covered = np.vstack((frac_photosphere, frac_spots)).T

#    print(frac_photosphere)
#    print(frac_spots)
#    print(frac_planet)
#    print()

#    lc_IR = frac_covered[:,0]*np.sum(stellar_flux_model_interp_Teff_IR) + \
#    frac_covered[:,1]*np.sum(stellar_flux_model_interp_spots_IR)
#    phase_max_ = lc_IR.argmax()
#    phase_min_ = lc_IR.argmin()
#
    flux_cube = np.zeros((number_of_rotation_phases, len(stellar_flux_model_interp_Teff_IR)))

    for ii in range(number_of_rotation_phases):
        flux_cube[ii,:] = frac_covered[ii,0]*stellar_flux_model_interp_Teff_IR + \
        frac_covered[ii,1]*stellar_flux_model_interp_spots_IR
#
    rot_phase_ = np.linspace(0.,360, number_of_rotation_phases)/360.

    return flux_cube _____no_output_____def make_diff_flux_(ii, tdepth_,flux_cube_orbit_1, flux_cube_orbit_2, flux_cube_orbit_3, flux_cube_orbit_4, idx_HST):

    if (ii+1 != 4) & (ii+1 != 8):
        itr_flux_ = (1.-0.*tdepth_)*np.mean(np.vstack((flux_cube_orbit_2,flux_cube_orbit_3)), axis = 0)
        oot_flux_ = np.mean(np.vstack((flux_cube_orbit_1,flux_cube_orbit_4)), axis = 0)
    if ii+1 == 4:
        itr_flux_ = (1.-0.*tdepth_)*np.mean(flux_cube_orbit_3, axis = 0)
        oot_flux_ = np.mean(np.vstack((flux_cube_orbit_1,flux_cube_orbit_3,flux_cube_orbit_4)), axis = 0)
    if ii+1 == 8:
        itr_flux_ = (1.-0.*tdepth_)*np.mean(np.vstack((flux_cube_orbit_3,flux_cube_orbit_4)), axis = 0)
        oot_flux_ = np.mean(np.vstack((flux_cube_orbit_1,flux_cube_orbit_2)), axis = 0)
#
    diff_flux_ = 1e6*(oot_flux_[idx_HST] - itr_flux_[idx_HST])/oot_flux_[idx_HST]
#    diff_flux_arr_[ii,:] = 1e6*(oot_flux_[idx_HST] - itr_flux_[idx_HST])/itr_flux_[idx_HST]

    return diff_flux______no_output_____def add_two_planet_transits_(ii, star_cube_t1, star_cube_t2, observations_per_orbit_):

    area_visible_disk = star_cube_t1[:,:,0][star_cube_t1[:,:,0] != 1.].size
#    rr_ = int(Rp_Rs*0.5*star_cube.shape[0])
    rr_ = int(0.5*np.ceil(np.sqrt(area_visible_disk*Rp_Rs**2)))
    center_ = int(0.5*star_cube.shape[0])
    pixels_per_obs_ = int(pixels_per_HST_visit_round_of_10/(observations_per_orbit_-1))
#    pixels_per_obs_ = 50 # FIXED
#
    if ii+1 != 8:
        for oo in range(observations_per_orbit_):
#            obs_2_indiv_ = center_-125 + (oo-5)*pixels_per_obs_
#            obs_3_indiv_ = center_+125 + (oo-5)*pixels_per_obs_
            obs_2_indiv_ = center_-25 + (oo-observations_per_orbit_)*pixels_per_obs_
            obs_3_indiv_ = center_+(-25+425) + (oo-observations_per_orbit_)*pixels_per_obs_
#            print(pixels_per_obs_, obs_2_indiv_, obs_3_indiv_)

            star_cube_t1[center_-rr_:center_+rr_, obs_2_indiv_-rr_:obs_2_indiv_+rr_, oo] = planet_id
            star_cube_t2[center_-rr_:center_+rr_, obs_3_indiv_-rr_:obs_3_indiv_+rr_, oo] = planet_id
#            print(rr_)
#            print(star_cube_t1.shape)
#            area_visible_disk = star_cube_t1[:,:,oo][star_cube_t1[:,:,oo] != 1.].size
#            area_planet = star_cube_t1[:,:,oo][star_cube_t1[:,:,oo] == planet_id].size
#            print(area_visible_disk, area_planet, area_planet/area_visible_disk)
#            print(star_cube_t1[center_-rr_:center_+rr_, obs_2_indiv_-rr_:obs_2_indiv_+rr_, oo].shape)
#        xxx
    else:
        for oo in range(observations_per_orbit_):
#            obs_3_indiv_ = center_-125 + (oo-5)*pixels_per_obs_
#            obs_4_indiv_ = center_+125  + (oo-5)*pixels_per_obs_
            obs_3_indiv_ = center_-25 + (oo-observations_per_orbit_)*pixels_per_obs_
            obs_4_indiv_ = center_+(-25+425) + (oo-observations_per_orbit_)*pixels_per_obs_

            star_cube_t1[center_-rr_:center_+rr_, obs_3_indiv_-rr_:obs_3_indiv_+rr_, oo] = planet_id
            star_cube_t2[center_-rr_:center_+rr_, obs_4_indiv_-rr_:obs_4_indiv_+rr_, oo] = planet_id

#   for oo in range(observations_per_orbit_):
#       fig = plt.figure(figsize=(10,5))
#       grid = plt.GridSpec(observations_per_orbit_, 2, wspace=0.3, hspace=0.3)
#       plt.subplot(grid[oo,0])
#       plt.imshow(star_cube_orbit_2[:,:,oo])

#       plt.subplot(grid[oo,1])
#       plt.imshow(star_cube_orbit_3[:,:,oo])
#       plt.show()

    return star_cube_t1, star_cube_t2_____no_output_____def add_one_planet_transits_(ii, star_cube, observations_per_orbit_):

    area_visible_disk = star_cube[:,:,0][star_cube[:,:,0] != 1.].size
#    rr_ = int(Rp_Rs*0.5*star_cube.shape[0])
    rr_ = int(0.5*np.ceil(np.sqrt(area_visible_disk*Rp_Rs**2)))
    center_ = int(0.5*star_cube.shape[0])
    pixels_per_obs_ = int(pixels_per_HST_visit_round_of_10/(observations_per_orbit_-1))
#    pixels_per_obs_ = 50 # FIXED
#
    for oo in range(observations_per_orbit_):
        obs_3_indiv_ = center_ + (oo-observations_per_orbit_)*pixels_per_obs_
        star_cube[center_-rr_:center_+rr_, obs_3_indiv_-rr_:obs_3_indiv_+rr_, oo] = planet_id

    return star_cube_____no_output_____def many_visits_random_photosphere(number_of_spots, area_covered, add_planet, number_of_visits_, nn, show_full_lc_, \
                                  plot_individual_visits_, show_rotated_star_map_, observations_per_orbit_):

#    number_of_visits_ = 8
    diff_flux_arr_ = np.zeros((number_of_visits_, len(idx_HST)))
    mean_subtracted_diff_flux_arr_ = np.zeros((number_of_visits_, len(idx_HST)))
    mean_subtracted_binned_spec_per_visit = np.zeros((number_of_visits_, 17))
    per_bin_errors_ = np.random.uniform(-30.,30, size = (number_of_visits_,17))

    star_map_all_visits_ = np.zeros((180, 360, number_of_visits_))

    photosphere_id = 0.25
    spots_id = 0.04
    planet_id = -1.

    phase_transit_ = np.zeros(number_of_visits_)

    for ii in range(number_of_visits_):
#
        spot_lat_ = -60. + 120*np.random.random(number_of_spots)
        spot_lon_ = -180 + 360.*np.random.random(number_of_spots)
        spot_sizes_ = spot_sizes(area_covered, number_of_spots, aspect_ratio)
        faculae_size = 1.5*spot_sizes_
        star_map = create_photosphere_map(spot_sizes_, faculae_size, spot_lon_, spot_lat_, photosphere_id, spots_id)
        star_map_all_visits_[:,:,ii] = star_map
#
        phase_ = 2.*Degrees_stellar_rotation_per_HST_orbit_*np.linspace(0,3,4) + np.random.uniform(357.5)
        number_of_rotation_phases = len(phase_)
#        print(ii, phase_)

        phase_transit_[ii] = phase_[0]

        pp_tmp_ = Degrees_stellar_rotation_per_HST_orbit_/observations_per_orbit_
        phases_per_orbit_ = np.zeros((len(phase_),observations_per_orbit_))
        for pp_ in range(len(phase_)):
            phases_per_orbit_[pp_,:] = phase_[pp_] + (pp_tmp_)*np.linspace(-observations_per_orbit_/2,\
                                                                           observations_per_orbit_/2,\
                                                                           observations_per_orbit_)
#
        do_plot_ = 'n'
        star_cube_orbit_1 = rotate_star(star_map,phases_per_orbit_[0,:],do_plot_,add_planet,photosphere_id,spots_id)
        star_cube_orbit_2 = rotate_star(star_map,phases_per_orbit_[1,:],do_plot_,add_planet,photosphere_id,spots_id)
        star_cube_orbit_3 = rotate_star(star_map,phases_per_orbit_[2,:],do_plot_,add_planet,photosphere_id,spots_id)
        star_cube_orbit_4 = rotate_star(star_map,phases_per_orbit_[3,:],do_plot_,add_planet,photosphere_id,spots_id)

        if (ii+1 != 4) & (ii+1 != 8):
            star_cube_orbit_2, star_cube_orbit_3 = \
            add_two_planet_transits_(ii, star_cube_orbit_2, star_cube_orbit_3, observations_per_orbit_)
        if ii+1 == 4:
            star_cube_orbit_3 = \
            add_one_planet_transits_(ii, star_cube_orbit_3, observations_per_orbit_)
        if ii+1 == 8:
            star_cube_orbit_3, star_cube_orbit_4 = \
            add_two_planet_transits_(ii, star_cube_orbit_3, star_cube_orbit_4, observations_per_orbit_)
#
#
        pp_ = observations_per_orbit_
        flux_cube_orbit_1 = make_flux_cube(star_cube_orbit_1, pp_, photosphere_id, spots_id, planet_id)
        flux_cube_orbit_2 = make_flux_cube(star_cube_orbit_2, pp_, photosphere_id, spots_id, planet_id)
        flux_cube_orbit_3 = make_flux_cube(star_cube_orbit_3, pp_, photosphere_id, spots_id, planet_id)
        flux_cube_orbit_4 = make_flux_cube(star_cube_orbit_4, pp_, photosphere_id, spots_id, planet_id)

        diff_flux_arr_[ii,:] = \
        make_diff_flux_(ii, tdepth_,flux_cube_orbit_1, flux_cube_orbit_2, flux_cube_orbit_3, flux_cube_orbit_4, \
                        idx_HST)

        mean_subtracted_diff_flux_arr_[ii,:] = diff_flux_arr_[ii,:] - np.nanmean(diff_flux_arr_[ii,:])
#
        bin_means,bin_edges,binnumber = \
        stats.binned_statistic(filter_IR[idx_HST],mean_subtracted_diff_flux_arr_[ii,:],statistic='mean', bins=17)

        bin_width = (bin_edges[1] - bin_edges[0])
        bin_centers_ = bin_edges[1:] - bin_width/2

        mean_subtracted_binned_spec_per_visit[ii,:] =  bin_means + 0.*per_bin_errors_[ii,:]
#
        if show_rotated_star_map_ == 'y':
            save_fig = 'n'
            aa = fig_rotated_star_map_(phase_, \
                                       star_cube_orbit_1, star_cube_orbit_2, star_cube_orbit_3, star_cube_orbit_4, \
                                       nn, ii, observations_per_orbit_, save_fig)
#
        if show_full_lc_ == 'y':
            save_fig = 'n'
            lc_transit_ = np.hstack((np.sum(flux_cube_orbit_1, axis = 1),
                                    np.sum(flux_cube_orbit_2, axis = 1),
                                    np.sum(flux_cube_orbit_3, axis = 1),
                                    np.sum(flux_cube_orbit_4, axis = 1)))

            phase_full_rotation = np.linspace(0.,359., 20)
            do_plot_
            star_cube_full_rotation = \
            rotate_star(star_map, phase_full_rotation, do_plot_, add_planet_, photosphere_id, spots_id)
            flux_cube_full_rotation = \
            make_flux_cube(star_cube_full_rotation, len(phase_full_rotation), photosphere_id, spots_id, planet_id)

            lc_full = np.zeros((flux_cube_full_rotation.shape[0]))
            for kk in range(len(lc_full)):
                lc_full[kk] = np.sum(flux_cube_full_rotation[kk,:])

            aa = fig_full_lc_(phase_full_rotation, lc_full/lc_transit_[0], \
                              phases_per_orbit_.flatten(), lc_transit_/lc_transit_[0], nn, ii, save_fig)
#
#
    averaged_spec_all_visits_ = np.nanmean(mean_subtracted_diff_flux_arr_, axis = 0)
    averaged_spec_all_visits_binned_ = np.nanmean(mean_subtracted_binned_spec_per_visit, axis = 0)
#
    if plot_individual_visits_ == 'y':
        save_fig = 'y'
        aa = fig_plot_individual_visits(number_of_visits_, filter_IR[idx_HST], mean_subtracted_diff_flux_arr_, \
                              bin_centers_, mean_subtracted_binned_spec_per_visit, nn, save_fig)
#
    plot_averaged_spec_all_visits_ = 'y'
    if plot_averaged_spec_all_visits_ == 'y':
        save_fig = 'y'
        aa=fig_plot_averaged_spec_all_visits_(bin_centers_, averaged_spec_all_visits_binned_, \
                                              normalized_HST_depths_, nn, save_fig)

#    plt.savefig('tmp_1.png')
    return mean_subtracted_diff_flux_arr_,mean_subtracted_binned_spec_per_visit,bin_centers_,\
star_map_all_visits_,phase_transit______no_output_____def many_visits_same_photosphere(star_map,add_planet,number_of_visits_,nn,show_full_lc_,lc_full,phase_full_rotation,\
                                plot_individual_visits_, show_rotated_star_map_, load_phase_transit, phase_load, \
                                observations_per_orbit_, add_noise_):

    diff_flux_arr_ = np.zeros((number_of_visits_, len(idx_HST)))
    mean_subtracted_diff_flux_arr_ = np.zeros((number_of_visits_, len(idx_HST)))
    mean_subtracted_binned_spec_per_visit = np.zeros((number_of_visits_, 17))
    per_bin_errors_ = np.random.uniform(-30.,30, size = (number_of_visits_,17))

    photosphere_id = 0.25
    spots_id = 0.04
    planet_id = -1.

    if load_phase_transit == 'n':
        phase_transit_ = np.zeros(number_of_visits_)

    for ii in range(number_of_visits_):

        if load_phase_transit == 'n':
#            phase_ = 0.67*np.linspace(0,3,4) + np.random.uniform(357.5)
            phase_ = 2.*Degrees_stellar_rotation_per_HST_orbit_*np.linspace(0,3,4) + np.random.uniform(357.5)
            phase_transit_[ii] = phase_[0]
        else:
#            phase_ = 0.67*np.linspace(0,3,4) + phase_load[ii]
            phase_ = 2.*Degrees_stellar_rotation_per_HST_orbit_*np.linspace(0,3,4) + phase_load[ii]
            phase_transit_ = phase_load
#
#
#        observations_per_orbit_ = 2
        pp_tmp_ = Degrees_stellar_rotation_per_HST_orbit_/observations_per_orbit_

        phases_per_orbit_ = np.zeros((len(phase_),observations_per_orbit_))
        for pp_ in range(len(phase_)):
            phases_per_orbit_[pp_,:] = phase_[pp_] + (pp_tmp_)*np.linspace(-observations_per_orbit_/2,\
                                                                           observations_per_orbit_/2,\
                                                                           observations_per_orbit_)
#        xxxx

        do_plot_ = 'n'
        star_cube_orbit_1 = rotate_star(star_map,phases_per_orbit_[0,:],do_plot_,add_planet,photosphere_id,spots_id)
        star_cube_orbit_2 = rotate_star(star_map,phases_per_orbit_[1,:],do_plot_,add_planet,photosphere_id,spots_id)
        star_cube_orbit_3 = rotate_star(star_map,phases_per_orbit_[2,:],do_plot_,add_planet,photosphere_id,spots_id)
        star_cube_orbit_4 = rotate_star(star_map,phases_per_orbit_[3,:],do_plot_,add_planet,photosphere_id,spots_id)
#
#
        if add_planet == 'y':
            if (ii+1 != 4) & (ii+1 != 8):
                star_cube_orbit_2, star_cube_orbit_3 = \
                add_two_planet_transits_(ii, star_cube_orbit_2, star_cube_orbit_3, observations_per_orbit_)
            if ii+1 == 4:
                star_cube_orbit_3 = \
                add_one_planet_transits_(ii, star_cube_orbit_3, observations_per_orbit_)
            if ii+1 == 8:
                star_cube_orbit_3, star_cube_orbit_4 = \
                add_two_planet_transits_(ii, star_cube_orbit_3, star_cube_orbit_4, observations_per_orbit_)
#
#

#        plt.figure(figsize=(10,20))
#        plt.subplot(421)
#        plt.imshow(star_cube_orbit_2[:,:,0], cmap='hot', vmin=-1., vmax=1.)

#        plt.subplot(422)
#        plt.imshow(star_cube_orbit_3[:,:,0], cmap='hot', vmin=-1., vmax=1.)

#        plt.subplot(423)
#        plt.imshow(star_cube_orbit_2[:,:,1], cmap='hot', vmin=-1., vmax=1.)

#        plt.subplot(424)
#        plt.imshow(star_cube_orbit_3[:,:,1], cmap='hot', vmin=-1., vmax=1.)

#        plt.subplot(425)
#        plt.imshow(star_cube_orbit_2[:,:,2], cmap='hot', vmin=-1., vmax=1.)

#        plt.subplot(426)
#        plt.imshow(star_cube_orbit_3[:,:,2], cmap='hot', vmin=-1., vmax=1.)

#        plt.subplot(427)
#        plt.imshow(star_cube_orbit_2[:,:,3], cmap='hot', vmin=-1., vmax=1.)

#        plt.subplot(428)
#        plt.imshow(star_cube_orbit_3[:,:,3], cmap='hot', vmin=-1., vmax=1.)
#        xxxx
        pp_ = observations_per_orbit_
        flux_cube_orbit_1 = make_flux_cube(star_cube_orbit_1, pp_, photosphere_id, spots_id, planet_id)
        flux_cube_orbit_2 = make_flux_cube(star_cube_orbit_2, pp_, photosphere_id, spots_id, planet_id)
        flux_cube_orbit_3 = make_flux_cube(star_cube_orbit_3, pp_, photosphere_id, spots_id, planet_id)
        flux_cube_orbit_4 = make_flux_cube(star_cube_orbit_4, pp_, photosphere_id, spots_id, planet_id)
#
#
        diff_flux_arr_[ii,:] = make_diff_flux_(ii, tdepth_,flux_cube_orbit_1, flux_cube_orbit_2, \
                                               flux_cube_orbit_3, flux_cube_orbit_4, idx_HST)
#
        mean_subtracted_diff_flux_arr_[ii,:] = diff_flux_arr_[ii,:] - 0.*np.nanmean(diff_flux_arr_[ii,:])
#

#        plt.plot(diff_flux_arr_[ii,:])
#        xxxx

        bin_means,bin_edges,binnumber = \
        stats.binned_statistic(filter_IR[idx_HST],mean_subtracted_diff_flux_arr_[ii,:],statistic='mean', bins=17)

        bin_width = (bin_edges[1] - bin_edges[0])
        bin_centers_ = bin_edges[1:] - bin_width/2

        if add_noise_ == 'y':
            mean_subtracted_binned_spec_per_visit[ii,:] =  bin_means + 1.*per_bin_errors_[ii,:]
        else:
            mean_subtracted_binned_spec_per_visit[ii,:] =  bin_means + 0.*per_bin_errors_[ii,:]
#
        if show_rotated_star_map_ == 'y':
            save_fig = 'n'
            aa = fig_rotated_star_map_(phase_, \
                                       star_cube_orbit_1, star_cube_orbit_2, star_cube_orbit_3, star_cube_orbit_4, \
                                       nn, ii, observations_per_orbit_, save_fig)
#
        if show_full_lc_ == 'y':
            save_fig = 'n'
            lc_transit_ = np.hstack((np.sum(flux_cube_orbit_1, axis = 1),
                                    np.sum(flux_cube_orbit_2, axis = 1),
                                    np.sum(flux_cube_orbit_3, axis = 1),
                                    np.sum(flux_cube_orbit_4, axis = 1)))

            aa = fig_full_lc_(phase_full_rotation, lc_full/lc_transit_[0], \
                              phases_per_orbit_.flatten(), lc_transit_/lc_transit_[0], nn, ii, save_fig)
#
#
    averaged_spec_all_visits_ = np.nanmean(mean_subtracted_diff_flux_arr_, axis = 0)
    averaged_spec_all_visits_binned_ = np.nanmean(mean_subtracted_binned_spec_per_visit, axis = 0)
#
#
    if plot_individual_visits_ == 'y':
        save_fig = 'n'
        aa = fig_plot_individual_visits(number_of_visits_, filter_IR[idx_HST], mean_subtracted_diff_flux_arr_, \
                              bin_centers_, mean_subtracted_binned_spec_per_visit, nn, save_fig)
#
    plot_averaged_spec_all_visits_ = 'y'
    if plot_averaged_spec_all_visits_ == 'y':
        save_fig = 'y'
        aa=fig_plot_averaged_spec_all_visits_(bin_centers_, averaged_spec_all_visits_binned_, \
                                              normalized_HST_depths_, nn, save_fig)
#
    return mean_subtracted_diff_flux_arr_, mean_subtracted_binned_spec_per_visit, bin_centers_, phase_transit______no_output_____load_chi_sq_ = 'n'

if load_chi_sq_ == 'y':
    reduced_chi_sq_ = np.load('reduced_chi_sq_42820.npy')

    min_idx_ = reduced_chi_sq_.argmin()
    spot_crossings_idx_ = np.asarray([])#0,2,3,5,7,17,21,27,30,35,39,43,44,45,46,47,62,64,65,66,74,84,85,91,98])
    print(reduced_chi_sq_, min(reduced_chi_sq_), max(reduced_chi_sq_))
    idx_, = np.where(reduced_chi_sq_ < 17.)
    #print(len(idx_), idx_)

    n_bins = int(max(reduced_chi_sq_) - min(reduced_chi_sq_))
#    n_bins_spot_crossings_ = int(max(reduced_chi_sq_[spot_crossings_idx_]) - min(reduced_chi_sq_[spot_crossings_idx_]))

    fig = plt.figure(figsize=(20,5))
    plt.hist(reduced_chi_sq_, bins = 20, fc = None, ec = 'b', hatch = '\\', alpha = 0.25, label = 'All simulations')
#    plt.hist(reduced_chi_sq_[spot_crossings_idx_], bins = n_bins_spot_crossings_//1, hatch = '/', fc = None, \
#             ec = 'r', alpha = 0.25, label = \
#             'Simulations with Spot Crossings (~' + str(int(100*len(spot_crossings_idx_)/len(reduced_chi_sq_))) + '%)')
    plt.xlabel('Reduced Chi-square', fontsize = 18)
    plt.ylabel('Number of simulations', fontsize = 18)
    plt.legend(loc = 'best', fontsize = 14)
    plt.show()_____no_output_____number_of_iterations_ = 1#25
number_of_visits_ = 1
observations_per_orbit_ = 4

star_cube = np.zeros((750,750,4,number_of_iterations_))
flux_cube = np.zeros((4, 2551, number_of_iterations_))
mean_subtracted_diff_flux_arr_all_iterations_ = np.zeros((number_of_iterations_,number_of_visits_,
                                                          len(filter_IR[idx_HST])))

star_map = np.zeros((180, 360, number_of_iterations_))

phase_transit_ = np.zeros((number_of_iterations_, number_of_visits_))

photosphere_ = 'random'
photosphere_ = 'same'
load_phase_transit, load_star_map = 'n', 'n'#'y', 'y'#
add_planet_ = 'y'#'y'
add_noise_ = 'n'#'y'

show_full_lc_ = 'y'#'n'
plot_individual_visits_ = 'y'
show_rotated_star_map_ = 'y'#'y'#

for ii in range(number_of_iterations_):

    if photosphere_ == 'random':

        mean_subtracted_diff_flux_arr_,mean_subtracted_binned_spec_per_visit,bin_centers_,\
        star_map_tmp,phase_transit_tmp=many_visits_random_photosphere(number_of_spots,area_covered,add_planet_,\
                                                                      number_of_visits_,ii,show_full_lc_, \
                                                                      plot_individual_visits_, \
                                                                      show_rotated_star_map_, \
                                                                      observations_per_orbit_)
        phase_transit_[ii, :] = phase_transit_tmp
        star_map[:,:,ii] = star_map_tmp

    if photosphere_ == 'same':

        spot_lat_ = -60. + 120*np.random.random(number_of_spots)
        spot_lon_ = -180 + 360.*np.random.random(number_of_spots)
        spot_sizes_ = spot_sizes(area_covered, number_of_spots, aspect_ratio)
        faculae_size = 1.5*spot_sizes_ # WIP

        photosphere_id = 0.25
        spots_id = 0.04
        planet_id = -1.

        do_plot_ = 'n'

        if (load_phase_transit == 'n') & (load_star_map == 'n'):
            star_map_same = \
            create_photosphere_map(spot_sizes_,faculae_size,spot_lon_,spot_lat_,photosphere_id,spots_id)
            phase_transit_load = -999.

        else:
            star_map_file = 'star_map_file_42820.npy'#.npy'#
            phase_file = 'phase_file_42820.npy'#.npy'#_041620.npy'
            star_map_load, phase_transit_load = np.load(star_map_file), np.load(phase_file)

            idx_ = -1# FOR WATER FEATURE IN "STAR_MAP_FILE.NPY"
            star_map_same, phase_transit_load = star_map_load[:,:,idx_], phase_transit_load[idx_,:]

        phase_full_rotation = np.linspace(0.,359., 10)

        do_plot_ = 'n'
        star_cube_full_rotation = \
        rotate_star(star_map_same, phase_full_rotation, do_plot_, add_planet_, photosphere_id, spots_id)
        flux_cube_full_rotation = \
        make_flux_cube(star_cube_full_rotation, len(phase_full_rotation), photosphere_id, spots_id, planet_id)

        lc_full = np.zeros((flux_cube_full_rotation.shape[0]))
        for kk in range(len(lc_full)):
            lc_full[kk] = np.sum(flux_cube_full_rotation[kk,:])

        mean_subtracted_diff_flux_arr_, mean_subtracted_binned_spec_per_visit, bin_centers_, phase_transit_tmp = \
        many_visits_same_photosphere(star_map_same, add_planet_, number_of_visits_, ii, show_full_lc_, lc_full, \
                                     phase_full_rotation, plot_individual_visits_, show_rotated_star_map_, \
                                     load_phase_transit, phase_transit_load, observations_per_orbit_,add_noise_)

        mean_subtracted_diff_flux_arr_all_iterations_[ii,:,:] = mean_subtracted_diff_flux_arr_

        phase_transit_[ii, :] = phase_transit_tmp

        star_map[:,:,ii] = star_map_same

    print('Iteration ', str(ii), '...done!')

save_output_ = 'n'

if save_output_ == 'y':
    today = date.today()
    star_map_file = 'star_map_file_' + str(today.month) + str(today.day) + str(today.year-2000) + '.npy'
    phase_file = 'phase_file_' + str(today.month) + str(today.day) + str(today.year-2000) + '.npy'
    np.save(star_map_file, star_map)
    np.save(phase_file, phase_transit_)_____no_output_____# xxx_____no_output_____mean_subtracted_binned_spec_per_visit = np.zeros((number_of_visits_, 17))
per_bin_errors_ = np.random.uniform(-25.,25., size = (number_of_visits_,17))

reduced_chi_sq_ = np.zeros((number_of_iterations_))

for jj in range(number_of_iterations_):
    mean_subtracted_diff_flux_arr_ITER_ = mean_subtracted_diff_flux_arr_all_iterations_[jj,:,:]
    averaged_spec_all_visits_ = np.nanmean(mean_subtracted_diff_flux_arr_ITER_, axis = 0)

    for vv in range(number_of_visits_):

        bin_means,bin_edges,binnumber = \
        stats.binned_statistic(filter_IR[idx_HST],mean_subtracted_diff_flux_arr_ITER_[vv,:],statistic='mean', bins=17)

        bin_width = (bin_edges[1] - bin_edges[0])
        bin_centers_ = bin_edges[1:] - bin_width/2

        add_noise_per_bin_ = 'n'#'y'
        if add_noise_per_bin_ == 'y':
            mean_subtracted_binned_spec_per_visit[vv,:] =  bin_means + 1.*per_bin_errors_[vv,:]
        else:
            mean_subtracted_binned_spec_per_visit[vv,:] =  bin_means + 0.*per_bin_errors_[vv,:]

    averaged_spec_all_visits_binned_ = np.nanmean(mean_subtracted_binned_spec_per_visit, axis = 0)

    obs = normalized_HST_depths_
    calc = averaged_spec_all_visits_binned_
    reduced_chi_sq_[jj] = np.sum(((obs - calc)/depth_err_)**2)

    fig = plt.figure(figsize=(20,5))
    plt.plot(filter_IR[idx_HST], averaged_spec_all_visits_, 'gray', alpha = 0.5)
    plt.plot(bin_centers_, averaged_spec_all_visits_binned_, 'rs-', ms = 7, lw = 15, alpha = 0.5, \
             label = 'Model; Reduced chisq = ' + str(np.round(reduced_chi_sq_,2)))
    plt.errorbar(bin_centers_, normalized_HST_depths_, yerr = 1.*depth_err_, fmt = 'bs-', ms = 7, \
                 lw = 15, alpha = 0.5)
#             label = 'K2-18 HST', lw = 15, alpha = 0.5)
    plt.legend(fontsize = 14)
    plt.show()
print(reduced_chi_sq_)
#today = date.today()
#np.save('reduced_chi_sq_' + str(today.month) + str(today.day) + str(today.year-2000) + '.npy', reduced_chi_sq_)_____no_output_____xxx_____no_output_____# ANIMATE ROTATION

%matplotlib qt

fig = plt.figure(figsize=(2,2))
aa = np.asarray(star_cube[:,:,0])
img_ = plt.imshow(aa, animated=True, cmap='hot', vmin=-1., vmax=1.)

def updatefig(i):
    aa = np.asarray(star_cube[:,:,i])
    img_.set_array(aa)
#    print(i)
    return img_,

anim = animation.FuncAnimation(fig, updatefig, interval=10, blit=True, save_count=number_of_rotation_phases)
plt.axis('off')
#plt.show()_____no_output_____Writer = animation.writers['ffmpeg']
writer = Writer(fps=20, metadata=dict(artist='Me'), bitrate=1800)
anim.save('rotate_.mp4', writer=writer)_____no_output_____
</code>


# Notebook from galleon/Evolutionary-Deep-Learning
Path: EDL_4_COEV_Regression.ipynb

_____no_output_____Original Source: https://github.com/DEAP/deap/blob/master/examples/ga/onemax_numpy.py

DEAP is free software: you can redistribute it and/or modify
   it under the terms of the GNU Lesser General Public License as
   published by the Free Software Foundation, either version 3 of
   the License, or (at your option) any later version.

   DEAP is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
   GNU Lesser General Public License for more details.
   You should have received a copy of the GNU Lesser General Public
   License along with DEAP. If not, see <http://www.gnu.org/licenses/>._____no_output_____

#@title Install DEAP
!pip install deap --quiet
     |██                              | 10kB 23.3MB/s eta 0:00:01
     |████▏                           | 20kB 17.9MB/s eta 0:00:01
     |██████▏                         | 30kB 9.9MB/s eta 0:00:01
     |████████▎                       | 40kB 8.2MB/s eta 0:00:01
     |██████████▍                     | 51kB 5.3MB/s eta 0:00:01
     |████████████▍                   | 61kB 6.2MB/s eta 0:00:01
     |██████████████▌                 | 71kB 6.2MB/s eta 0:00:01
     |████████████████▋               | 81kB 6.1MB/s eta 0:00:01
     |██████████████████▋             | 92kB 6.1MB/s eta 0:00:01
     |████████████████████▊           | 102kB 6.5MB/s eta 0:00:01
     |██████████████████████▉         | 112kB 6.5MB/s eta 0:00:01
     |████████████████████████▉       | 122kB 6.5MB/s eta 0:00:01
     |███████████████████████████     | 133kB 6.5MB/s eta 0:00:01
     |█████████████████████████████   | 143kB 6.5MB/s eta 0:00:01
     |███████████████████████████████ | 153kB 6.5MB/s eta 0:00:01
     |████████████████████████████████| 163kB 6.5MB/s
#@title Imports
import operator
import math
import random
import sys
import time

import numpy as np
from deap import algorithms
from deap import base
from deap import creator
from deap import tools
from deap import gp

from IPython.display import clear_output_____no_output_____#@title Setup Fitness Criteria
#GA
creator.create("FitnessMax", base.Fitness, weights=(1.0,))
creator.create("IndGA", list, fitness=creator.FitnessMax)
#GP
creator.create("FitnessMin", base.Fitness, weights=(-1.0,))
creator.create("Individual", gp.PrimitiveTree, fitness=creator.FitnessMin)_____no_output_____#@title Use Boston Housing dataset as input regression set
from sklearn.datasets import load_boston
X, y = load_boston(return_X_y=True)
X = np.swapaxes(X,0,1)
inputs = X.shape[0]_____no_output_____

# **Boston Housing Features**
* CRIM: Per capita crime rate by town
* ZN: Proportion of residential land zoned for lots over 25,000 sq. ft
* INDUS: Proportion of non-retail business acres per town
* CHAS: Charles River dummy variable (= 1 if tract bounds river; 0 otherwise)
* NOX: Nitric oxide concentration (parts per 10 million)
* RM: Average number of rooms per dwelling
* AGE: Proportion of owner-occupied units built prior to 1940
* DIS: Weighted distances to five Boston employment centers
* RAD: Index of accessibility to radial highways
* TAX: Full-value property tax rate per $10,000

* PTRATIO: Pupil-teacher ratio by town
* B: 1000(Bk — 0.63)², where Bk is the proportion of [people of African American descent] by town
* LSTAT: Percentage of lower status of the population
* TARGET = MEDV: Median value of owner-occupied homes in $1000s
_____no_output_____

#@title Code to Inspect Dataset
import pandas as pd
boston_dataset = load_boston()
boston = pd.DataFrame(boston_dataset.data, columns=boston_dataset.feature_names)
boston.head()_____no_output_____#@title Add Genetic Operators to Toolbox for GA feature selection
toolbox_ga = base.Toolbox()
toolbox_ga.register("float", random.uniform, -1, 1)
toolbox_ga.register("individual", tools.initRepeat, creator.IndGA, toolbox_ga.float, inputs)
toolbox_ga.register("population", tools.initRepeat, list, toolbox_ga.individual)

toolbox_ga.register("select", tools.selTournament, tournsize=3)
toolbox_ga.register("mate", tools.cxTwoPoint)
toolbox_ga.register("mutate", tools.mutGaussian, mu=0, sigma=0.01, indpb=0.05)_____no_output_____def protectedDiv(left, right):
  with np.errstate(divide='ignore',invalid='ignore'):
    x = np.divide(left, right)
    if isinstance(x, np.ndarray):
      x[np.isinf(x)] = 1
      x[np.isnan(x)] = 1
    elif np.isinf(x) or np.isnan(x):
      x = 1
  return x

#@title Create Set of Operators
pset = gp.PrimitiveSet("MAIN", inputs)
pset.addPrimitive(np.add, 2, name="vadd")
pset.addPrimitive(np.subtract, 2, name="vsub")
pset.addPrimitive(np.multiply, 2, name="vmul")
pset.addPrimitive(protectedDiv, 2)
pset.addPrimitive(np.negative, 1, name="vneg")
#pset.addPrimitive(np.cos, 1, name="vcos")
#pset.addPrimitive(np.sin, 1, name="vsin")
#pset.addEphemeralConstant("rand101", lambda: random.randrange(-1000,1000))

toolbox_gp = base.Toolbox()
toolbox_gp.register("expr", gp.genHalfAndHalf, pset=pset, min_=1, max_=2)
toolbox_gp.register("individual", tools.initIterate, creator.Individual, toolbox_gp.expr)
toolbox_gp.register("population", tools.initRepeat, list, toolbox_gp.individual)
toolbox_gp.register("compile", gp.compile, pset=pset)

def evalSymbReg(individual, points):
    # Transform the tree expression in a callable function
    func = toolbox_gp.compile(expr=individual)
    # Evaluate the mean squared error between the expression
    p = np.expand_dims(points, axis=1)
    x = X * np.asarray(p)
    diff = math.sqrt(np.sum((func(*x.tolist()) - y)**2))
    return diff,

toolbox_gp.register("evaluate", evalSymbReg, points=[x/10. for x in range(-10,10)])
toolbox_gp.register("select", tools.selTournament, tournsize=3)
toolbox_gp.register("mate", gp.cxOnePoint)
toolbox_gp.register("expr_mut", gp.genFull, min_=0, max_=2)
toolbox_gp.register("mutate", gp.mutUniform, expr=toolbox_gp.expr_mut, pset=pset)

toolbox_gp.decorate("mate", gp.staticLimit(key=operator.attrgetter("height"), max_value=5))
toolbox_gp.decorate("mutate", gp.staticLimit(key=operator.attrgetter("height"), max_value=5))_____no_output_____#@title Code to Plot the Expression Tree
import matplotlib.pyplot as plt
import networkx as nx

def plot_expression(individual):
  options = {"node_size": 500, "alpha": 0.8}
  nodes, edges, labels = gp.graph(individual)

  g = nx.Graph()
  g.add_nodes_from(nodes)
  g.add_edges_from(edges)

  pos = nx.spring_layout(g)
  nx.draw_networkx_nodes(g, pos, **options)
  nx.draw_networkx_edges(g, pos, width=1.0, alpha=0.5)
  nx.draw_networkx_labels(g, pos, labels, font_size=9, font_color='k')
  plt.show()_____no_output_____#@title Run the Evolution
GA_GEN, GP_GEN, BASE_POP = 1, 10, 10000
pop_ga = toolbox_ga.population(n=BASE_POP*GA_GEN)
pop_gp = toolbox_gp.population(n=BASE_POP*GP_GEN)

stats = tools.Statistics(lambda ind: ind.fitness.values)
stats.register("avg", np.mean)
stats.register("std", np.std)
stats.register("min", np.min)
stats.register("max", np.max)

logbook = tools.Logbook()
logbook.header = "gen", "type", "evals", "std", "min", "avg", "max"

best_ga = tools.selRandom(pop_ga, 1)[0]
best_gp = tools.selRandom(pop_gp, 1)[0]

for ind in pop_gp:
    ind.fitness.values = toolbox_gp.evaluate(ind, points=best_ga)

for ind in pop_ga:
    ind.fitness.values = toolbox_gp.evaluate(best_gp, points=ind)

record = stats.compile(pop_ga)
logbook.record(gen=0, type='ga', evals=len(pop_ga), **record)

record = stats.compile(pop_gp)
logbook.record(gen=0, type='gp', evals=len(pop_gp), **record)

_____no_output_____print(logbook.stream)

CXPB, MUTPB, NGEN = 0.5, 0.2, 100

done = False
# Begin the evolution
for g in range(1, NGEN):
    if done: break

    if (g+1) % GA_GEN == 0:
      # Select and clone the offspring
      off_ga = toolbox_ga.select(pop_ga, len(pop_ga))
      off_ga = [toolbox_ga.clone(ind) for ind in off_ga]

      # Apply crossover and mutation
      for ind1, ind2 in zip(off_ga[::2], off_ga[1::2]):
          if random.random() < CXPB:
              toolbox_ga.mate(ind1, ind2)
              del ind1.fitness.values
              del ind2.fitness.values

      for ind in off_ga:
          if random.random() < MUTPB:
              toolbox_ga.mutate(ind)
              del ind.fitness.values

      pop_ga = off_ga

    if (g+1) % GP_GEN == 0:
      off_gp = toolbox_gp.select(pop_gp, len(pop_gp))
      off_gp = [toolbox_gp.clone(ind) for ind in off_gp]

      for ind1, ind2 in zip(off_gp[::2], off_gp[1::2]):
          if random.random() < CXPB:
              toolbox_gp.mate(ind1, ind2)
              del ind1.fitness.values
              del ind2.fitness.values

      for ind in off_gp:
          if random.random() < MUTPB:
              toolbox_gp.mutate(ind)
              del ind.fitness.values

      # Replace the old population by the offspring
      pop_gp = off_gp

    # Evaluate the individuals with an invalid fitness
    for ind in pop_gp:
        ind.fitness.values = toolbox_gp.evaluate(ind, points=best_ga)
    for ind in pop_ga:
        ind.fitness.values = toolbox_gp.evaluate(best_gp, points=ind)

    record = stats.compile(pop_ga)
    best_ga_min = record["min"]
    logbook.record(gen=g, type='ga', evals=len(pop_ga), **record)

    record = stats.compile(pop_gp)
    best_gp_min = record["min"]
    logbook.record(gen=g, type='gp', evals=len(pop_gp), **record)
    #print(logbook.stream)
    print(record)

    best_ga = tools.selBest(pop_ga, 1)[0]
    best_gp = tools.selBest(pop_gp, 1)[0]

    if best_gp_min < 135:
        done = True
        print("Sovled")

    if (g+1) % 1 == 0:
      clear_output()
      print(logbook.stream)
      plot_expression(best_gp)
      output = ["%.2f" % e for e in best_ga]
      print(f"Best GA {output}")
      time.sleep(1)

print("Best individual GA is %s, %s" % (best_ga, best_ga.fitness.values))
print("Best individual GP is %s, %s" % (best_gp, best_gp.fitness.values))
plot_expression(best_gp)7  	ga  	10000 	186429     	181.891	11623      	5.00766e+06
7  	gp  	100000	4.79252e+07	132.819	438389     	1.09956e+10