# Notebook from hwiks/SpaceFluff
Path: DataViz.ipynb

# Data visualization_____no_output_____Here there are three different functions for getting a quick view to the opinions on a specific image in a certain workflow_____no_output_____

<code>
import pandas as pd
import json
import seaborn as sns
import matplotlib.pyplot as plt
import numpy as np
from PIL import Image
from pylab import *_____no_output_____filename_classify = '/home/anna/Desktop/SUNDIAL/images/beta_space-fluff_classifications_clean.csv'
filename_onthego = '/home/anna/Desktop/SUNDIAL/images/beta_space-fluff_onthego_clean.csv'
filename_hardcore = '/home/anna/Desktop/SUNDIAL/images/beta_space-fluff_hardcore_clean.csv'_____no_output_____data_classify = pd.read_csv(filename_classify)
data_onthego = pd.read_csv(filename_onthego)
data_hardcore = pd.read_csv(filename_hardcore)_____no_output_____def answers_count(file, taskvalue, answer):
    howmany_answers = 0
    for line in file[taskvalue]:
        if str(line).lower() == answer.lower():
            howmany_answers += 1
    return howmany_answers

def general_opinion_onthego(image_ID):

    image = '/home/anna/Desktop/SUNDIAL/images/beta/'+image_ID
    im = Image.open(image)
    imshow(im)

    file = data_onthego
    image_classifications = file.loc[file['subject_name_onthego'] == image_ID]

    total = len(image_classifications)
    galaxies = answers_count(image_classifications, 'taskvalue_T0_onthego', 'galaxy')
    clusters = answers_count(image_classifications, 'taskvalue_T0_onthego', 'Group of objects (cluster)')
    other = answers_count(image_classifications, 'taskvalue_T0_onthego', 'Something else/empty center')

    print('Here are the stats for the image: ', image_ID)

    print('Total numeber of votes: ', total)

    print('Number of votes for:')

    print('Galaxy: ', galaxies)
    print('Cluster: ', clusters)
    print('Something else: ', other)

def general_opinion_classify(image_ID):

    image = '/home/anna/Desktop/SUNDIAL/images/beta/'+image_ID
    im = Image.open(image)
    imshow(im)

    file = data_classify
    image_classifications = file.loc[file['subject_name_classify'] == image_ID]

    total = len(image_classifications)
    galaxies = answers_count(image_classifications, 'taskvalue_T0_classify', 'galaxy')
    clusters = answers_count(image_classifications, 'taskvalue_T0_classify', 'Group of objects (cluster)')
    other = answers_count(image_classifications, 'taskvalue_T0_classify', 'Something else/empty center')
    fluffs = answers_count(image_classifications, 'taskvalue_T1_classify', 'fluffy')
    brights = answers_count(image_classifications, 'taskvalue_T1_classify', 'bright')
    scattered = answers_count(image_classifications, 'taskvalue_T2_classify', 'Scattered light')
    small_star = answers_count(image_classifications, 'taskvalue_T2_classify', 'looks like a small star')
    nothing = answers_count(image_classifications, 'taskvalue_T2_classify', 'nothing: background too bright or galaxy too faint')
    other_other = answers_count(image_classifications, 'taskvalue_T2_classify', 'something else')

    print('Here are the stats for the image: ', image_ID)

    print('Total numeber of votes: ', total)

    print('Number of votes for:')

    print('Galaxy: ', galaxies)
    if galaxies > 0:
        print('\t Fluffy: ', fluffs)
        print('\t Bright: ', brights)
    print('Cluster: ', clusters)
    print('Something else: ', other)
    if other > 0:
        print('\t Scattered light: ', scattered)
        print('\t Looking like a small star: ', small_star)
        print('\t Nothing: ', nothing)
        print('\t Something else: ', other_other)

def general_opinion_hardcore(image_ID):

    image = '/home/anna/Desktop/SUNDIAL/images/beta/'+image_ID
    im = Image.open(image)
    imshow(im)

    file = data_hardcore
    image_classifications = file.loc[file['subject_name'] == image_ID]

    total = len(image_classifications)

    galaxies = answers_count(image_classifications, 'taskvalue_T0', 'galaxy')
    clusters = answers_count(image_classifications, 'taskvalue_T0', 'Group of objects (cluster)')
    other = answers_count(image_classifications, 'taskvalue_T0', 'Something else/empty center')

    red = answers_count(image_classifications, 'taskvalue_T1', 'Red/yellow')
    blue = answers_count(image_classifications, 'taskvalue_T1', 'white/blue')
    impossible = answers_count(image_classifications, 'taskvalue_T1', 'impossible to say')

    fluffs = answers_count(image_classifications, 'taskvalue_T2', 'fluffy')
    brights = answers_count(image_classifications, 'taskvalue_T2', 'bright')

    nocore = answers_count(image_classifications, 'taskvalue_T3', 'no/unsure')
    bright_core= answers_count(image_classifications, 'taskvalue_T3', 'yes, a bright point')
    bulge = answers_count(image_classifications, 'taskvalue_T3', 'yes, a bulge')

    rounds = answers_count(image_classifications, 'taskvalue_T4', 'round')
    elliptical = answers_count(image_classifications, 'taskvalue_T4', 'elliptical')
    disturbed = answers_count(image_classifications, 'taskvalue_T4', 'distorted/disturbed')

    dense = answers_count(image_classifications, 'taskvalue_T5', 'smooth and dense')
    fuzzy = answers_count(image_classifications, 'taskvalue_T5', 'smooth and fuzzy')
    clumpy = answers_count(image_classifications, 'taskvalue_T5', 'clumpy and/or featured')

    scattered = answers_count(image_classifications, 'taskvalue_T9', 'Scattered light')
    small_star = answers_count(image_classifications, 'taskvalue_T9', 'looks like a small star')
    nothing = answers_count(image_classifications, 'taskvalue_T9', 'nothing: background too bright or galaxy too faint')
    other_other = answers_count(image_classifications, 'taskvalue_T9', 'something else')

    print('Here are the stats for the image: ', image_ID)

    print('Total numeber of votes: ', total)

    print('Number of votes for:')

    print('Galaxy: ', galaxies)
    if galaxies > 0:
        print('\t The color:')
        print('\t \t Red/yellow: ', red)
        print('\t \t Blue/white: ', blue)
        print('\t \t Impossible to say: ', impossible)

        print('\t The perceived consistency: ')
        print('\t Fluffy: ', fluffs)
        print('\t Bright: ', brights)

        print('\t The core:')
        print('\t \t No core: ', nocore)
        print('\t \t Bright core: ', bright_core)
        print('\t \t Bulge: ', bulge)

        print('\t The shape:')
        print('\t \t Round: ', rounds)
        print('\t \t Elliptical: ', elliptical)
        print('\t \t Distorted/disturbed: ', disturbed)

        print('\t The texture:')
        print('\t \t Smooth and dense: ', dense)
        print('\t \t Smooth and fuzzy: ', fuzzy)
        print('\t \t Clumpy and/or featured: ', clumpy)

    print('Cluster: ', clusters)
    print('Something else: ', other)
    if other > 0:
        print('\t Scattered light: ', scattered)
        print('\t Looking like a small star: ', small_star)
        print('\t Nothing: ', nothing)
        print('\t Something else: ', other_other)    _____no_output_____general_opinion_classify('UDGcand_15_insp.png')Here are the stats for the image:  UDGcand_15_insp.png
Total numeber of votes:  11
Number of votes for:
Galaxy:  11
	 Fluffy:  4
	 Bright:  7
Cluster:  0
Something else:  0
general_opinion_onthego('UDGcand_15_insp.png')Here are the stats for the image:  UDGcand_15_insp.png
Total numeber of votes:  6
Number of votes for:
Galaxy:  6
Cluster:  0
Something else:  0
general_opinion_hardcore('UDGcand_15_insp.png')Here are the stats for the image:  UDGcand_15_insp.png
Total numeber of votes:  4
Number of votes for:
Galaxy:  2
	 The color:
	 	 Red/yellow:  2
	 	 Blue/white:  0
	 	 Impossible to say:  0
	 The perceived consistency:
	 Fluffy:  0
	 Bright:  2
	 The core:
	 	 No core:  0
	 	 Bright core:  1
	 	 Bulge:  1
	 The shape:
	 	 Round:  2
	 	 Elliptical:  0
	 	 Distorted/disturbed:  0
	 The texture:
	 	 Smooth and dense:  2
	 	 Smooth and fuzzy:  0
	 	 Clumpy and/or featured:  0
Cluster:  2
Something else:  0
def bars(answers, how_many):
    fig = plt.figure()
    ax = fig.add_axes([0,0,1,1])
    ax.set_title('Answers', size = 20)
    ax.bar(answers, how_many, color = ['orange','lightseagreen', 'turquoise'])
    plt.show()

def bar_plot(file, taskvalue):

    other = answers_count(file, taskvalue,  'Something else/empty center')
    clusters = answers_count(file,  taskvalue,  'Group of objects (cluster)' )
    galaxies = answers_count(file,  taskvalue,  'Galaxy' )

    answers = ['Something else/empty center', 'Group of objects (cluster)',  'Galaxy']
    how_many = [other, clusters, galaxies]
    print(how_many)
    bars(answers, how_many)
    _____no_output_____bar_plot(data_onthego, 'taskvalue_T0_onthego')[403, 684, 1258]
bar_plot(data_classify, 'taskvalue_T0_classify')[551, 1306, 2064]
bar_plot(data_hardcore, 'taskvalue_T0')[261, 808, 737]

</code>


# Notebook from djbsmith/dmu_products
Path: dmu1/dmu1_ml_ELAIS-N2/2_Merging.ipynb

# ELAIS-N2 master catalogue

This notebook presents the merge of the various pristine catalogues to produce HELP mater catalogue on ELAIS-N2._____no_output_____

from herschelhelp_internal import git_version
print("This notebook was run with herschelhelp_internal version: \n{}".format(git_version()))
import datetime
print("This notebook was executed on: \n{}".format(datetime.datetime.now()))This notebook was run with herschelhelp_internal version:
0246c5d (Thu Jan 25 17:01:47 2018 +0000) [with local modifications]
This notebook was executed on:
2018-02-18 00:53:05.376795
%matplotlib inline
#%config InlineBackend.figure_format = 'svg'

import matplotlib.pyplot as plt
plt.rc('figure', figsize=(10, 6))

import os
import time

from astropy import units as u
from astropy.coordinates import SkyCoord
from astropy.table import Column, Table
import numpy as np
from pymoc import MOC

from herschelhelp_internal.masterlist import merge_catalogues, nb_merge_dist_plot, specz_merge
from herschelhelp_internal.utils import coords_to_hpidx, ebv, gen_help_id, inMoc_____no_output_____TMP_DIR = os.environ.get('TMP_DIR', "./data_tmp")
OUT_DIR = os.environ.get('OUT_DIR', "./data")
SUFFIX = os.environ.get('SUFFIX', time.strftime("_%Y%m%d"))

try:
    os.makedirs(OUT_DIR)
except FileExistsError:
    pass_____no_output_____

## I - Reading the prepared pristine catalogues_____no_output_____

wfc = Table.read("{}/INT-WFC.fits".format(TMP_DIR))
rcs = Table.read("{}/RCSLenS.fits".format(TMP_DIR))
ps1 = Table.read("{}/PS1.fits".format(TMP_DIR))
sparcs = Table.read("{}/SpARCS.fits".format(TMP_DIR))
swire= Table.read("{}/SWIRE.fits".format(TMP_DIR))_____no_output_____

## II - Merging tables

We first merge the optical catalogues and then add the infrared ones: WFC, DXS, SpARCS, HSC, PS1, SERVS, SWIRE.

At every step, we look at the distribution of the distances to the nearest source in the merged catalogue to determine the best crossmatching radius._____no_output_____### WFC_____no_output_____

master_catalogue = wfc
master_catalogue['wfc_ra'].name = 'ra'
master_catalogue['wfc_dec'].name = 'dec'_____no_output_____

### Add RCSLenS_____no_output_____

nb_merge_dist_plot(
    SkyCoord(master_catalogue['ra'], master_catalogue['dec']),
    SkyCoord(rcs['rcs_ra'], rcs['rcs_dec'])
)_____no_output_____# Given the graph above, we use 0.8 arc-second radius
master_catalogue = merge_catalogues(master_catalogue, rcs, "rcs_ra", "rcs_dec", radius=0.8*u.arcsec)_____no_output_____

### Add PanSTARRS_____no_output_____

nb_merge_dist_plot(
    SkyCoord(master_catalogue['ra'], master_catalogue['dec']),
    SkyCoord(ps1['ps1_ra'], ps1['ps1_dec'])
)_____no_output_____# Given the graph above, we use 0.8 arc-second radius
master_catalogue = merge_catalogues(master_catalogue, ps1, "ps1_ra", "ps1_dec", radius=0.8*u.arcsec)_____no_output_____

### Add SpARCS_____no_output_____

nb_merge_dist_plot(
    SkyCoord(master_catalogue['ra'], master_catalogue['dec']),
    SkyCoord(sparcs['sparcs_ra'], sparcs['sparcs_dec'])
)_____no_output_____# Given the graph above, we use 0.8 arc-second radius
master_catalogue = merge_catalogues(master_catalogue, sparcs, "sparcs_ra", "sparcs_dec", radius=0.8*u.arcsec)_____no_output_____

### Add SWIRE_____no_output_____

nb_merge_dist_plot(
    SkyCoord(master_catalogue['ra'], master_catalogue['dec']),
    SkyCoord(swire['swire_ra'], swire['swire_dec'])
)_____no_output_____# Given the graph above, we use 1 arc-second radius
master_catalogue = merge_catalogues(master_catalogue, swire, "swire_ra", "swire_dec", radius=1.*u.arcsec)_____no_output_____

### Cleaning

When we merge the catalogues, astropy masks the non-existent values (e.g. when a row comes only from a catalogue and has no counterparts in the other, the columns from the latest are masked for that row). We indicate to use NaN for masked values for floats columns, False for flag columns and -1 for ID columns._____no_output_____

for col in master_catalogue.colnames:
    if "m_" in col or "merr_" in col or "f_" in col or "ferr_" in col or "stellarity" in col:
        master_catalogue[col].fill_value = np.nan
    elif "flag" in col:
        master_catalogue[col].fill_value = 0
    elif "id" in col:
        master_catalogue[col].fill_value = -1

master_catalogue = master_catalogue.filled()_____no_output_____master_catalogue[:10].show_in_notebook()_____no_output_____

## III - Merging flags and stellarity

Each pristine catalogue contains a flag indicating if the source was associated to a another nearby source that was removed during the cleaning process.  We merge these flags in a single one._____no_output_____

flag_cleaned_columns = [column for column in master_catalogue.colnames
                        if 'flag_cleaned' in column]

flag_column = np.zeros(len(master_catalogue), dtype=bool)
for column in flag_cleaned_columns:
    flag_column |= master_catalogue[column]

master_catalogue.add_column(Column(data=flag_column, name="flag_cleaned"))
master_catalogue.remove_columns(flag_cleaned_columns)_____no_output_____

Each pristine catalogue contains a flag indicating the probability of a source being a Gaia object (0: not a Gaia object, 1: possibly, 2: probably, 3: definitely).  We merge these flags taking the highest value._____no_output_____

flag_gaia_columns = [column for column in master_catalogue.colnames
                     if 'flag_gaia' in column]

master_catalogue.add_column(Column(
    data=np.max([master_catalogue[column] for column in flag_gaia_columns], axis=0),
    name="flag_gaia"
))
master_catalogue.remove_columns(flag_gaia_columns)_____no_output_____

Each prisitine catalogue may contain one or several stellarity columns indicating the probability (0 to 1) of each source being a star.  We merge these columns taking the highest value._____no_output_____

stellarity_columns = [column for column in master_catalogue.colnames
                      if 'stellarity' in column]

print(", ".join(stellarity_columns))wfc_stellarity, rcs_stellarity, sparcs_stellarity, swire_stellarity
# We create an masked array with all the stellarities and get the maximum value, as well as its
# origin.  Some sources may not have an associated stellarity.
stellarity_array = np.array([master_catalogue[column] for column in stellarity_columns])
stellarity_array = np.ma.masked_array(stellarity_array, np.isnan(stellarity_array))

max_stellarity = np.max(stellarity_array, axis=0)
max_stellarity.fill_value = np.nan

no_stellarity_mask = max_stellarity.mask

master_catalogue.add_column(Column(data=max_stellarity.filled(), name="stellarity"))

stellarity_origin = np.full(len(master_catalogue), "NO_INFORMATION", dtype="S20")
stellarity_origin[~no_stellarity_mask] = np.array(stellarity_columns)[np.argmax(stellarity_array, axis=0)[~no_stellarity_mask]]

master_catalogue.add_column(Column(data=stellarity_origin, name="stellarity_origin"))

master_catalogue.remove_columns(stellarity_columns)_____no_output_____

## IV - Adding E(B-V) column_____no_output_____

master_catalogue.add_column(
    ebv(master_catalogue['ra'], master_catalogue['dec'])
)_____no_output_____

## V.a - Adding HELP unique identifiers and field columns_____no_output_____

master_catalogue.add_column(Column(gen_help_id(master_catalogue['ra'], master_catalogue['dec']),
                                   name="help_id"))
master_catalogue.add_column(Column(np.full(len(master_catalogue), "ELAIS-N2", dtype='<U18'),
                                   name="field"))_____no_output_____# Check that the HELP Ids are unique
if len(master_catalogue) != len(np.unique(master_catalogue['help_id'])):
    print("The HELP IDs are not unique!!!")
else:
    print("OK!")OK!

## V.b - Adding Spec-z_____no_output_____

specz =  Table.read("../../dmu23/dmu23_ELAIS-N2/data/ELAIS-N2-specz-v2.1.fits")_____no_output_____nb_merge_dist_plot(
    SkyCoord(master_catalogue['ra'], master_catalogue['dec']),
    SkyCoord(specz['ra'] * u.deg, specz['dec'] * u.deg)
)_____no_output_____master_catalogue = specz_merge(master_catalogue, specz, radius=1. * u.arcsec)_____no_output_____

## VI - Choosing between multiple values for the same filter

_____no_output_____### CFHT Megacam SpARCS vs RCSLenS
SpARCS apears to be significantly deeper and contains both total and aperture magnitudes so we take SpARCS over RCSLenS if both are available

| Survey  | Bands     |
|---------|-----------|
| SpARCS  | ugrz      |
| RCSLenS | grizy     |_____no_output_____

megacam_origin = Table()
megacam_origin.add_column(master_catalogue['help_id'])_____no_output_____
megacam_stats = Table()
megacam_stats.add_column(Column(data=['u','g','r','i','z','y'], name="Band"))
for col in [ "SpARCS", "RCSLenS"]:
    megacam_stats.add_column(Column(data=np.full(6, 0), name="{}".format(col)))
    megacam_stats.add_column(Column(data=np.full(6, 0), name="use {}".format(col)))/opt/anaconda3/envs/herschelhelp_internal/lib/python3.6/site-packages/numpy/core/numeric.py:301: FutureWarning: in the future, full(6, 0) will return an array of dtype('int64')
  format(shape, fill_value, array(fill_value).dtype), FutureWarning)
megacam_bands = ['u','g','r','i','z','y'] # Lowercase naming convention (k is Ks)
for band in ['g', 'r', 'z']: #the bands in both catalogues

    # Megacam total flux
    has_sparcs = ~np.isnan(master_catalogue['f_cfht_megacam_' + band])
    has_rcs = ~np.isnan(master_catalogue['f_rcs_' + band])

    use_sparcs = has_sparcs
    use_rcs = has_rcs & ~has_sparcs

    f_megacam = np.full(len(master_catalogue), np.nan)
    f_megacam[use_sparcs] = master_catalogue['f_cfht_megacam_' + band][use_sparcs]
    f_megacam[use_rcs] = master_catalogue['f_rcs_' + band][use_rcs]

    ferr_megacam = np.full(len(master_catalogue), np.nan)
    ferr_megacam[use_sparcs] = master_catalogue['ferr_cfht_megacam_' + band][use_sparcs]
    ferr_megacam[use_rcs] = master_catalogue['ferr_rcs_' + band][use_rcs]

    m_megacam = np.full(len(master_catalogue), np.nan)
    m_megacam[use_sparcs] = master_catalogue['m_cfht_megacam_' + band][use_sparcs]
    m_megacam[use_rcs] = master_catalogue['m_rcs_' + band][use_rcs]

    merr_megacam = np.full(len(master_catalogue), np.nan)
    merr_megacam[use_sparcs] = master_catalogue['merr_cfht_megacam_' + band][use_sparcs]
    merr_megacam[use_rcs] = master_catalogue['merr_rcs_' + band][use_rcs]

    flag_megacam = np.full(len(master_catalogue), np.nan)
    flag_megacam[use_sparcs] = master_catalogue['flag_cfht_megacam_' + band][use_sparcs]
    flag_megacam[use_rcs] = master_catalogue['flag_rcs_' + band][use_rcs]

    master_catalogue.add_column(Column(data=f_megacam, name="f_megacam_" + band))
    master_catalogue.add_column(Column(data=ferr_megacam, name="ferr_megacam_" + band))
    master_catalogue.add_column(Column(data=m_megacam, name="m_megacam_" + band))
    master_catalogue.add_column(Column(data=merr_megacam, name="merr_megacam_" + band))
    master_catalogue.add_column(Column(data=flag_megacam, name="flag_megacam_" + band))

    old_columns = ['f_cfht_megacam_' + band,
                               'ferr_cfht_megacam_' + band,
                               'm_cfht_megacam_' + band,
                               'merr_cfht_megacam_' + band,
                               'flag_cfht_megacam_' + band,
                               'f_rcs_' + band,
                               'ferr_rcs_' + band,
                               'm_rcs_' + band,
                               'merr_rcs_' + band,
                               'flag_rcs_' + band]

    master_catalogue.remove_columns(old_columns)

    origin = np.full(len(master_catalogue), '     ', dtype='<U5')
    origin[use_sparcs] = "SpARCS"
    origin[use_rcs] = "RCSLenS"

    megacam_origin.add_column(Column(data=origin, name= 'f_megacam_' + band ))

    megacam_stats['RCSLenS'][megacam_stats['Band'] == band] = np.sum(has_rcs)
    megacam_stats['SpARCS'][megacam_stats['Band'] == band] = np.sum(has_sparcs)
    megacam_stats['use RCSLenS'][megacam_stats['Band'] == band] = np.sum(use_rcs)
    megacam_stats['use SpARCS'][megacam_stats['Band'] == band] = np.sum(use_sparcs)
_____no_output_____#No aperture fluxes in rcs so just rename all the sparcs columns
for col in master_catalogue.colnames:
    if '_cfht_megacam_' in col:
        master_catalogue[col].name = col.replace('_cfht_megacam_','_megacam_')

    if ('_rcs_' in col) and ('_ap_rcs_' not in col):
        master_catalogue[col].name = col.replace('_rcs_','_megacam_')

    if '_ap_rcs_' in col:
        master_catalogue.remove_columns(col)_____no_output_____megacam_stats.show_in_notebook()_____no_output_____
megacam_origin.write("{}/elais-n2_megacam_fluxes_origins{}.fits".format(OUT_DIR, SUFFIX), overwrite=True)_____no_output_____

## VII.a Wavelength domain coverage

We add a binary `flag_optnir_obs` indicating that a source was observed in a given wavelength domain:

- 1 for observation in optical;
- 2 for observation in near-infrared;
- 4 for observation in mid-infrared (IRAC).

It's an integer binary flag, so a source observed both in optical and near-infrared by not in mid-infrared would have this flag at 1 + 2 = 3.

*Note 1: The observation flag is based on the creation of multi-order coverage maps from the catalogues, this may not be accurate, especially on the edges of the coverage.*

*Note 2: Being on the observation coverage does not mean having fluxes in that wavelength domain. For sources observed in one domain but having no flux in it, one must take into consideration de different depths in the catalogue we are using.*_____no_output_____

wfc_moc = MOC(filename="../../dmu0/dmu0_INTWFC/data/en2_intwfc_v2.1_HELP_coverage_MOC.fits")
sparcs_moc = MOC(filename="../../dmu0/dmu0_SpARCS/data/SpARCS_HELP_ELAIS-N2_MOC.fits")
rcs_moc = MOC(filename="../../dmu0/dmu0_RCSLenS/data/RCSLenS_ELAIS-N2_MOC.fits")
ps1_moc = MOC(filename="../../dmu0/dmu0_PanSTARRS1-3SS/data/PanSTARRS1-3SS_ELAIS-N2_MOC.fits")
swire_moc = MOC(filename="../../dmu0/dmu0_DataFusion-Spitzer/data/DF-SWIRE_ELAIS-N2_MOC.fits")
_____no_output_____was_observed_optical = inMoc(
    master_catalogue['ra'], master_catalogue['dec'],
    wfc_moc + sparcs_moc + ps1_moc)

#There are seemingly no MIR observations in ELAIS N2
#was_observed_nir = inMoc(
#    master_catalogue['ra'], master_catalogue['dec'],
#
#)

was_observed_mir = inMoc(
    master_catalogue['ra'], master_catalogue['dec'],
    swire_moc
)_____no_output_____master_catalogue.add_column(
    Column(
        1 * was_observed_optical +  4 * was_observed_mir, #2 * was_observed_nir +
        name="flag_optnir_obs")
)_____no_output_____

## VII.b Wavelength domain detection

We add a binary `flag_optnir_det` indicating that a source was detected in a given wavelength domain:

- 1 for detection in optical;
- 2 for detection in near-infrared;
- 4 for detection in mid-infrared (IRAC).

It's an integer binary flag, so a source detected both in optical and near-infrared by not in mid-infrared would have this flag at 1 + 2 = 3.

*Note 1: We use the total flux columns to know if the source has flux, in some catalogues, we may have aperture flux and no total flux.*

To get rid of artefacts (chip edges, star flares, etc.) we consider that a source is detected in one wavelength domain when it has a flux value in **at least two bands**. That means that good sources will be excluded from this flag when they are on the coverage of only one band._____no_output_____

# SpARCS is a catalogue of sources detected in r (with fluxes measured at
# this prior position in the other bands).  Thus, we are only using the r
# CFHT band.
# Check to use catalogue flags from HSC and PanSTARRS.
nb_optical_flux = (
    1 * ~np.isnan(master_catalogue['f_wfc_u']) +
    1 * ~np.isnan(master_catalogue['f_wfc_g']) +
    1 * ~np.isnan(master_catalogue['f_wfc_r']) +
    1 * ~np.isnan(master_catalogue['f_wfc_i']) +
    1 * ~np.isnan(master_catalogue['f_wfc_z']) +
    1 * ~np.isnan(master_catalogue['f_gpc1_g']) +
    1 * ~np.isnan(master_catalogue['f_gpc1_r']) +
    1 * ~np.isnan(master_catalogue['f_gpc1_i']) +
    1 * ~np.isnan(master_catalogue['f_gpc1_z']) +
    1 * ~np.isnan(master_catalogue['f_gpc1_y']) +
    1 * ~np.isnan(master_catalogue['f_megacam_u']) +
    1 * ~np.isnan(master_catalogue['f_megacam_g']) +
    1 * ~np.isnan(master_catalogue['f_megacam_r']) +
    1 * ~np.isnan(master_catalogue['f_megacam_i']) +
    1 * ~np.isnan(master_catalogue['f_megacam_z']) +
    1 * ~np.isnan(master_catalogue['f_megacam_y'])
)

nb_nir_flux = (
    1 * np.zeros(len(master_catalogue), dtype=bool)
)

nb_mir_flux = (
    1 * ~np.isnan(master_catalogue['f_irac_i1']) +
    1 * ~np.isnan(master_catalogue['f_irac_i2']) +
    1 * ~np.isnan(master_catalogue['f_irac_i3']) +
    1 * ~np.isnan(master_catalogue['f_irac_i4'])
)_____no_output_____has_optical_flux = nb_optical_flux >= 2
has_nir_flux = nb_nir_flux >= 2
has_mir_flux = nb_mir_flux >= 2

master_catalogue.add_column(
    Column(
        1 * has_optical_flux + 2 * has_nir_flux + 4 * has_mir_flux,
        name="flag_optnir_det")
)_____no_output_____

## VIII - Cross-identification table

We are producing a table associating to each HELP identifier, the identifiers of the sources in the pristine catalogue. This can be used to easily get additional information from them._____no_output_____

master_catalogue['help_id',
                 'wfc_id',
                 'rcs_id',
                 'swire_intid',
                 'sparcs_intid',
                 'ps1_id'].write(
    "{}/master_list_cross_ident_elais-n2{}.fits".format(OUT_DIR, SUFFIX), overwrite=True)
master_catalogue.remove_columns(['wfc_id',
                                 'rcs_id',
                                 'swire_intid',
                                 'sparcs_intid',
                                 'ps1_id'])_____no_output_____

## IX - Adding HEALPix index

We are adding a column with a HEALPix index at order 13 associated with each source._____no_output_____

master_catalogue.add_column(Column(
    data=coords_to_hpidx(master_catalogue['ra'], master_catalogue['dec'], order=13),
    name="hp_idx"
))_____no_output_____

## IX - Saving the catalogue_____no_output_____

columns = ["help_id", "field", "ra", "dec", "hp_idx"]

bands = [column[5:] for column in master_catalogue.colnames if 'f_ap' in column]
for band in bands:
    columns += ["f_ap_{}".format(band), "ferr_ap_{}".format(band),
                "m_ap_{}".format(band), "merr_ap_{}".format(band),
                "f_{}".format(band), "ferr_{}".format(band),
                "m_{}".format(band), "merr_{}".format(band),
                #"flag_{}".format(band)
               ]

#columns += ['m_rcs_g', 'merr_rcs_g', 'f_rcs_g', 'ferr_rcs_g',
#            'm_rcs_r', 'merr_rcs_r', 'f_rcs_r', 'ferr_rcs_r',
#            'm_rcs_i', 'merr_rcs_i', 'f_rcs_i', 'ferr_rcs_i',
#            'm_rcs_z', 'merr_rcs_z', 'f_rcs_z', 'ferr_rcs_z',
#            'm_rcs_y', 'merr_rcs_y', 'f_rcs_y', 'ferr_rcs_y']

columns += [ 'm_megacam_i', 'ferr_megacam_i', 'merr_megacam_i', 'flag_megacam_i', 'f_megacam_i',
             'm_megacam_y', 'ferr_megacam_y', 'merr_megacam_y', 'flag_megacam_y', 'f_megacam_y']

columns += ["stellarity", "stellarity_origin", "flag_cleaned", "flag_merged", "flag_gaia", "flag_optnir_obs", "flag_optnir_det", "ebv",
           'zspec_association_flag', 'zspec_qual', 'zspec']_____no_output_____# We check for columns in the master catalogue that we will not save to disk.
print("Missing columns: {}".format(set(master_catalogue.colnames) - set(columns)))Missing columns: {'flag_megacam_g', 'flag_gpc1_z', 'specz_id', 'flag_wfc_i', 'flag_megacam_u', 'flag_gpc1_y', 'flag_megacam_r', 'flag_wfc_g', 'flag_gpc1_r', 'flag_megacam_z', 'flag_gpc1_i', 'flag_wfc_z', 'flag_irac_i1', 'flag_wfc_r', 'flag_irac_i3', 'flag_irac_i2', 'flag_wfc_u', 'flag_gpc1_g', 'flag_irac_i4'}
master_catalogue[columns].write("{}/master_catalogue_elais-n2{}.fits".format(OUT_DIR, SUFFIX), overwrite=True)_____no_output_____


# Notebook from ShepherdCode/Soars2021
Path: Notebooks/ORF_CNN_133.alien3.ipynb

# ORF recognition by CNN

Alien with libcudnn8 after clear cache._____no_output_____

<code>
import time
t = time.time()
time.strftime('%Y-%m-%d %H:%M:%S %Z', time.localtime(t))_____no_output_____PC_SEQUENCES=10000   # how many protein-coding sequences
NC_SEQUENCES=10000   # how many non-coding sequences
PC_TESTS=1000
NC_TESTS=1000
RNA_LEN=32            # how long is each sequence
CDS_LEN=16            # min CDS len to be coding
ALPHABET=4          # how many different letters are possible
INPUT_SHAPE_2D = (RNA_LEN,ALPHABET,1) # Conv2D needs 3D inputs
INPUT_SHAPE = (RNA_LEN,ALPHABET) # Conv1D needs 2D inputs
FILTERS = 16   # how many different patterns the model looks for
NEURONS = 16
DROP_RATE = 0.0
WIDTH = 3   # how wide each pattern is, in bases
STRIDE_2D = (1,1)  # For Conv2D how far in each direction
STRIDE = 1 # For Conv1D, how far between pattern matches, in bases
EPOCHS=100  # how many times to train on all the data
SPLITS=3  # SPLITS=3 means train on 2/3 and validate on 1/3
FOLDS=3  # train the model this many times (range 1 to SPLITS)_____no_output_____import sys
IN_COLAB = False
try:
    from google.colab import drive
    IN_COLAB = True
except:
    pass
if IN_COLAB:
    print("On Google CoLab, mount cloud-local file, get our code from GitHub.")
    PATH='/content/drive/'
    #drive.mount(PATH,force_remount=True)  # hardly ever need this
    #drive.mount(PATH)    # Google will require login credentials
    DATAPATH=PATH+'My Drive/data/'  # must end in "/"
    import requests
    r = requests.get('https://raw.githubusercontent.com/ShepherdCode/Soars2021/master/SimTools/RNA_describe.py')
    with open('RNA_describe.py', 'w') as f:
        f.write(r.text)
    from RNA_describe import ORF_counter
    from RNA_describe import Random_Base_Oracle
    r = requests.get('https://raw.githubusercontent.com/ShepherdCode/Soars2021/master/SimTools/RNA_prep.py')
    with open('RNA_prep.py', 'w') as f:
        f.write(r.text)
    from RNA_prep import prepare_inputs_len_x_alphabet
else:
        print("CoLab not working. On my PC, use relative paths.")
        DATAPATH='data/'  # must end in "/"
        sys.path.append("..") # append parent dir in order to use sibling dirs
        from SimTools.RNA_describe import ORF_counter,Random_Base_Oracle
        from SimTools.RNA_prep import prepare_inputs_len_x_alphabet

MODELPATH="BestModel"  # saved on cloud instance and lost after logout
#MODELPATH=DATAPATH+MODELPATH  # saved on Google Drive but requires loginCoLab not working. On my PC, use relative paths.
from os import listdir
import csv
from zipfile import ZipFile

import numpy as np
import pandas as pd
from scipy import stats  # mode

from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import KFold
from sklearn.model_selection import cross_val_score

from keras.models import Sequential
from keras.layers import Dense,Embedding,Dropout
from keras.layers import Conv1D,Conv2D
from keras.layers import Flatten,MaxPooling1D,MaxPooling2D
from keras.losses import BinaryCrossentropy
# tf.keras.losses.BinaryCrossentropy

import matplotlib.pyplot as plt
from matplotlib import colors
mycmap = colors.ListedColormap(['red','blue'])  # list color for label 0 then 1
np.set_printoptions(precision=2)
2021-08-12 08:23:21.727487: I tensorflow/stream_executor/platform/default/dso_loader.cc:53] Successfully opened dynamic library libcudart.so.11.0
rbo=Random_Base_Oracle(RNA_LEN,True)
pc_all,nc_all = rbo.get_partitioned_sequences(CDS_LEN,10) # just testing
pc_all,nc_all = rbo.get_partitioned_sequences(CDS_LEN,PC_SEQUENCES+PC_TESTS)
print("Use",len(pc_all),"PC seqs")
print("Use",len(nc_all),"NC seqs")It took 27 trials to reach 10 per class.
It took 48295 trials to reach 11000 per class.
Use 11000 PC seqs
Use 11000 NC seqs
# Describe the sequences
def describe_sequences(list_of_seq):
    oc = ORF_counter()
    num_seq = len(list_of_seq)
    rna_lens = np.zeros(num_seq)
    orf_lens = np.zeros(num_seq)
    for i in range(0,num_seq):
        rna_len = len(list_of_seq[i])
        rna_lens[i] = rna_len
        oc.set_sequence(list_of_seq[i])
        orf_len = oc.get_max_orf_len()
        orf_lens[i] = orf_len
    print ("Average RNA length:",rna_lens.mean())
    print ("Average ORF length:",orf_lens.mean())

print("Simulated sequences prior to adjustment:")
print("PC seqs")
describe_sequences(pc_all)
print("NC seqs")
describe_sequences(nc_all)Simulated sequences prior to adjustment:
PC seqs
Average RNA length: 32.0
Average ORF length: 19.638272727272728
NC seqs
Average RNA length: 32.0
Average ORF length: 2.844
pc_train=pc_all[:PC_SEQUENCES]
nc_train=nc_all[:NC_SEQUENCES]
pc_test=pc_all[PC_SEQUENCES:]
nc_test=nc_all[NC_SEQUENCES:]_____no_output_____# Use code from our SimTools library.
X,y = prepare_inputs_len_x_alphabet(pc_train,nc_train,ALPHABET) # shuffles
print("Data ready.")Data ready.
def make_DNN():
    print("make_DNN")
    print("input shape:",INPUT_SHAPE)
    dnn = Sequential()
    #dnn.add(Embedding(input_dim=INPUT_SHAPE,output_dim=INPUT_SHAPE))
    dnn.add(Conv1D(filters=FILTERS,kernel_size=WIDTH,strides=STRIDE,padding="same",
            input_shape=INPUT_SHAPE))
    dnn.add(Conv1D(filters=FILTERS,kernel_size=WIDTH,strides=STRIDE,padding="same"))
    dnn.add(MaxPooling1D())
    dnn.add(Conv1D(filters=FILTERS,kernel_size=WIDTH,strides=STRIDE,padding="same"))
    dnn.add(Conv1D(filters=FILTERS,kernel_size=WIDTH,strides=STRIDE,padding="same"))
    dnn.add(MaxPooling1D())
    dnn.add(Flatten())
    dnn.add(Dense(NEURONS,activation="sigmoid",dtype=np.float32))
    dnn.add(Dropout(DROP_RATE))
    dnn.add(Dense(1,activation="sigmoid",dtype=np.float32))
    dnn.compile(optimizer='adam',
                loss=BinaryCrossentropy(from_logits=False),
                metrics=['accuracy'])   # add to default metrics=loss
    dnn.build(input_shape=INPUT_SHAPE)
    #ln_rate = tf.keras.optimizers.Adam(learning_rate = LN_RATE)
    #bc=tf.keras.losses.BinaryCrossentropy(from_logits=False)
    #model.compile(loss=bc, optimizer=ln_rate, metrics=["accuracy"])
    return dnn
model = make_DNN()
print(model.summary())make_DNN
input shape: (32, 4)
Model: "sequential"
_________________________________________________________________
Layer (type)                 Output Shape              Param #
=================================================================
conv1d (Conv1D)              (None, 32, 16)            208
_________________________________________________________________
conv1d_1 (Conv1D)            (None, 32, 16)            784
_________________________________________________________________
max_pooling1d (MaxPooling1D) (None, 16, 16)            0
_________________________________________________________________
conv1d_2 (Conv1D)            (None, 16, 16)            784
_________________________________________________________________
conv1d_3 (Conv1D)            (None, 16, 16)            784
_________________________________________________________________
max_pooling1d_1 (MaxPooling1 (None, 8, 16)             0
_________________________________________________________________
flatten (Flatten)            (None, 128)               0
_________________________________________________________________
dense (Dense)                (None, 16)                2064
_________________________________________________________________
dropout (Dropout)            (None, 16)                0
_________________________________________________________________
dense_1 (Dense)              (None, 1)                 17
=================================================================
Total params: 4,641
Trainable params: 4,641
Non-trainable params: 0
_________________________________________________________________
None
from keras.callbacks import ModelCheckpoint
def do_cross_validation(X,y):
    cv_scores = []
    fold=0
    mycallbacks = [ModelCheckpoint(
        filepath=MODELPATH, save_best_only=True,
        monitor='val_accuracy', mode='max')]
    splitter = KFold(n_splits=SPLITS)  # this does not shuffle
    for train_index,valid_index in splitter.split(X):
        if fold < FOLDS:
            fold += 1
            X_train=X[train_index] # inputs for training
            y_train=y[train_index] # labels for training
            X_valid=X[valid_index] # inputs for validation
            y_valid=y[valid_index] # labels for validation
            print("MODEL")
            # Call constructor on each CV. Else, continually improves the same model.
            model = model = make_DNN()
            print("FIT")  # model.fit() implements learning
            start_time=time.time()
            history=model.fit(X_train, y_train,
                    epochs=EPOCHS,
                    verbose=1,  # ascii art while learning
                    callbacks=mycallbacks,   # called at end of each epoch
                    validation_data=(X_valid,y_valid))
            end_time=time.time()
            elapsed_time=(end_time-start_time)
            print("Fold %d, %d epochs, %d sec"%(fold,EPOCHS,elapsed_time))
            # print(history.history.keys())  # all these keys will be shown in figure
            pd.DataFrame(history.history).plot(figsize=(8,5))
            plt.grid(True)
            plt.gca().set_ylim(0,1) # any losses > 1 will be off the scale
            plt.show()
_____no_output_____do_cross_validation(X,y)MODEL
make_DNN
input shape: (32, 4)
FIT
Epoch 1/100
from keras.models import load_model
X,y = prepare_inputs_len_x_alphabet(pc_test,nc_test,ALPHABET)
best_model=load_model(MODELPATH)
scores = best_model.evaluate(X, y, verbose=0)
print("The best model parameters were saved during cross-validation.")
print("Best was defined as maximum validation accuracy at end of any epoch.")
print("Now re-load the best model and test it on previously unseen data.")
print("Test on",len(pc_test),"PC seqs")
print("Test on",len(nc_test),"NC seqs")
print("%s: %.2f%%" % (best_model.metrics_names[1], scores[1]*100))
The best model parameters were saved during cross-validation.
Best was defined as maximum validation accuracy at end of any epoch.
Now re-load the best model and test it on previously unseen data.
Test on 1000 PC seqs
Test on 1000 NC seqs
accuracy: 89.90%
from sklearn.metrics import roc_curve
from sklearn.metrics import roc_auc_score
ns_probs = [0 for _ in range(len(y))]
bm_probs = best_model.predict(X)
ns_auc = roc_auc_score(y, ns_probs)
bm_auc = roc_auc_score(y, bm_probs)
ns_fpr, ns_tpr, _ = roc_curve(y, ns_probs)
bm_fpr, bm_tpr, _ = roc_curve(y, bm_probs)
plt.plot(ns_fpr, ns_tpr, linestyle='--', label='Guess, auc=%.4f'%ns_auc)
plt.plot(bm_fpr, bm_tpr, marker='.', label='Model, auc=%.4f'%bm_auc)
plt.title('ROC')
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.legend()
plt.show()
print("%s: %.2f%%" %('AUC',bm_auc*100.0))
_____no_output_____t = time.time()
time.strftime('%Y-%m-%d %H:%M:%S %Z', time.localtime(t))_____no_output_____
</code>


# Notebook from nmirson-ml/clusterai_2020
Path: clases/clase_11/clusterai2020_clase11_autoencoders_genomics.ipynb

____
__Universidad Tecnologica Nacional Buenos Aires__

__Ingenieria Industrial__

__Ciencia de Datos - Curso I5521 - Turno Sabado Mañana__

__Docente: Martin Palazzo__

__source: https://github.com/martinepalazzo__

__Tema: Reduccion de la dimensionalidad__
_________no_output_____

# liver 1
# renal 0_____no_output_____

### Import libraries_____no_output_____

# basics
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import tensorflow as tf
import numpy as np

#sklearn
from sklearn.cluster import AgglomerativeClustering
from sklearn.metrics import silhouette_samples, silhouette_score
from sklearn.feature_selection import VarianceThreshold
from sklearn.model_selection import train_test_split
from sklearn.decomposition import PCA
from sklearn.manifold import TSNE

#tensorflow keras
import keras
from keras.layers import Activation
from keras.layers.normalization import BatchNormalization
from keras import models
from keras import layers
from keras.layers import Input, Dense
from keras.models import Model
from keras import regularizers
from keras.layers import Dropout
from keras.utils.vis_utils import plot_model_____no_output_____

check if we are using GPU or CPU_____no_output_____

from tensorflow.python.client import device_lib
print(device_lib.list_local_devices())[name: "/device:CPU:0"
device_type: "CPU"
memory_limit: 268435456
locality {
}
incarnation: 9594344404223489304
, name: "/device:XLA_CPU:0"
device_type: "XLA_CPU"
memory_limit: 17179869184
locality {
}
incarnation: 13741651735321663797
physical_device_desc: "device: XLA_CPU device"
]

### Import Dataset_____no_output_____

from google.colab import files
uploaded = files.upload()_____no_output_____#uploaded = files.upload()_____no_output_____data = pd.read_csv('./liver_kidney_expseq.zip', delimiter = ',', index_col=0,compression='zip' )
print('Shape:', data.shape)
data.head()Shape: (812, 1027)
x = data.iloc[:, 1:]
y = data.iloc[:,0]_____no_output_____

Perform Train-Test split_____no_output_____

xtr00, xte0, ytr0, yte0 = train_test_split(x, y, test_size=0.3, random_state=42)
#ytr = ytr0.sort_values(by = [0])
ytr = ytr0
xtr0 = xtr00.reindex(ytr.index)
ytr = ytr.values
yte = yte0.values
ytr[ytr == 0] = -1
yte[yte == 0] = -1_____no_output_____

MinMax Scaler between 0 and 1_____no_output_____

from sklearn.preprocessing import MinMaxScaler
scaler = MinMaxScaler()
xtr = pd.DataFrame(scaler.fit_transform(xtr0.values))
xte = pd.DataFrame(scaler.transform(xte0))_____no_output_____xtr.shape_____no_output_____sns.heatmap(xtr)
plt.xlabel('Genes')
plt.ylabel('Patients')_____no_output_____

# **Dimension Reduction**

- PCA
- kPCA
- Autoencoders

We will learn a low dimensional latent space $z$ with different methods

$$
z = f(x)
$$

_____no_output_____

latent_dim = 5_____no_output_____

## Principal Component Analysis_____no_output_____Implement a PCA of the training data for visualization and exploration. To apply PCA first perform eigen decomposition of the co-variate matrix. Then project the input data by using a projection matrix obtained from the top P eigenvectors associated to the top P eigen values of the eigen decomposition.

$$
\Sigma = \upsilon \lambda \upsilon^{-1} \rightarrow \beta_{dp} \rightarrow \langle X .\beta \rangle = z
$$_____no_output_____

#
from sklearn.decomposition import PCA
pca = PCA(n_components=latent_dim)
pca.fit(xtr)
ztr_pca = pca.transform(xtr)
zte_pca = pca.transform(xte)_____no_output_____# check the cumulative variance within the preserved components
np.sum(pca.explained_variance_ratio_)_____no_output_____sns.barplot(np.arange(0,latent_dim),pca.explained_variance_ratio_)
plt.title('Explained variance per principal component')
plt.show()/usr/local/lib/python3.6/dist-packages/seaborn/_decorators.py:43: FutureWarning: Pass the following variables as keyword args: x, y. From version 0.12, the only valid positional argument will be `data`, and passing other arguments without an explicit keyword will result in an error or misinterpretation.
  FutureWarning
sns.heatmap(ztr_pca)
plt.xlabel('Latent variables (PCA principal components)')
plt.ylabel('Patients')_____no_output_____sns.set_style("white")
sns.set_context("talk")
sns.set_style("ticks")
ax1 = plt.subplot()
sns.scatterplot(x=xtr_pca[(ytr == 1).ravel(),1], y=xtr_pca[(ytr == 1).ravel(),0], alpha = 0.8, s= 170, legend='full', label = 'liver')
sns.scatterplot(x=xtr_pca[(ytr == -1).ravel(),1], y=xtr_pca[(ytr == -1).ravel(),0],alpha = 0.8, s= 170, legend='full', label = 'kidney')
ax1.legend(bbox_to_anchor=(-0.2, 1), loc=2, borderaxespad=0.,framealpha=1, frameon=True)
ax1.set_xlabel("PC 1")
ax1.set_ylabel("PC 2")
ax1.figure.set_size_inches( (8,6) )
ax1.set_title("PCA of training data")
plt.yticks(rotation=45)
plt.legend()
plt.show()_____no_output_____

# Kernel-PCA

Compute the eigen decomposition of the kernel matrix

$$
K = n U^{-1} \Lambda U \rightarrow Z_{np} = KU^{*}
$$

by using the Gaussian Kernel
$$
k_g(x_i, x_j) = \text{exp}\left ( \frac{\left \| x_i-x_j \right \|^{2}}{2\sigma ^{2}} \right )
$$

which is equivalent to the Radial Basis Function (RBF) Kernel

$$
k_{rbf}(x_i, x_j) = \text{exp}\left (\gamma {\left \| x_i-x_j \right \|^{2}}\right )
$$_____no_output_____

from sklearn.decomposition import KernelPCA
kpca = KernelPCA(n_components=latent_dim, kernel='rbf',gamma= 0.001)
ztr_kpca = kpca.fit_transform(xtr)
zte_kpca = kpca.transform(xte)_____no_output_____sns.heatmap(ztr_kpca)
plt.xlabel('Latent variables (kPCA principal components)')
plt.ylabel('Patients')_____no_output_____

# Autoencoder
Here an autoencoder function is defined. The architecture can be changed if it is necessary. The Autoencoder is composed by two functions, the encoder

$$
z = f(x)
$$

and the decoder

$$
\hat{x} = q(z)
$$

and the autoencoder model is

$$
\hat{x} = q(z) = q(f(x))
$$

trained to minimize the mean squared error between the input and reconstructed vectors

$$
L(x , \hat{x}) = L(x , q(f(x))) = \left \| x - \hat{x} \right \|^{2}
$$
_____no_output_____

def ae_arch_0(encoding_dim, epochs, bachs, l1_reg, l2_reg, lrate, xtr, xte):

    'Autoencoder function adapted from the paper'
    'A pan-cancer somatic mutation embedding using autoencoders.  Palazzo, et. Al. BMC Bioinformatics 20, 655 (2019). https://doi.org/10.1186/s12859-019-3298-z '

    earlstop = keras.callbacks.EarlyStopping(monitor='val_loss', min_delta=0.0001, patience=10,  mode='auto', baseline=None, restore_best_weights=False, verbose=1)
    rlrop = keras.callbacks.ReduceLROnPlateau(monitor='val_loss', min_delta=0.001, patience=10, verbose=1, mode='auto')

    ################### Optimizer ###################
    optim = keras.optimizers.Adam(lr=lrate)

    ################### MODEL ARCHITECTURE ###############

    # 00) INPUT
    inputdim = Input(shape=(xtr.shape[1],))

    # 01) DENSE LAYER
    encode_i = Dense(200, kernel_regularizer=regularizers.l2(l2_reg), activation = 'relu')(inputdim)

    # 02) BATCH NORM
    bn_i = BatchNormalization()(encode_i)

    # 03) DENSE LAYER
    encode_ii = Dense(100, activation = 'relu')(bn_i)

    latent = Dense(encoding_dim, kernel_regularizer = regularizers.l2(l2_reg), activation = 'relu')(encode_ii)

    # 04) DECODER
    #decode_i = Dense(100, activation = 'relu')(latent)
    decode_ii = Dense(200,activation='relu')(latent)

    # 05) DECODER
    decode_iii = Dense(xtr.shape[1],activation='linear')(decode_ii)

    # 06) AUTOENCODER
    autoencoder = Model(inputdim, decode_iii)
    encoder = Model(inputdim, latent)

    #encoded_input5 = Input(shape=(encoding_dim5,))
    #decoder_layer5 = autoencoder5.layers[-3]
    #decoder5 = Model(x3, decoder_layer5(x3))

    autoencoder.summary()
    tf.keras.utils.plot_model(autoencoder, show_shapes=True)
    keras.utils.plot_model(autoencoder, show_shapes=True)
    compression_factor = float(np.shape(xtr)[1]) / encoding_dim
    print("Compression factor: %s" % compression_factor)

    ################## COMPILE AND FIT MODEL #############
    autoencoder.compile(optimizer=optim, loss='mse')
    autoencoder.fit(xtr, xtr,epochs=epochs, batch_size=bachs,  shuffle=True,  validation_data=(xte, xte), callbacks = [earlstop, rlrop])

    histval = autoencoder.history.history['val_loss']
    histtra = autoencoder.history.history['loss']

    return encoder, histtra, histval_____no_output_____

Train an autoencoder model. Store the encoder function and the training loss scores._____no_output_____

encoder_val, hist_tra, hist_val = ae_arch_0(latent_dim,50,32,0.001, 0.003, 0.0001, xtr, xte)Model: "functional_33"
_________________________________________________________________
Layer (type)                 Output Shape              Param #
=================================================================
input_9 (InputLayer)         [(None, 1026)]            0
_________________________________________________________________
dense_40 (Dense)             (None, 200)               205400
_________________________________________________________________
batch_normalization_8 (Batch (None, 200)               800
_________________________________________________________________
dense_41 (Dense)             (None, 100)               20100
_________________________________________________________________
dense_42 (Dense)             (None, 5)                 505
_________________________________________________________________
dense_43 (Dense)             (None, 200)               1200
_________________________________________________________________
dense_44 (Dense)             (None, 1026)              206226
=================================================================
Total params: 434,231
Trainable params: 433,831
Non-trainable params: 400
_________________________________________________________________
Compression factor: 205.2
Epoch 1/50
18/18 [==============================] - 0s 25ms/step - loss: 1.0452 - val_loss: 1.1354
Epoch 2/50
18/18 [==============================] - 0s 9ms/step - loss: 0.9681 - val_loss: 1.0638
Epoch 3/50
18/18 [==============================] - 0s 8ms/step - loss: 0.8954 - val_loss: 0.9955
Epoch 4/50
18/18 [==============================] - 0s 8ms/step - loss: 0.8267 - val_loss: 0.9314
Epoch 5/50
18/18 [==============================] - 0s 8ms/step - loss: 0.7627 - val_loss: 0.8722
Epoch 6/50
18/18 [==============================] - 0s 7ms/step - loss: 0.7042 - val_loss: 0.8181
Epoch 7/50
18/18 [==============================] - 0s 8ms/step - loss: 0.6506 - val_loss: 0.7680
Epoch 8/50
18/18 [==============================] - 0s 7ms/step - loss: 0.6013 - val_loss: 0.7215
Epoch 9/50
18/18 [==============================] - 0s 8ms/step - loss: 0.5566 - val_loss: 0.6804
Epoch 10/50
18/18 [==============================] - 0s 8ms/step - loss: 0.5147 - val_loss: 0.6381
Epoch 11/50
18/18 [==============================] - 0s 8ms/step - loss: 0.4762 - val_loss: 0.6031
Epoch 12/50
18/18 [==============================] - 0s 8ms/step - loss: 0.4406 - val_loss: 0.5675
Epoch 13/50
18/18 [==============================] - 0s 9ms/step - loss: 0.4077 - val_loss: 0.5359
Epoch 14/50
18/18 [==============================] - 0s 8ms/step - loss: 0.3773 - val_loss: 0.5058
Epoch 15/50
18/18 [==============================] - 0s 8ms/step - loss: 0.3488 - val_loss: 0.4795
Epoch 16/50
18/18 [==============================] - 0s 8ms/step - loss: 0.3224 - val_loss: 0.4539
Epoch 17/50
18/18 [==============================] - 0s 9ms/step - loss: 0.2981 - val_loss: 0.4281
Epoch 18/50
18/18 [==============================] - 0s 8ms/step - loss: 0.2755 - val_loss: 0.4070
Epoch 19/50
18/18 [==============================] - 0s 8ms/step - loss: 0.2544 - val_loss: 0.3857
Epoch 20/50
18/18 [==============================] - 0s 8ms/step - loss: 0.2350 - val_loss: 0.3675
Epoch 21/50
18/18 [==============================] - 0s 8ms/step - loss: 0.2173 - val_loss: 0.3494
Epoch 22/50
18/18 [==============================] - 0s 8ms/step - loss: 0.2007 - val_loss: 0.3334
Epoch 23/50
18/18 [==============================] - 0s 8ms/step - loss: 0.1852 - val_loss: 0.3188
Epoch 24/50
18/18 [==============================] - 0s 8ms/step - loss: 0.1713 - val_loss: 0.3054
Epoch 25/50
18/18 [==============================] - 0s 8ms/step - loss: 0.1583 - val_loss: 0.2906
Epoch 26/50
18/18 [==============================] - 0s 7ms/step - loss: 0.1463 - val_loss: 0.2794
Epoch 27/50
18/18 [==============================] - 0s 7ms/step - loss: 0.1352 - val_loss: 0.2680
Epoch 28/50
18/18 [==============================] - 0s 8ms/step - loss: 0.1248 - val_loss: 0.2577
Epoch 29/50
18/18 [==============================] - 0s 7ms/step - loss: 0.1155 - val_loss: 0.2497
Epoch 30/50
18/18 [==============================] - 0s 8ms/step - loss: 0.1069 - val_loss: 0.2396
Epoch 31/50
18/18 [==============================] - 0s 7ms/step - loss: 0.0989 - val_loss: 0.2301
Epoch 32/50
18/18 [==============================] - 0s 8ms/step - loss: 0.0918 - val_loss: 0.2235
Epoch 33/50
18/18 [==============================] - 0s 8ms/step - loss: 0.0849 - val_loss: 0.2192
Epoch 34/50
18/18 [==============================] - 0s 7ms/step - loss: 0.0788 - val_loss: 0.2124
Epoch 35/50
18/18 [==============================] - 0s 9ms/step - loss: 0.0732 - val_loss: 0.2051
Epoch 36/50
18/18 [==============================] - 0s 9ms/step - loss: 0.0681 - val_loss: 0.2003
Epoch 37/50
18/18 [==============================] - 0s 8ms/step - loss: 0.0633 - val_loss: 0.1960
Epoch 38/50
18/18 [==============================] - 0s 7ms/step - loss: 0.0590 - val_loss: 0.1911
Epoch 39/50
18/18 [==============================] - 0s 8ms/step - loss: 0.0550 - val_loss: 0.1873
Epoch 40/50
18/18 [==============================] - 0s 7ms/step - loss: 0.0514 - val_loss: 0.1841
Epoch 41/50
18/18 [==============================] - 0s 8ms/step - loss: 0.0481 - val_loss: 0.1807
Epoch 42/50
18/18 [==============================] - 0s 8ms/step - loss: 0.0451 - val_loss: 0.1774
Epoch 43/50
18/18 [==============================] - 0s 9ms/step - loss: 0.0422 - val_loss: 0.1743
Epoch 44/50
18/18 [==============================] - 0s 8ms/step - loss: 0.0397 - val_loss: 0.1719
Epoch 45/50
18/18 [==============================] - 0s 8ms/step - loss: 0.0375 - val_loss: 0.1695
Epoch 46/50
18/18 [==============================] - 0s 9ms/step - loss: 0.0354 - val_loss: 0.1673
Epoch 47/50
18/18 [==============================] - 0s 7ms/step - loss: 0.0334 - val_loss: 0.1646
Epoch 48/50
18/18 [==============================] - 0s 8ms/step - loss: 0.0317 - val_loss: 0.1619
Epoch 49/50
18/18 [==============================] - 0s 8ms/step - loss: 0.0304 - val_loss: 0.1605
Epoch 50/50
18/18 [==============================] - 0s 8ms/step - loss: 0.0288 - val_loss: 0.1588

Plot the training and validation Loss._____no_output_____

plt.plot(hist_val, c = 'g', label = "Validation", marker = 'o', alpha = 0.5)
plt.plot(hist_tra, c = 'b', label = "Training", marker = 'o', alpha = 0.5)
plt.title("Autoencoder Loss during training epochs")
plt.legend()
plt.xlabel('epochs')
plt.ylabel('Loss (MSE)')
plt.show()_____no_output_____

Encode training and validation samples by using the Encoder function to obtain its projection on the latent space._____no_output_____

ztr_ae = encoder_val.predict(xtr)
zte_ae = encoder_val.predict(xte)_____no_output_____

Visualize the training samples by class (1 or -1) in the latent space._____no_output_____

sns.heatmap(ztr_ae)
plt.xlabel('Latent variables (Autoencoder)')
plt.ylabel('Patients')_____no_output_____

# Visualize _____no_output_____

ztr_ae_tsne = TSNE(n_components=2).fit_transform(ztr_ae)
ztr_kpca_tsne = TSNE(n_components=2).fit_transform(ztr_kpca)
ztr_pca_tsne = TSNE(n_components=2).fit_transform(ztr_pca)_____no_output_____f, ax = plt.subplots(1,3)
f.set_size_inches(12,4)
sns.scatterplot(x = ztr_ae_tsne[(ytr).ravel() == -1,0], y = ztr_ae_tsne[ytr.ravel() == -1,1], ax= ax[0])
sns.scatterplot(x = ztr_ae_tsne[ytr.ravel() == 1,0], y = ztr_ae_tsne[ytr.ravel() == 1,1], ax= ax[0])
ax[0].set_title("AE")
sns.scatterplot(x = ztr_kpca_tsne[(ytr).ravel() == -1,0], y = ztr_kpca_tsne[ytr.ravel() == -1,1], ax= ax[1])
sns.scatterplot(x = ztr_kpca_tsne[ytr.ravel() == 1,0], y = ztr_kpca_tsne[ytr.ravel() == 1,1], ax= ax[1])
ax[1].set_title("KPCA")
sns.scatterplot(x = ztr_pca_tsne[(ytr).ravel() == -1,0], y = ztr_pca_tsne[ytr.ravel() == -1,1], ax= ax[2])
sns.scatterplot(x = ztr_pca_tsne[ytr.ravel() == 1,0], y = ztr_pca_tsne[ytr.ravel() == 1,1], ax= ax[2])
ax[2].set_title("PCA")
plt.show()_____no_output_____

# Clustering using the latent space

Perform clustering on the learned latent space $z$. _____no_output_____

from sklearn.cluster import KMeans
from sklearn.metrics.cluster import adjusted_rand_score

rand_index_pca = []
rand_index_kpca = []
rand_index_ae = []

# iterate for different number of clusters
for clust in range(2,10):
  kmeans_pca = KMeans(n_clusters=clust, random_state=0).fit(ztr_pca)
  kmeans_kpca = KMeans(n_clusters=clust, random_state=0).fit(ztr_kpca)
  kmeans_ae = KMeans(n_clusters=clust, random_state=0).fit(ztr_ae)

  # measure clustering performance
  rand_index_pca.append(adjusted_rand_score(ytr, kmeans_pca.labels_))
  rand_index_kpca.append(adjusted_rand_score(ytr, kmeans_kpca.labels_))
  rand_index_ae.append(adjusted_rand_score(ytr, kmeans_ae.labels_))_____no_output_____plt.plot(np.arange(2,10), rand_index_ae, label = 'ae', marker = 'o')
plt.plot(np.arange(2,10), rand_index_pca, label = 'pca', marker = 'o')
plt.plot(np.arange(2,10), rand_index_kpca, label = 'kpca', marker = 'o')
plt.legend()
plt.ylabel('rand index')
plt.xlabel('clusters')
plt.title('Clustering performance using latent space')_____no_output__________no_output__________no_output_____


# Notebook from MasonGrabowski/ModSimPy
Path: code/salmon.ipynb

# Modeling and Simulation in Python

Case Study: Predicting salmon returns

This case study is based on a ModSim student project by Josh Deng and Erika Lu.

Copyright 2017 Allen Downey

License: [Creative Commons Attribution 4.0 International](https://creativecommons.org/licenses/by/4.0)
_____no_output_____

# Configure Jupyter so figures appear in the notebook
%matplotlib inline

# Configure Jupyter to display the assigned value after an assignment
%config InteractiveShell.ast_node_interactivity='last_expr_or_assign'

# import functions from the modsim.py module
from modsim import *_____no_output_____

### Can we predict salmon populations?

Each year the [U.S. Atlantic Salmon Assessment Committee](https://www.nefsc.noaa.gov/USASAC/Reports/USASAC2018-Report-30-2017-Activities.pdf) reports estimates of salmon populations in oceans and rivers in the northeastern United States.  The reports are useful for monitoring changes in these populations, but they generally do not include predictions.

The goal of this case study is to model year-to-year changes in population, evaluate how predictable these changes are, and estimate the probability that a particular population will increase or decrease in the next 10 years.

As an example, I'll use data from page 18 of the 2017 report, which provides population estimates for the Narraguagus and Sheepscot Rivers in Maine.

![USASAC_Report_2017_Page18](data/USASAC_Report_2017_Page18.png)

At the end of this notebook, I make some suggestions for extracting data from a PDF document automatically, but for this example I will keep it simple and type it in.

Here are the population estimates for the Narraguagus River:_____no_output_____

pops = [2749, 2845, 4247, 1843, 2562, 1774, 1201, 1284, 1287, 2339, 1177, 962, 1176, 2149, 1404, 969, 1237, 1615, 1201];_____no_output_____

To get this data into a Pandas Series, I'll also make a range of years to use as an index._____no_output_____

years = range(1997, 2016)_____no_output_____

And here's the series._____no_output_____

pop_series = TimeSeries(pops, index=years, dtype=float)_____no_output_____

Here's what it looks like:_____no_output_____

def plot_population(series):
    plot(series, label='Estimated population')
    decorate(xlabel='Year',
             ylabel='Population estimate',
             title='Narraguacus River',
             ylim=[0, 5000])

plot_population(pop_series)_____no_output_____

## Modeling changes

To see how the population changes from year-to-year, I'll use `diff` to compute the absolute difference between each year and the next.

`shift` adjusts the result so each change aligns with the year it happened._____no_output_____

abs_diffs = np.ediff1d(pop_series, np.nan)_____no_output_____

We can compute relative differences by dividing by the original series elementwise._____no_output_____

rel_diffs = abs_diffs / pop_series_____no_output_____

Or we can use the `modsim` function `compute_rel_diff`:_____no_output_____

rel_diffs = compute_rel_diff(pop_series)_____no_output_____

These relative differences are observed annual net growth rates.  So let's drop the `NaN` and save them._____no_output_____

rates = rel_diffs.dropna()_____no_output_____

A simple way to model this system is to draw a random value from this series of observed rates each year.  We can use the NumPy function `choice` to make a random choice from a series._____no_output_____

np.random.choice(rates)_____no_output_____

## Simulation

Now we can simulate the system by drawing random growth rates from the series of observed rates.

I'll start the simulation in 2015._____no_output_____

t_0 = 2015
p_0 = pop_series[t_0]_____no_output_____

Create a `System` object with variables `t_0`, `p_0`, `rates`, and `duration=10` years.

The series of observed rates is one big parameter of the model._____no_output_____

system = System(t_0=t_0,
                p_0=p_0,
                duration=10,
                rates=rates)_____no_output_____

Write an update functon that takes as parameters `pop`, `t`, and `system`.
It should choose a random growth rate, compute the change in population, and return the new population._____no_output_____

# Solution goes here_____no_output_____

Test your update function and run it a few times_____no_output_____

update_func1(p_0, t_0, system)_____no_output_____

Here's a version of `run_simulation` that stores the results in a `TimeSeries` and returns it._____no_output_____

def run_simulation(system, update_func):
    """Simulate a queueing system.

    system: System object
    update_func: function object
    """
    t_0 = system.t_0
    t_end = t_0 + system.duration

    results = TimeSeries()
    results[t_0] = system.p_0

    for t in linrange(t_0, t_end):
        results[t+1] = update_func(results[t], t, system)

    return results_____no_output_____

Use `run_simulation` to run generate a prediction for the next 10 years.

The plot your prediction along with the original data.  Your prediction should pick up where the data leave off._____no_output_____

# Solution goes here_____no_output_____

To get a sense of how much the results vary, we can run the model several times and plot all of the results._____no_output_____

def plot_many_simulations(system, update_func, iters):
    """Runs simulations and plots the results.

    system: System object
    update_func: function object
    iters: number of simulations to run
    """
    for i in range(iters):
        results = run_simulation(system, update_func)
        plot(results, color='gray', linewidth=5, alpha=0.1)_____no_output_____

The plot option `alpha=0.1` makes the lines semi-transparent, so they are darker where they overlap.

Run `plot_many_simulations` with your update function and `iters=30`.  Also plot the original data._____no_output_____

# Solution goes here_____no_output_____

The results are highly variable: according to this model, the population might continue to decline over the next 10 years, or it might recover and grow rapidly!

It's hard to say how seriously we should take this model.  There are many factors that influence salmon populations that are not included in the model.  For example, if the population starts to grow quickly, it might be limited by resource limits, predators, or fishing.  If the population starts to fall, humans might restrict fishing and stock the river with farmed fish.

So these results should probably not be considered useful predictions.  However, there might be something useful we can do, which is to estimate the probability that the population will increase or decrease in the next 10 years.  _____no_output_____## Distribution of net changes

To describe the distribution of net changes, write a function called `run_many_simulations` that runs many simulations, saves the final populations in a `ModSimSeries`, and returns the `ModSimSeries`.
_____no_output_____

def run_many_simulations(system, update_func, iters):
    """Runs simulations and report final populations.

    system: System object
    update_func: function object
    iters: number of simulations to run

    returns: series of final populations
    """
    # FILL THIS IN_____no_output_____# Solution goes here_____no_output_____

Test your function by running it with `iters=5`._____no_output_____

run_many_simulations(system, update_func1, 5)_____no_output_____

Now we can run 1000 simulations and describe the distribution of the results._____no_output_____

last_pops = run_many_simulations(system, update_func1, 1000)
last_pops.describe()_____no_output_____

If we substract off the initial population, we get the distribution of changes._____no_output_____

net_changes = last_pops - p_0
net_changes.describe()_____no_output_____

The median is negative, which indicates that the population decreases more often than it increases.

We can be more specific by counting the number of runs where `net_changes` is positive._____no_output_____

np.sum(net_changes > 0)_____no_output_____

Or we can use `mean` to compute the fraction of runs where `net_changes` is positive._____no_output_____

np.mean(net_changes > 0)_____no_output_____

And here's the fraction where it's negative._____no_output_____

np.mean(net_changes < 0)_____no_output_____

So, based on observed past changes, this model predicts that the population is more likely to decrease than increase over the next 10 years, by about 2:1._____no_output_____## A refined model

There are a few ways we could improve the model.

1.  It looks like there might be cyclic behavior in the past data, with a period of 4-5 years.  We could extend the model to include this effect.

2.  Older data might not be as relevant for prediction as newer data, so we could give more weight to newer data.

The second option is easier to implement, so let's try it.

I'll use `linspace` to create an array of "weights" for the observed rates.  The probability that I choose each rate will be proportional to these weights.

The weights have to add up to 1, so I divide through by the total._____no_output_____

weights = linspace(0, 1, len(rates))
weights /= sum(weights)
plot(weights)
decorate(xlabel='Index into the rates array',
         ylabel='Weight')_____no_output_____

I'll add the weights to the `System` object, since they are parameters of the model._____no_output_____

system.weights = weights_____no_output_____

We can pass these weights as a parameter to `np.random.choice` (see the [documentation](https://docs.scipy.org/doc/numpy/reference/generated/numpy.random.choice.html))_____no_output_____

np.random.choice(system.rates, p=system.weights)_____no_output_____

Write an update function that takes the weights into account._____no_output_____

# Solution goes here_____no_output_____

Use `plot_many_simulations` to plot the results._____no_output_____

# Solution goes here_____no_output_____

Use `run_many_simulations` to collect the results and `describe` to summarize the distribution of net changes._____no_output_____

# Solution goes here_____no_output_____

Does the refined model have much effect on the probability of population decline?_____no_output_____

# Solution goes here_____no_output_____

## Extracting data from a PDF document

The following section uses PyPDF2 to get data from a PDF document.  It uses features we have not seen yet, so don't worry if it doesn't all make sense.

The PyPDF2 package provides functions to read PDF documents and get the data.

If you don't already have it installed, and you are using Anaconda, you can install it by running the following command in a Terminal or Git Bash:

```
conda install -c conda-forge pypdf2
```_____no_output_____

import PyPDF2_____no_output_____

The 2017 report is in the data directory._____no_output_____

pdfFileObj = open('data/USASAC2018-Report-30-2017-Activities-Page11.pdf', 'rb')_____no_output_____

The `PdfFileReader` object knows how to read PDF documents._____no_output_____

pdfReader = PyPDF2.PdfFileReader(pdfFileObj)_____no_output_____

This file contains only one page._____no_output_____

pdfReader.numPages_____no_output_____

`getPage` selects the only page in the document._____no_output_____

page = pdfReader.getPage(0)
page.extractText()_____no_output_____

The following function iterates through the lines on the page, removes whitespace, and ignores lines that contain only whitespace._____no_output_____

def iter_page(page):
    for item in page.extractText().splitlines():
        item = item.strip()
        if item:
            yield item_____no_output_____

The following function gets the next `n` pages from the page._____no_output_____

def next_n(iterable, n):
    """Get the next n items from an iterable."""
    return [next(iterable) for i in range(n)]_____no_output_____

We skip the text at the top of the page._____no_output_____

t = iter_page(page)
discard = next_n(t, 8)_____no_output_____

The next 7 strings are the column headings of the table._____no_output_____

columns = next_n(t, 7)_____no_output_____

Create an empty `Dataframe` with the column headings._____no_output_____

df = pd.DataFrame(columns=columns)
df_____no_output_____

Get the next 19 lines of the table._____no_output_____

for i in range(19):
    year = int(next(t))
    data = next_n(t, 7)
    df.loc[year] = data_____no_output_____

The last line in the table gets messed up, so I'll do that one by hand._____no_output_____

df.loc[2017] = ['363', '663', '13', '2', '1041', '806', '235'] _____no_output_____

Here's the result._____no_output_____

df_____no_output_____

In general, reading tables from PDF documents is fragile and error-prone.  Sometimes it is easier to just type it in._____no_output_____


# Notebook from DavidLeoni/juprova
Path: jupman-tests.ipynb

import jupman;
jupman.init()_____no_output_____

# Jupman Tests

In this page we put some tests for Jupyter. The page Title has one sharp, the Sections always have two sharps.

## Sezione 1

bla bla

## Sezione 2

Subsections always have three sharps

### Subsection 1
bla bla

### Subsection 2
bla bla
_____no_output_____## Download links

Files manually put in `_static` :

* Download [trial.odt](_static/trial.odt)
* Download [trial.pdf](_static/trial.pdf)

Files in arbitrary folder position :

* Download [requirements.txt](requirements.txt)

NOTE: download links are messy, [see issue 8](https://github.com/DavidLeoni/jupman/issues/8)

_____no_output_____## Info/Warning Boxes

Until there is an info/warning extension for Markdown/CommonMark (see this issue), such boxes can be created by using HTML
 elements like this:_____no_output_____

**Note:** This is an info!

_____no_output_____

**Note:** This is a warn!

_____no_output_____For this to work reliably, you should obey the following guidelines:

* The class attribute has to be either "alert alert-info" or "alert alert-warning", other values will not be converted correctly.
* No further attributes are allowed.
* For compatibility with CommonMark, you should add an empty line between the
 start tag and the beginning of the content.
_____no_output_____
## Math

For math stuff, [see npshpinx docs](https://nbsphinx.readthedocs.io/en/0.2.14/markdown-cells.html#Equations)

Here we put just some equation to show it behaves fine in Jupman

This is infinity: $\infty$ _____no_output_____## Unicode

Unicode characters should display an HTML, but with latex you might have problems, and need to manually map characters in conf.py

You should see a star in a black circle: ✪

You should see a check: ✓

table characters: │ ├ └ ─_____no_output_____## Image

### SVG Images

SVG images work in notebook, but here it is commented since it breaks Latex, [see issue](https://github.com/DavidLeoni/jupman/issues/1)

```
![An image](img/cc-by.svg)
```

This one also doesn't works (and shows ugly code in the notebook anyway)

```
from IPython.display import SVG
SVG(filename='img/cc-by.svg')
```
_____no_output_____### PNG Images

![A PNG image](_static/img/notebook_icon.png)_____no_output_____### Inline images - pure markdown_____no_output_____    Bla ![A PNG image](_static/img/notebook_icon.png)  bli blo_____no_output_____Bla ![A PNG image](_static/img/notebook_icon.png)  bli blo_____no_output_____### Inline images - markdown and img_____no_output_____    bla  bli blo_____no_output_____
bla  bli blo_____no_output_____### Img class

If we pass a class, it will to be present in the website:

This   should be inline_____no_output_____## Expressions list

Highlighting **does** work both in Jupyter and Sphinx

Three quotes, multiple lines - Careful: put **exactly 4 spaces** indentation

1.  ```python
    [2,3,1] != "[2,3,1]"
    ```
1.  ```python
    [4,8,12] == [2*2,"4*2",6*2]
    ```
1.  ```python
    [][:] == []
    ```_____no_output_____Three quotes, multiple lines, more compact - works in Jupyter,  **doesn't** in Sphinx

1.  ```python
    [2,3,1] != "[2,3,1]"```
1.  ```python
    [4,8,12] == [2*2,"4*2",6*2]```
1.  ```python
    [][:] == []```_____no_output_____Highlighting **doesn't** work in Jupyter neither in Sphinx:

Three quotes, single line

1. ```python [2,3,1] != ["2",3,1]```
1. ```python [4,8,12] == [2*2,"4*2",6*2]```
1. ```python [][:] == "[]"```

Single quote, single line

1. `python [2,3,1] != ["2",3,1]`
1. `python [4,8,12] == [2*2,"4*2",6*2]`
1. `python [][:] == "[]"`
_____no_output_____## Toggable cells

There are various ways to have toggable cells, for example for showing exercise solutions. Some of the solutions just don't work. Preferred solution for now is with Javascript

### Toggling with Javascript (PREFERRED)

* Works in MarkDown
* Works while in Jupyter
* Works in HTML
* Does not show in Latex (which might be a good point, if you intend to put somehow solutions at the end of the document)
* NOTE: after creating the text to see the results you have to run the initial cell with jupman.init (as for the toc)
* NOTE: you can't use Markdown block code since of Sept 2017 doesn't show well in HTML output

    # FIRST SOLUTION
    question = raw_input("What?")
    answers = random.randint(1,8)
    if question == "":
        sys.exit()

    # SECOND SOLUTION
    question = raw_input("SECOND SOLUTION")
    answers = random.randint(1,8)
    if question == "":
        sys.exit()

_____no_output_____## HTML details in Markdown, code tag

* Works while in Jupyter
* Doesn't work in HTML output
* as of Sept Oct 2017, not yet supported in Microsoft browsers

Click here to see the code

    question = raw_input("What?")
    answers = random.randint(1,8)
    if question == "":
        sys.exit()

_____no_output_____### HTML details in Markdown, Markdown mixed code

* Works while in Jupyter
* Doesn't work in HTML output
* as of Sept Oct 2017, not yet supported in Microsoft browsers

Click here to see the code

```python
    question = raw_input("What?")
    answers = random.randint(1,8)
    if question == "":
        sys.exit()
```

_____no_output_____### HTML details in HTML, raw NBConvert Format

* Doesn't work in Jupyter
* Works in HTML output
    * NOTE: as of Sept Oct 2017, not yet supported in Microsoft browsers
* Doesn't show at all in PDF output
_____no_output_____

Some other Markdown cell aftwerwards ...._____no_output_____## Files in templates

Since Dec 2019 they are not accessible [see issue 10](https://github.com/DavidLeoni/jupman/issues/10), but it is not a great problem, you can always put a link to Github, see for example [exam-yyyy-mm-dd.ipynb](https://github.com/DavidLeoni/jupman/tree/master/_templates/exam/exam-yyyy-mm-dd.ipynb)_____no_output_____## Python tutor

There are various ways to embed Python tutor, first we put the recommended one._____no_output_____###  jupman.pytut_____no_output_____**RECOMMENDED**: You can put a call to `jupman.pytut()` at the end of a cell, and the cell code will magically appear in python tutor in the output (except the call to `pytut()` of course).
Does not need internet  connection._____no_output_____

x = [5,8,4,10,30,20,40,50,60,70,20,30]
y= {3:9}
z = [x]

jupman.pytut()_____no_output_____

**jupman.pytut scope**: BEWARE of variables which were initialized in previous cells, they WILL NOT be available in Python Tutor:_____no_output_____

w = 8_____no_output_____x =  w + 5
jupman.pytut()Traceback (most recent call last):
  File "/home/da/Da/prj/jupman/prj/jupman.py", line 2305, in _runscript
    self.run(script_str, user_globals, user_globals)
  File "/usr/lib/python3.5/bdb.py", line 431, in run
    exec(cmd, globals, locals)
  File "<string>", line 2, in <module>
NameError: name 'w' is not defined

**jupman.pytut window overflow**: When too much right space is taken, it might be difficult to scroll:
_____no_output_____

x = [3,2,5,2,42,34,2,4,34,2,3,4,23,4,23,4,2,34,23,4,23,4,23,4,234,34,23,4,23,4,23,4,2]

jupman.pytut()_____no_output_____x =  w + 5
jupman.pytut()Traceback (most recent call last):
  File "/home/da/Da/prj/jupman/prj/jupman.py", line 2305, in _runscript
    self.run(script_str, user_globals, user_globals)
  File "/usr/lib/python3.5/bdb.py", line 431, in run
    exec(cmd, globals, locals)
  File "<string>", line 2, in <module>
NameError: name 'w' is not defined

**jupman.pytut execution:** Some cells might execute in Jupyter but not so well in Python Tutor, due to [its inherent limitations](https://github.com/pgbovine/OnlinePythonTutor/blob/master/unsupported-features.md):_____no_output_____

x = 0
for i in range(10000):
    x += 1
print(x)
jupman.pytut()10000

**jupman.pytut infinite loops**: Since execution occurs first in Jupyter and then in Python tutor, if you have an infinite loop no Python Tutor instance will be spawned:

```python
while True:
    pass

jupman.pytut()
```_____no_output_____**jupman.pytut() resizability:** long vertical and horizontal expansion should work:_____no_output_____

x = {0:'a'}
for i in range(1,30):
    x[i] = x[i-1]+str(i*10000)
jupman.pytut()_____no_output_____

**jupman.pytut cross arrows**: With multiple visualizations, arrows shouldn't cross from one to the other even if underlying script is loaded multiple times (relates to visualizerIdOverride)_____no_output_____

x = [1,2,3]

jupman.pytut()_____no_output_____

**jupman.pytut print output**: With only one line of print, Print output panel shouldn't be too short:_____no_output_____

print("hello")

jupman.pytut()hello
y = [1,2,3,4]

jupman.pytut()_____no_output_____

### HTML magics

Another option is to directly paste Python Tutor iframe in the cells, and use Jupyter `%%HTML` magics command.

HTML should be available both in notebook and website - of course, requires an internet connection.

Beware: you need the HTTP**S** !_____no_output_____

%%HTML

_____no_output_____

### NBTutor_____no_output_____To show Python Tutor in notebooks, there is already a jupyter extension called [NBTutor](https://github.com/lgpage/nbtutor) , afterwards you can use magic `%%nbtutor` to show the interpreter.

Unfortunately, it doesn't show in the generated HTML :-/
_____no_output_____

%reload_ext nbtutor_____no_output_____%%nbtutor

for x in range(1,4):
    print("ciao")
x=5
y=7
x +y
ciao
ciao
ciao

## Stripping answers

For stripping answers examples, see [jupyter-intro/jupyter-intro-sol](jupyter-intro/jupyter-intro-sol.ipynb). For explanation, see [usage](usage.ipynb#Tags-to-strip)_____no_output_____## Formatting problems_____no_output_____### Characters per line

Python standard for code has limit to 79, many styles have 80 (see [Wikipedia](https://en.wikipedia.org/wiki/Characters_per_line))

We can keep 80:

```
--------------------------------------------------------------------------------
```

```python
--------------------------------------------------------------------------------
```

Errors hold 75 dashes:

Plain:
```
---------------------------------------------------------------------------
ZeroDivisionError                         Traceback (most recent call last)
<ipython-input-15-9e1622b385b6> in <module>()
----> 1 1/0

ZeroDivisionError: division by zero

```

As Python markup:
```python
---------------------------------------------------------------------------
ZeroDivisionError                         Traceback (most recent call last)
<ipython-input-15-9e1622b385b6> in <module>()
----> 1 1/0

ZeroDivisionError: division by zero
```_____no_output_____

len('---------------------------------------------------------------------------')_____no_output_____

On website this **may** display a scroll bar, because it will actually print `'` apexes plus the dashes_____no_output_____

'-'*80_____no_output_____

This should **not** display a scrollbar:_____no_output_____

'-'*78_____no_output_____

This should **not** display a scrollbar:_____no_output_____

print('-'*80)--------------------------------------------------------------------------------

### Very large input

In Jupyter: default behaviour, show scrollbar

On the website: should expand in horizontal as much as it wants, the rationale is that for input code since it may be printed to PDF you should always manually put line breaks._____no_output_____

# line with an exceedingly long comment line with an exceedingly long comment line with an exceedingly long comment line with an exceedingly long comment line with an exceedingly long comment line with an exceedingly long comment

# line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment line with an an out-of-this-world long comment

_____no_output_____

**Very long HTML** (and long code line)

Should expand in vertical as much as it wants._____no_output_____

%%HTML

See full screen

_____no_output_____

### Very long output

In Jupyter: by clicking, you can collapse

On the website: a scrollbar should appear_____no_output_____

for x in range(150):
    print('long output ...', x)long output ... 0
long output ... 1
long output ... 2
long output ... 3
long output ... 4
long output ... 5
long output ... 6
long output ... 7
long output ... 8
long output ... 9
long output ... 10
long output ... 11
long output ... 12
long output ... 13
long output ... 14
long output ... 15
long output ... 16
long output ... 17
long output ... 18
long output ... 19
long output ... 20
long output ... 21
long output ... 22
long output ... 23
long output ... 24
long output ... 25
long output ... 26
long output ... 27
long output ... 28
long output ... 29
long output ... 30
long output ... 31
long output ... 32
long output ... 33
long output ... 34
long output ... 35
long output ... 36
long output ... 37
long output ... 38
long output ... 39
long output ... 40
long output ... 41
long output ... 42
long output ... 43
long output ... 44
long output ... 45
long output ... 46
long output ... 47
long output ... 48
long output ... 49
long output ... 50
long output ... 51
long output ... 52
long output ... 53
long output ... 54
long output ... 55
long output ... 56
long output ... 57
long output ... 58
long output ... 59
long output ... 60
long output ... 61
long output ... 62
long output ... 63
long output ... 64
long output ... 65
long output ... 66
long output ... 67
long output ... 68
long output ... 69
long output ... 70
long output ... 71
long output ... 72
long output ... 73
long output ... 74
long output ... 75
long output ... 76
long output ... 77
long output ... 78
long output ... 79
long output ... 80
long output ... 81
long output ... 82
long output ... 83
long output ... 84
long output ... 85
long output ... 86
long output ... 87
long output ... 88
long output ... 89
long output ... 90
long output ... 91
long output ... 92
long output ... 93
long output ... 94
long output ... 95
long output ... 96
long output ... 97
long output ... 98
long output ... 99
long output ... 100
long output ... 101
long output ... 102
long output ... 103
long output ... 104
long output ... 105
long output ... 106
long output ... 107
long output ... 108
long output ... 109
long output ... 110
long output ... 111
long output ... 112
long output ... 113
long output ... 114
long output ... 115
long output ... 116
long output ... 117
long output ... 118
long output ... 119
long output ... 120
long output ... 121
long output ... 122
long output ... 123
long output ... 124
long output ... 125
long output ... 126
long output ... 127
long output ... 128
long output ... 129
long output ... 130
long output ... 131
long output ... 132
long output ... 133
long output ... 134
long output ... 135
long output ... 136
long output ... 137
long output ... 138
long output ... 139
long output ... 140
long output ... 141
long output ... 142
long output ... 143
long output ... 144
long output ... 145
long output ... 146
long output ... 147
long output ... 148
long output ... 149