# Notebook from perryisbusy/My-Pandas-Playground
Path: Section 2.ipynb

# operator _____no_output_____

# Solve really complex mathematicalexpression using Python
2 + 2
# 3 + 3_____no_output_____5
10
-23
0_____no_output_____3.14
9.99
-113.23819_____no_output_____"Python crash course"
'Sarcramento'_____no_output_____"139.99A@#$%"_____no_output_____type("139.99A@#$%")_____no_output_____""_____no_output_____type("")_____no_output_____True_____no_output_____False_____no_output_____5 < 7_____no_output_____10 > 12_____no_output_____None_____no_output_____type(None)_____no_output_____5 + 3_____no_output_____6 ** 4_____no_output_____3.14 + 7.96_____no_output_____# concatenation
"race" + "car"_____no_output_____10 - 5_____no_output_____"haha " * 4_____no_output_____

# PEMDAS
# Parentheses Exponents Multiplication Division Addition Subtraction
3 + 4 * 5
# (3 + 4) * 5_____no_output_____

15 / 4_____no_output_____10 / 2_____no_output_____# floor division
10 // 3_____no_output_____14 // 3_____no_output_____# get a reminder
14 % 3_____no_output_____# Equality operator
1 == 1.0_____no_output_____"Hello" == 'hello'_____no_output_____"Hello" == "Hello "_____no_output_____True == True_____no_output_____False == False_____no_output_____# Inequality operator --> !=
# 5 <> 10 # wrong
5 != 10_____no_output_____5 < 8_____no_output_____5 <= 8_____no_output_____5 <= 8_____no_output_____

# Variable_____no_output_____

age = 29
age_____no_output_____age + 5_____no_output_____price = 19.99
first_name = "Boris"
profession = "Data Analyst"
is_handsome = True
# snake_case_____no_output_____first_name + " is awesome"_____no_output_____age = age + 5_____no_output_____profession = "Data Engineer" + " Data Analyst"_____no_output_____profession_____no_output_____age_____no_output_____

# Built-in functions_____no_output_____

len(profession)_____no_output_____str(3.14)_____no_output_____int("10")_____no_output_____float("9.384")_____no_output_____type(profession)_____no_output_____type(False)_____no_output_____type(None)_____no_output_____type(10) == type(10.0)_____no_output_____type(10) == type("10")_____no_output_____

# Custom Function_____no_output_____

# Declare a function that accepts a temperature in Celsius and returns it in Fahrenheit

def  convert_to_fahrenheit(celsius_temp = 0):
    product = celsius_temp * 1.8
    product += 32
    return product_____no_output_____convert_to_fahrenheit(30)_____no_output_____len(str(convert_to_fahrenheit(30)))_____no_output_____convert_to_fahrenheit(14)_____no_output_____convert_to_fahrenheit(celsius_temp = 24)_____no_output_____convert_to_fahrenheit()_____no_output_____profession = "Developer"_____no_output_____profession.upper()
profession.lower()_____no_output_____"DEVELOPER".casefold()_____no_output_____profession.swapcase()_____no_output_____

# string is immutable - incapable of change
# list is mutable - capable_____no_output_____

profession_____no_output_____profession[0]_____no_output_____profession[0] = "d"_____no_output_____"once upon a time".title()_____no_output_____"once upon a time".capitalize()_____no_output_____job = "        Date Analyst        "_____no_output_____job.strip()
job.rstrip()
job.lstrip()_____no_output_____job.index("D")_____no_output_____# string chaining
job.replace("e", "a").strip()_____no_output_____"a" in job_____no_output_____job.find("x")_____no_output_____job.index("x")_____no_output_____animal = "rhinoceros"_____no_output_____animal.startswith("rhino")_____no_output_____# case sensitive
animal.endswith("oS")_____no_output_____"a" in job_____no_output_____"fuck" not in animal_____no_output_____"d1112313".isdigit()_____no_output_____

# List
# Mutable data strucuture that holds an ordered collection of elements_____no_output_____

party_attendess = ["Michael", "Freddy", "Jason"]_____no_output_____party_attendess_____no_output_____len(party_attendess)_____no_output_____presidents = ["Washington", "Jefferson"]_____no_output_____presidents.append("Madison")_____no_output_____presidents_____no_output_____popcorn = ["Salted", "Unsalted", "Caramel"]_____no_output_____# popcorn.pop()
popcorn.pop(2)_____no_output_____popcorn.remove("dd")_____no_output_____popcorn.index("Surger")_____no_output_____popcorn.insert(2, "Caramel")_____no_output_____popcorn_____no_output_____popcorn.count("Surger")_____no_output_____popcorn.count("Salted")_____no_output_____popcorn.reverse()_____no_output_____popcorn_____no_output_____"Caramel" in popcorn_____no_output_____"Cat" in "caterpillar"_____no_output_____tmp = popcorn.copy()_____no_output_____tmp_____no_output_____

# List can contain everything, boolean, string, float, integer_____no_output_____

lst = [popcorn, "caterpillar", 123, True, None, {"a":1, "b":2}, (8, 9, 10)]_____no_output_____lst_____no_output_____

# Index positions and slicing_____no_output_____

spiderman = "Spiderman"_____no_output_____spiderman_____no_output_____for item in spiderman:
    print(item, end="")Spiderman"Ω≈˜µåß©˙∆˚¬…æœ∑´®†¥¨ˆøπ“‘«÷√".isprintable()_____no_output_____spiderman[-len(spiderman)]_____no_output_____superheroes = ["Batman", "Superman", "Wolverine", "Ironman", "Arnold Schwarzenegger"]_____no_output_____len(superheroes)_____no_output_____for location, item in enumerate(superheroes):
    print(location, item)0 Batman
1 Superman
2 Wolverine
3 Ironman
4 Arnold Schwarzenegger
# up to but not including
superheroes[1:3]_____no_output_____spiderman[1:3]_____no_output_____# 500 doesn't exits
superheroes[2:500]_____no_output_____spiderman[3:]_____no_output_____superheroes[:]_____no_output_____spiderman[3:-2]_____no_output_____

# Dictionaries
# An unordered collection of key-value pairs
# IT is mutable_____no_output_____

# key must be unique and value can be duplicate
meau = {
    "Filet Mignon" : 29.99,
    "Big Mac" : 3.99,
    "Pizza" : 0.99,
    "Salmon" : 29.99
}_____no_output_____meau_____no_output_____len(meau)_____no_output_____meau["Filet Mignon"]_____no_output_____meau["salmon"]_____no_output_____meau.get("Tuna", 0)_____no_output_____for key, value in meau.items():
    print(key, value)Filet Mignon 29.99
Big Mac 3.99
Pizza 0.99
Salmon 29.99
meau["Burrito"] = 13.99_____no_output_____meau["Chicken Tender"] = None_____no_output_____meau.pop("Big Mac")_____no_output_____"Big Mac" in meau_____no_output_____"Pizza" in meau_____no_output_____"Bubble Gum" not in meau_____no_output_____meau.keys()
meau.values()_____no_output_____13.99 in meau.values()_____no_output_____"Hot Pockets" in meau.keys()_____no_output_____meau.update({"Nacho Fries" : 1.99})_____no_output_____meau_____no_output_____meau.popitem()_____no_output_____laptop = ["Macbook Pro 2015", "Macbook Pro 2019", "Thinkpad T460s", "Thinkpad T490s"]_____no_output_____price = [500]_____no_output_____laptop_price = dict.fromkeys(laptop)_____no_output_____laptop_price_____no_output_____


# Notebook from javierpastorfernandez/stardist
Path: examples/2D/1_data.ipynb

**In case of problems or questions, please first check the list of [Frequently Asked Questions (FAQ)](https://stardist.net/docs/faq.html).**_____no_output_____

from __future__ import print_function, unicode_literals, absolute_import, division
import numpy as np
import matplotlib
matplotlib.rcParams["image.interpolation"] = None
import matplotlib.pyplot as plt
%matplotlib inline
%config InlineBackend.figure_format = 'retina'

from glob import glob
from tqdm import tqdm
from tifffile import imread
from csbdeep.utils import Path, download_and_extract_zip_file

from stardist import fill_label_holes, relabel_image_stardist, random_label_cmap
from stardist.matching import matching_dataset

np.random.seed(42)
lbl_cmap = random_label_cmap()_____no_output_____

# Data

This notebook demonstrates how the training data for *StarDist* should look like and whether the annotated objects can be appropriately described by star-convex polygons.

The training data that needs to be provided for StarDist consists of corresponding pairs of raw images and pixelwise annotated ground truth images (masks), where every pixel has a unique integer value indicating the object id (or 0 for background).

For this demo we will download the file `dsb2018.zip` that contains the respective train and test images with associated ground truth labels as used in [our paper](https://arxiv.org/abs/1806.03535).
They are a subset of the `stage1_train` images from the Kaggle 2018 Data Science Bowl, which are [available in full](https://data.broadinstitute.org/bbbc/BBBC038/) from the [Broad Bioimage Benchmark Collection](https://data.broadinstitute.org/bbbc/)._____no_output_____

download_and_extract_zip_file(
    url       = 'https://github.com/stardist/stardist/releases/download/0.1.0/dsb2018.zip',
    targetdir = 'data',
    verbose   = 1,
)Files found, nothing to download.
X = sorted(glob('data/dsb2018/train/images/*.tif'))
Y = sorted(glob('data/dsb2018/train/masks/*.tif'))
assert all(Path(x).name==Path(y).name for x,y in zip(X,Y))_____no_output_____

Load only a small subset_____no_output_____

X, Y = X[:10], Y[:10]_____no_output_____X = list(map(imread,X))
Y = list(map(imread,Y))_____no_output_____

# Example image_____no_output_____

i = min(4, len(X)-1)
img, lbl = X[i], fill_label_holes(Y[i])
assert img.ndim in (2,3)
img = img if img.ndim==2 else img[...,:3]
# assumed axes ordering of img and lbl is: YX(C)_____no_output_____plt.figure(figsize=(16,10))
plt.subplot(121); plt.imshow(img,cmap='gray');   plt.axis('off'); plt.title('Raw image')
plt.subplot(122); plt.imshow(lbl,cmap=lbl_cmap); plt.axis('off'); plt.title('GT labels')
None;_____no_output_____

# Fitting ground-truth labels with star-convex polygons_____no_output_____

n_rays = [2**i for i in range(2,8)]
scores = []
for r in tqdm(n_rays):
    Y_reconstructed = [relabel_image_stardist(lbl, n_rays=r) for lbl in Y]
    mean_iou = matching_dataset(Y, Y_reconstructed, thresh=0, show_progress=False).mean_true_score
    scores.append(mean_iou)100%|██████████| 6/6 [00:05<00:00,  1.03it/s]
plt.figure(figsize=(8,5))
plt.plot(n_rays, scores, 'o-')
plt.xlabel('Number of rays for star-convex polygon')
plt.ylabel('Reconstruction score (mean intersection over union)')
plt.title("Accuracy of ground truth reconstruction (should be > 0.8 for a reasonable number of rays)")
None;_____no_output_____plt.figure(figsize=(8,5))
plt.plot(n_rays, scores, 'o-')
plt.xlabel('Number of rays for star-convex polygon')
plt.ylabel('Reconstruction score (mean intersection over union)')
plt.title("Accuracy of ground truth reconstruction (should be > 0.8 for a reasonable number of rays)")
None;_____no_output_____

## Example image reconstructed with various number of rays_____no_output_____

fig, ax = plt.subplots(2,3, figsize=(16,11))
for a,r in zip(ax.flat,n_rays):
    a.imshow(relabel_image_stardist(lbl, n_rays=r), cmap=lbl_cmap)
    a.set_title('Reconstructed (%d rays)' % r)
    a.axis('off')
plt.tight_layout();_____no_output_____


# Notebook from jackhmiller/My-DS-Blog
Path: _notebooks/2020-01-23-GL.ipynb

# Group Lasso

- toc: false
- branch: master
- badges: true
- comments: true
- image: images/Lasso.png
- hide: false
- search_exclude: false_____no_output_____It is frequently the case when dealing with high dimensional datasets that there are more variables than observations but we only expect a small fragment of the variables to be truly meaningful. To compensate for such occurances and enhance our ability to generalize our resulting model, it is common to employ regularization techniques which are also used to reduce overfitting in other settings. The most common examples are L1 (Lasso) and L2 (Ridge) regularization. Lasso, in particular, causes sparsity for weights. It provides sparse solutions, because it will send to zero some of the β coefficients (the least related with the response variable). The effect of this penalization can be controlled using the λ parameter. A large λ value provides solutions where the penalization has a greater importance, and thus there are more zeros among the β coefficients.

_____no_output_____However, there are many regression problems dealing with high dimensional data in which the covariates have a natural group structure, and it is desirable to have all coefficients within a group become nonzero (or zero) simultaneously. For example, in biostatistics it is common to deal with genetic datasets in which predictors are grouped into genetical pathways. In stock market analysis one can group companies from the same business segment. In climate data one can group different regions… And lasso provides individual sparse solutions, not group sparse. A more general leading example is when we have qualitative factors among our predictors. We typically code their levels using a set of dummy variables or contrasts, and would want to include or exclude this group of variables together. For such a scenario, we have a technique called the Group Lasso, which accounts for a natural grouped structure of predictors while causing sparsity for weights. _____no_output_____In the image above, we can see a visual depiction of the contours of the L1, group lasso, and L2 penalties respectively (Source: Ming Yuan, "Model selection and estimation in regression with grouped variables," Statistical Methodology 68:1 (Feb 2006). Specifically, we can see how the group lasso (middle image) incorporates elements of both the L1 and L2 penalties. _____no_output_____Lets get into the math of the Group Lasso. Consider a linear regression model involving J groups of covariates, where j = 1,..., J, and the vector $\mathbb{Z}_j \in \mathbb{R}^{pj}$ represents the covariates of group j. Our goal is to predict a real valued response $Y \in \mathbb{R}$ based on the collection of covariates ($\mathbb{Z}_1, ..., \mathbb{Z}_J $. A linear model for the regression function $\mathbb{E}(Y|\mathbb{Z})$ $\sum_{j=1}^{J}\mathbb{Z}_{j}^{T}\theta_j$ where $\theta_j \in \mathbb{R}^{pj}$ represents a group of $p_j$ regression coefficients._____no_output_____Given a collection of N samples $\left \{ (y_i, z_{i1}, z_{i2},..., z_{iJ}) \right \}_{i=1}^{N}$ the group lasso solves the convex problem:
$$ \underset{\theta_j\in\mathbb{R}^{pj}}{min}\frac{1}{2}\left \| y-\sum_{j=1}^{J}\mathbb{Z}_{j}^{T}\theta_j \right \|_{2}^{2} + \lambda\sum_{j=1}^{J}\left \| \theta_j \right \|_2 $$
where $\left \| \theta_j \right \|_2$ is the euclidean norm of the vector $\theta_j$._____no_output_____This is a group generalization of the lasso, with the properties:
- depending on $\lambda \geq 0$, either the entire vector $\theta_j$ will be zero, or all its elements will be nonzero
- when $p_j=1$ (continuous variables), then we have $\left \| \theta_j \right \|_2 = \left | \theta_j \right |$, so if all of the groups are singletons, the optimization problem reduces to ordinary lasso. _____no_output_____We can solve the group lasso problem using block coordinate descent. Here is a proof showing that we can solve it iteratively, for $j = 1 · · · , J$:_____no_output_____We start with our optimization problem:
$$ \underset{\theta_j\in\mathbb{R}^{pj}}{min}\frac{1}{2}\left \| r_j-\mathbb{Z}_j^T\theta_j \right \|_2^2 + \lambda\left \| \theta_j \right \|_2 $$
Where $r_j = y - \sum_{k\neq j}^{}Z_k^T\theta_k$_____no_output_____With a bit of manipulation we get $$ -Z_j^T(y-\sum_{j=1}^{J}Z_j\theta_j)+\lambda(s(\frac{\theta_j}{\left \| \theta_j \right \|_{2}})) $$_____no_output_____Using the definition of $r_j$ we can solve for $\theta$:
$$ \theta_j = (Z_j^TZ_j + \frac{\lambda}{\left \| \theta_j \right \|_2})^{-1}Z_j^Tr_j $$_____no_output_____Or $$ \theta_j = S_{\frac{\lambda}{\left \| \theta_j \right \|^2}}(\frac{Z_{j}^{T}r_j}{Z_{j}^{T}Z_j}) $$_____no_output_____
_____no_output_____We will now implement the block coordinate proximal gradient descent to solve the group lasso problem presented above. We will be using the kaggle classic boston housing dataset, where our independent variable is price, and our dependent variables are a combination of categorical (number of bed/bath) and continous (price/sq feet) features. Our first step is to create dummy variables corresponding to the categorical variables. To avoid multicollinearity issues we use 0 bedrooms, 1 bathroom, and short sale as baselines, respectively. To improve results we standardize our data, and will use a $\lambda$ value of 0.012._____no_output_____

from sklearn.preprocessing import MinMaxScaler
import itertools_____no_output_____bh = pd.read_csv("boston_housing.csv")_____no_output_____

Create dummy variables_____no_output_____

rooms = pd.get_dummies(bh.Bedrooms, prefix='Bedrooms')
baths = pd.get_dummies(bh.Bathrooms, prefix='Bath')
status = pd.get_dummies(bh.Status, prefix='Status')_____no_output_____df = pd.concat([bh, rooms, baths, status], axis=1)_____no_output_____df.drop(['Bathrooms', 'Bedrooms', 'Status'], axis=1, inplace=True)_____no_output_____

Normalize data_____no_output_____

scaler = MinMaxScaler()
df['Price'] = scaler.fit_transform(df['Price'].values.reshape(-1,1))
df['PriceSF'] = scaler.fit_transform(df['PriceSF'].values.reshape(-1,1))_____no_output_____df.head()_____no_output_____y = df['Price'].values_____no_output_____features = df.drop('Price', axis=1)_____no_output_____groups = ['PriceSF', 'Bed', 'Bath', 'Status']_____no_output_____

Defining our soft thresholding function for PGM, and our loss function. _____no_output_____

def soft_threshold(x, gamma=lamda):
    for i, val in enumerate(x):
        if val > gamma:
            x[i] = 1-(lamda/abs(val-gamma))
        elif val  < gamma:
            x[i] = 1-(lamda/abs(val+gamma))
        elif (val <= gamma) and (val>= -gamma):
            x[i] = 0
    return x

def loss(b, l=0.012):
    temp_coeffs = [[beta]*i for beta, i in zip(b, group_lengths)]
    coeff_vector = np.array(list(itertools.chain(*temp_coeffs)))
    f_x = np.sum(0.5*(y - np.dot(features.values,coeff_vector))**2)
    penalty = l*sum([i**2 for i in b])

    return f_x + penalty

def create_b_not_vector(b, i):
    not_group_lengths = [j for k, j in enumerate(group_lengths) if k != i]
    temp_coeffs = [[beta]*i for beta, i in zip(b, not_group_lengths)]
    return np.array(list(itertools.chain(*temp_coeffs)))_____no_output_____slices = [(0,1), (1, 6), (6, 10), (10, 13)]
b = np.random.rand(len(features.columns))
lamda = 0.012
losses = [loss(b)]
iterations = 1
for iteration in range(200):
    for sliced in slices:
        Z = features[features.columns[sliced[0]:sliced[1]]]
        Z_cols = Z.columns
        Z_not = features.loc[:, [feat for feat in features.columns if feat not in Z_cols]]
        b_not = [i for j, i in enumerate(b) if j not in range(sliced[0], sliced[1])]
        r = y - np.dot(Z_not, b_not)
        a = b[sliced[0]:sliced[1]] - np.sum((-Z.values.T*(r-np.dot(Z.values, b[sliced[0]:sliced[1]]))), axis=1)
        b[sliced[0]:sliced[1]] = soft_threshold(a)

    f_x = np.sum(0.5*(y - np.dot(features.values,b))**2)
    penalty = lamda*sum([i**2 for i in b])
    losses.append(f_x + penalty)
    iterations += 1_____no_output_____

And there you have it_____no_output_____


# Notebook from gvvynplaine/scipy-2017-codegen-tutorial
Path: notebooks/23-lambdify-Tc99m.ipynb

## Using lambdify for plotting expressions
The syntethic isotope Technetium-99m is used in medical diagnostics ([scintigraphy](https://en.wikipedia.org/wiki/Nuclear_medicine)):
$$
^{99m}Tc \overset{\lambda_1}{\longrightarrow} \,^{99}Tc \overset{\lambda_2}{\longrightarrow} \,^{99}Ru \\
\lambda_1 = 3.2\cdot 10^{-5}\,s^{-1} \\
\lambda_2 = 1.04 \cdot 10^{-13}\,s^{-1} \\
$$
SymPy can solve the differential equations describing the amounts versus time analytically.
Let's denote the concentrations of each isotope $x(t),\ y(t)\ \&\ z(t)$ respectively._____no_output_____

import sympy as sym
sym.init_printing()_____no_output_____symbs = t, l1, l2, x0, y0, z0 = sym.symbols('t lambda_1 lambda_2 x0 y0 z0', real=True, nonnegative=True)
funcs = x, y, z = [sym.Function(s)(t) for s in 'xyz']
inits = [f.subs(t, 0) for f in funcs]
diffs = [f.diff(t) for f in funcs]
exprs = -l1*x, l1*x - l2*y, l2*y
eqs = [sym.Eq(diff, expr) for diff, expr in zip(diffs, exprs)]
eqs_____no_output_____solutions = sym.dsolve(eqs)
solutions_____no_output_____

now we need to determine the integration constants from the intial conditions:_____no_output_____

integration_constants = set.union(*[sol.free_symbols for sol in solutions]) - set(symbs)
integration_constants_____no_output_____initial_values = [sol.subs(t, 0) for sol in solutions]
initial_values_____no_output_____const_exprs = sym.solve(initial_values, integration_constants)
const_exprs_____no_output_____analytic = [sol.subs(const_exprs) for sol in solutions]
analytic_____no_output_____

## Exercise: Create a function from a symbolic expression
We want to plot the time evolution of x, y & z from the above analytic expression (called ``analytic`` above):_____no_output_____

from math import log10
import numpy as np
year_s = 365.25*24*3600
tout = np.logspace(0, log10(3e6*year_s), 500)  # 1 s to 3 million years_____no_output_____%load_ext scipy2017codegen.exercise_____no_output_____

*Use either the *``%exercise``* or *``%load``* magic to get the exercise / solution respecitvely:*

Replace **???** so that `f(t)` evaluates $x(t),\ y(t)\ \&\ z(t)$. Hint: use the right hand side of the equations in ``analytic`` (use the attribute ``rhs`` of the items in ``anayltic``):_____no_output_____

# %exercise exercise_Tc99.py
xyz_num = sym.lambdify([t, l1, l2, *inits], [eq.rhs for eq in analytic])
yout = xyz_num(tout, 3.2e-5, 1.04e-13, 1, 0, 0)_____no_output_____import matplotlib.pyplot as plt
%matplotlib inline
fig, ax = plt.subplots(1, 1, figsize=(14, 4))
ax.loglog(tout.reshape((tout.size, 1)), np.array(yout).T)
ax.legend(['$^{99m}Tc$', '$^{99}Tc$', '$^{99}Ru$'])
ax.set_xlabel('Time / s')
ax.set_ylabel('Concentration / a.u.')
_ = ax.set_ylim([1e-11, 2])_____no_output_____


# Notebook from adam-m-jcbs/Simmy
Path: IPyRoot/SimmyDev.ipynb

A scratch notebook for testing and developing `simmy`._____no_output_____

from titan import TitanConfig
from subchandra import SCConfig
#This means that modules will be automatically reloaded when changed.
#Makes for swift exporatory development, probably a performance hit if using well-tested code.
%load_ext autoreload
%autoreload 2_____no_output_____

# Simulation & SimulationGrid_____no_output_____### SimConfig_____no_output_____##### XRB Config_____no_output_____##### SCConfig_____no_output_____

#Test SCConfig/SimConfig
test_model_dir = '/home/ajacobs/Research/Projects/Simmy/IPyRoot/SCTestGrid/10044-090-210-4lev-full-512'

#Init from existing test
test_model_config = SCConfig(test_model_dir)
test_model_config.printConfig()Inputs Configuration
Description: Configuration of the inputs file.  This is the file passed
        to the Maestro executable that sets various Maestro parameters,
        configures the simulation, and provides the location of initial model
        data.
Fields:
    im_file
        Description: Initial model file with data to be read into the Maestro basestate.
        Current value: "sub_chandra.M_WD-1.00.M_He-0.045.hse.C.10240"
    job_name
        Description: Description of the simulation.
        Current value: "512^3 base grid, T_core = 10^7, T_base = 210 MK -- M_WD=1.0, M_He=0.04"
    max_levs
        Description: Number of levels the AMR will refine to.
        Current value: 4
    coarse_res
        Description: Resolution of the base (coarsest) level
        Current value: 512
    anelastic_cutoff
        Description: Density cutoff below which the Maestro velocity constraint is simplified to the anelastic constraint.
        Current value: 64000.0
    octant
        Description: Boolean that sets if an octant or full star should be modeled.
        Current value: .false.
    dim
        Description: Dimensionality of the problem.
        Current value: 3
    physical_size
        Description: Sidelength in cm of the square domain.
        Current value: 1500000000.0
    plot_deltat
        Description: Time interval in s at which to save pltfiles.
        Current value: 10.0d0
    mini_plot_deltat
        Description: Time interval in s at which to save minipltfiles.
        Current value: 0.2d0
    chk_int
        Description: Timestep interval at which to save chkpoint files.
        Current value: 10
    plot_base_name
        Description: Basename for pltfiles. Pltfiles will be saved with this name plus their timestep.
        Current value: "10044-090-210-4lev-full-512_plt"
    mini_plot_base_name
        Description: Basename for minipltfiles. Minipltfiles will be saved with this name plus their timestep.
        Current value: "10044-090-210-4lev-full-512_miniplt"
    check_base_name
        Description: Basename for checkpoint files. Chkfiles will be saved with this name plus their timestep.
        Current value: "10044-090-210-4lev-full-512_chk"
    bc_lo
        Description: Integer flag for the lower (x=y=z=0) boundary
        Current value: 12
    bc_hi
        Description: Integer flag for the hi (x=y=z=max) boundary
        Current value: 12

Initial Model Configuration
Description: Configures the initial model for this simulation.  This
        corresponds to the _params file used by init1d to build an initial 1D
        model to be mapped to the 3D domain.  The data from this model are also
        stored.
Fields:
    nx
        Description: Resolution (number of cells) of the 1D model, should match Maestro base state resolution.
        Current value: 10240
    M_tot
        Description: Mass of the WD core in M_sol.
        Current value: 1.0
    M_He
        Description: Mass of He envelope in M_sol.
        Current value: 0.0445
    delta
        Description: Transition delta from core to envelope in cm.
        Current value: 2000000.0
    temp_core
        Description: Isothermal core temperature in K.
        Current value: 90000000.0
    temp_base
        Description: Temperature at the base of the He envelope in K.
        Current value: 210000000.0
    xmin
        Description: Spatial coordinate in cm the model starts at.
        Current value: 0.0
    xmax
        Description: Spatial coordinate in cm of the last cell, should match the sidelength of domain in octant simulation, half sidelength for full star.
        Current value: 750000000.0
    mixed_co_wd
        Description: Boolean that sets if core is C/O or just C.
        Current value: .false.
    low_density_cutoff
        Description: Density floor in the initial model (NOT for the 3D Maestro domain).
        Current value: 1.d-4
    temp_fluff
        Description: Temperature floor, will also be temperature when below density floor.
        Current value: 7.5d7
    smallt
        Description: An unused parameter that used to be like temp_fluff.
        Current value: 1.d6
    radius
        Description: NumPy array of initial model radius in cm.
        Current value: [  3.66210938e+04   1.09863281e+05   1.83105469e+05 ...,   7.49816895e+08
   7.49890137e+08   7.49963379e+08]
    density
        Description: NumPy array of initial model density in g/cm^3.
        Current value: [  4.04783862e+07   4.04783745e+07   4.04783511e+07 ...,   4.83128894e-03
   4.81266153e-03   4.79410982e-03]
    temperature
        Description: NumPy array of initial model temperature in K.
        Current value: [ 90000000.  90000000.  90000000. ...,  75000000.  75000000.  75000000.]
    pressure
        Description: NumPy array of initial model pressure in dyn/cm^2.
        Current value: [  6.09564571e+24   6.09564325e+24   6.09563834e+24 ...,   7.98193462e+16
   7.98192591e+16   7.98191723e+16]
    soundspeed
        Description: NumPy array of initial model sound speed in cm/s.
        Current value: [  0.00000000e+00   4.58558611e+08   4.58558562e+08 ...,   4.69352862e+09
   4.70260012e+09   4.71168726e+09]
    entropy
        Description: NumPy array of initial model specific entropy in erg/(g*K).
        Current value: [  0.00000000e+00   6.82762373e+07   6.82762427e+07 ...,   8.82233068e+11
   8.85642789e+11   8.89064988e+11]
    species
        Description: NumPy 2D array of initial model species mass fractions.
        Current value: [[  1.00000000e-10   1.00000000e-10   1.00000000e-10 ...,   1.00000000e+00
    1.00000000e+00   1.00000000e+00]
 [  1.00000000e+00   1.00000000e+00   1.00000000e+00 ...,   1.00000000e-10
    1.00000000e-10   1.00000000e-10]
 [  1.00000000e-10   1.00000000e-10   1.00000000e-10 ...,   1.00000000e-10
    1.00000000e-10   1.00000000e-10]]

#Init from scratch test
test_model_dir = '/home/ajacobs/Research/Projects/Simmy/IPyRoot/SCTestGrid/10044-090-210-4lev-debug'

#Design
#  + Get baseline inputs and im ConfigRecs
#  + Make factory method smart enough to use this to gen initial model and finish off initialization of ConfigRecs

#Build up initial model ConfigRecord baseline
im_rec = SCConfig.genIMConfigRec()
im_rec.setField('M_tot', '1.0')
im_rec.setField('M_He', '0.0445')
im_rec.setField('delta', '2000000.0')
im_rec.setField('temp_core', '90000000.0')
im_rec.setField('temp_base', '210000000.0')
im_rec.setField('im_exe', '/home/ajacobs/Codebase/Maestro/Util/initial_models/sub_chandra/init_1d.Linux.gfortran.debug.exe')
print(im_rec)

#Build up inputs ConfigRecord baseline
inputs_rec = SCConfig.genInputsConfigRec()
inputs_rec.setField('max_levs', 2)
inputs_rec.setField('coarse_res', 128)
print(inputs_rec)

#Create the Config object and print
test_model_config = SCConfig(test_model_dir, config_recs=[im_rec,inputs_rec])Initial Model Configuration
Description: Configures the initial model for this simulation.  This
        corresponds to the _params file used by init1d to build an initial 1D
        model to be mapped to the 3D domain.  The data from this model are also
        stored.
Fields:
    im_exe
        Description: Full path to the initial model builder.
        Current value: /home/ajacobs/Codebase/Maestro/Util/initial_models/sub_chandra/init_1d.Linux.gfortran.debug.exe
    nx
        Description: Resolution (number of cells) of the 1D model, should match Maestro base state resolution.
        Current value: None
    M_tot
        Description: Mass of the WD core in M_sol.
        Current value: 1.0
    M_He
        Description: Mass of He envelope in M_sol.
        Current value: 0.0445
    delta
        Description: Transition delta from core to envelope in cm.
        Current value: 2000000.0
    temp_core
        Description: Isothermal core temperature in K.
        Current value: 90000000.0
    temp_base
        Description: Temperature at the base of the He envelope in K.
        Current value: 210000000.0
    xmin
        Description: Spatial coordinate in cm the model starts at.
        Current value: 0.0
    xmax
        Description: Spatial coordinate in cm of the last cell, should match the sidelength of domain in octant simulation, half sidelength for full star.
        Current value: None
    mixed_co_wd
        Description: Boolean that sets if core is C/O or just C.
        Current value: .false.
    low_density_cutoff
        Description: Density floor in the initial model (NOT for the 3D Maestro domain).
        Current value: 1.d-4
    temp_fluff
        Description: Temperature floor, will also be temperature when below density floor.
        Current value: 7.5d7
    smallt
        Description: An unused parameter that used to be like temp_fluff.
        Current value: 1.d6
    radius
        Description: NumPy array of initial model radius in cm.
        Current value: None
    density
        Description: NumPy array of initial model density in g/cm^3.
        Current value: None
    temperature
        Description: NumPy array of initial model temperature in K.
        Current value: None
    pressure
        Description: NumPy array of initial model pressure in dyn/cm^2.
        Current value: None
    soundspeed
        Description: NumPy array of initial model sound speed in cm/s.
        Current value: None
    entropy
        Description: NumPy array of initial model specific entropy in erg/(g*K).
        Current value: None
    species
        Description: NumPy 2D array of initial model species mass fractions.
        Current value: None

Inputs Configuration
Description: Configuration of the inputs file.  This is the file passed
        to the Maestro executable that sets various Maestro parameters,
        configures the simulation, and provides the location of initial model
        data.
Fields:
    im_file
        Description: Initial model file with data to be read into the Maestro basestate.
        Current value: None
    drdxfac
        Description: 5
        Current value: 5
    job_name
        Description: Description of the simulation.
        Current value: None
    max_levs
        Description: Number of levels the AMR will refine to.
        Current value: 2
    coarse_res
        Description: Resolution of the base (coarsest) level
        Current value: 128
    anelastic_cutoff
        Description: Density cutoff below which the Maestro velocity constraint is simplified to the anelastic constraint.
        Current value: None
    octant
        Description: Boolean that sets if an octant or full star should be modeled.
        Current value: .false.
    dim
        Description: Dimensionality of the problem.
        Current value: 3
    physical_size
        Description: Sidelength in cm of the square domain.
        Current value: None
    plot_deltat
        Description: Time interval in s at which to save pltfiles.
        Current value: 5.0
    mini_plot_deltat
        Description: Time interval in s at which to save minipltfiles.
        Current value: 0.2
    chk_int
        Description: Timestep interval at which to save chkpoint files.
        Current value: 10
    plot_base_name
        Description: Basename for pltfiles. Pltfiles will be saved with this name plus their timestep.
        Current value: None
    mini_plot_base_name
        Description: Basename for minipltfiles. Minipltfiles will be saved with this name plus their timestep.
        Current value: None
    check_base_name
        Description: Basename for checkpoint files. Chkfiles will be saved with this name plus their timestep.
        Current value: None
    bc_lo
        Description: Integer flag for the lower (x=y=z=0) boundary
        Current value: None
    bc_hi
        Description: Integer flag for the hi (x=y=z=max) boundary
        Current value: None

/home/ajacobs/Codebase/Maestro/Util/initial_models/sub_chandra/init_1d.Linux.gfortran.debug.exe /home/ajacobs/Research/Projects/Simmy/IPyRoot/SCTestGrid/10044-090-210-4lev-debug/model/_params.10044-090-210-4lev-debug

# Machine_____no_output_____### RunConfig_____no_output_____##### TitanConfig_____no_output_____

#Test TitanConfig
config_dict = {}
config_dict['nodes'] = 128
test_config = TitanConfig('/home/ajacobs/Research/Projects/Simmy/IPyRoot/TestModel', config_dict)

#Tests to formalize
#    + Successfully creates file_____no_output_____

##### ICER Config_____no_output_____##### xrb Config_____no_output_____


# Notebook from phuijse/EVIC2020
Path: EVIC2020.ipynb

# Deep probabilistic models with applications in astronomy

We will review the basics of inference with probabilistic models, the more recent "deep" probabilistic models and how to implement them using the [Pyro probabilistic programming library](http://docs.pyro.ai/en/stable/getting_started.html). After that we will have a hands-on experience training probabilistic models to analyze time series from astronomical survey projects.

- This notebook lives at: https://github.com/phuijse/EVIC2020
    - git clone https://github.com/phuijse/EVIC2020
    - conda env create -f environment.yml
- [Open in google colab](https://colab.research.google.com/drive/1-RfsaAUnQ6foX6yGHGbbp_e_P5zR1dVv?usp=sharing)
- [More material and code examples on this topic](https://github.com/magister-informatica-uach/INFO320)
- [Lecture slides on neural networks (in spanish)](https://docs.google.com/presentation/d/1IJ2n8X4w8pvzNLmpJB-ms6-GDHWthfsJTFuyUqHfXg8/edit?usp=sharing)

Author: Pablo Huijse, phuijse at inf dot uach dot cl_____no_output_____

%matplotlib notebook
import matplotlib.pyplot as plt
import numpy as np
from IPython.display import YouTubeVideo
from tqdm.notebook import tqdm
import corner

import torch
import pyro
print(f"Torch version: {torch.__version__}")
print(f"Pyro version: {pyro.__version__}")

%load_ext autoreload
%autoreload 2
from astro_utils import plot_params, plot_lc, plot_lc_folded, featurize_lc, plot_lc_features, make_train_plots_____no_output_____

# Pyro basics_____no_output_____Example: Bayesian linear regression

$$
y_i = w x_i + b +  \epsilon_i
$$

with $N$ observations $(x_i, y_i)$, Gaussian noise and Gaussian priors for $w$ and  $b$

The generative process in this case is

- Sample $w \sim \mathcal{N}(\mu_w, \sigma_w^2)$
- Sample $b \sim \mathcal{N}(\mu_b, \sigma_b^2)$
- For $i=1,2,\ldots, N$
    - Sample $y_i \sim \mathcal{N}(w x_i + b, \sigma_\epsilon^2)$

where $\mu_w, \sigma_w, \mu_b, \sigma_b, \sigma_\epsilon$ are hyperparameters
_____no_output_____
## Writing a model

To write the model we use the submodules and primitives

- `pyro.distributions` to define prior/likelihoods
- `pyro.sample` to define random variables (RV): Expects name, distribution and optionally observations
- `pyro.plate` for conditionally independent RV: Expects name, and size_____no_output_____

from pyro.distributions import Normal, Uniform

def model(x, y=None):
    w = pyro.sample("w", Normal(0.0, 1.0))
    b = pyro.sample("b", Normal(0.0, 1.0))
    s = pyro.sample("s", Uniform(0.0, 1.0))
    with pyro.plate('dataset', size=len(x)):
        return pyro.sample("y", Normal(x*w + b, s), obs=y)_____no_output_____

We can use

- `pyro.infer.Predictive`

to draw samples from the model_____no_output_____

predictive = pyro.infer.Predictive(model, num_samples=500)

hatx = np.linspace(-6, 6, num=100).astype('float32')
apriori_samples = predictive(torch.from_numpy(hatx))

# Plot samples from the priors
figure = corner.corner(np.stack([apriori_samples[var].detach().numpy()[:, 0] for var in ['b', 'w', 's']]).T,
                       smooth=1., labels=["bias", "weight", "noise_std"], bins=20,
                       quantiles=[0.16, 0.5, 0.84],
                       show_titles=True, title_kwargs={"fontsize": 12})

# Plot posterior predictive of y given x
y_trace = apriori_samples["y"].detach().numpy()
med = np.median(y_trace, axis=0)
qua = np.quantile(y_trace, (0.05, 0.95), axis=0)

fig, ax = plt.subplots(figsize=(7, 3), tight_layout=True)
ax.plot(hatx, y_trace.T);_____no_output_____

## Inference

In the bayesian setting we want the posterior distribution

$$
p(\theta | \mathcal{D}) = \frac{p(\mathcal{D}|\theta) p(\theta)}{\int_\theta p(\mathcal{D}|\theta) p(\theta)}
$$

where $\mathcal{D}$ is our dataset and $\theta = (w, b)$

For complex models the posterior is intractable. So we either do

- MCMC: Train a Markov chain to generate samples as if they came from the actual posterior: Sampling based
- Variational Inference: Choose a more simple posterior that is similar to the actual posterior: Optimization based

_____no_output_____### Variational Inference

Propose an approximate (simple) posterior $q_\phi(\theta)$, e.g. factorized Gaussian

Optimize $\phi$ so that $q_\phi$ approximates $p(\theta|\mathcal{D})$

This is typically done by maximizing a lower bound on the evidence

$$
\mathcal{ELBO}(\phi) = \mathbb{E}_{\theta \sim q_\phi}[ \log p(\mathcal{D}|\theta)] - \text{KL}[q_\phi(\theta)|p(\theta)]
$$

- Maximize the likelihood of the model
- Minimize the distance between the approximate posterior and the prior

Once $q$ has been trained we use it as a replacement for $p(\theta|\mathcal{D})$ to calculate the **posterior predictive distribution**

$$
p(\mathbf{y}|\mathbf{x}, \mathcal{D}) = \int p(\mathbf{y}|\mathbf{x}, \theta) p(\theta| \mathcal{D}) \,d\theta
$$

_____no_output_____## VI with Pyro

We use `pyro.infer.SVI` to perform **Stochastic Variational Inference**, which expects

- A generative model
- An approximate posterior (guide)
- Cost function: Typically ELBO
- Optimizer: How to optimize the ELBO, typically gradient descent based

We can use the `pyro.infer.autoguide` to create approximate posteriors from predefined recipes, for example a factorized Gaussian posterior (`AutoDiagonalNormal`)_____no_output_____

pyro.enable_validation(True) # Useful for debugging
pyro.clear_param_store()

# Create a guide (approximate posterior)
from pyro.infer.autoguide import AutoDiagonalNormal as approx_posterior
guide = approx_posterior(model)

# Stochastic Variational Inference
svi = pyro.infer.SVI(model=model,
                     guide=guide,
                     loss=pyro.infer.Trace_ELBO(),
                     optim=pyro.optim.ClippedAdam({"lr": 1e-2}))_____no_output_____

Lets consider a dataset with two observations $\mathcal{D} = \{ (-2, -2), (2, 2) \}$

`svi.step(x, y)` performs a gradient ascent step to maximize the ELBO_____no_output_____

fig, ax = plt.subplots(1, 3, figsize=(10, 2.5), dpi=80, tight_layout=True)

nepochs = 1000
loss = np.zeros(shape=(nepochs, ))
params = np.zeros(shape=(nepochs, 2, 3))

# Observed data
x = torch.tensor([-2., 2.])
y = torch.tensor([-2., 2.])

for k in tqdm(range(nepochs)):
    loss[k] = svi.step(x, y)

    phi = [param.detach().numpy() for param in guide.parameters()]
    params[k, 0, :] = phi[0] # Locations
    params[k, 1, :] = phi[1] # Scales
    if np.mod(k, 10) == 0:
        plot_params(ax, k+1, loss, params)
        fig.canvas.draw()_____no_output_____

Again we can use `pyro.infer.Predictive` to draw samples from the model

This time we use the guide to sample $w$ and $b$_____no_output_____

predictive = pyro.infer.Predictive(model,
                                   guide=guide,
                                   num_samples=1000)

posterior_samples = predictive(torch.from_numpy(hatx))

# Plot posterior of w,  b and s
figure = corner.corner(np.stack([posterior_samples[var].detach().numpy()[:, 0] for var in ['b', 'w', 's']]).T,
                       smooth=1., labels=["bias", "weight", "noise_std"], bins=20,
                       quantiles=[0.16, 0.5, 0.84],
                       show_titles=True, title_kwargs={"fontsize": 12})

# Plot posterior predictive of y given x
y_trace = posterior_samples["y"].detach().numpy()
med = np.median(y_trace, axis=0)
qua = np.quantile(y_trace, (0.05, 0.95), axis=0)

fig, ax = plt.subplots(figsize=(7, 3), tight_layout=True)
ax.plot(hatx, med)
ax.fill_between(hatx, qua[0], qua[1], alpha=0.5);

ax.errorbar(x.numpy(), y.numpy(), yerr=2*posterior_samples['s'].median().item(),
           fmt='none', c='k', zorder=100);
_____no_output_____

# PyTorch basics_____no_output_____Artificial Neuron: Linear regressor plus activation function (e.g. sigmoid -> Logistic regression)

$$
\hat y = g\left(\sum_i w_i x_i + b\right)
$$

Multilayer perceptron: Several fully connected layers concatenated

$$
\begin{align}
f_i &=   b_i + \sum_{j=1}^H w_{ij} h_j  \nonumber \\
&=  b_i + \sum_{j=1}^H w_{ij} g \left( b_j + \sum_{d=1}^D w_{jd} x_d  \right) \nonumber
\end{align}
$$

_____no_output_____Neural network model in pytorch

- We write a class that inherits from `torch.nn.Module`
- The constructor `__init__(self, args):` define the layers, e.g. fully-connected (`Linear`), convolutional (`Conv1D`, `Conv2D`)
- The function `forward(self, x):` defines how layers are connected_____no_output_____

class MultiLayerPerceptron(torch.nn.Module):

    def __init__(self, D, H, O):
        super(type(self), self).__init__()
        self.hidden_layer = torch.nn.Linear(D, H) # WX + B
        self.output_layer = torch.nn.Linear(H, O)
        self.activation = torch.nn.Sigmoid()

    def forward(self, x):
        z = self.activation(self.hidden_layer(x))
        return self.output_layer(z)_____no_output_____

# Variational Autoencoder (VAE) [(Kingma et al., 2014)](https://arxiv.org/abs/1312.6114)

Non-linear latent variable model

_____no_output_____- Latent variables: $z \sim \mathcal{N}(0, I)$
- Observed variable: $x|z \sim \mathcal{N}(\hat \mu, \hat \sigma^2)$

We use a neural network $f_\theta$ to model a non-linear function $\hat \mu (z)$

We want to infer the latent given the observed

$$
p(z|x) = \frac{p(x|z) p(z)}{\int p(x|z) p(z) dz}
$$

But this is not tractable so we use a variational approximation (factorized Gaussian)

$$
\begin{align}
q_\phi(z|x) &= \mathcal{N}(\mu(x), \sigma(x)^2) \nonumber \\
&= \mu(x) + \sigma(x) \epsilon, \quad \epsilon \sim \mathcal{N}(0, I) \nonumber
\end{align}
$$

where a neural network $g_\phi$ are used to model the non-linear functions $\mu(x)$ and $\sigma(x)$

We amortize the parameters of $q$ using the neural network, making inference very efficient

Note: The VAE is not a full bayesian neural network, the parameters of the neural net are point estimates_____no_output_____

from pyro.distributions import Normal
import torch.nn as nn

class Encoder(torch.nn.Module):
    def __init__(self, data_dim, latent_dim, hidden_dim):
        super(type(self), self).__init__()
        self.hidden = nn.Linear(data_dim, hidden_dim)
        self.z_loc = nn.Linear(hidden_dim, latent_dim)
        self.z_scale = nn.Linear(hidden_dim, latent_dim)
        self.activation = nn.Softplus()

    def forward(self, x):
        h = self.activation(self.hidden(x))
        return self.z_loc(h), self.activation(self.z_scale(h))

class Decoder(torch.nn.Module):
    def __init__(self, data_dim, latent_dim, hidden_dim):
        super(type(self), self).__init__()
        self.hidden = nn.Linear(latent_dim, hidden_dim)
        self.x_loc = nn.Linear(hidden_dim, data_dim)
        self.activation = nn.Softplus()

    def forward(self, z):
        h = self.activation(self.hidden(z))
        return self.x_loc(h)

class VariationalAutoEncoder(torch.nn.Module):

    def __init__(self, data_dim, latent_dim, hidden_dim, sigmax):
        super(VariationalAutoEncoder, self).__init__()
        self.encoder = Encoder(data_dim, latent_dim, hidden_dim)
        self.decoder = Decoder(data_dim, latent_dim, hidden_dim)
        self.latent_dim = latent_dim
        self.data_dim = data_dim
        self.sigmax = sigmax

    def model(self, x):
        pyro.module("decoder", self.decoder)
        with pyro.plate("data", size=x.shape[0]):
            # p(z)
            z_loc = torch.zeros(x.shape[0], self.latent_dim, device=x.device)
            z_scale = torch.ones(x.shape[0], self.latent_dim, device=x.device)
            z = pyro.sample("latent", Normal(z_loc, z_scale).to_event(1))
            # p(x|z)
            x_loc = self.decoder.forward(z)
            pyro.sample("observed", Normal(x_loc, self.sigmax).to_event(1), obs=x)
            return x_loc

    def guide(self, x):
        pyro.module("encoder", self.encoder)
        with pyro.plate("data", size=x.shape[0]):
            # q(z|x)
            z_loc, z_scale  = self.encoder.forward(x)
            pyro.sample("latent", Normal(z_loc, z_scale).to_event(1))_____no_output_____

# Astronomical time series

- **Light curve:** Time series of a star's flux (brightness)
- We will consider two optical passbands
- The "apparent" brightness is estimated through **Photometry**
- Main tool for variable star studies

_____no_output_____

import bz2
import pickle

with bz2.BZ2File("lcdata.pbz2", 'r') as f:
    lcs, periods, labels = pickle.load(f)

fig, ax = plt.subplots(figsize=(7, 3), tight_layout=True)
plot_lc(ax, lcs[0])_____no_output_____

- Light curve "features": Irregular sampling, gaps in observations, [heteroscedastic](https://en.wikipedia.org/wiki/Heteroscedasticity) noise
- **Variable stars**: Brightness change in time either regularly or stochastically
- Some variable stars are radial pulsators. They expand/heat and contract/cool regularly. Examples: RR Lyrae and Cepheid_____no_output_____

YouTubeVideo('sXJBrRmHPj8')_____no_output_____

If we know the pulsation period we can use the **epoch folding transformation** to obtain a phase diagram

_____no_output_____

def fold(time, period):
    """
    returns phase = time/period - floor(time/period)
    """
    return np.mod(time, period)/period

fig, ax = plt.subplots(figsize=(7, 3), tight_layout=True)
plot_lc_folded(ax, lcs[0], periods[0])_____no_output_____

# Training a VAE for periodic light curves_____no_output_____Very simple preprocessing and data preparation routine_____no_output_____

pha_interp = np.linspace(0, 1, num=40)
mag_interp = np.zeros(shape=(len(lcs), 2, len(pha_interp)))
err_interp = np.zeros(shape=(len(lcs), 2, len(pha_interp)))

for k, (lc, period) in enumerate(zip(lcs, periods)):
    mag_interp[k], err_interp[k], stats = featurize_lc(lc, period, pha_interp)

fig, ax = plt.subplots(figsize=(7, 3), tight_layout=True)
plot_lc_features(ax, pha_interp, mag_interp[0], err_interp[0])_____no_output_____from sklearn.preprocessing import LabelEncoder
from torch.utils.data import TensorDataset, DataLoader, Subset

# reproducibility
torch.manual_seed(12345);
np.random.seed(12345)

fig, ax = plt.subplots(figsize=(7, 2), tight_layout=True)
ax.hist(labels, bins=np.arange(-0.5, 5.5, step=1), rwidth=0.8);

le = LabelEncoder()
labels_int = le.fit_transform(labels)
# Create light curve dataset from numpy arrays
lc_dataset = TensorDataset(torch.from_numpy(mag_interp.astype('float32')),
                           torch.from_numpy(err_interp.astype('float32'))
                           )

# Generate data loaders
idx = np.random.permutation(len(lcs))
train_loader = DataLoader(dataset=Subset(lc_dataset, idx[len(lcs)//5:]),
                          batch_size=32, shuffle=True)

valid_loader = DataLoader(dataset=Subset(lc_dataset, idx[:len(lcs)//5]), batch_size=512)_____no_output_____

Define a VAE that receives two inputs (r and g) and generates two outputs_____no_output_____

from pyro.distributions import Normal
import torch.nn as nn

class VariationalAutoEncoder(torch.nn.Module):

    def __init__(self, data_dim, latent_dim, hidden_dim):
        super(VariationalAutoEncoder, self).__init__()
        self.encoder = Encoder(data_dim, latent_dim, hidden_dim)
        self.decoder = Decoder(data_dim, latent_dim, hidden_dim)
        self.latent_dim = latent_dim
        self.data_dim = data_dim

    def model(self, mags, errs):
        pyro.module("decoder", self.decoder)
        with pyro.plate("minibatch", size=mags.shape[0]):
            # p(z)
            z_loc = torch.zeros(mags.shape[0], self.latent_dim, device=mags.device)
            z_scale = torch.ones(mags.shape[0], self.latent_dim, device=mags.device)
            z = pyro.sample("latent", Normal(z_loc, z_scale).to_event(1))
            # p(x|z)
            x_loc1, x_loc2 = self.decoder.forward(z)
            pyro.sample("observed_g", Normal(x_loc1, errs[:, 0, :]).to_event(1), obs=mags[:, 0, :])
            pyro.sample("observed_r", Normal(x_loc2, errs[:, 1, :]).to_event(1), obs=mags[:, 1, :])
            return x_loc1, x_loc2

    def guide(self, mags, errs):
        pyro.module("encoder", self.encoder)
        with pyro.plate("minibatch", size=mags.shape[0]):
            # q(z|x)
            z_loc, z_scale  = self.encoder.forward(mags)
            pyro.sample("latent", Normal(z_loc, z_scale).to_event(1))_____no_output_____class Encoder(torch.nn.Module):
    def __init__(self, data_dim, latent_dim, hidden_dim):
        super(type(self), self).__init__()
        self.hidden1_g = nn.Linear(data_dim, hidden_dim)
        self.hidden1_r = nn.Linear(data_dim, hidden_dim)
        self.hidden2 = nn.Linear(hidden_dim*2, hidden_dim)
        self.z_loc = nn.Linear(hidden_dim, latent_dim)
        self.z_scale = nn.Linear(hidden_dim, latent_dim)
        self.activation = nn.Softplus()

    def forward(self, x):
        hg = self.activation(self.hidden1_g(x[:, 0, :]))
        hr = self.activation(self.hidden1_r(x[:, 1, :]))
        h = torch.cat([hg, hr], dim=1)
        h = self.activation(self.hidden2(h))
        return self.z_loc(h), self.activation(self.z_scale(h))

class Decoder(torch.nn.Module):
    def __init__(self, data_dim, latent_dim, hidden_dim):
        super(type(self), self).__init__()
        self.hidden1 = nn.Linear(latent_dim, hidden_dim)
        self.hidden2 = nn.Linear(hidden_dim, hidden_dim)
        self.x_locg = nn.Linear(hidden_dim, data_dim)
        self.x_locr = nn.Linear(hidden_dim, data_dim)
        self.activation = nn.Softplus()

    def forward(self, z):
        h = self.activation(self.hidden1(z))
        h = self.activation(self.hidden2(h))
        return self.x_locg(h), self.x_locr(h)_____no_output_____

Model and SVI object definitions and training loop_____no_output_____

pyro.enable_validation(True)
pyro.clear_param_store()

vae = VariationalAutoEncoder(data_dim=40, latent_dim=2, hidden_dim=20).cuda()

svi = pyro.infer.SVI(model=vae.model,
                     guide=vae.guide,
                     optim=pyro.optim.Adam({"lr": 1e-3}),
                     loss=pyro.infer.Trace_ELBO())

fig, ax = plt.subplots(1, 2, figsize=(8, 3), tight_layout=True)

nepochs = 100
epoch_loss = np.zeros(shape=(nepochs, 2))
for nepoch in tqdm(range(nepochs)):

    # Training loop
    for mags, errs in train_loader:
        epoch_loss[nepoch, 0] += svi.step(mags.cuda(), errs.cuda())
    epoch_loss[nepoch, 0] /= len(train_loader.dataset)

    # Validation loop
    for mags, errs in valid_loader:
        epoch_loss[nepoch, 1] += svi.evaluate_loss(mags.cuda(), errs.cuda())
    epoch_loss[nepoch, 1] /= len(valid_loader.dataset)

    # Plot latent space and learning curves
    Z = torch.cat((vae.encoder(lc_dataset.tensors[0].cuda())), dim=1)
    Z = Z.detach().cpu().numpy()
    make_train_plots(ax, nepoch, Z, labels_int, epoch_loss)
    fig.canvas.draw()_____no_output_____

## Input and reconstructions_____no_output_____

vae = vae.cpu()

fig, ax = plt.subplots(2, 10, figsize=(8, 2), tight_layout=True)

mags, errs = next(iter(valid_loader))
z_loc, z_scale = vae.encoder.forward(mags)
mug, mur = vae.decoder(z_loc)
for k, (c, mu) in enumerate(zip(['g', 'r'], [mug, mur])):
    for i in range(10):
        ax[k, i].plot(mags.numpy()[i, k, :], c=c)
        ax[k, i].plot(mu[i].detach().numpy(), ls='--', c=c)
        ax[k, i].invert_yaxis();
        ax[k, i].axis('off')_____no_output_____

## Latent space visualization_____no_output_____

fig, ax = plt.subplots(1, 1, figsize=(8, 6), tight_layout=True)

for k, name in zip(np.unique(labels_int), np.unique(labels)):
    mask = labels_int == k
    ax.errorbar(x=Z[mask, 0], y=Z[mask, 1],
                xerr=Z[mask, 2], yerr=Z[mask, 3],
                fmt='none', alpha=0.5, label=name)
ax.legend(loc=4)
ax.set_xlim([-5., 5.])
ax.set_ylim([-5., 5.]);_____no_output_____

## Latent space interpolation_____no_output_____

M = 10
z_plot = np.linspace(-5., 5., num=M)

fig, ax = plt.subplots(M, M, figsize=(8, 6), tight_layout=True)
for i in range(M):
    for j in range(M):
        z = torch.tensor(np.array([z_plot[j], z_plot[M-1-i]]), dtype=torch.float32)
        mug, mur = vae.decoder.forward(z)
        mug = mug.detach().numpy()
        mur = mur.detach().numpy()
        ax[i, j].plot(mug, lw=2, c='g')
        ax[i, j].plot(mur, lw=2, c='r')
        ax[i, j].invert_yaxis();
        ax[i, j].axis('off')_____no_output_____

# Interested in Astroinformatics?

_____no_output_____