# Notebook from murfrosoft/webscraping
Path: ST-TOS Jammer scraping.ipynb

Goal: Scrape Star Trek ratings from JammersReviews.com and plot the average ratings across each series, season by season._____no_output_____

import pandas as pd
import requests
from bs4 import BeautifulSoup as bs_____no_output_____url = 'https://www.jammersreviews.com/st-tos/'
page = requests.get(url)
soup = bs(page.content, 'html.parser')

episodeList = []
seasonList = []
titleList = []
ratingList = []

episode = 1
for h in soup.find_all('h3'):
    if('Season' in h.text):
        season = h.text.split(' ')[1]
        #print('*',season)
        setTag = h.next_sibling.next_sibling
        shows = setTag.find_all('li')
        for show in shows:
            title = show.a.text
            if not show.find('img'):
                rating = '0'
            else:
                rating = show.img['alt'].split(' ')[0]
            episodeList.append(episode)
            seasonList.append(int(season))
            titleList.append(title)
            ratingList.append(float(rating))
            #print(episode, season, title, rating)
            episode += 1
            #print(movie)

# Create a dataframe
frame = pd.DataFrame({'episode':episodeList,'season':seasonList,'title':titleList, 'rating':ratingList})
frame.head()_____no_output_____avrate = []
seas = [1,2,3]
for season in range(1,4):
    avrate.append(sum(frame[frame['season'] == season]['rating'])/len(frame[frame['season'] == season]))
_tos = pd.DataFrame({'average':avrate,'season':seas})
_tos.plot.barh(x='season',y='average',rot=0,title='TOS Average Rating',legend=False, xlim=(2.0,3.5), figsize=(5,1));_____no_output_____url = 'https://www.jammersreviews.com/st-voy/'
page = requests.get(url)
soup = bs(page.content, 'html.parser')

episodeList = []
seasonList = []
titleList = []
ratingList = []

episode = 1
for h in soup.find_all('h3'):
    if('Season' in h.text):
        season = h.text.split(' ')[1]
        #print('*',season)
        setTag = h.next_sibling.next_sibling
        shows = setTag.find_all('li')
        for show in shows:
            title = show.a.text
            if 'Season Recap' in title:
                continue
            if not show.find('img'):
                rating = '0'
            else:
                rating = show.img['alt'].split(' ')[0]
            episodeList.append(episode)
            seasonList.append(int(season))
            titleList.append(title)
            ratingList.append(float(rating))
            #print(episode, season, title, rating)
            episode += 1
            #print(movie)

# Create a dataframe
frame = pd.DataFrame({'episode':episodeList,'season':seasonList,'title':titleList, 'rating':ratingList})
frame.head()_____no_output_____avrate = []
seas = [1,2,3,4,5,6,7]
for season in range(1,8):
    avrate.append(sum(frame[frame['season'] == season]['rating'])/len(frame[frame['season'] == season]))
_voy = pd.DataFrame({'average':avrate,'season':seas})
#frame.plot.bar(x='episode', y='rating', rot=0)
_voy.plot.barh(x='season',y='average',rot=0,title='Voyager Average Rating',legend=False, xlim=(2.0,3.5), figsize=(5,2));_____no_output_____url = 'https://www.jammersreviews.com/st-ds9/'
page = requests.get(url)
soup = bs(page.content, 'html.parser')

episodeList = []
seasonList = []
titleList = []
ratingList = []

episode = 1
for h in soup.find_all('h3'):
    if('Season' in h.text):
        season = h.text.split(' ')[1]
        #print('*',season)
        setTag = h.next_sibling.next_sibling
        shows = setTag.find_all('li')
        for show in shows:
            title = show.a.text
            if 'Season Recap' in title:
                continue
            if not show.find('img'):
                rating = '0'
            else:
                rating = show.img['alt'].split(' ')[0]
            episodeList.append(episode)
            seasonList.append(int(season))
            titleList.append(title)
            ratingList.append(float(rating))
            #print(episode, season, title, rating)
            episode += 1
            #print(movie)

# Create a dataframe
frame = pd.DataFrame({'episode':episodeList,'season':seasonList,'title':titleList, 'rating':ratingList})
frame.head()_____no_output_____avrate = []
seas = [1,2,3,4,5,6,7]
for season in range(1,8):
    avrate.append(sum(frame[frame['season'] == season]['rating'])/len(frame[frame['season'] == season]))
_ds9 = pd.DataFrame({'average':avrate,'season':seas})
#frame.plot.bar(x='episode', y='rating', rot=0)
_ds9.plot.barh(x='season',y='average',rot=0,title='DS9 Average Rating',legend=False, xlim=(2.0,3.5), figsize=(5,2));_____no_output_____url = 'https://www.jammersreviews.com/st-tng/'
page = requests.get(url)
soup = bs(page.content, 'html.parser')

episodeList = []
seasonList = []
titleList = []
ratingList = []

episode = 1
for h in soup.find_all('h3'):
    if('Season' in h.text):
        season = h.text.split(' ')[1]
        #print('*',season)
        setTag = h.next_sibling.next_sibling
        shows = setTag.find_all('li')
        for show in shows:
            title = show.a.text
            if 'Season Recap' in title:
                continue
            if not show.find('img'):
                rating = '0'
            else:
                rating = show.img['alt'].split(' ')[0]
            episodeList.append(episode)
            seasonList.append(int(season))
            titleList.append(title)
            ratingList.append(float(rating))
            #print(episode, season, title, rating)
            episode += 1
            #print(movie)

# Create a dataframe
frame = pd.DataFrame({'episode':episodeList,'season':seasonList,'title':titleList, 'rating':ratingList})
frame.head()_____no_output_____avrate = []
seas = [1,2,3,4,5,6,7]
for season in range(1,8):
    avrate.append(sum(frame[frame['season'] == season]['rating'])/len(frame[frame['season'] == season]))
_tng = pd.DataFrame({'average':avrate,'season':seas})
#frame.plot.bar(x='episode', y='rating', rot=0)
_tng.plot.barh(x='season',y='average',rot=0,title='TNG Average Rating',legend=False, xlim=(2.0,3.5), figsize=(5,2));_____no_output_____url = 'https://www.jammersreviews.com/st-ent/'
page = requests.get(url)
soup = bs(page.content, 'html.parser')

episodeList = []
seasonList = []
titleList = []
ratingList = []

episode = 1
for h in soup.find_all('h3'):
    if('Season' in h.text):
        season = h.text.split(' ')[1]
        #print('*',season)
        setTag = h.next_sibling.next_sibling
        shows = setTag.find_all('li')
        for show in shows:
            title = show.a.text
            if 'Season Recap' in title:
                continue
            if not show.find('img'):
                rating = '0'
            else:
                rating = show.img['alt'].split(' ')[0]
            episodeList.append(episode)
            seasonList.append(int(season))
            titleList.append(title)
            ratingList.append(float(rating))
            #print(episode, season, title, rating)
            episode += 1
            #print(movie)

# Create a dataframe
frame = pd.DataFrame({'episode':episodeList,'season':seasonList,'title':titleList, 'rating':ratingList})
frame.head()_____no_output_____avrate = []
seas = [1,2,3,4]
for season in range(1,5):
    avrate.append(sum(frame[frame['season'] == season]['rating'])/len(frame[frame['season'] == season]))
_ent = pd.DataFrame({'average':avrate,'season':seas})
#frame.plot.bar(x='episode', y='rating', rot=0)
_ent.plot.barh(x='season',y='average',rot=0,title='Enterprise Average Rating',legend=False, xlim=(2.0,3.5), figsize=(4,1));_____no_output_____url = 'https://www.jammersreviews.com/st-dsc/'
page = requests.get(url)
soup = bs(page.content, 'html.parser')

episodeList = []
seasonList = []
titleList = []
ratingList = []

episode = 1
for h in soup.find_all('h3'):
    if('Season' in h.text):
        season = h.text.split(' ')[1]
        #print('*',season)
        setTag = h.next_sibling.next_sibling
        shows = setTag.find_all('li')
        for show in shows:
            title = show.a.text
            if 'Season Recap' in title:
                continue
            if not show.find('img'):
                rating = '0'
            else:
                rating = show.img['alt'].split(' ')[0]
            episodeList.append(episode)
            seasonList.append(int(season))
            titleList.append(title)
            ratingList.append(float(rating))
            #print(episode, season, title, rating)
            episode += 1
            #print(movie)

# Create a dataframe
frame = pd.DataFrame({'episode':episodeList,'season':seasonList,'title':titleList, 'rating':ratingList})
frame.head()_____no_output_____avrate = []
seas = [1,2]
for season in range(1,3):
    avrate.append(sum(frame[frame['season'] == season]['rating'])/len(frame[frame['season'] == season]))
_dis = pd.DataFrame({'average':avrate,'season':seas})
#frame.plot.bar(x='episode', y='rating', rot=0)
_dis.plot.barh(x='season',y='average',rot=0,title='Discovery Average Rating',legend=False, xlim=(2.0,3.5), figsize=(4,0.5));_____no_output______dis.plot.barh(x='season',y='average',rot=0,title='Discovery Average Rating',legend=False, xlim=(2.0,3.5), figsize=(5,0.4))
_ent.plot.barh(x='season',y='average',rot=0,title='Enterprise Average Rating',legend=False, xlim=(2.0,3.5), figsize=(5,0.9))
_voy.plot.barh(x='season',y='average',rot=0,title='Voyager Average Rating',legend=False, xlim=(2.0,3.5), figsize=(5,1.8))
_ds9.plot.barh(x='season',y='average',rot=0,title='DS9 Average Rating',legend=False, xlim=(2.0,3.5), figsize=(5,1.8))
_tng.plot.barh(x='season',y='average',rot=0,title='TNG Average Rating',legend=False, xlim=(2.0,3.5), figsize=(5,1.8))
_tos.plot.barh(x='season',y='average',rot=0,title='TOS Average Rating',legend=False, xlim=(2.0,3.5), figsize=(5,.7));_____no_output_____


# Notebook from proTao/LearningBayes
Path: code/chap09soln.ipynb

# Think Bayes: Chapter 9

This notebook presents code and exercises from Think Bayes, second edition.

Copyright 2016 Allen B. Downey

MIT License: https://opensource.org/licenses/MIT_____no_output_____

from __future__ import print_function, division

% matplotlib inline
import warnings
warnings.filterwarnings('ignore')

import math
import numpy as np

from thinkbayes2 import Pmf, Cdf, Suite, Joint
import thinkplot_____no_output_____

## Improving Reading Ability

From DASL(http://lib.stat.cmu.edu/DASL/Stories/ImprovingReadingAbility.html)

> An educator conducted an experiment to test whether new directed reading activities in the classroom will help elementary school pupils improve some aspects of their reading ability. She arranged for a third grade class of 21 students to follow these activities for an 8-week period. A control classroom of 23 third graders followed the same curriculum without the activities. At the end of the 8 weeks, all students took a Degree of Reading Power (DRP) test, which measures the aspects of reading ability that the treatment is designed to improve.

> Summary statistics on the two groups of children show that the average score of the treatment class was almost ten points higher than the average of the control class. A two-sample t-test is appropriate for testing whether this difference is statistically significant. The t-statistic is 2.31, which is significant at the .05 level.

_____no_output_____I'll use Pandas to load the data into a DataFrame._____no_output_____

import pandas as pd

df = pd.read_csv('drp_scores.csv', skiprows=21, delimiter='\t')
df.head()_____no_output_____

And use `groupby` to compute the means for the two groups._____no_output_____

grouped = df.groupby('Treatment')
for name, group in grouped:
    print(name, group.Response.mean())Control 41.5217391304
Treated 51.4761904762

The `Normal` class provides a `Likelihood` function that computes the likelihood of a sample from a normal distribution._____no_output_____

from scipy.stats import norm
from thinkbayes2 import EvalNormalPdf

class Normal(Suite, Joint):

    def Likelihood(self, data, hypo):
        """

        data: sequence of test scores
        hypo: mu, sigma
        """
        mu, sigma = hypo
        likes = EvalNormalPdf(data, mu, sigma)
        return np.prod(likes)_____no_output_____

The prior distributions for `mu` and `sigma` are uniform._____no_output_____

mus = np.linspace(20, 80, 101)
sigmas = np.linspace(5, 30, 101)_____no_output_____

I use `itertools.product` to enumerate all pairs of `mu` and `sigma`._____no_output_____

from itertools import product

control = Normal(product(mus, sigmas))
data = df[df.Treatment=='Control'].Response
control.Update(data)_____no_output_____

After the update, we can plot the probability of each `mu`-`sigma` pair as a contour plot._____no_output_____

thinkplot.Contour(control, pcolor=True)
thinkplot.Config(xlabel='mu', ylabel='sigma')_____no_output_____

And then we can extract the marginal distribution of `mu`_____no_output_____

pmf_mu0 = control.Marginal(0)
thinkplot.Pdf(pmf_mu0)
thinkplot.Config(xlabel='mu', ylabel='Pmf')_____no_output_____

And the marginal distribution of `sigma`_____no_output_____

pmf_sigma0 = control.Marginal(1)
thinkplot.Pdf(pmf_sigma0)
thinkplot.Config(xlabel='sigma', ylabel='Pmf')_____no_output_____

**Exercise:** Run this analysis again for the control group.  What is the distribution of the difference between the groups?  What is the probability that the average "reading power" for the treatment group is higher?  What is the probability that the variance of the treatment group is higher?_____no_output_____

# Solution

treated = Normal(product(mus, sigmas))
data = df[df.Treatment=='Treated'].Response
treated.Update(data)_____no_output_____# Solution

# Here's the posterior joint distribution for the treated group

thinkplot.Contour(treated, pcolor=True)
thinkplot.Config(xlabel='mu', ylabel='Pmf')_____no_output_____# Solution

# The marginal distribution of mu

pmf_mu1 = treated.Marginal(0)
thinkplot.Pdf(pmf_mu1)
thinkplot.Config(xlabel='mu', ylabel='Pmf')_____no_output_____# Solution

# The marginal distribution of sigma

pmf_sigma1 = treated.Marginal(1)
thinkplot.Pdf(pmf_sigma1)
thinkplot.Config(xlabel='sigma', ylabel='Pmf')_____no_output_____# Solution

# Now we can compute the distribution of the difference between groups

pmf_diff = pmf_mu1 - pmf_mu0
pmf_diff.Mean(), pmf_diff.MAP()_____no_output_____# Solution

# And CDF_diff(0), which is the probability that the difference is <= 0

pmf_diff = pmf_mu1 - pmf_mu0
cdf_diff = pmf_diff.MakeCdf()
thinkplot.Cdf(cdf_diff)
cdf_diff[0]_____no_output_____# Solution

# Or we could directly compute the probability that mu is
# greater than mu2

pmf_mu1.ProbGreater(pmf_mu0)_____no_output_____# Solution

# Finally, here's the probability that the standard deviation
# in the treatment group is higher.

pmf_sigma1.ProbGreater(pmf_sigma0)_____no_output_____# It looks like there is a high probability that the mean of
# the treatment group is higher, and the most likely size of
# the effect is 9-10 points.

# It looks like the variance of the treated group is substantially
# smaller, which suggests that the treatment might be helping
# low scorers more than high scorers._____no_output_____

## Paintball_____no_output_____Suppose you are playing paintball in an indoor arena 30 feet
wide and 50 feet long.  You are standing near one of the 30 foot
walls, and you suspect that one of your opponents has taken cover
nearby.  Along the wall, you see several paint spatters, all the same
color, that you think your opponent fired recently.

The spatters are at 15, 16, 18, and 21 feet, measured from the
lower-left corner of the room.  Based on these data, where do you
think your opponent is hiding?

Here's the Suite that does the update.  It uses `MakeLocationPmf`,
defined below._____no_output_____

class Paintball(Suite, Joint):
    """Represents hypotheses about the location of an opponent."""

    def __init__(self, alphas, betas, locations):
        """Makes a joint suite of parameters alpha and beta.

        Enumerates all pairs of alpha and beta.
        Stores locations for use in Likelihood.

        alphas: possible values for alpha
        betas: possible values for beta
        locations: possible locations along the wall
        """
        self.locations = locations
        pairs = [(alpha, beta)
                 for alpha in alphas
                 for beta in betas]
        Suite.__init__(self, pairs)

    def Likelihood(self, data, hypo):
        """Computes the likelihood of the data under the hypothesis.

        hypo: pair of alpha, beta
        data: location of a hit

        Returns: float likelihood
        """
        alpha, beta = hypo
        x = data
        pmf = MakeLocationPmf(alpha, beta, self.locations)
        like = pmf.Prob(x)
        return like_____no_output_____def MakeLocationPmf(alpha, beta, locations):
    """Computes the Pmf of the locations, given alpha and beta.

    Given that the shooter is at coordinates (alpha, beta),
    the probability of hitting any spot is inversely proportionate
    to the strafe speed.

    alpha: x position
    beta: y position
    locations: x locations where the pmf is evaluated

    Returns: Pmf object
    """
    pmf = Pmf()
    for x in locations:
        prob = 1.0 / StrafingSpeed(alpha, beta, x)
        pmf.Set(x, prob)
    pmf.Normalize()
    return pmf_____no_output_____def StrafingSpeed(alpha, beta, x):
    """Computes strafing speed, given location of shooter and impact.

    alpha: x location of shooter
    beta: y location of shooter
    x: location of impact

    Returns: derivative of x with respect to theta
    """
    theta = math.atan2(x - alpha, beta)
    speed = beta / math.cos(theta)**2
    return speed_____no_output_____

The prior probabilities for `alpha` and `beta` are uniform._____no_output_____

alphas = range(0, 31)
betas = range(1, 51)
locations = range(0, 31)

suite = Paintball(alphas, betas, locations)
suite.UpdateSet([15, 16, 18, 21])_____no_output_____

To visualize the joint posterior, I take slices for a few values of `beta` and plot the conditional distributions of `alpha`.  If the shooter is close to the wall, we can be somewhat confident of his position.  The farther away he is, the less certain we are._____no_output_____

locations = range(0, 31)
alpha = 10
betas = [10, 20, 40]
thinkplot.PrePlot(num=len(betas))

for beta in betas:
    pmf = MakeLocationPmf(alpha, beta, locations)
    pmf.label = 'beta = %d' % beta
    thinkplot.Pdf(pmf)

thinkplot.Config(xlabel='Distance',
                ylabel='Prob')_____no_output_____

Here are the marginal posterior distributions for `alpha` and `beta`._____no_output_____

marginal_alpha = suite.Marginal(0, label='alpha')
marginal_beta = suite.Marginal(1, label='beta')

print('alpha CI', marginal_alpha.CredibleInterval(50))
print('beta CI', marginal_beta.CredibleInterval(50))

thinkplot.PrePlot(num=2)

thinkplot.Cdf(Cdf(marginal_alpha))
thinkplot.Cdf(Cdf(marginal_beta))

thinkplot.Config(xlabel='Distance',
                ylabel='Prob')alpha CI (14, 21)
beta CI (5, 31)

To visualize the joint posterior, I take slices for a few values of `beta` and plot the conditional distributions of `alpha`.  If the shooter is close to the wall, we can be somewhat confident of his position.  The farther away he is, the less certain we are._____no_output_____

betas = [10, 20, 40]
thinkplot.PrePlot(num=len(betas))

for beta in betas:
    cond = suite.Conditional(0, 1, beta)
    cond.label = 'beta = %d' % beta
    thinkplot.Pdf(cond)

thinkplot.Config(xlabel='Distance',
                ylabel='Prob')_____no_output_____

Another way to visualize the posterio distribution: a pseudocolor plot of probability as a function of `alpha` and `beta`._____no_output_____

thinkplot.Contour(suite.GetDict(), contour=False, pcolor=True)

thinkplot.Config(xlabel='alpha',
                ylabel='beta',
                axis=[0, 30, 0, 20])_____no_output_____

Here's another visualization that shows posterior credible regions._____no_output_____

d = dict((pair, 0) for pair in suite.Values())

percentages = [75, 50, 25]
for p in percentages:
    interval = suite.MaxLikeInterval(p)
    for pair in interval:
        d[pair] += 1

thinkplot.Contour(d, contour=False, pcolor=True)
thinkplot.Text(17, 4, '25', color='white')
thinkplot.Text(17, 15, '50', color='white')
thinkplot.Text(17, 30, '75')

thinkplot.Config(xlabel='alpha',
                   ylabel='beta',
                   legend=False)_____no_output_____

**Exercise:** From [John D. Cook](http://www.johndcook.com/blog/2010/07/13/lincoln-index/)

"Suppose you have a tester who finds 20 bugs in your program. You want to estimate how many bugs are really in the program. You know there are at least 20 bugs, and if you have supreme confidence in your tester, you may suppose there are around 20 bugs. But maybe your tester isn't very good. Maybe there are hundreds of bugs. How can you have any idea how many bugs there are? There’s no way to know with one tester. But if you have two testers, you can get a good idea, even if you don’t know how skilled the testers are.

Suppose two testers independently search for bugs. Let k1 be the number of errors the first tester finds and k2 the number of errors the second tester finds. Let c be the number of errors both testers find.  The Lincoln Index estimates the total number of errors as k1 k2 / c [I changed his notation to be consistent with mine]."

So if the first tester finds 20 bugs, the second finds 15, and they find 3 in common, we estimate that there are about 100 bugs.  What is the Bayesian estimate of the number of errors based on this data?_____no_output_____

# Solution

from scipy.special import binom as choose

def binom(k, n, p):
    """Computes the rest of the binomial PMF.

    k: number of hits
    n: number of attempts
    p: probability of a hit
    """
    return p**k * (1-p)**(n-k)

class Lincoln(Suite, Joint):
    """Represents hypotheses about the number of errors."""

    def Likelihood(self, data, hypo):
        """Computes the likelihood of the data under the hypothesis.

        hypo: n, p1, p2
        data: k1, k2, c
        """
        n, p1, p2 = hypo
        k1, k2, c = data

        part1 = choose(n, k1) * binom(k1, n, p1)
        part2 = choose(k1, c) * choose(n-k1, k2-c) * binom(k2, n, p2)
        return part1 * part2_____no_output_____# Solution

data = 20, 15, 3
probs = np.linspace(0, 1, 31)
hypos = []
for n in range(32, 350):
    for p1 in probs:
        for p2 in probs:
            hypos.append((n, p1, p2))

suite = Lincoln(hypos)
suite.Update(data)_____no_output_____# Solution

n_marginal = suite.Marginal(0)
thinkplot.Pmf(n_marginal, label='n')
thinkplot.Config(xlabel='number of bugs',
                   ylabel='PMF')_____no_output_____# Solution

print('post mean n', n_marginal.Mean())
print('MAP n', n_marginal.MAP())post mean n 105.457426627
MAP n 72

**Exercise:** The GPS problem.  According to [Wikipedia]()

￼
> GPS included a (currently disabled) feature called Selective Availability (SA) that adds intentional, time varying errors of up to 100 meters (328 ft) to the publicly available navigation signals. This was intended to deny an enemy the use of civilian GPS receivers for precision weapon guidance.
> [...]
> Before it was turned off on May 2, 2000, typical SA errors were about 50 m (164 ft) horizontally and about 100 m (328 ft) vertically.[10] Because SA affects every GPS receiver in a given area almost equally, a fixed station with an accurately known position can measure the SA error values and transmit them to the local GPS receivers so they may correct their position fixes. This is called Differential GPS or DGPS. DGPS also corrects for several other important sources of GPS errors, particularly ionospheric delay, so it continues to be widely used even though SA has been turned off. The ineffectiveness of SA in the face of widely available DGPS was a common argument for turning off SA, and this was finally done by order of President Clinton in 2000.

Suppose it is 1 May 2000, and you are standing in a field that is 200m square.  You are holding a GPS unit that indicates that your location is 51m north and 15m west of a known reference point in the middle of the field.

However, you know that each of these coordinates has been perturbed by a "feature" that adds random errors with mean 0 and standard deviation 30m.

1) After taking one measurement, what should you believe about your position?

Note: Since the intentional errors are independent, you could solve this problem independently for X and Y.  But we'll treat it as a two-dimensional problem, partly for practice and partly to see how we could extend the solution to handle dependent errors.

You can start with the code in gps.py.

2) Suppose that after one second the GPS updates your position and reports coordinates (48, 90).  What should you believe now?

3) Suppose you take 8 more measurements and get:

    (11.903060613102866, 19.79168669735705)
    (77.10743601503178, 39.87062906535289)
    (80.16596823095534, -12.797927542984425)
    (67.38157493119053, 83.52841028148538)
    (89.43965206875271, 20.52141889230797)
    (58.794021026248245, 30.23054016065644)
    (2.5844401241265302, 51.012041625783766)
    (45.58108994142448, 3.5718287379754585)

At this point, how certain are you about your location?_____no_output_____

# Solution

from thinkbayes2 import EvalNormalPdf

class Gps(Suite, Joint):
    """Represents hypotheses about your location in the field."""

    def Likelihood(self, data, hypo):
        """Computes the likelihood of the data under the hypothesis.

        hypo:
        data:
        """
        std = 30
        meanx, meany = hypo
        x, y = data
        like = EvalNormalPdf(x, meanx, std)
        like *= EvalNormalPdf(y, meany, std)
        return like_____no_output_____# Solution

from itertools import product

coords = np.linspace(-100, 100, 101)
joint = Gps(product(coords, coords))
joint.Update((51, -15))_____no_output_____# Solution

joint.Update((48, 90))_____no_output_____# Solution

pairs = [(11.903060613102866, 19.79168669735705),
             (77.10743601503178, 39.87062906535289),
             (80.16596823095534, -12.797927542984425),
             (67.38157493119053, 83.52841028148538),
             (89.43965206875271, 20.52141889230797),
             (58.794021026248245, 30.23054016065644),
             (2.5844401241265302, 51.012041625783766),
             (45.58108994142448, 3.5718287379754585)]

joint.UpdateSet(pairs)_____no_output_____# Solution

thinkplot.PrePlot(2)
pdfx = joint.Marginal(0)
pdfy = joint.Marginal(1)
thinkplot.Pdf(pdfx, label='posterior x')
thinkplot.Pdf(pdfy, label='posterior y')_____no_output_____# Solution

print(pdfx.Mean(), pdfx.Std())
print(pdfy.Mean(), pdfy.Std())53.1957132094 9.48680494799
31.0728627918 9.48683298048

**Exercise:** [The Flea Beetle problem from DASL](http://lib.stat.cmu.edu/DASL/Datafiles/FleaBeetles.html)

Datafile Name: Flea Beetles

Datafile Subjects: Biology

Story Names: Flea Beetles

Reference: Lubischew, A.A. (1962) On the use of discriminant functions in taxonomy. Biometrics, 18, 455-477. Also found in: Hand, D.J., et al. (1994) A Handbook of Small Data Sets, London: Chapman & Hall, 254-255.

Authorization: Contact Authors

Description: Data were collected on the genus of flea beetle Chaetocnema, which contains three species: concinna (Con), heikertingeri (Hei), and heptapotamica (Hep). Measurements were made on the width and angle of the aedeagus of each beetle. The goal of the original study was to form a classification rule to distinguish the three species.

Number of cases: 74

Variable Names:

Width: The maximal width of aedeagus in the forpart (in microns)

Angle: The front angle of the aedeagus (1 unit = 7.5 degrees)

Species: Species of flea beetle from the genus Chaetocnema
_____no_output_____Suggestions:

1. Plot CDFs for the width and angle data, broken down by species, to get a visual sense of whether the normal distribution is a good model.

2. Use the data to estimate the mean and standard deviation for each variable, broken down by species.

3. Given a joint posterior distribution for `mu` and `sigma`, what is the likelihood of a given datum?

4. Write a function that takes a measured width and angle and returns a posterior PMF of species.

5. Use the function to classify each of the specimens in the table and see how many you get right.
_____no_output_____

import pandas as pd

df = pd.read_csv('flea_beetles.csv', delimiter='\t')
df.head()_____no_output_____def plot_cdfs(df, col):
    for name, group in df.groupby('Species'):
        cdf = Cdf(group[col], label=name)
        thinkplot.Cdf(cdf)

    thinkplot.Config(xlabel=col, legend=True, loc='lower right')_____no_output_____plot_cdfs(df, 'Width')_____no_output_____plot_cdfs(df, 'Angle')_____no_output_____from thinkbayes2 import EvalNormalPdf

class Beetle(Suite, Joint):

    def Likelihood(self, data, hypo):
        """
        data: sequence of measurements
        hypo: mu, sigma
        """
        mu, sigma = hypo
        likes = EvalNormalPdf(data, mu, sigma)
        return np.prod(likes)

    def PredictiveProb(self, data):
        """Compute the posterior total probability of a datum.

        data: sequence of measurements
        """
        total = 0
        for (mu, sigma), prob in self.Items():
            likes = norm.pdf(data, mu, sigma)
            total += prob * np.prod(likes)
        return total_____no_output_____from itertools import product

def MakeWidthSuite(data):
    mus = np.linspace(115, 160, 51)
    sigmas = np.linspace(1, 10, 51)
    suite = Beetle(product(mus, sigmas))
    suite.Update(data)
    return suite_____no_output_____groups = df.groupby('Species')_____no_output_____for name, group in groups:
    suite = MakeWidthSuite(group.Width)
    thinkplot.Contour(suite)
    print(name, suite.PredictiveProb(137))Con 0.0200863522822
Hei 0.00368097211374
Hep 0.0867791556717
def MakeAngleSuite(data):
    mus = np.linspace(8, 16, 101)
    sigmas = np.linspace(0.1, 2, 101)
    suite = Beetle(product(mus, sigmas))
    suite.Update(data)
    return suite_____no_output_____for name, group in groups:
    suite = MakeAngleSuite(group.Angle)
    thinkplot.Contour(suite)
    print(name, suite.PredictiveProb(13))Con 0.20957983351
Hei 0.181327548441
Hep 0.0106017571682
class Species:

    def __init__(self, name, suite_width, suite_angle):
        self.name = name
        self.suite_width = suite_width
        self.suite_angle = suite_angle

    def __str__(self):
        return self.name

    def Likelihood(self, data):
        width, angle = data
        like1 = self.suite_width.PredictiveProb(width)
        like2 = self.suite_angle.PredictiveProb(angle)
        return like1 * like2_____no_output_____species = {}

for name, group in groups:
    suite_width = MakeWidthSuite(group.Width)
    suite_angle = MakeAngleSuite(group.Angle)
    species[name] = Species(name, suite_width, suite_angle)_____no_output_____species['Con'].Likelihood((145, 14))_____no_output_____class Classifier(Suite):

    def Likelihood(self, data, hypo):
        return hypo.Likelihood(data)_____no_output_____suite = Classifier(species.values())
suite.Print()Hep 0.333333333333
Hei 0.333333333333
Con 0.333333333333
suite.Update((145, 14))
suite.Print()Hep 0.00116844449591
Hei 0.000689941368376
Con 0.998141614136


# Notebook from qiaochen/VeloRep
Path: notebooks/dentategyrus/.ipynb_checkpoints/model-dentategyrus-baseline-checkpoint.ipynb

<code>
import time
import sys, os
import pickle
import numpy as np
import scvelo as scv
import scanpy
import scipy
import torch
import torch.nn as nn
import torch.nn.functional as F

from torch import optim
from veloproj import *
from veloproj.model import leastsq_pt
from IPython import display
from matplotlib import pyplot as plt
import anndata_____no_output_____parser = get_parser()
args = parser.parse_args(args=['--lr', '1e-5',
                               '--n-epochs', '20000',
                               '--g-rep-dim', '100',
                               '--k-dim', '100',
                               '--model-name', 'baseline_tmp.cpt',
                               '--exp-name', 'baseline_dentategyrus',
                               '--device', 'cuda:0',
                               '--gumbsoft_tau', '5',
                               '--nb_g_src', "X",
                              ])
_____no_output_____torch.manual_seed(args.seed)
torch.cuda.manual_seed(args.seed)
np.random.seed(args.seed)
torch.backends.cudnn.deterministic = True

device = torch.device(args.device if args.device.startswith('cuda') and torch.cuda.is_available() else "cpu")_____no_output_____cluster_edges = [("OPC", "OL")]
k_cluster = "clusters"
EXP_NAME = args.exp_name
exp_metrics = {}_____no_output_____def main_AE(args, model, lr=args.lr, weight_decay=args.weight_decay, save_name="tmp.cpt"):
    optimizer = optim.AdamW(model.parameters(), lr=lr, weight_decay=weight_decay)
    lr = args.lr
    i, losses = 0, [sys.maxsize]
    min_loss = losses[-1]
    model_saved = False

    model.train()
    while i < args.n_epochs:
        i += 1
        loss = train_step_AE([tensor_s, tensor_u], model, optimizer, xyids=[0, 1], device=device)
        losses.append(loss)
        if i % args.log_interval == 0:
            if losses[-1] < min_loss:
                min_loss = losses[-1]
                torch.save(model.state_dict(), os.path.join(args.output, save_name))
                model_saved = True
            else:
                if model_saved:
                    model.load_state_dict(torch.load(os.path.join(args.output, save_name)))
                    model = model.to(device)
                lr *= args.lr_decay
                for param_group in optimizer.param_groups:
                    param_group['lr'] = lr

            print("Train Epoch: {:2d}/{:2d} \tLoss: {:.6f}"
                  .format(i, args.n_epochs, losses[-1]))

    plt.plot(losses[1:])
    if losses[-1] < min_loss:
        torch.save(model.state_dict(), os.path.join(args.output, save_name))
    return model_____no_output_____adata = scv.datasets.dentategyrus()
scv.utils.show_proportions(adata)
scv.pp.filter_and_normalize(adata, min_shared_counts=30, n_top_genes=2000)
scv.pp.moments(adata, n_pcs=30, n_neighbors=30)
print(adata)Abundance of ['spliced', 'unspliced']: [0.9 0.1]
Filtered out 11019 genes that are detected 30 counts (shared).
Normalized count data: X, spliced, unspliced.
Extracted 2000 highly variable genes.
Logarithmized X.
computing neighbors
    finished (0:00:14) --> added
    'distances' and 'connectivities', weighted adjacency matrices (adata.obsp)
computing moments based on connectivities
    finished (0:00:00) --> added
    'Ms' and 'Mu', moments of un/spliced abundances (adata.layers)
AnnData object with n_obs × n_vars = 2930 × 2000
    obs: 'clusters', 'age(days)', 'clusters_enlarged', 'initial_size_spliced', 'initial_size_unspliced', 'initial_size', 'n_counts'
    var: 'means', 'dispersions', 'dispersions_norm', 'highly_variable'
    uns: 'clusters_colors', 'pca', 'neighbors'
    obsm: 'X_umap', 'X_pca'
    varm: 'PCs'
    layers: 'ambiguous', 'spliced', 'unspliced', 'Ms', 'Mu'
    obsp: 'distances', 'connectivities'
scv.tl.velocity(adata, vkey='stc_velocity', mode="stochastic")
scv.tl.velocity_graph(adata, vkey='stc_velocity')
scv.tl.velocity_confidence(adata, vkey='stc_velocity')
scv.pl.velocity_embedding_stream(adata, vkey="stc_velocity", basis='X_umap', color=k_cluster,
                                title='ScVelo Stochastic Mode')
exp_metrics["stc_mode"] = evaluate(adata, cluster_edges, k_cluster, "stc_velocity")computing velocities
    finished (0:00:00) --> added
    'stc_velocity', velocity vectors for each individual cell (adata.layers)
computing velocity graph
    finished (0:00:06) --> added
    'stc_velocity_graph', sparse matrix with cosine correlations (adata.uns)
--> added 'stc_velocity_length' (adata.obs)
--> added 'stc_velocity_confidence' (adata.obs)
--> added 'stc_velocity_confidence_transition' (adata.obs)
computing velocity embedding
    finished (0:00:00) --> added
    'stc_velocity_umap', embedded velocity vectors (adata.obsm)
spliced = adata.layers['Ms']
unspliced = adata.layers['Mu']
tensor_s, tensor_u = torch.FloatTensor(spliced).to(device), torch.FloatTensor(unspliced).to(device)
tensor_x = torch.FloatTensor(adata.X.toarray()).to(device)
tensor_v = torch.FloatTensor(adata.layers['stc_velocity']).to(device)_____no_output_____def expBaseAE(adata, exp_metrics):
    n_cells, n_genes = adata.X.shape
    in_dim = n_genes
    z_dim = args.z_dim
    h_dim = args.h_dim

    model = get_baseline_AE(in_dim, z_dim, h_dim).to(device)
    model = main_AE(args, model, save_name=f"baseAE_{args.model_name}")
    model.eval()
    with torch.no_grad():
        x = model.encoder(tensor_x)
        s = model.encoder(tensor_s)
        u = model.encoder(tensor_u)

        _, gamma, _ = leastsq_pt(s, u,
                                    device=device,
                                    fit_offset=True,
                                    perc=[5, 95]
                                    )
        v = (u - gamma * s).cpu().numpy()
        x = x.cpu().numpy()
        s = s.cpu().numpy()
        u = u.cpu().numpy()

    adata = new_adata(adata, x, s, u, v, g_basis=args.nb_g_src)
    scv.tl.velocity_graph(adata, vkey='new_velocity')
    scv.pl.velocity_embedding_stream(adata, vkey="new_velocity", basis='X_umap', color=['clusters', 'age(days)'],
                                    title="Baseline AutoEncoder",
                                    )
    scv.tl.velocity_confidence(adata, vkey='new_velocity')
    exp_metrics['Baseline AutoEncoder'] = evaluate(adata, cluster_edges, k_cluster, "new_velocity")

expBaseAE(adata, exp_metrics)Train Epoch: 100/20000 	Loss: 19.579882
Train Epoch: 200/20000 	Loss: 11.325132
Train Epoch: 300/20000 	Loss: 8.389540
Train Epoch: 400/20000 	Loss: 6.872127
Train Epoch: 500/20000 	Loss: 5.988616
Train Epoch: 600/20000 	Loss: 5.336883
Train Epoch: 700/20000 	Loss: 4.882497
Train Epoch: 800/20000 	Loss: 4.519815
Train Epoch: 900/20000 	Loss: 4.252237
Train Epoch: 1000/20000 	Loss: 3.962225
Train Epoch: 1100/20000 	Loss: 3.749236
Train Epoch: 1200/20000 	Loss: 3.548991
Train Epoch: 1300/20000 	Loss: 3.423876
Train Epoch: 1400/20000 	Loss: 3.264456
Train Epoch: 1500/20000 	Loss: 3.137578
Train Epoch: 1600/20000 	Loss: 3.024992
Train Epoch: 1700/20000 	Loss: 2.924335
Train Epoch: 1800/20000 	Loss: 2.818689
Train Epoch: 1900/20000 	Loss: 2.735250
Train Epoch: 2000/20000 	Loss: 2.658336
Train Epoch: 2100/20000 	Loss: 2.579005
Train Epoch: 2200/20000 	Loss: 2.508111
Train Epoch: 2300/20000 	Loss: 2.445522
Train Epoch: 2400/20000 	Loss: 2.382178
Train Epoch: 2500/20000 	Loss: 2.318239
Train Epoch: 2600/20000 	Loss: 2.254217
Train Epoch: 2700/20000 	Loss: 2.220867
Train Epoch: 2800/20000 	Loss: 2.148023
Train Epoch: 2900/20000 	Loss: 2.095773
Train Epoch: 3000/20000 	Loss: 2.040852
Train Epoch: 3100/20000 	Loss: 2.005140
Train Epoch: 3200/20000 	Loss: 1.947642
Train Epoch: 3300/20000 	Loss: 2.152724
Train Epoch: 3400/20000 	Loss: 1.913978
Train Epoch: 3500/20000 	Loss: 1.863033
Train Epoch: 3600/20000 	Loss: 1.839899
Train Epoch: 3700/20000 	Loss: 1.780931
Train Epoch: 3800/20000 	Loss: 1.746152
Train Epoch: 3900/20000 	Loss: 1.698262
Train Epoch: 4000/20000 	Loss: 1.666705
Train Epoch: 4100/20000 	Loss: 1.632216
Train Epoch: 4200/20000 	Loss: 1.579696
Train Epoch: 4300/20000 	Loss: 1.542365
Train Epoch: 4400/20000 	Loss: 1.505603
Train Epoch: 4500/20000 	Loss: 1.461632
Train Epoch: 4600/20000 	Loss: 1.429857
Train Epoch: 4700/20000 	Loss: 1.399434
Train Epoch: 4800/20000 	Loss: 1.380595
Train Epoch: 4900/20000 	Loss: 1.328523
Train Epoch: 5000/20000 	Loss: 1.291257
Train Epoch: 5100/20000 	Loss: 1.259162
Train Epoch: 5200/20000 	Loss: 1.224472
Train Epoch: 5300/20000 	Loss: 1.193033
Train Epoch: 5400/20000 	Loss: 1.160571
Train Epoch: 5500/20000 	Loss: 1.176616
Train Epoch: 5600/20000 	Loss: 1.150990
Train Epoch: 5700/20000 	Loss: 1.106392
Train Epoch: 5800/20000 	Loss: 1.107601
Train Epoch: 5900/20000 	Loss: 1.134836
Train Epoch: 6000/20000 	Loss: 1.084926
Train Epoch: 6100/20000 	Loss: 1.059580
Train Epoch: 6200/20000 	Loss: 1.032829
Train Epoch: 6300/20000 	Loss: 1.005506
Train Epoch: 6400/20000 	Loss: 0.989259
Train Epoch: 6500/20000 	Loss: 0.967748
Train Epoch: 6600/20000 	Loss: 0.946309
Train Epoch: 6700/20000 	Loss: 0.938827
Train Epoch: 6800/20000 	Loss: 0.895469
Train Epoch: 6900/20000 	Loss: 0.879807
Train Epoch: 7000/20000 	Loss: 0.860672
Train Epoch: 7100/20000 	Loss: 0.841361
Train Epoch: 7200/20000 	Loss: 0.897386
Train Epoch: 7300/20000 	Loss: 0.820061
Train Epoch: 7400/20000 	Loss: 0.805441
Train Epoch: 7500/20000 	Loss: 0.785111
Train Epoch: 7600/20000 	Loss: 0.781603
Train Epoch: 7700/20000 	Loss: 0.754027
Train Epoch: 7800/20000 	Loss: 0.740493
Train Epoch: 7900/20000 	Loss: 0.772454
Train Epoch: 8000/20000 	Loss: 0.732052
Train Epoch: 8100/20000 	Loss: 0.718889
Train Epoch: 8200/20000 	Loss: 0.723836
Train Epoch: 8300/20000 	Loss: 0.706040
Train Epoch: 8400/20000 	Loss: 0.694043
Train Epoch: 8500/20000 	Loss: 0.687886
Train Epoch: 8600/20000 	Loss: 0.685514
Train Epoch: 8700/20000 	Loss: 0.679205
Train Epoch: 8800/20000 	Loss: 0.633767
Train Epoch: 8900/20000 	Loss: 0.670673
Train Epoch: 9000/20000 	Loss: 0.623324
Train Epoch: 9100/20000 	Loss: 0.643469
Train Epoch: 9200/20000 	Loss: 0.640262
Train Epoch: 9300/20000 	Loss: 0.631938
Train Epoch: 9400/20000 	Loss: 0.734420
Train Epoch: 9500/20000 	Loss: 0.626137
Train Epoch: 9600/20000 	Loss: 0.662020
Train Epoch: 9700/20000 	Loss: 0.695955
Train Epoch: 9800/20000 	Loss: 0.635731
Train Epoch: 9900/20000 	Loss: 0.651402
Train Epoch: 10000/20000 	Loss: 0.624515
Train Epoch: 10100/20000 	Loss: 0.646017
Train Epoch: 10200/20000 	Loss: 0.623063
Train Epoch: 10300/20000 	Loss: 0.624553
Train Epoch: 10400/20000 	Loss: 0.628742
Train Epoch: 10500/20000 	Loss: 0.699380
Train Epoch: 10600/20000 	Loss: 0.637846
Train Epoch: 10700/20000 	Loss: 0.759253
Train Epoch: 10800/20000 	Loss: 0.648112
Train Epoch: 10900/20000 	Loss: 0.720772
Train Epoch: 11000/20000 	Loss: 0.657437
Train Epoch: 11100/20000 	Loss: 0.698014
Train Epoch: 11200/20000 	Loss: 0.702930
Train Epoch: 11300/20000 	Loss: 0.724398
Train Epoch: 11400/20000 	Loss: 0.635794
Train Epoch: 11500/20000 	Loss: 0.629414
Train Epoch: 11600/20000 	Loss: 0.634217
Train Epoch: 11700/20000 	Loss: 0.655395
Train Epoch: 11800/20000 	Loss: 0.632502
Train Epoch: 11900/20000 	Loss: 0.633241
Train Epoch: 12000/20000 	Loss: 0.621870
Train Epoch: 12100/20000 	Loss: 0.668807
Train Epoch: 12200/20000 	Loss: 0.638453
Train Epoch: 12300/20000 	Loss: 0.633410
Train Epoch: 12400/20000 	Loss: 0.674989
Train Epoch: 12500/20000 	Loss: 0.656147
Train Epoch: 12600/20000 	Loss: 0.637898
Train Epoch: 12700/20000 	Loss: 0.632677
Train Epoch: 12800/20000 	Loss: 0.657174
Train Epoch: 12900/20000 	Loss: 0.641346
Train Epoch: 13000/20000 	Loss: 0.633637
Train Epoch: 13100/20000 	Loss: 0.633938
Train Epoch: 13200/20000 	Loss: 0.631873
Train Epoch: 13300/20000 	Loss: 0.656787
Train Epoch: 13400/20000 	Loss: 0.639717
Train Epoch: 13500/20000 	Loss: 0.633655
Train Epoch: 13600/20000 	Loss: 0.656558
Train Epoch: 13700/20000 	Loss: 0.639115
Train Epoch: 13800/20000 	Loss: 0.636354
Train Epoch: 13900/20000 	Loss: 0.631104
Train Epoch: 14000/20000 	Loss: 0.621689
Train Epoch: 14100/20000 	Loss: 0.634780
Train Epoch: 14200/20000 	Loss: 0.636610
Train Epoch: 14300/20000 	Loss: 0.636960
Train Epoch: 14400/20000 	Loss: 0.654920
Train Epoch: 14500/20000 	Loss: 0.657338
Train Epoch: 14600/20000 	Loss: 0.634948
Train Epoch: 14700/20000 	Loss: 0.630930
Train Epoch: 14800/20000 	Loss: 0.652882
Train Epoch: 14900/20000 	Loss: 0.630275
Train Epoch: 15000/20000 	Loss: 0.634958
Train Epoch: 15100/20000 	Loss: 0.640666
Train Epoch: 15200/20000 	Loss: 0.635893
Train Epoch: 15300/20000 	Loss: 0.618788
Train Epoch: 15400/20000 	Loss: 0.644462
Train Epoch: 15500/20000 	Loss: 0.633792
Train Epoch: 15600/20000 	Loss: 0.633874
Train Epoch: 15700/20000 	Loss: 0.644783
Train Epoch: 15800/20000 	Loss: 0.631295
Train Epoch: 15900/20000 	Loss: 0.645190
Train Epoch: 16000/20000 	Loss: 0.629622
Train Epoch: 16100/20000 	Loss: 0.635998
Train Epoch: 16200/20000 	Loss: 0.638503
Train Epoch: 16300/20000 	Loss: 0.642419
Train Epoch: 16400/20000 	Loss: 0.633912
Train Epoch: 16500/20000 	Loss: 0.630397
Train Epoch: 16600/20000 	Loss: 0.632762
Train Epoch: 16700/20000 	Loss: 0.629281
Train Epoch: 16800/20000 	Loss: 0.639436
Train Epoch: 16900/20000 	Loss: 0.633702
Train Epoch: 17000/20000 	Loss: 0.633870
Train Epoch: 17100/20000 	Loss: 0.631418
Train Epoch: 17200/20000 	Loss: 0.638571
Train Epoch: 17300/20000 	Loss: 0.635252
Train Epoch: 17400/20000 	Loss: 0.634360
Train Epoch: 17500/20000 	Loss: 0.632353
Train Epoch: 17600/20000 	Loss: 0.636352
Train Epoch: 17700/20000 	Loss: 0.630548
Train Epoch: 17800/20000 	Loss: 0.646740
Train Epoch: 17900/20000 	Loss: 0.635927
Train Epoch: 18000/20000 	Loss: 0.637583
Train Epoch: 18100/20000 	Loss: 0.648844
Train Epoch: 18200/20000 	Loss: 0.643328
Train Epoch: 18300/20000 	Loss: 0.656029
Train Epoch: 18400/20000 	Loss: 0.632069
Train Epoch: 18500/20000 	Loss: 0.633117
Train Epoch: 18600/20000 	Loss: 0.635500
Train Epoch: 18700/20000 	Loss: 0.637722
Train Epoch: 18800/20000 	Loss: 0.631469
Train Epoch: 18900/20000 	Loss: 0.631498
Train Epoch: 19000/20000 	Loss: 0.633956
Train Epoch: 19100/20000 	Loss: 0.631589
Train Epoch: 19200/20000 	Loss: 0.630608
Train Epoch: 19300/20000 	Loss: 0.630776
Train Epoch: 19400/20000 	Loss: 0.631227
Train Epoch: 19500/20000 	Loss: 0.636715
Train Epoch: 19600/20000 	Loss: 0.633428
Train Epoch: 19700/20000 	Loss: 0.631257
Train Epoch: 19800/20000 	Loss: 0.632042
Train Epoch: 19900/20000 	Loss: 0.630990
Train Epoch: 20000/20000 	Loss: 0.630774
computing neighbors
    finished (0:00:00) --> added
    'distances' and 'connectivities', weighted adjacency matrices (adata.obsp)
computing moments based on connectivities
    finished (0:00:00) --> added
    'Ms' and 'Mu', moments of un/spliced abundances (adata.layers)
computing velocity graph
    finished (0:00:01) --> added
    'new_velocity_graph', sparse matrix with cosine correlations (adata.uns)
computing velocity embedding
    finished (0:00:00) --> added
    'new_velocity_umap', embedded velocity vectors (adata.obsm)
def expAblationCohAgg(adata, exp_metrics):
    n_cells, n_genes = adata.X.shape
    in_dim = n_genes
    z_dim = args.z_dim
    h_dim = args.h_dim
    g_basis = args.nb_g_src
    model = get_ablation_CohAgg(
                                adata,
                                in_dim,
                                z_dim,
                                h_dim,
                                g_basis,
                                device)
    model = main_AE(args, model, save_name=f"CohAgg_{args.model_name}")
    model.eval()
    with torch.no_grad():
        x = model.encoder(tensor_x)
        s = model.encoder(tensor_s)
        u = model.encoder(tensor_u)

        _, gamma, _ = leastsq_pt(s, u,
                                    device=device,
                                    fit_offset=True,
                                    perc=[5, 95]
                                    )
        v = (u - gamma * s).cpu().numpy()
        x = x.cpu().numpy()
        s = s.cpu().numpy()
        u = u.cpu().numpy()

    adata = new_adata(adata, x, s, u, v, g_basis=args.nb_g_src)
    scv.tl.velocity_graph(adata, vkey='new_velocity')
    scv.pl.velocity_embedding_stream(adata, vkey="new_velocity", basis='X_umap', color=['clusters', 'age(days)'],
                                    title="Ablation with GCN Only",
                                    )
    scv.tl.velocity_confidence(adata, vkey='new_velocity')
    exp_metrics['Ablation GCN Only'] = evaluate(adata, cluster_edges, k_cluster, "new_velocity")

expAblationCohAgg(adata, exp_metrics)computing neighbors
    finished (0:00:00) --> added
    'distances' and 'connectivities', weighted adjacency matrices (adata.obsp)
computing moments based on connectivities
    finished (0:00:00) --> added
    'Ms' and 'Mu', moments of un/spliced abundances (adata.layers)
Train Epoch: 100/20000 	Loss: 2.996022
Train Epoch: 200/20000 	Loss: 2.871044
Train Epoch: 300/20000 	Loss: 2.651241
Train Epoch: 400/20000 	Loss: 2.308443
Train Epoch: 500/20000 	Loss: 1.900360
Train Epoch: 600/20000 	Loss: 1.546577
Train Epoch: 700/20000 	Loss: 1.319960
Train Epoch: 800/20000 	Loss: 1.207195
Train Epoch: 900/20000 	Loss: 1.149219
Train Epoch: 1000/20000 	Loss: 1.102230
Train Epoch: 1100/20000 	Loss: 1.044342
Train Epoch: 1200/20000 	Loss: 0.963348
Train Epoch: 1300/20000 	Loss: 0.857648
Train Epoch: 1400/20000 	Loss: 0.750121
Train Epoch: 1500/20000 	Loss: 0.676032
Train Epoch: 1600/20000 	Loss: 0.638435
Train Epoch: 1700/20000 	Loss: 0.619636
Train Epoch: 1800/20000 	Loss: 0.607002
Train Epoch: 1900/20000 	Loss: 0.595287
Train Epoch: 2000/20000 	Loss: 0.582556
Train Epoch: 2100/20000 	Loss: 0.567861
Train Epoch: 2200/20000 	Loss: 0.550732
Train Epoch: 2300/20000 	Loss: 0.530731
Train Epoch: 2400/20000 	Loss: 0.507717
Train Epoch: 2500/20000 	Loss: 0.481952
Train Epoch: 2600/20000 	Loss: 0.454344
Train Epoch: 2700/20000 	Loss: 0.426534
Train Epoch: 2800/20000 	Loss: 0.400763
Train Epoch: 2900/20000 	Loss: 0.378804
Train Epoch: 3000/20000 	Loss: 0.361079
Train Epoch: 3100/20000 	Loss: 0.346883
Train Epoch: 3200/20000 	Loss: 0.334777
Train Epoch: 3300/20000 	Loss: 0.323708
Train Epoch: 3400/20000 	Loss: 0.312983
Train Epoch: 3500/20000 	Loss: 0.302267
Train Epoch: 3600/20000 	Loss: 0.291601
Train Epoch: 3700/20000 	Loss: 0.281081
Train Epoch: 3800/20000 	Loss: 0.270839
Train Epoch: 3900/20000 	Loss: 0.261106
Train Epoch: 4000/20000 	Loss: 0.251915
Train Epoch: 4100/20000 	Loss: 0.243271
Train Epoch: 4200/20000 	Loss: 0.235192
Train Epoch: 4300/20000 	Loss: 0.227626
Train Epoch: 4400/20000 	Loss: 0.220422
Train Epoch: 4500/20000 	Loss: 0.213601
Train Epoch: 4600/20000 	Loss: 0.207063
Train Epoch: 4700/20000 	Loss: 0.200894
Train Epoch: 4800/20000 	Loss: 0.194946
Train Epoch: 4900/20000 	Loss: 0.189262
Train Epoch: 5000/20000 	Loss: 0.184027
Train Epoch: 5100/20000 	Loss: 0.179137
Train Epoch: 5200/20000 	Loss: 0.174763
Train Epoch: 5300/20000 	Loss: 0.170866
Train Epoch: 5400/20000 	Loss: 0.167395
Train Epoch: 5500/20000 	Loss: 0.164348
Train Epoch: 5600/20000 	Loss: 0.161607
Train Epoch: 5700/20000 	Loss: 0.159112
Train Epoch: 5800/20000 	Loss: 0.156805
Train Epoch: 5900/20000 	Loss: 0.154591
Train Epoch: 6000/20000 	Loss: 0.152343
Train Epoch: 6100/20000 	Loss: 0.150020
Train Epoch: 6200/20000 	Loss: 0.147641
Train Epoch: 6300/20000 	Loss: 0.145040
Train Epoch: 6400/20000 	Loss: 0.142267
Train Epoch: 6500/20000 	Loss: 0.139285
Train Epoch: 6600/20000 	Loss: 0.136084
Train Epoch: 6700/20000 	Loss: 0.132773
Train Epoch: 6800/20000 	Loss: 0.129269
Train Epoch: 6900/20000 	Loss: 0.125831
Train Epoch: 7000/20000 	Loss: 0.122534
Train Epoch: 7100/20000 	Loss: 0.119376
Train Epoch: 7200/20000 	Loss: 0.116519
Train Epoch: 7300/20000 	Loss: 0.113886
Train Epoch: 7400/20000 	Loss: 0.111453
Train Epoch: 7500/20000 	Loss: 0.109273
Train Epoch: 7600/20000 	Loss: 0.107232
Train Epoch: 7700/20000 	Loss: 0.105713
Train Epoch: 7800/20000 	Loss: 0.103516
Train Epoch: 7900/20000 	Loss: 0.101787
Train Epoch: 8000/20000 	Loss: 0.099892
Train Epoch: 8100/20000 	Loss: 0.098184
Train Epoch: 8200/20000 	Loss: 0.096393
Train Epoch: 8300/20000 	Loss: 0.094677
Train Epoch: 8400/20000 	Loss: 0.093240
Train Epoch: 8500/20000 	Loss: 0.091218
Train Epoch: 8600/20000 	Loss: 0.089476
Train Epoch: 8700/20000 	Loss: 0.088022
Train Epoch: 8800/20000 	Loss: 0.086383
Train Epoch: 8900/20000 	Loss: 0.084989
Train Epoch: 9000/20000 	Loss: 0.083748
Train Epoch: 9100/20000 	Loss: 0.082627
Train Epoch: 9200/20000 	Loss: 0.081372
Train Epoch: 9300/20000 	Loss: 0.080393
Train Epoch: 9400/20000 	Loss: 0.079663
Train Epoch: 9500/20000 	Loss: 0.078815
Train Epoch: 9600/20000 	Loss: 0.077693
Train Epoch: 9700/20000 	Loss: 0.076763
Train Epoch: 9800/20000 	Loss: 0.076004
Train Epoch: 9900/20000 	Loss: 0.074911
Train Epoch: 10000/20000 	Loss: 0.074181
Train Epoch: 10100/20000 	Loss: 0.073464
Train Epoch: 10200/20000 	Loss: 0.072980
Train Epoch: 10300/20000 	Loss: 0.071727
Train Epoch: 10400/20000 	Loss: 0.071124
Train Epoch: 10500/20000 	Loss: 0.070902
Train Epoch: 10600/20000 	Loss: 0.070388
Train Epoch: 10700/20000 	Loss: 0.068698
Train Epoch: 10800/20000 	Loss: 0.068103
Train Epoch: 10900/20000 	Loss: 0.067358
Train Epoch: 11000/20000 	Loss: 0.066451
Train Epoch: 11100/20000 	Loss: 0.065805
Train Epoch: 11200/20000 	Loss: 0.065322
Train Epoch: 11300/20000 	Loss: 0.065143
Train Epoch: 11400/20000 	Loss: 0.064424
Train Epoch: 11500/20000 	Loss: 0.065906
Train Epoch: 11600/20000 	Loss: 0.063539
Train Epoch: 11700/20000 	Loss: 0.063328
Train Epoch: 11800/20000 	Loss: 0.063990
Train Epoch: 11900/20000 	Loss: 0.062512
Train Epoch: 12000/20000 	Loss: 0.061714
Train Epoch: 12100/20000 	Loss: 0.061307
Train Epoch: 12200/20000 	Loss: 0.060795
Train Epoch: 12300/20000 	Loss: 0.061024
Train Epoch: 12400/20000 	Loss: 0.060157
Train Epoch: 12500/20000 	Loss: 0.060745
Train Epoch: 12600/20000 	Loss: 0.060713
Train Epoch: 12700/20000 	Loss: 0.059844
Train Epoch: 12800/20000 	Loss: 0.059825
Train Epoch: 12900/20000 	Loss: 0.059514
Train Epoch: 13000/20000 	Loss: 0.059265
Train Epoch: 13100/20000 	Loss: 0.058468
Train Epoch: 13200/20000 	Loss: 0.057965
Train Epoch: 13300/20000 	Loss: 0.058630
Train Epoch: 13400/20000 	Loss: 0.057921
Train Epoch: 13500/20000 	Loss: 0.058573
Train Epoch: 13600/20000 	Loss: 0.057102
Train Epoch: 13700/20000 	Loss: 0.056638
Train Epoch: 13800/20000 	Loss: 0.056939
Train Epoch: 13900/20000 	Loss: 0.056660
Train Epoch: 14000/20000 	Loss: 0.057939
Train Epoch: 14100/20000 	Loss: 0.056383
Train Epoch: 14200/20000 	Loss: 0.057083
Train Epoch: 14300/20000 	Loss: 0.056454
Train Epoch: 14400/20000 	Loss: 0.056464
Train Epoch: 14500/20000 	Loss: 0.056579
Train Epoch: 14600/20000 	Loss: 0.056239
Train Epoch: 14700/20000 	Loss: 0.056249
Train Epoch: 14800/20000 	Loss: 0.056262
Train Epoch: 14900/20000 	Loss: 0.057102
Train Epoch: 15000/20000 	Loss: 0.057781
Train Epoch: 15100/20000 	Loss: 0.056375
Train Epoch: 15200/20000 	Loss: 0.057047
Train Epoch: 15300/20000 	Loss: 0.056671
Train Epoch: 15400/20000 	Loss: 0.056451
Train Epoch: 15500/20000 	Loss: 0.056579
Train Epoch: 15600/20000 	Loss: 0.056642
Train Epoch: 15700/20000 	Loss: 0.056877
Train Epoch: 15800/20000 	Loss: 0.057714
Train Epoch: 15900/20000 	Loss: 0.056339
Train Epoch: 16000/20000 	Loss: 0.056775
Train Epoch: 16100/20000 	Loss: 0.056100
Train Epoch: 16200/20000 	Loss: 0.056460
Train Epoch: 16300/20000 	Loss: 0.056667
Train Epoch: 16400/20000 	Loss: 0.056737
Train Epoch: 16500/20000 	Loss: 0.056197
Train Epoch: 16600/20000 	Loss: 0.056592
Train Epoch: 16700/20000 	Loss: 0.056588
Train Epoch: 16800/20000 	Loss: 0.056967
Train Epoch: 16900/20000 	Loss: 0.056475
Train Epoch: 17000/20000 	Loss: 0.056430
Train Epoch: 17100/20000 	Loss: 0.056856
Train Epoch: 17200/20000 	Loss: 0.056907
Train Epoch: 17300/20000 	Loss: 0.056958
Train Epoch: 17400/20000 	Loss: 0.056853
Train Epoch: 17500/20000 	Loss: 0.056311
Train Epoch: 17600/20000 	Loss: 0.057349
Train Epoch: 17700/20000 	Loss: 0.056504
Train Epoch: 17800/20000 	Loss: 0.056339
Train Epoch: 17900/20000 	Loss: 0.056141
Train Epoch: 18000/20000 	Loss: 0.056991
Train Epoch: 18100/20000 	Loss: 0.056559
Train Epoch: 18200/20000 	Loss: 0.056542
Train Epoch: 18300/20000 	Loss: 0.056515
Train Epoch: 18400/20000 	Loss: 0.056660
Train Epoch: 18500/20000 	Loss: 0.056138
Train Epoch: 18600/20000 	Loss: 0.056576
Train Epoch: 18700/20000 	Loss: 0.056225
Train Epoch: 18800/20000 	Loss: 0.056885
Train Epoch: 18900/20000 	Loss: 0.056534
Train Epoch: 19000/20000 	Loss: 0.056100
Train Epoch: 19100/20000 	Loss: 0.056173
Train Epoch: 19200/20000 	Loss: 0.056147
Train Epoch: 19300/20000 	Loss: 0.057261
Train Epoch: 19400/20000 	Loss: 0.056663
Train Epoch: 19500/20000 	Loss: 0.056055
Train Epoch: 19600/20000 	Loss: 0.056677
Train Epoch: 19700/20000 	Loss: 0.056700
Train Epoch: 19800/20000 	Loss: 0.056156
Train Epoch: 19900/20000 	Loss: 0.056142
Train Epoch: 20000/20000 	Loss: 0.057165
computing neighbors
    finished (0:00:00) --> added
    'distances' and 'connectivities', weighted adjacency matrices (adata.obsp)
computing moments based on connectivities
    finished (0:00:00) --> added
    'Ms' and 'Mu', moments of un/spliced abundances (adata.layers)
computing velocity graph
    finished (0:00:01) --> added
    'new_velocity_graph', sparse matrix with cosine correlations (adata.uns)
computing velocity embedding
    finished (0:00:00) --> added
    'new_velocity_umap', embedded velocity vectors (adata.obsm)
def expAblationAttComb(adata, exp_metrics):
    from sklearn.decomposition import PCA
    n_cells, n_genes = adata.X.shape
    z_dim = args.z_dim
    g_rep_dim = args.g_rep_dim
    h_dim = args.h_dim
    k_dim = args.k_dim
    gb_tau = args.gumbsoft_tau
    G_embeddings = PCA(n_components=g_rep_dim).fit_transform(adata.X.T.toarray())

    model = get_ablation_attcomb(
        z_dim,
        n_genes,
        n_cells,
        h_dim,
        k_dim,
        G_embeddings,
        g_rep_dim,
        gb_tau,
        device)

    model = main_AE(args, model, save_name=f"AttComb_{args.model_name}")
    model.eval()
    with torch.no_grad():
        x = model.encoder(tensor_x)
        s = model.encoder(tensor_s)
        u = model.encoder(tensor_u)

        _, gamma, _ = leastsq_pt(s, u,
                                    device=device,
                                    fit_offset=True,
                                    perc=[5, 95]
                                    )
        v = (u - gamma * s).cpu().numpy()
        x = x.cpu().numpy()
        s = s.cpu().numpy()
        u = u.cpu().numpy()

    adata = new_adata(adata, x, s, u, v, g_basis=args.nb_g_src)
    scv.tl.velocity_graph(adata, vkey='new_velocity')
    scv.pl.velocity_embedding_stream(adata, vkey="new_velocity", basis='X_umap', color=['clusters', 'age(days)'],
                                    title="Ablation with Attentive Combination Only",
                                    )
    scv.tl.velocity_confidence(adata, vkey='new_velocity')
    exp_metrics['Ablation AttComb Only'] = evaluate(adata, cluster_edges, k_cluster, "new_velocity")

expAblationAttComb(adata, exp_metrics)Train Epoch: 100/20000 	Loss: 22.296110
Train Epoch: 200/20000 	Loss: 11.209204
Train Epoch: 300/20000 	Loss: 8.055142
Train Epoch: 400/20000 	Loss: 6.474950
Train Epoch: 500/20000 	Loss: 5.562144
Train Epoch: 600/20000 	Loss: 5.017074
Train Epoch: 700/20000 	Loss: 4.585483
Train Epoch: 800/20000 	Loss: 4.285267
Train Epoch: 900/20000 	Loss: 3.956163
Train Epoch: 1000/20000 	Loss: 3.746126
Train Epoch: 1100/20000 	Loss: 3.576847
Train Epoch: 1200/20000 	Loss: 3.497398
Train Epoch: 1300/20000 	Loss: 3.345078
Train Epoch: 1400/20000 	Loss: 3.258979
Train Epoch: 1500/20000 	Loss: 3.169951
Train Epoch: 1600/20000 	Loss: 3.096140
Train Epoch: 1700/20000 	Loss: 3.039300
Train Epoch: 1800/20000 	Loss: 2.979802
Train Epoch: 1900/20000 	Loss: 2.939360
Train Epoch: 2000/20000 	Loss: 2.893918
Train Epoch: 2100/20000 	Loss: 2.836920
Train Epoch: 2200/20000 	Loss: 2.812449
Train Epoch: 2300/20000 	Loss: 2.765800
Train Epoch: 2400/20000 	Loss: 2.739450
Train Epoch: 2500/20000 	Loss: 2.695289
Train Epoch: 2600/20000 	Loss: 2.667977
Train Epoch: 2700/20000 	Loss: 2.635406
Train Epoch: 2800/20000 	Loss: 2.606824
Train Epoch: 2900/20000 	Loss: 2.583778
Train Epoch: 3000/20000 	Loss: 2.561750
Train Epoch: 3100/20000 	Loss: 2.540195
Train Epoch: 3200/20000 	Loss: 2.519508
Train Epoch: 3300/20000 	Loss: 2.489537
Train Epoch: 3400/20000 	Loss: 2.469573
Train Epoch: 3500/20000 	Loss: 2.449957
Train Epoch: 3600/20000 	Loss: 2.429441
Train Epoch: 3700/20000 	Loss: 2.412853
Train Epoch: 3800/20000 	Loss: 2.392382
Train Epoch: 3900/20000 	Loss: 2.372402
Train Epoch: 4000/20000 	Loss: 2.358211
Train Epoch: 4100/20000 	Loss: 2.341208
Train Epoch: 4200/20000 	Loss: 2.325586
Train Epoch: 4300/20000 	Loss: 2.316036
Train Epoch: 4400/20000 	Loss: 2.337067
Train Epoch: 4500/20000 	Loss: 2.414153
Train Epoch: 4600/20000 	Loss: 2.307628
Train Epoch: 4700/20000 	Loss: 2.297759
Train Epoch: 4800/20000 	Loss: 2.279346
Train Epoch: 4900/20000 	Loss: 2.266170
Train Epoch: 5000/20000 	Loss: 2.254791
Train Epoch: 5100/20000 	Loss: 2.241108
Train Epoch: 5200/20000 	Loss: 2.229543
Train Epoch: 5300/20000 	Loss: 2.218495
Train Epoch: 5400/20000 	Loss: 2.206671
Train Epoch: 5500/20000 	Loss: 2.200698
Train Epoch: 5600/20000 	Loss: 2.186262
Train Epoch: 5700/20000 	Loss: 2.177135
Train Epoch: 5800/20000 	Loss: 2.165068
Train Epoch: 5900/20000 	Loss: 2.154455
Train Epoch: 6000/20000 	Loss: 2.145957
Train Epoch: 6100/20000 	Loss: 2.137772
Train Epoch: 6200/20000 	Loss: 2.129022
Train Epoch: 6300/20000 	Loss: 2.116606
Train Epoch: 6400/20000 	Loss: 2.108815
Train Epoch: 6500/20000 	Loss: 2.101003
Train Epoch: 6600/20000 	Loss: 2.090555
Train Epoch: 6700/20000 	Loss: 2.080659
Train Epoch: 6800/20000 	Loss: 2.069031
Train Epoch: 6900/20000 	Loss: 2.063165
Train Epoch: 7000/20000 	Loss: 2.053384
Train Epoch: 7100/20000 	Loss: 2.047582
Train Epoch: 7200/20000 	Loss: 2.032883
Train Epoch: 7300/20000 	Loss: 2.025606
Train Epoch: 7400/20000 	Loss: 2.018063
Train Epoch: 7500/20000 	Loss: 2.010695
Train Epoch: 7600/20000 	Loss: 1.999934
Train Epoch: 7700/20000 	Loss: 1.992181
Train Epoch: 7800/20000 	Loss: 1.985742
Train Epoch: 7900/20000 	Loss: 1.976691
Train Epoch: 8000/20000 	Loss: 1.969319
Train Epoch: 8100/20000 	Loss: 1.960726
Train Epoch: 8200/20000 	Loss: 1.953407
Train Epoch: 8300/20000 	Loss: 1.944151
Train Epoch: 8400/20000 	Loss: 1.942530
Train Epoch: 8500/20000 	Loss: 1.929456
Train Epoch: 8600/20000 	Loss: 1.922021
Train Epoch: 8700/20000 	Loss: 1.918418
Train Epoch: 8800/20000 	Loss: 1.910591
Train Epoch: 8900/20000 	Loss: 1.902623
Train Epoch: 9000/20000 	Loss: 1.895047
Train Epoch: 9100/20000 	Loss: 1.888137
Train Epoch: 9200/20000 	Loss: 1.880993
Train Epoch: 9300/20000 	Loss: 1.872991
Train Epoch: 9400/20000 	Loss: 1.867524
Train Epoch: 9500/20000 	Loss: 1.858583
Train Epoch: 9600/20000 	Loss: 1.851774
Train Epoch: 9700/20000 	Loss: 1.846951
Train Epoch: 9800/20000 	Loss: 1.837829
Train Epoch: 9900/20000 	Loss: 1.831614
Train Epoch: 10000/20000 	Loss: 1.823735
Train Epoch: 10100/20000 	Loss: 1.819651
Train Epoch: 10200/20000 	Loss: 1.811622
Train Epoch: 10300/20000 	Loss: 1.806599
Train Epoch: 10400/20000 	Loss: 1.797688
Train Epoch: 10500/20000 	Loss: 1.790483
Train Epoch: 10600/20000 	Loss: 1.785919
Train Epoch: 10700/20000 	Loss: 1.778986
Train Epoch: 10800/20000 	Loss: 1.773682
Train Epoch: 10900/20000 	Loss: 1.767894
Train Epoch: 11000/20000 	Loss: 1.762872
Train Epoch: 11100/20000 	Loss: 1.754915
Train Epoch: 11200/20000 	Loss: 1.749223
Train Epoch: 11300/20000 	Loss: 1.743148
Train Epoch: 11400/20000 	Loss: 1.737233
Train Epoch: 11500/20000 	Loss: 1.734197
Train Epoch: 11600/20000 	Loss: 1.727782
Train Epoch: 11700/20000 	Loss: 1.720003
Train Epoch: 11800/20000 	Loss: 1.716374
Train Epoch: 11900/20000 	Loss: 1.711648
Train Epoch: 12000/20000 	Loss: 1.707833
Train Epoch: 12100/20000 	Loss: 1.698742
Train Epoch: 12200/20000 	Loss: 1.694128
Train Epoch: 12300/20000 	Loss: 1.687910
Train Epoch: 12400/20000 	Loss: 1.681852
Train Epoch: 12500/20000 	Loss: 1.678009
Train Epoch: 12600/20000 	Loss: 1.671663
Train Epoch: 12700/20000 	Loss: 1.667063
Train Epoch: 12800/20000 	Loss: 1.687084
Train Epoch: 12900/20000 	Loss: 1.662932
Train Epoch: 13000/20000 	Loss: 1.657005
Train Epoch: 13100/20000 	Loss: 1.652357
Train Epoch: 13200/20000 	Loss: 1.647759
Train Epoch: 13300/20000 	Loss: 1.642066
Train Epoch: 13400/20000 	Loss: 1.637426
Train Epoch: 13500/20000 	Loss: 1.639685
Train Epoch: 13600/20000 	Loss: 1.635154
Train Epoch: 13700/20000 	Loss: 1.631344
Train Epoch: 13800/20000 	Loss: 1.627799
Train Epoch: 13900/20000 	Loss: 1.623474
Train Epoch: 14000/20000 	Loss: 1.619106
Train Epoch: 14100/20000 	Loss: 1.614887
Train Epoch: 14200/20000 	Loss: 1.610781
Train Epoch: 14300/20000 	Loss: 1.606496
Train Epoch: 14400/20000 	Loss: 1.602101
Train Epoch: 14500/20000 	Loss: 1.598194
Train Epoch: 14600/20000 	Loss: 1.595981
Train Epoch: 14700/20000 	Loss: 1.591599
Train Epoch: 14800/20000 	Loss: 1.587035
Train Epoch: 14900/20000 	Loss: 1.585607
Train Epoch: 15000/20000 	Loss: 1.580394
Train Epoch: 15100/20000 	Loss: 1.577409
Train Epoch: 15200/20000 	Loss: 1.572983
Train Epoch: 15300/20000 	Loss: 1.571503
Train Epoch: 15400/20000 	Loss: 1.562941
Train Epoch: 15500/20000 	Loss: 1.557573
Train Epoch: 15600/20000 	Loss: 1.552610
Train Epoch: 15700/20000 	Loss: 1.550323
Train Epoch: 15800/20000 	Loss: 1.547808
Train Epoch: 15900/20000 	Loss: 1.539433
Train Epoch: 16000/20000 	Loss: 1.536536
Train Epoch: 16100/20000 	Loss: 1.531693
Train Epoch: 16200/20000 	Loss: 1.529480
Train Epoch: 16300/20000 	Loss: 1.523062
Train Epoch: 16400/20000 	Loss: 1.519822
Train Epoch: 16500/20000 	Loss: 1.516006
Train Epoch: 16600/20000 	Loss: 1.514253
Train Epoch: 16700/20000 	Loss: 1.508911
Train Epoch: 16800/20000 	Loss: 1.506323
Train Epoch: 16900/20000 	Loss: 1.501095
Train Epoch: 17000/20000 	Loss: 1.499678
Train Epoch: 17100/20000 	Loss: 1.493802
Train Epoch: 17200/20000 	Loss: 1.490988
Train Epoch: 17300/20000 	Loss: 1.485268
Train Epoch: 17400/20000 	Loss: 1.481854
Train Epoch: 17500/20000 	Loss: 1.480696
Train Epoch: 17600/20000 	Loss: 1.475615
Train Epoch: 17700/20000 	Loss: 1.471059
Train Epoch: 17800/20000 	Loss: 1.468042
Train Epoch: 17900/20000 	Loss: 1.463781
Train Epoch: 18000/20000 	Loss: 1.461234
Train Epoch: 18100/20000 	Loss: 1.461626
Train Epoch: 18200/20000 	Loss: 1.460261
Train Epoch: 18300/20000 	Loss: 1.454122
Train Epoch: 18400/20000 	Loss: 1.451583
Train Epoch: 18500/20000 	Loss: 1.452980
Train Epoch: 18600/20000 	Loss: 1.450934
Train Epoch: 18700/20000 	Loss: 1.446020
Train Epoch: 18800/20000 	Loss: 1.442387
Train Epoch: 18900/20000 	Loss: 1.438720
Train Epoch: 19000/20000 	Loss: 1.439665
Train Epoch: 19100/20000 	Loss: 1.436789
Train Epoch: 19200/20000 	Loss: 1.440140
Train Epoch: 19300/20000 	Loss: 1.436736
Train Epoch: 19400/20000 	Loss: 1.431831
Train Epoch: 19500/20000 	Loss: 1.430558
Train Epoch: 19600/20000 	Loss: 1.426027
Train Epoch: 19700/20000 	Loss: 1.425290
Train Epoch: 19800/20000 	Loss: 1.427494
Train Epoch: 19900/20000 	Loss: 1.424927
Train Epoch: 20000/20000 	Loss: 1.424482
computing neighbors
    finished (0:00:00) --> added
    'distances' and 'connectivities', weighted adjacency matrices (adata.obsp)
computing moments based on connectivities
    finished (0:00:00) --> added
    'Ms' and 'Mu', moments of un/spliced abundances (adata.layers)
computing velocity graph
    finished (0:00:01) --> added
    'new_velocity_graph', sparse matrix with cosine correlations (adata.uns)
computing velocity embedding
    finished (0:00:00) --> added
    'new_velocity_umap', embedded velocity vectors (adata.obsm)
def expPCA(adata, exp_metrics):
    from sklearn.decomposition import PCA
    pca = PCA(n_components=args.z_dim)
    x, s, u, v = sklearn_decompose(pca,
                                   tensor_x.detach().cpu().numpy(),
                                   tensor_s.detach().cpu().numpy(),
                                   tensor_u.detach().cpu().numpy(),
                                   tensor_v.detach().cpu().numpy()
                                  )

    adata = new_adata(adata, x, s, u, v, g_basis=args.nb_g_src)
    scv.tl.velocity_graph(adata, vkey='new_velocity')
    scv.pl.velocity_embedding_stream(adata, vkey="new_velocity", basis='X_umap', color=['clusters', 'age(days)'],
                                    title="Principle Component Analysis",
                                    )
    scv.tl.velocity_confidence(adata, vkey='new_velocity')
    exp_metrics['Baseline PCA'] = evaluate(adata, cluster_edges, k_cluster, "new_velocity")

expPCA(adata, exp_metrics)computing neighbors
    finished (0:00:00) --> added
    'distances' and 'connectivities', weighted adjacency matrices (adata.obsp)
computing moments based on connectivities
    finished (0:00:00) --> added
    'Ms' and 'Mu', moments of un/spliced abundances (adata.layers)
computing velocity graph
    finished (0:00:01) --> added
    'new_velocity_graph', sparse matrix with cosine correlations (adata.uns)
computing velocity embedding
    finished (0:00:00) --> added
    'new_velocity_umap', embedded velocity vectors (adata.obsm)
def expFA(adata, exp_metrics):
    from sklearn.decomposition import FactorAnalysis
    method = FactorAnalysis(n_components=args.z_dim)
    x, s, u, v = sklearn_decompose(method,
                                   tensor_x.detach().cpu().numpy(),
                                   tensor_s.detach().cpu().numpy(),
                                   tensor_u.detach().cpu().numpy(),
                                   tensor_v.detach().cpu().numpy()
                                  )

    adata = new_adata(adata, x, s, u, v, g_basis=args.nb_g_src)
    scv.tl.velocity_graph(adata, vkey='new_velocity')
    scv.pl.velocity_embedding_stream(adata, vkey="new_velocity", basis='X_umap', color=['clusters', 'age(days)'],
                                    title="Factor Analysis",
                                    )
    scv.tl.velocity_confidence(adata, vkey='new_velocity')
    exp_metrics['Baseline FA'] = evaluate(adata, cluster_edges, k_cluster, "new_velocity")

expFA(adata, exp_metrics)computing neighbors
    finished (0:00:00) --> added
    'distances' and 'connectivities', weighted adjacency matrices (adata.obsp)
computing moments based on connectivities
    finished (0:00:00) --> added
    'Ms' and 'Mu', moments of un/spliced abundances (adata.layers)
computing velocity graph
    finished (0:00:01) --> added
    'new_velocity_graph', sparse matrix with cosine correlations (adata.uns)
computing velocity embedding
    finished (0:00:00) --> added
    'new_velocity_umap', embedded velocity vectors (adata.obsm)
with open("{}.pkl".format(EXP_NAME), 'wb') as out_file:
    pickle.dump(exp_metrics, out_file)_____no_output_____
</code>


# Notebook from EltonCN/evolvepy
Path: examples/3-CNF-SAT.ipynb

import evolvepy as ep
import numpy as np
import matplotlib.pyplot as plt_____no_output_____

# 3-CNF-SAT_____no_output_____In this exemple we are going to see an exemple of Conjunctive Normal Form Satisfability. In this example we will show the CNF with 3 literals clauses. Although the code can cover the problem for N-CNF-SAT._____no_output_____# Setting the expression_____no_output_____First of all we are going to write a exemple expression. They are a combination of clauses joined by the boolean symbol "*" which stands for AND. Those clauses are combinations of literals joined by the boolean symbol "+" which stands for OR and each literal can be negated with the symbol "!" which stands for NOT.

We will generate a random expression._____no_output_____

from itertools import chain

# List with all lower and upper case characters and their negations
alphabet = [chr(i) for i in chain(range(65, 91), range(97, 123))]
not_alphabet = ["!"+chr(i) for i in chain(range(65, 91), range(97, 123))]
alphabet = alphabet+not_alphabet
_____no_output_____n_clausule = len(alphabet) - 2_____no_output_____# Here we define the logical expression, it need to be contained in parenthesis and the variables.
expression = ""
for i in range(n_clausule):
    expression += "("

    variables = np.random.choice(alphabet, 3)
    clause = "+".join(variables)

    expression += clause
    expression += ")"
    expression += "*"

expression = expression[:-1] # Remove last "*"
expression
_____no_output_____

# Evaluator

To define the evaluator, let's start by defining a class to parse the expression and evaluate the individuals:_____no_output_____

class CNF_SAT_3:
    def __init__(self, expression:str):
        variables = {}

        # Process expression
        expression = expression.replace("(", "")
        expression = expression.replace(")", "")

        expressions = expression.split("*")

        expressions = list(map(lambda x: x.split("+"),  expressions))
        self._expressions = expressions

        # Get variables
        variables = np.asarray(self._expressions).flatten()
        variables = np.char.strip(variables, chars="!")
        variables = np.unique(variables)

        self._variables = variables
        self._n_variable = len(variables)

        self._variables_table = {self._variables[i]:i for i in range(self._n_variable)} #"a" -> 1
        self._inverse_table = {v: k for k, v in self._variables_table.items()} #1 -> "a"

        self._n_test = 0

    @property
    def n_variable(self) -> int:
        return self._n_variable

    @property
    def n_test(self) -> int:
        return self._n_test

    def evaluate(self, individuals:np.ndarray) -> np.ndarray:
        individual = individuals[0]["chr0"]
        score = 0

        for expression in self._expressions:
            for variable in expression:
                if variable[0] == '!':
                    if individual[self._variables_table[variable[1]]]:
                        score += 1
                        break
                elif individual[self._variables_table[variable]]:
                    score += 1
                    break

        self._n_test += 1

        return score

    def print_individual(self, individual:np.ndarray) -> None:
        fitness = self.evaluate([individual])

        print("Individual's fitness:", fitness, "/", len(self._expressions))
        print("Variables:")

        for i in range(self._n_variable):
            value = individual["chr0"][i]
            variable = self._inverse_table[i]

            print(variable+":", value)
        _____no_output_____

Our fitness function will be the problem evaluation, which returns the number of true clauses. In this way, we can make our population move towards a better and better solution with each generation._____no_output_____

problem = CNF_SAT_3(expression)
fitness_function = problem.evaluate_____no_output_____

Here we define the evaluator object for the model with the function we defined above:_____no_output_____

from evolvepy.evaluator import FunctionEvaluator

evaluator = FunctionEvaluator(fitness_function, mode=FunctionEvaluator.PYTHON)_____no_output_____

## Generator

Here we will define the generator for the problem_____no_output_____First, we need to define a descriptor with a boolean (true/false) chromosome_____no_output_____

# Here we have individual with n_variables chromosomes of boolean type.
# There is no need to define the chromosome varition range
descriptor = ep.generator.Descriptor(problem.n_variable, types=[bool])

print(descriptor.dtype)[('chr0', '?', (52,))]

Then we will use a BinaryMutationLayer to mutate the individual in this context and a CombineLayer
to apply the selection funtions. To see details of those layers and funtions you can read prior notebooks._____no_output_____

from evolvepy.generator import CombineLayer, Generator
from evolvepy.generator.mutation import BinaryMutationLayer, bit_mutation
from evolvepy.generator.crossover import one_point
from evolvepy.generator.selection import tournament

combine = CombineLayer(tournament, one_point)
mutation = BinaryMutationLayer(bit_mutation, 0.8, 0.4)

combine.next = mutation

gen = Generator(descriptor=descriptor, first_layer=combine, last_layer=mutation)_____no_output_____

# Run and Results

Let's evolve our population and see the results_____no_output_____

n_generations = 50
n_individual = 10
evolver = ep.Evolver(gen, evaluator, n_individual)
hist, last_population = evolver.evolve(n_generations)_____no_output_____best = last_population[np.argmax(hist[-1])]
best_____no_output_____import matplotlib.pyplot as plt

plt.plot(hist.max(axis=1))
plt.xlabel("Generation")
plt.ylabel("Fitness")
plt.title("Evolution History")
plt.show()_____no_output_____

Looking at the evolution history, we can see that the algorithm was able to quickly converge to the solution.

![Evolution History](3-CNF-SAT-Figure1.jpg)_____no_output_____By comparing the number of times the problem was evaluated, with the number of possible combinations of variables, we can see that he managed to find the solution with some efficiency:_____no_output_____

problem.n_test/(2**problem.n_variable)_____no_output_____

We can print all the variables of the solution:_____no_output_____

problem.print_individual(best)Individual's fitness: 102 / 102
Variables:
A: False
B: True
C: True
D: False
E: False
F: False
G: False
H: True
I: True
J: True
K: False
L: True
M: False
N: True
O: True
P: True
Q: True
R: True
S: True
T: True
U: True
V: True
W: False
X: True
Y: False
Z: True
a: True
b: True
c: False
d: False
e: True
f: True
g: True
h: True
i: True
j: True
k: True
l: True
m: False
n: True
o: True
p: True
q: True
r: True
s: True
t: True
u: True
v: False
w: True
x: False
y: True
z: False


# Notebook from mortazavilab/swan_vis
Path: tutorials/Getting started.ipynb

#  Getting started: initializing, adding data, and saving your SwanGraph _____no_output_____First, if you haven't already, make sure to [install Swan](https://github.com/fairliereese/swan_vis/wiki#installation).
After installing, you'll be able to run Swan from Python.

Then, download the data and the reference transcriptome annotation from [here](http://crick.bio.uci.edu/freese/swan_files_example/). The bash commands to do so are given below.

The main workflow to get started with Swan consists of:
1. [Adding a reference transcriptome (optional)](#add_trans)
2. Adding a transcriptome for your samples
    * [From a GTF](#add_gtf)
    * [From a TALON db](#add_db)
3. [Adding datasets and their expression values](#add_ab)
4. [Adding metadata to your datasets](#add_meta)

Other sections:
* [Example data download](#data_download)
* [Starting and initializing your SwanGraph](#init)
* [Saving and loading your SwanGraph](#save_load)

This page can also be read from top to bottom, just know that you may be running things more than once!

For information on the file formats needed to use Swan, please read the [file format specifications FAQ](https://freese.gitbook.io/swan/faqs/file_formats).

_____no_output_____##  Download example data_____no_output_____This data is the data used in the [Swan publication](https://academic.oup.com/bioinformatics/article/37/9/1322/5912931)

Run this block in your bash terminal
```bash
mkdir data
mkdir figures
cd data/

# download files
wget http://crick.bio.uci.edu/freese/swan_files.tgz

# expand files
tar -xzf swan_files.tgz
mv swan_files/* .
rm -r swan_files/

# download reference annotation
wget ftp://ftp.ebi.ac.uk/pub/databases/gencode/Gencode_human/release_29/gencode.v29.annotation.gtf.gz
gunzip gencode.v29.annotation.gtf.gz

cd ../
```_____no_output_____Alternatively, just run on a smaller example, chr20.

Run this block in your bash terminal

```bash
mkdir data
mkdir figures
cd data/

# download files
wget http://crick.bio.uci.edu/freese/swan_files_example.tgz

# expand files
tar -xzf swan_files_example.tar.gz
mv swan_files_example/* .
rm -r swan_files_example/

cd ../
```_____no_output_____## Starting up Swan and initializing your SwanGraph_____no_output_____The rest of the code in this tutorial should be run in using Python_____no_output_____Initialize an empty SwanGraph and add the transcriptome annotation to the SwanGraph._____no_output_____

import swan_vis as swan

# initialize a new SwanGraph
sg = swan.SwanGraph() _____no_output_____

**Note:** to initialize a SwanGraph in single-cell mode (which will avoid calculating percent isoform use \[pi\] numbers for each cell), use the following code:

```python
sg = swan.SwanGraph(sc=True)
```_____no_output_____

annot_gtf = 'data/gencode.v29.annotation.gtf'
data_gtf = 'data/all_talon_observedOnly.gtf'
ab_file = 'data/all_talon_abundance_filtered.tsv'
talon_db = 'data/talon.db'
pass_list = 'data/all_pass_list.csv'
meta = 'data/metadata.tsv'_____no_output_____

## Adding a reference transcriptome_____no_output_____

# add an annotation transcriptome
sg.add_annotation(annot_gtf)
Adding annotation to the SwanGraph

## Adding transcript models from a GTF_____no_output_____Add all filtered transcript models to the SwanGraph._____no_output_____

# add a dataset's transcriptome and abundance information to
# the SwanGraph
sg.add_transcriptome(data_gtf)
Adding transcriptome to the SwanGraph

## Adding datasets and their abundance_____no_output_____Use an abundance matrix with columns for each desired dataset to add datasets to the SwanGraph._____no_output_____

# add each dataset's abundance information to the SwanGraph
sg.add_abundance(ab_file)
Adding abundance for datasets hepg2_1, hepg2_2, hffc6_1, hffc6_2, hffc6_3 to SwanGraph.

##  Saving and loading your SwanGraph_____no_output_____Following this, you can save your SwanGraph so you can easily work with it again without re-adding all the data._____no_output_____

# save the SwanGraph as a Python pickle file
sg.save_graph('swan')Saving graph as swan.p

And you can reload the graph again._____no_output_____

# load up a saved SwanGraph from a pickle file
sg = swan.read('swan.p')Read in graph from swan.p

##  Adding transcript models from a TALON DB_____no_output_____Swan is also directly compatible with TALON databases and can pull transcript models directly from them. You can also optionally pass in a list of isoforms from [`talon_filter_transcripts`](https://github.com/mortazavilab/TALON#talon_filter) to filter your input transcript models._____no_output_____

# for this new example, create a new empty SwanGraph
sg = swan.SwanGraph()

# and add the annotation transcriptome to it
sg.add_annotation(annot_gtf)

# add transcriptome from TALON db
sg.add_transcriptome(talon_db, pass_list=pass_list)

# add each dataset's abundance information to the SwanGraph
sg.add_abundance(ab_file)
Adding annotation to the SwanGraph

Adding transcriptome to the SwanGraph

Adding abundance for datasets hepg2_1, hepg2_2, hffc6_1, hffc6_2, hffc6_3 to SwanGraph.

##  Adding metadata_____no_output_____Swan provides functionality to perform tests and plotting on the basis of metadata categories._____no_output_____

sg.add_metadata(meta)/Users/fairliereese/miniconda3/lib/python3.7/site-packages/anndata/_core/anndata.py:794: UserWarning:
AnnData expects .obs.index to contain strings, but got values like:
    [0, 1, 2, 3, 4]

    Inferred to be: integer

  value_idx = self._prep_dim_index(value.index, attr)
sg.adata.obs_____no_output_____# save the SwanGraph as a Python pickle file
sg.save_graph('data/swan')
sg.save_graph('swan')
sg.save_graph('data/swan_files_full')
# sg.save_graph('data/swan_back')Saving graph as data/swan.p
Saving graph as swan.p
Saving graph as data/swan_files_full.p
Saving graph as data/swan_back.p
print(sg.adata.layers['counts'][:5, :5])
print(sg.adata.layers['tpm'][:5, :5])
print(sg.adata.layers['pi'][:5, :5])[[ 98.  43.   4.  23.   0.]
 [207.  66.   6.  52.   0.]
 [100. 148.   0.  82.   0.]
 [108. 191.   0.  98.   0.]
 [ 91. 168.   2. 106.   0.]]
[[196.13847    86.06076     8.005652   46.032497    0.       ]
 [243.97517    77.789185    7.071744   61.28845     0.       ]
 [131.32097   194.35504     0.        107.6832      0.       ]
 [137.06158   242.39594     0.        124.37069     0.       ]
 [147.9865    273.20584     3.2524502 172.37987     0.       ]]
[[100.       100.       100.       100.         0.      ]
 [ 99.519226 100.        60.000004 100.         0.      ]
 [ 98.039215 100.         0.       100.         0.      ]
 [ 99.08257  100.         0.       100.         0.      ]
 [100.       100.       100.       100.         0.      ]]
df = swan.calc_pi(sg.adata, sg.t_df, obs_col='dataset')_____no_output_____print(sg.adata.layers['counts'][:5, :5])
print(sg.adata.layers['tpm'][:5, :5])
print(sg.adata.layers['pi'][:5, :5])[[ 98.  43.   4.  23.   0.]
 [207.  66.   6.  52.   0.]
 [100. 148.   0.  82.   0.]
 [108. 191.   0.  98.   0.]
 [ 91. 168.   2. 106.   0.]]
[[196.13847    86.06076     8.005652   46.032497    0.       ]
 [243.97517    77.789185    7.071744   61.28845     0.       ]
 [131.32097   194.35504     0.        107.6832      0.       ]
 [137.06158   242.39594     0.        124.37069     0.       ]
 [147.9865    273.20584     3.2524502 172.37987     0.       ]]
[[100.       100.       100.       100.         0.      ]
 [ 99.519226 100.        60.000004 100.         0.      ]
 [ 98.039215 100.         0.       100.         0.      ]
 [ 99.08257  100.         0.       100.         0.      ]
 [100.       100.       100.       100.         0.      ]]


# Notebook from arfc/I2CNER
Path: simulated_scenarios/new_tech-new_nuclear/output/plotter.ipynb

<code>
import numpy as np
import csv
import pandas as pd
import matplotlib.pyplot as plt
import matplotlib
from collections import OrderedDict
%matplotlib inline

fname='elc_output.csv'
colfilter = ['Process','Period','PV']
xlocs = [2013]
xlocs=xlocs+list(np.arange(2020,2101,10))
a = pd.read_csv(fname, delimiter=';', usecols = colfilter)
a = a[colfilter]
a=a.groupby(['Process', 'Period']).sum() #sum values for same period, process
a=a.reset_index()
a.set_index('Period', inplace=True)

procs = a.Process.unique()
newyrs=pd.Series(range(2013,2101))
pltdict=OrderedDict()

for prc in procs:
    b=a.loc[a['Process'] == prc].reindex(newyrs,fill_value=0)
    pltdict[prc]=b['PV']
#print(procs)

ordlegend=['Hydro','Nuclear',
           'Coal','Oil', 'LNG',
           'SOFC',#'CCS-Gas',
           'Geothermal','Solar','Emerging Solar',
           'Onshore Wind'#,'Offshore Fixed',#,'Offshore Floating',
           #'Li-ion'
          ]
colours = ['aqua','orangered',
           'black', 'saddlebrown', 'slategray',
           'salmon',#'tan',
           'green','yellow','maroon',
           'blue'#,'dodgerblue',#,'teal',
          #'violet'
          ]

gridRow=2
gridColumn=6
legendCols=5

fig = plt.figure(figsize=(20,10))
gs = fig.add_gridspec(gridRow,gridColumn)
ax1 = fig.add_subplot(gs[0,2:4])
ax1.set_title('Electricity supplied for demand',fontsize='18')
ax2 = fig.add_subplot(gs[1,1:3])
ax2.set_title('Capacity of supply & storage',fontsize='18')
ax3 = fig.add_subplot(gs[1,3:5])
ax3.set_title('Emission sources',fontsize='18')

fig.tight_layout()
fig.subplots_adjust(hspace=0.4, wspace=0.35)

pltyrs=range(2013,2101)
ax1.stackplot(pltyrs,  pltdict['FTE-DIMEHYD13'],pltdict['FTE-DIMEONUC13']+pltdict['FTE-DIMENUC27'],
             pltdict['ELCTECOA'],pltdict['ELCTEOIL'], pltdict['ELCTELNG']+pltdict['FTE-DIMECC17'],
              pltdict['SOFC30'], #pltdict['FTE-DIMECCSG22'],
             pltdict['ELCTEGEO'],
           pltdict['ELCTEPVS']+pltdict['FTE-DIMEPVN17'],pltdict['FTE-DIMESOL17'],
               pltdict['ELCTEWND']+pltdict['FTE-DIMEWON17'],#pltdict['FTE-DIMEWOFX17'],#pltdict['FTE-DIMEWOFL19'],
              #pltdict['FTE-ELCSTG17'],
             labels=ordlegend, colors=colours)
ax1.tick_params(axis="x", labelsize=10)
ax1.tick_params(axis="y", labelsize=12)
ax1.get_yaxis().set_major_formatter(matplotlib.ticker.FuncFormatter(lambda x, p: format(int(x), ',')))

ax1.set_xlim((2013, 2100))
ax1.legend(loc='upper center', bbox_to_anchor=(0.5, -0.05), shadow=True, ncol=legendCols)
#ax1.set_xlabel('Years',fontsize='18')
ax1.set_ylabel('Electricity supplied (GWh)',fontsize='16')
ax1.set_xticks(ticks=xlocs)

fname='co2_emi_sources.csv'
colfilter = ['Process','Period','PV']

cdf = pd.read_csv(fname, delimiter=';', usecols = colfilter)
cdf = cdf[colfilter]
cdf=cdf.groupby(['Process', 'Period']).sum() #sum values for same period, process
cdf=cdf.reset_index()
cdf.set_index('Period', inplace=True)

procs2 = cdf.Process.unique()
pltdict2={}

for prc in procs2:
    c=cdf.loc[cdf['Process'] == prc].reindex(newyrs,fill_value=0)
    pltdict2[prc]=c['PV']
#print(procs2)

fname='net_cap.csv'
colfilter = ['Process','Period','PV']

capdf = pd.read_csv(fname, delimiter=';', usecols = colfilter)
capdf = capdf[colfilter]
capdf=capdf.groupby(['Process', 'Period']).sum() #sum values for same period, process
capdf=capdf.reset_index()
capdf.set_index('Period', inplace=True)

procs3 = capdf.Process.unique()
pltdict4={}

for prc in procs3:
    c=capdf.loc[capdf['Process'] == prc].reindex(newyrs,fill_value=0)
    pltdict4[prc]=c['PV']
#print(procs3)

ordlegend2=['Hydro','Nuclear',
           'Coal','Oil', 'LNG',
           'PEM Electrolyzer','SOFC',
           'Geothermal','Solar','Emerging Solar',
           'Onshore Wind'
          ]

colours2 = ['aqua','orangered',
           'black', 'saddlebrown', 'slategray',
           'indigo','salmon',
           'green','yellow','maroon',
           'blue'
          ]

pltyrs2=range(2013,2101)
l2 = ax2.stackplot(pltyrs2, pltdict4['ELCTEHYD'],pltdict4['ELCTENUC']+pltdict4['NEWNUC27'],
             pltdict4['ELCTECOA'],pltdict4['ELCTEOIL'], pltdict4['ELCTELNG']+pltdict4['ELCCC17'],
             pltdict4['ECPEM22'],pltdict4['SOFC30'],
           pltdict4['ELCTEGEO'],
           pltdict4['ELCTEPVS']+pltdict4['ELCPVN17'],pltdict4['EMSOL17'],
               pltdict4['ELCTEWND']+pltdict4['ELCWON17'],
             labels=ordlegend2, colors=colours2)
ax2.set_xlim((2013, 2100))
ax2.legend(loc='upper center', bbox_to_anchor=(0.5, -0.05), shadow=True, ncol=legendCols-1)
ax2.set_ylabel('Nameplate Capacity (GW)',fontsize='16')
ax2.set_xticks(ticks=xlocs)

ordlegend3=['Hydro','Nuclear',
           'Coal','Oil', 'LNG',
           'PEM Electrolyzer','SOFC',
           'Geothermal','Solar','Emerging Solar',
           'Onshore Wind'
          ]
colours3 = ['aqua','orangered',
           'black', 'saddlebrown', 'slategray',
           'indigo','salmon',
           'green','yellow','maroon',
           'blue'
          ]
pltyrs3=range(2013,2101)
ax3.stackplot(pltyrs3, pltdict2['ELCTEHYD'],pltdict2['ELCTENUC']+pltdict2['NEWNUC27'],
             pltdict2['ELCTECOA'],pltdict2['ELCTEOIL'], pltdict2['ELCTELNG']+pltdict2['ELCCC17'],
              pltdict2['FTE-EPEMH22'],pltdict2['SOFC30'],
             pltdict2['ELCTEGEO'],
           pltdict2['ELCTEPVS']+pltdict2['ELCPVN17'],pltdict2['EMSOL17'],
               pltdict2['ELCTEWND']+pltdict2['ELCWON17'],
             labels=ordlegend3, colors=colours3)
ax3.set_xlim((2013, 2100))
ax3.set_ylabel('Carbon Emissions (million t)',fontsize='16')
ax3.legend(loc='upper center', bbox_to_anchor=(0.5, -0.05), shadow=True, ncol=legendCols-1)
ax3.set_xticks(ticks=xlocs)

fig.savefig('newtechs_nuc.png',bbox_inches='tight', dpi=300)_____no_output_____fname='selc_sources.csv'
colfilter = ['Process','Period','PV']

sdf = pd.read_csv(fname, delimiter=';', usecols = colfilter)
sdf = sdf[colfilter]
sdf=sdf.groupby(['Process', 'Period']).sum() #sum values for same period, process
sdf=sdf.reset_index()
sdf.set_index('Period', inplace=True)

procs4 = sdf.Process.unique()
pltdict4={}
stgperc={}
stgsum=0

for prc in procs4:
    s=sdf.loc[sdf['Process'] == prc].reindex(newyrs,fill_value=0)
    pltdict4[prc]=s['PV']
    stgsum+=(pltdict4[prc].sum())

print('Total storage electricity:',stgsum)
for prc in procs4:
    stgperc[prc]=(pltdict4[prc].sum())/stgsum
    print('Process:',prc)
    print('Percentage:',stgperc[prc]*100)Total storage electricity: 29733489.646427125
Process: FTE-SIMECC17
Percentage: 1.4608879034788544
Process: FTE-SIMEHYD17
Percentage: 2.880227434577915
Process: FTE-SIMENUC27
Percentage: 13.367098190041018
Process: FTE-SIMEONUC17
Percentage: 0.6438748770647388
Process: FTE-SIMEPVN17
Percentage: 26.17929358409376
Process: FTE-SIMESOL17
Percentage: 35.754970684981465
Process: FTE-SIMEWON17
Percentage: 19.713647325762246
fname='h2_sources.csv'
colfilter = ['Process','Period','PV']

hdf = pd.read_csv(fname, delimiter=';', usecols = colfilter)
hdf = hdf[colfilter]
hdf=hdf.groupby(['Process', 'Period']).sum() #sum values for same period, process
hdf=hdf.reset_index()
hdf.set_index('Period', inplace=True)

procs5 = hdf.Process.unique()
pltdict5={}
h2perc={}
h2sum=0

for prc in procs5:
    h=hdf.loc[hdf['Process'] == prc].reindex(newyrs,fill_value=0)
    pltdict5[prc]=h['PV']
    h2sum+=(pltdict5[prc].sum())

print('Total H2:',h2sum)
for prc in procs5:
    h2perc[prc]=(pltdict5[prc].sum())/h2sum
    print('Process:',prc)
    print('Percentage:',h2perc[prc]*100)
_____no_output_____'''
ordlegend4=['Hydro','Nuclear',
            'LNG',
           'Solar','Emerging Solar',
           'Onshore Wind'
          ]
colours4 = ['aqua','orangered',
           'slategray',
           'yellow','maroon',
           'blue'
          ]
pltyrs4=range(2013,2101)
ax4.stackplot(pltyrs4, pltdict4['FTE-SIMEHYD17'],pltdict4['FTE-SIMEONUC17']+pltdict4['FTE-SIMENUC27'],
             pltdict4['FTE-SIMECC17'],
           pltdict4['FTE-SIMEPVN17'],pltdict4['FTE-SIMESOL17'],
               pltdict4['FTE-SIMEWON17'],
             labels=ordlegend4, colors=colours4)
ax4.set_xlim((2013, 2100))
ax4.set_ylabel('Electricity diverted to storage (GWh)',fontsize='18')
ax4.legend(loc='upper center', bbox_to_anchor=(0.5, -0.05), shadow=True, ncol=6)
ax4.set_xticks(ticks=xlocs)
ax4.tick_params(axis="x", labelsize=10)
ax4.tick_params(axis="y", labelsize=12)
ax4.get_yaxis().set_major_formatter(matplotlib.ticker.FuncFormatter(lambda x, p: format(int(x), ',')))
ordlegend5=['PEM Electrolyzer']
colours5 = ['indigo']
pltyrs5=range(2013,2101)
ax5.stackplot(pltyrs5,
             pltdict5['FTE-EPEMH22'],
                       labels=ordlegend5, colors=colours5)
ax5.set_xlim((2013, 2100))
ax5.set_ylabel('Hydrogen Sources (GWh)',fontsize='18')
ax5.legend(loc='upper center', bbox_to_anchor=(0.5, -0.05), shadow=True, ncol=1)
ax5.set_xticks(ticks=xlocs)
ax5.tick_params(axis="x", labelsize=10)
ax5.tick_params(axis="y", labelsize=12)
ax5.get_yaxis().set_major_formatter(matplotlib.ticker.FuncFormatter(lambda x, p: format(int(x), ',')))
'''Total H2: 24264064.48228824
Process: FTE-EPEMH22
Percentage: 100.0

</code>


# Notebook from chekirou/projet3i025_3
Path: geneticalgorithm_maxone.ipynb

# Algorithme génétique pour le problème du max-one

*nicolas.bredeche(at)upmc.fr (autre contributeur: Arthur Pajot, Paul Ecoffet)*

**UE 3i015**: *Introduction à l'Intelligence Artificielle et la Recherche Opérationelle*, L3, Sorbonne Univ.

Mise à jour:
* 2017-03-20 - première version
* 2018-03-29 - ajout d'une représentation graphique, découpage en cellule

_____no_output_____

#!/usr/bin/env python

%matplotlib inline
%load_ext Cython

import numpy as np
import matplotlib as mpl
import matplotlib.pyplot as pl

import operator
import copy
import datetime

from random import random, randint, sample

verbose = False
The Cython extension is already loaded. To reload it, use:
  %reload_ext Cython
# Graph-related functions

def plot(x,y,xMin,xMax,yMin,yMax):
    colors = "red" #np.random.rand(len(x))
    area = np.pi * (5 * np.ones(1))**2
    pl.xlim(xMin, xMax)
    pl.ylim(yMin, yMax)
    pl.scatter( np.asarray(x), np.asarray(y), s=area, c=colors, alpha=0.5)
    pl.show()

def plotIndividuals(datapop,xMin=0,xMax=1,yMin=0,yMax=1):
    xCoord = []
    yCoord = []

    for e in datapop:
        xCoord.append(e[0])
        yCoord.append(e[1])

    plot(xCoord,yCoord,xMin,xMax,yMin,yMax)_____no_output_____# Evolution-related functions

def evalue(individu):
    fitness = 0
    for e in individu:
        if e == 1:
            fitness = fitness + 1
    return fitness

def mute(individu,pMute):
    nouvelIndividu = []
    for e in individu:
        if random() < pMute:
            nouvelIndividu.append( (e + 1) % 2 )
        else:
            nouvelIndividu.append( e )
    return nouvelIndividu
_____no_output_____#
# MAIN
#

def main():
    # stats

    if verbose == True:
        print "budget évaluation   :", evaluations
        print "nombre de paramètres:", tailleInd

        print ""
        print "taille population: ", taillePop
        print "taille tournoi: ", tailleTournoi
        print "P_mutation:", Pmutation
        print ""

    # initialisation des individus

    population = []

    for i in range(taillePop):
        individu = []
        for j in range(tailleInd):
            individu.append( randint(0,1) )
        population.append([individu,0]) # liste de tuple: N*(genome[],fitness)

    # boucle principale

    it = 0
    stop = False # en cas de succès
    nbGen = 0

    data = []

    while it + taillePop <= evaluations and stop == False:

        if verbose == True:
            print "# Generation", nbGen,":",

        # évalue les individus, et mémorise le meilleur. Selection: (mu=1+4)-ES

        for individu in population:
            individu[1] = evalue(individu[0])

        # affiche la population

        meilleureFitness = 0
        meilleurIndividu = []
        for individu in population:
            #print individu[0],"- fitness: ",individu[1]
            if individu[1] > meilleureFitness:
                meilleureFitness = individu[1]
                meilleurIndividu = individu[:]
        if verbose == True:
            print ''.join(str(i) for i in meilleurIndividu[0]),"- fitness: ",meilleureFitness,"/",tailleInd

        data.append([it+taillePop,meilleureFitness])

        if meilleureFitness == tailleInd:
            if verbose == True:
                print "Succès!"
            stop = True
            break

        # construit une nouvelle population

        nouvellePopulation = []

        for i in range(taillePop):

            # sélectionne K individus (au hasard) pour le tournoi
            tournoi = sample(list(range(taillePop)),tailleTournoi) # renvoi les index de K individus parmis N

            # sélectionne le meilleur parmis les K individus (classe par ordre croissant, renvoi le dernier élément selon le fitness)
            parent = population[ sorted(tournoi, key=lambda x:population[x][1])[-1] ][0]

            # crée un nouvel individu par mutation de l'individu parent, et ajoute à la nouvelle population
            nouvelIndividu = mute(parent,Pmutation)
            nouvellePopulation.append([nouvelIndividu,0])

        # passe a la génération suivante

        population = nouvellePopulation[:]
        it = it + taillePop
        nbGen += 1

    # affichage

    plotIndividuals(data,0,evaluations,0,tailleInd)
    print "Meilleur solution:", meilleurIndividu
_____no_output_____

# Problème du max-one

**Données du problème**
* représentation: une solution candidate définie comme une suite finie de 0 et de 1 (ici: 30 valeurs)
* objectif: la fonction fitness calcule une distance de Hamming entre la solution candidate et la solution optimale (ici: l'objectif est artificiellement fixé comme une suite de 1)
* algorithme: algorithme génétique avec opérateur de mutation, sélection par tournoi
* le *nombre d'évaluation* et la *taille des individus* sont fixés. Ce sont des contraintes du problème.

**Degrés de liberté**
* les valeurs de la *taille de la population*, la *taille du tournoi*, et la *probabilité de mutation par gène* sont à choisir

**Exercice**
* modifiez et observez l'influence des valeurs de *taillePop*, *tailleTournoi* et *Pmutation* sur la vitesse de convergence. Attention, il s'agit d'une algorithme stochastique, donc deux exécutions peuvent mener à des résultats différents._____no_output_____

print "\nProblème MAX-ONE\n"

# contraintes du problème (budget et représentation)

evaluations = 300
tailleInd = 30

# paramètres à régler (modifiable)

taillePop = 10
tailleTournoi = 8
Pmutation = 1.0/tailleInd

main()
Problème MAX-ONE