# Notebook from Ezibenroc/calibration_analysis
Path: dahu/blas/calibration_generation.ipynb

# Dgemm trace of a real and complete HPL run_____no_output_____

import io
import os
import zipfile
import pandas
import yaml
import datetime
import re
from plotnine import *
import plotnine
plotnine.options.figure_size = (12, 8)
import warnings
warnings.simplefilter(action='ignore') # removing annoying warning
import json

def read_csv(archive_name, csv_name, columns=None, filter_func=lambda x: x, usecols=None):
    archive = zipfile.ZipFile(archive_name)
    df = pandas.read_csv(io.BytesIO(filter_func(archive.read(csv_name))), names=columns)
    df.columns = df.columns.str.strip()
    df['jobid'] = int(get_yaml(archive_name, 'info.yaml')['jobid'])
    return df

def get_yaml(archive_name, yaml_name):
    archive = zipfile.ZipFile(archive_name)
    return yaml.load(io.BytesIO(archive.read(yaml_name)))

def get_json(archive_name, json_name):
    archive = zipfile.ZipFile(archive_name)
    return json.load(io.BytesIO(archive.read(json_name)))_____no_output_____def __compute_rank_mapping(archive_name, exp_id):
    '''
    Compute the mapping rank-core, based on mpirun output (thanks to the option --report-bindings).
    '''
    history = get_json(archive_name, 'history.json')
    mpirun = [cmd for cmd in history if cmd['command'].startswith('mpirun') and len(cmd['hostnames']) == 1]
    mpirun = mpirun[exp_id]['stderr'].split('\n')
    regex = re.compile('\[(?P<hostname>[a-z]+-\d+\.[a-z]+\.grid5000\.fr):\d+\] MCW rank (?P<rank>\d+) bound to socket (?P<socket>\d+)\[core (?P<core>\d+)')
    mapping = []
    for line in mpirun:
        match = regex.search(line)
        if match:
            mapping.append({'rank': int(match.group('rank')),
                            'hostname': match.group('hostname'),
                            'socket': int(match.group('socket')),
                            'core': int(match.group('core'))
                           })
    df = pandas.DataFrame(mapping)[['rank', 'hostname', 'socket', 'core']]
    df['node'] = df['hostname'].str[len('dahu-'):-len('.grenoble.grid5000.fr')].astype(int)
    df['cpu'] = df['node']*2 + df['socket'] - 2
    return df.sort_values(by='rank').reset_index(drop=True)

def compute_rank_mapping(archive_name):
    nb_exp = len(read_csv(archive_name, get_yaml(archive_name, 'info.yaml')['expfile'][0]))
    mappings = [__compute_rank_mapping(archive_name, i) for i in range(nb_exp)]
    ref = mappings[0]
    for df in mappings[1:]:
        assert ref.equals(df)
    return ref_____no_output_____hpl_archive = 'calibration_generation/2/grenoble_2020-02-13_1916176.zip'
calibration_archives = {
    'calibration_generation/2/grenoble_2020-02-13_1916194.zip': 'hpl trace',
    'calibration_generation/1/grenoble_2020-02-12_1916052.zip': 'uniform product',
    'calibration_generation/1/grenoble_2020-02-12_1916072.zip': 'uniform product, fixed K',
    'calibration_generation/1/grenoble_2020-02-12_1916054.zip': 'uniform sizes',
}

def __read_hpl_archive(archive_name, exp_id):
    blas_col = ['file', 'line', 'function', 'rank', 'timestamp', 'duration', 'm', 'n', 'k']
    df = read_csv(archive_name, 'trace_blas_%d.csv' % exp_id, columns=blas_col)
    print('Read trace n°%d from %s: %6d lines' % (exp_id, archive_name, len(df)))
    df['start'] = df.timestamp
    df['end'] = df.timestamp + df.duration
    df = df[['jobid', 'function', 'start', 'end', 'rank', 'm', 'n', 'k', 'duration']]
    df['rank+1'] = df['rank'] + 1
    df['function'] = df['function'].str.strip()
    df['exp_id'] = exp_id
    df['block_size'] = read_csv(archive_name, get_yaml(archive_name, 'info.yaml')['expfile'][0]).iloc[exp_id].block_size
    mapping = compute_rank_mapping(archive_name)
    df = df.set_index('rank').join(mapping.set_index('rank')).reset_index()
    return df

def read_hpl_archive(archive_name):
    nb_exp = len(read_csv(archive_name, get_yaml(archive_name, 'info.yaml')['expfile'][0]))
    return pandas.concat([__read_hpl_archive(archive_name, i) for i in range(nb_exp)])

hpl_trace = read_hpl_archive(hpl_archive)

hpl_trace = hpl_trace[hpl_trace['function'] == 'dgemm']
hpl_trace.head()Read trace n°0 from calibration_generation/2/grenoble_2020-02-13_1916176.zip: 248679 lines
hpl_trace[['jobid', 'hostname', 'node']].drop_duplicates().sort_values(by=['jobid', 'node']).reset_index(drop=True)_____no_output_____

## HPL result_____no_output_____

res = read_csv(hpl_archive, 'results.csv')
res[['jobid', 'matrix_size', 'block_size', 'time', 'gflops']].sort_values(by='matrix_size')_____no_output_____

## Checking the parameters_____no_output_____

read_csv(hpl_archive, 'exp_hpl.csv')_____no_output_____

## Prediction error on the duration_____no_output_____

import cashew
print(cashew.__git_version__)
import cashew.archive_extraction as ae
import cashew.linear_regression as lrca8a64e6735b60bc60c49342320befcdeaf55f90
blas = {}
for archive, mode in calibration_archives.items():
    tmp = ae.read_archive(archive, 'result.csv')
    lr.compute_variable_products(tmp, 'mnk')
    blas[mode] = tmp
blas.keys()_____no_output_____models = {
    mode: lr.regression(df, 'duration', ['mnk', 'mn', 'mk', 'nk'])
    for mode, df in blas.items()
}
for mode, reg in models.items():
    perf = [proc['avg_gflops'] for proc in reg]
    avg = 1/(sum([1/p for p in perf])/len(perf))  # harmonic mean for gflops
    print('%.2f Gflops | %s' % (avg, mode))

models = {mode: pandas.DataFrame(reg) for mode, reg in models.items()}26.13 Gflops | hpl trace
26.10 Gflops | uniform product
26.16 Gflops | uniform product, fixed K
26.37 Gflops | uniform sizes
hpl_trace['pred'] = 0
hpl_trace['mnk'] = hpl_trace.m * hpl_trace.n * hpl_trace.k
hpl_trace['mn'] = hpl_trace.m * hpl_trace.n
hpl_trace['mk'] = hpl_trace.m * hpl_trace.k
hpl_trace['nk'] = hpl_trace.n * hpl_trace.k

traces = []
for mode, all_regs in models.items():
    new_trace = hpl_trace.copy()
    for cpu in new_trace['cpu'].unique():
        all_regs['cpu_id'] = 2*all_regs['node'] + all_regs['cpu'] - 2
        reg = all_regs[all_regs['cpu_id'] == cpu]
        assert len(reg) == 1
        reg = list(reg.iterrows())[0][1]
        mask = new_trace['cpu'] == cpu
        new_trace.loc[mask, 'pred'] += reg['intercept']
        for prod in ['mnk', 'mn', 'mk', 'nk']:
            new_trace.loc[mask, 'pred'] += reg[prod] * new_trace[mask][prod]
    new_trace['mode'] = mode
    traces.append(new_trace)
traces = pandas.concat(traces)_____no_output_____duration_thresh = 0.1_____no_output_____traces['error'] = (traces['pred'] - traces['duration']) / traces['duration']
traces['abs_error'] = traces['error'].abs()_____no_output_____traces[traces.duration > duration_thresh]['error'].describe()_____no_output_____traces['error_perc'] = traces['error']*100
plot = ggplot(traces[traces.duration > duration_thresh], aes(x='error_perc')) + geom_histogram() + theme_bw()
plot = plot + facet_wrap('mode', nrow=4) + xlab('Prediction error (%)')
plot = plot + ggtitle('Distribution of the prediction error in the dgemm calls longer than %.2f seconds' % duration_thresh)
plot = plot + geom_vline(xintercept=0)
plot_____no_output_____traces['cpu_id'] = traces['cpu']
for node in sorted(traces['node'].unique()):
    tmp = traces[(traces.duration > duration_thresh) & (traces['node'] == node)]
    hostname = tmp.iloc[0]['hostname']
    plot = ggplot(tmp) + aes(x='error_perc') + geom_histogram() + theme_bw()
    plot = plot + facet_wrap(['mode', 'cpu'], ncol=2, labeller='label_both') + xlab('Prediction error (%)')
    plot = plot + ggtitle('Distribution of the prediction error in the dgemm calls longer than %.2f seconds (%s)' % (duration_thresh, hostname))
    plot = plot + geom_vline(xintercept=0)
    plot = plot + expand_limits(x=(-20, 20))
    print(plot)_____no_output_____

## Monitoring_____no_output_____

def get_monitoring(archive):
    info = ae.read_yaml(archive, 'info.yaml')
    start_exp = pandas.to_datetime(info['timestamp']['run_exp']['start'])
    monitoring = ae.read_archive_csv(archive, 'monitoring.csv')
    cluster = info['cluster']
    site = info['site']
    monitoring['node'] = monitoring['hostname'].str[len(cluster)+1:-(len(site)+len('..grid5000.fr'))].astype(int)
    monitoring['timestamp'] = pandas.to_datetime(monitoring['timestamp'])
    monitoring = monitoring[monitoring['timestamp'] >= start_exp]
    monitoring['start'] = monitoring['timestamp'] - start_exp
    return monitoring

def my_melt(df, prefix, idcol):
    result = []
    columns = [col for col in df.columns if col.startswith(prefix)]
    for col in columns:
        tmp = df[idcol].copy()
        tmp[prefix] = df[col]
        tmp['group'] = int(col[len(prefix):])
        result.append(tmp)
    return pandas.concat(result)

def get_temp_freq(archive, mode):
    monitoring = get_monitoring(archive)
    platform = ae.get_platform(archive)
    mapping = ae.platform_to_cpu_mapping(platform)
    temperature = my_melt(monitoring, 'temperature_core_', ['hostname', 'start', 'node'])
    temperature['cpu'] = temperature.apply(lambda row: mapping[row.group], axis=1)
    frequency = my_melt(monitoring, 'frequency_core_', ['hostname', 'start', 'node'])
    frequency = frequency[frequency['group'] <= max(mapping.keys())]  # removing the cores with largest IDs (they are not real cores, just hyperthreads)
    frequency['cpu'] = frequency.apply(lambda row: mapping[row.group], axis=1)
    frequency['frequency_core_'] *= 1e-9
    temperature['mode'] = mode
    frequency['mode'] = mode
    return temperature, frequency_____no_output_____monitorings = [get_temp_freq(hpl_archive, 'HPL run')]
for archive, mode in calibration_archives.items():
    monitorings.append(get_temp_freq(archive, mode))

temperature = pandas.concat(t[0] for t in monitorings)
frequency   = pandas.concat(t[1] for t in monitorings)_____no_output_____max_time = 120
ggplot(temperature[temperature['start'] <= pandas.to_timedelta(max_time, unit='s')]) +\
    aes(x='start', y='temperature_core_', color='factor(cpu)') +\
    geom_point(shape=1) +\
    facet_wrap('mode') +\
    theme_bw() +\
    xlab('Timestamp') +\
    ylab('Temperature (°C)') +\
    labs(color='CPU') +\
    ggtitle('Temperature evolution (first %d seconds)' % max_time)_____no_output_____ggplot(frequency[frequency['start'] <= pandas.to_timedelta(max_time, unit='s')]) +\
    aes(x='start', y='frequency_core_', color='factor(cpu)') +\
    geom_point(shape=1) +\
    facet_wrap('mode') +\
    theme_bw() +\
    xlab('Timestamp') +\
    ylab('Frequency (GHz)') +\
    labs(color='CPU') +\
    ggtitle('Temperature evolution (first %d seconds)' % max_time) +\
    geom_hline(yintercept=2.4, linetype='dotted')_____no_output_____


# Notebook from alexaldr/curso_de_python
Path: notebooks/curso_de_python_alunos_parte_02.ipynb

# Bem vindos à segunda parte do curso intensivo de

![Python logo](https://www.python.org/static/community_logos/python-logo-generic.svg)_____no_output_____##**PANDAS** é uma biblioteca Python que fornece ferramentas de análise de dados e estruturas de dados de alta performance e *fáceis de usar*. Por ser a principal e mais completa biblioteca para estes objetivos, **pandas** é fundamental para análise de dados com Python.

---_____no_output_____## As estruturas de dados primárias nos pandas são implementadas como duas classes:

* **Série**: é uma única coluna. Um DataFrame contém uma ou mais séries e um nome para cada série.
* **DataFrame**: você pode imaginar como uma tabela de dados relacionais, com linhas e colunas nomeadas.

_____no_output_____

# Importações necessárias
import pandas as pd

# Pandas já trás várias outras bibliotecas embutidas dentro dele, como MatplotLib e NumPy_____no_output_____

# Series
Uma Series é como um array unidimensional, uma lista de valores. Toda Series possui um índice, o _index_, que dá rótulos a cada elemento da lista. Abaixo criamos uma Series _notas_, o _index_ desta Series é a coluna à esquerda, que vai de 0 a 4 neste caso, que o pandas criou automaticamente, já que não especificamos uma lista de rótulos._____no_output_____

notas = pd.Series([2,7,5,10,6])
notas
# serie1 = pd.Series(["San Francisco", "San Jose", "Sacrament"])
# serie2 = pd.Series([852469, 1015785, 485199])_____no_output_____notas.values_____no_output_____notas.index_____no_output_____notas[2]_____no_output_____# Definimos então um index
notas = pd.Series([7, 7, 5, 10, 5, 5], index=["Wilton", "Amanda", "Hamilton", "Júlia", "Carla", "Samuel"])
notas_____no_output_____notas["Amanda"]_____no_output_____

### Pandas fornece métodos que fornecem informações estatísticas sobre os valores:_____no_output_____

print("Média:", notas.mean())
print("Desvio padrão:", notas.std())_____no_output_____notas.describe()_____no_output_____notas_histograma = notas.hist()_____no_output_____

### Flexibilidade para aplicar expressões e funções matemáticas do numpy diretamente sobre a série:_____no_output_____

# sobre todos os campos numéricos
notas**2_____no_output_____import numpy as np

np.log(notas) # logaritmo natural_____no_output_____np.max(notas)
# notas.max()_____no_output_____

# DataFrames
É uma estrutura bidimensional de dados, como uma planilha._____no_output_____

# Usando um dicionário como entrada dos dados
df = pd.DataFrame({"Aluno" :    ["Wilton", "Amanda", "Hamilton", "Júlia", "Carla", "Samuel"],
                   "Faltas" :   [3, 1, 2, 1, 2, 2],
                   "Prova" :    [2, 7, 5, 10, 6, 7],
                   "Seminário": [8.5, 7.5, 9.0, 7.5, 8.0, 7.5],
                   "Sexo":      ["H", "M", "H", "M", "M", "H"]
                   })
df_____no_output_____# Qual o tipo de cada coluna?
df.dtypes_____no_output_____df.columns_____no_output_____df["Aluno"]_____no_output_____df.sort_values(by="Seminário", ascending=False)_____no_output_____df.describe()_____no_output_____# Selecionar pelo index
df.loc[2]  # Linha_____no_output_____df.loc[2, "Aluno"]  # Célula_____no_output_____df.loc[df.index, ["Aluno", "Seminário"]]_____no_output_____df.loc[df.index, ["Aluno", "Seminário"]].sort_values(by="Seminário", ascending=False)_____no_output_____# Critério de seleção
df[df["Seminário"] > 8.0]
# df["Seminário"] > 8.0_____no_output_____df[(df["Seminário"] > 7.0) & (df["Prova"] >= 7)] # & é um operador bitwise sobre as condições True e False (1 e 0)_____no_output_____df.hist(column="Faltas")_____no_output_____import matplotlib as plt

plt.pyplot.hist(df["Faltas"])_____no_output_____df.plot.bar()_____no_output_____df.plot.line(x="Aluno")_____no_output_____df.plot.area(x="Aluno", y=["Prova", "Seminário"])_____no_output_____

# Pandas
### Vimos como manipular dados que foram criados em tempo de execução.
Acontece que, na maioria das vezes, queremos analisar dados que já estão prontos e disponíveis._____no_output_____## Sites que fornecem datasets para download:
* http://www.dados.df.gov.br/
* https://brasil.io/datasets
* http://dados.gov.br/dataset
* https://dadosabertos.camara.leg.br/
* http://www.transparencia.mg.gov.br/dados-abertos_____no_output_____

import pandas as pd_____no_output_____

# Importação de dataframes_____no_output_____

# A exclamação antes executa como comando do console
!pip3 install wget_____no_output_____# Baixar o arquivo primeiro
!pip3 install wget

import wget

url = "https://github.com/alexaldr/curso_de_python/blob/master/dataframes/cursos-prouni.csv?raw=true"

file_name = wget.download(url, "dataframe_prouni.csv")

# ler o arquivo para um dataframe
df2 = pd.read_csv("dataframe_prouni.csv")
df2.sample(3)_____no_output_____# dataframe que vamos trabalhar
df = pd.read_csv("https://github.com/alexaldr/curso_de_python/blob/master/dataframes/capes-bolsas.zip?raw=true",
                 sep=",",
                 compression="zip"
                 )
# pd.read_csv
# pd.read_excel
# pd.read_html
# pd.read_json
# pd.read_sql
df.sample(3)_____no_output_____# algumas vezes pode ser necessário incluir outros parâmetros
df_prouni = pd.read_csv("https://github.com/alexaldr/curso_de_python/blob/master/dataframes/pda-bolsistas-prouni-2018.CSV?raw=true",
                        # error_bad_lines=False,
                        encoding="cp1252",
                        sep=";"
                        )
df_prouni.sample(3)_____no_output_____df = pd.read_csv("https://github.com/alexaldr/curso_de_python/blob/master/dataframes/forest-fires-in-brazil.zip?raw=true",
                #  error_bad_lines=False,
                 encoding="cp1252",
                 sep=",",
                 compression="zip"
                 )
df.head(3)_____no_output_____# salvar arquivos grandes em disco primeiro
import requests

url = "https://images.pexels.com/photos/1181671/pexels-photo-1181671.jpeg?crop=entropy&cs=srgb&dl=adult-coder-developer-1181671.jpg&fit=crop&fm=jpg&h=1281&w=1920"
response = requests.get(url, stream=True)

# verifica se houve resposta 200 e salva o arquivo em chuncks (pedaços)
if response.status_code == 200:
    with open('wallpaper.jpg', 'wb') as handle:
        for block in response.iter_content(1024):
            handle.write(block)
        print("Arquivo salvo com sucesso!")
        print("Photo by Christina Morillo (from Pexels)\n"+
              "https://www.pexels.com/photo/python-book-1181671/")
else:
    print("Impossível salvar o arquivo!\n"+
          f"Erro {response.status_code}")

# A título de curiosidade
# Exibir imagens no Jupyter Notebook
from IPython.display import Image, display
display(Image(filename="wallpaper.jpg", width=700))_____no_output_____

# Manipulação de dataframes_____no_output_____

# número de linhas e colunas
df.shape_____no_output_____# nomes das colunas
df.columns_____no_output_____# tipos das colunas
df.dtypes_____no_output_____# mudar o tipo de uma coluna
df["Codigo IES"] = df["Codigo IES"].astype("object")_____no_output_____df.head()_____no_output_____df.tail()_____no_output_____df.sample(5)_____no_output_____# axis=0:   linhas
# axis=0:   colunas
# inplace:  modifica o próprio dataframe
df.drop("Unnamed: 0", axis=1, inplace=True)
# del df["Unnamed: 0"]
df.head()_____no_output_____# se estiver desordenado
df.sort_values("Ano", inplace=True)
df.head()_____no_output_____df.describe()_____no_output_____df["MESTRADO"].describe()_____no_output_____df["UF"].value_counts()_____no_output_____# filtros
df[df["Ano"]>=2018].sample(5)_____no_output_____# primeiras cinco linhas + colunas 1 à 6
df.iloc[0:5, 1:7]

# df.iloc[136100:-5, 8:4:-1]_____no_output_____# primeiras linhas + colunas por label
df.loc[1:5, ["Ano", "IES", "UF"]]
# Ordem?_____no_output_____df.sort_index(inplace=True)
df.head()_____no_output_____# soma dos campos com valores nulos (True=1)
df.isnull().sum()_____no_output_____# substitui nulos (NaN) pelo valor passado
df.fillna(0)_____no_output_____# exlui linhas ou colunas que possuem algum valor nulo
df.dropna(inplace=True)_____no_output_____df.isnull().sum()
# df.shape_____no_output_____# df.loc[df["IES"]=="UNIMONTES"].loc[df["Área Conhecimento"]=="SAÚDE E BIOLÓGICAS"].sort_values("Ano").sample(5)
unimontes = df.loc[df["IES"]=="UNIMONTES"]
unimontes.sample(5)_____no_output_____# Podemos excluir colunas que não fazem mais sentido para o novo dataframe
unimontes.drop("Região", axis=1, inplace=True)
unimontes.head(3)_____no_output_____# Change the index to be based on the 'id' column

unimontes["index_1"] = list(i for i in range(0, unimontes.shape[0]))
unimontes.head(3)_____no_output_____# insert at index
unimontes.insert(2, "index_2", list(i for i in range(0, unimontes.shape[0])) )
unimontes.head(3)_____no_output_____unimontes.set_index("ind", inplace=True)
unimontes.head(3)_____no_output_____unimontes.to_excel("bolsas_capes_unimontes.xls", encoding="cp1252")_____no_output_____unimontes.rename(str.upper, axis="columns", inplace=True)
# unimontes = unimontes.rename(columns=(lambda x: x.replace(" ", "_"))).copy()
# unimontes = unimontes.rename(columns={"UF":"uf"}).copy()

unimontes.columns_____no_output_____# recriar o dataframe unimontes
unimontes = df.loc[df["IES"]=="UNIMONTES"]
unimontes.sample(3)_____no_output_____# listagem dos valores únicos da coluna
unimontes["Área Conhecimento"].unique()_____no_output_____# contagem dos valores únicos das colunas
unimontes.nunique()_____no_output_____# IES?
unimontes.groupby("Área Conhecimento")["IES"].count()

# for i in (unimontes.groupby("Área Conhecimento")["IES"]):
#     print(*i)_____no_output_____# IES, UF, Ano
unimontes.groupby("Área Conhecimento")["IES"].count() == unimontes.groupby("Área Conhecimento")["Iniciação Científica"].count()_____no_output_____list(unimontes["Área Conhecimento"].value_counts()) == list(unimontes.groupby("Área Conhecimento")["IES"].count().sort_values(ascending=False))_____no_output_____

# Visualização de dataframes_____no_output_____

# unimontes.groupby("Área Conhecimento")["IES"].count().plot.bar()

unimontes_areas = unimontes.groupby("Área Conhecimento")["IES"].count()
unimontes_areas.plot.pie()_____no_output_____unimontes_areas.plot.line()_____no_output_____unimontes_areas.plot.area()_____no_output_____unimontes_areas.plot.bar()_____no_output_____unimontes_areas.plot.barh()_____no_output_____unimontes_areas.sort_values().plot.barh()_____no_output_____unimontes_areas.sort_values().plot.barh(figsize=(8, 8))_____no_output_____unimontes_areas.sort_values().plot.barh(figsize=(8, 8),
                                        title="Áreas de Pesquisa na Unimontes",
                                        # grid=True,
                                        legend="Gráfico de barras horizontal",
                                        style=[":", "--"],
                                        fontsize=12,
                                        colors = ["#89ABCD", "#789ABC", "#6789AB", "#56789A",
                                                  "#456789", "#345678", "#234567", "#123456"],
                                        )_____no_output_____

# Matplotlib_____no_output_____###Se uma imagem vale mais que mil palavras, então diga isso com uma imagem.
_(Millôr Fernandes)______no_output_____

# Curiosidade
from PIL import Image, ImageDraw, ImageFont

img = Image.new("RGB", (800, 100), color = (0, 0, 0))

fnt = ImageFont.truetype(
    "/usr/local/lib/python3.6/dist-packages/matplotlib/mpl-data/fonts/ttf/STIXGeneralBol.ttf", 40)
d = ImageDraw.Draw(img)
d.text((40,25), "Uma imagem vale mais que mil palavras!", font=fnt, fill=(255, 255, 0))
d.rectangle([(5, 5), (795, 95)], fill=None, outline="blue", )

img.save("text_to_image.png")
img_____no_output_____# importação do módulo
import matplotlib.pyplot as plt
# from matplotlib import pyplot as plt_____no_output_____# suficiente: uma série
y = [1, 1, 2, 3, 5, 8, 13, 21, 34, 55]

plt.plot(y)
plt.show()_____no_output_____x = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
y = [1, 1, 2, 3, 5, 8, 13, 21, 34, 55]

plt.plot(x, y)
plt.show()_____no_output_____x = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
y = [1, 1, 2, 3, 5, 8, 13, 21, 34, 55]

# rótulos eixo x e y
plt.xlabel('Número de ordem')
plt.ylabel('Série de Fibonacci')

plt.plot(x, y)
plt.show()_____no_output_____

#Format Strings
>    fmt = "[color][marker][line]"

Serve para formatar a cor, estilo dos pontos e estilo da linha.

**Colors**

* "b": blue
* "g": green
* "r": red
* "c": cyan
* "m": magenta
* "y": yellow
* "k": black
* "w": white

**Markers**
* ".": point marker
* ",": pixel marker
* "o": circle marker
* "v": triangle_down marker
* "^": triangle_up marker
* "<": triangle_left marker
* ">": triangle_right marker
* "1": tri_down marker
* "2": tri_up marker
* "3": tri_left marker
* "4": tri_right marker
* "s": square marker
* "p": pentagon marker
* "\*": star marker
* "h": hexagon1 marker
* "H": hexagon2 marker
* "+": plus marker
* "X": x marker
* "x": thin_x marker
* "D": diamond marker
* "d": thin_diamond marker
* "|": vline marker
* "_": hline marker

**Line Styles**
* "-": solid line style
* "--": dashed line style
* "-.": dash-dot line style
* ":": dotted line style_____no_output_____

x = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
y = [1, 1, 2, 3, 5, 8, 13, 21, 34, 55]

plt.xlabel('Número de ordem')
plt.ylabel('Série de Fibonacci')

plt.plot(x, y, "rs:")
plt.show()_____no_output_____x = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
y = [1, 1, 2, 3, 5, 8, 13, 21, 34, 55]

plt.xlabel('Número de ordem')
plt.ylabel('Série de Fibonacci')

# valores do eixo x e y
plt.xticks(x)
plt.yticks(y)

plt.plot(x, y, "rs:")
plt.show()_____no_output_____x = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
y = [1, 1, 2, 3, 5, 8, 13, 21, 34, 55]

plt.xlabel('Número de ordem')
plt.ylabel('Série de Fibonacci')

plt.xticks(x)

# limitar eixo y
plt.yticks(list(i for i in y if i >=3 or i==1))

plt.plot(x, y, "rs:")
plt.show()_____no_output_____x = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
y = [1, 1, 2, 3, 5, 8, 13, 21, 34, 55]

plt.xlabel('Número de ordem')
plt.ylabel('Série de Fibonacci')

plt.xticks(x)
plt.yticks(list(i for i in y if i >=3 or i==1))

# grid
plt.grid(True, which="both", axis="both")

plt.plot(x, y, "rs:")
plt.show()_____no_output_____x = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
y = [1, 1, 2, 3, 5, 8, 13, 21, 34, 55]

plt.xlabel('Número de ordem')
plt.ylabel('Série de Fibonacci')

plt.xticks(x)
plt.yticks(list(i for i in y if i >=3 or i==1))

plt.grid(True, which="both", axis="both")

# escala igualitária x vs y
plt.xlim(0, 60)
plt.ylim(0, 60)

plt.plot(x, y, "rs:")
plt.show()_____no_output_____x = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
y = [1, 1, 2, 3, 5, 8, 13, 21, 34, 55]

plt.xlabel('Número de ordem')
plt.ylabel('Série de Fibonacci')

plt.xticks(x)
plt.yticks(list(i for i in y if i >=3 or i==1))

plt.grid(True, which="both", axis="both")

plt.xlim(0, 60)
plt.ylim(0, 60)

# Título
plt.title("Primeiros números da série de Fibonacci")

plt.plot(x, y, "rs:")
plt.show()_____no_output_____plt.bar(x, y)
plt.xticks(x)
plt.show()_____no_output_____plt.barh(x, y)
plt.yticks(x)
plt.show()_____no_output_____plt.scatter(x,y)
plt.show()_____no_output_____plt.pie(y)
plt.show()_____no_output_____print(*plt.style.available, sep="\n")_____no_output_____y = [1, 1, 2, 1, 7, 8, 3, 4, 3, 5, 7, 1, 3, 2, 1, 9]
plt.hist(y)
plt.show()_____no_output_____y = [1, 1, 2, 1, 7, 8, 3, 4, 3, 5, 7, 1, 3, 2, 1, 9]
plt.xticks(y)
plt.hist(y, bins=range(0,len(y))) # alinhados a esquerda
plt.show()_____no_output_____import numpy as np

y = [1, 1, 2, 1, 7, 8, 3, 4, 3, 5, 7, 1, 3, 2, 1, 9]
plt.xticks(y)

plt.hist(y, bins=np.arange(len(y))-0.5) # alinhados ao centro
plt.show()_____no_output_____import numpy as np

t = np.arange(0.0, 1.0, 0.01)

plt.plot(t, np.sin(2*np.pi*t))
plt.plot(t, np.cos(2*np.pi*t))

plt.show()_____no_output_____x = [1, 2, 3, 4, 5, 6, 7]
y = [5, 7, 7, 5, 3, 3, 5]

x2 = [1, 2, 3, 4, 5, 6, 7]
y2 = [5, 3, 3, 5, 7, 7, 5]

plt.plot(x, y, label="Linha 01", )
plt.plot(x2, y2, label="Linha 02")

plt.xlabel("Número de ordem")
plt.ylabel("Variável importante")

plt.title("Gráfico com duas séries")
# adicionar legenda
plt.legend()
plt.show()_____no_output_____# infinito ou carinha?
x = [1, 2, 3, 4, 5, 6, 7]
y = [5, 7, 7, 5, 3, 3, 5]

x2 = [1, 2, 3, 4, 5, 6, 7]
y2 = [5, 3, 3, 5, 7, 7, 5]

plt.plot(x, y, label="Linha 01", )
plt.plot(x2, y2, label="Linha 02")

# adicionais
# plt.plot([1.5, 2, 2.5, 3, 3.5],[7, 7.5, 7.5, 7.5, 7], "k-.")
# plt.plot([4.5, 5, 5.5, 6, 6.5],[7, 7.5, 7.5, 7.5, 7], "k-.")
# plt.plot([2.5, 5.5], [5, 5], "go")
# plt.plot(4, 3, "r^")
# plt.plot([2, 3, 4, 5, 6], [2.5, 2, 2, 2, 2.5], "r-")

plt.xlabel("Número de ordem")
plt.ylabel("Variável importante")

plt.title("Gráfico com duas séries")

plt.show()_____no_output_____

### [Clique aqui](https://docs.python.org/3/library/stdtypes.html#printf-style-string-formatting) para ver outras opções de conversão em formatação de strings, conforme exemplo abaixo._____no_output_____

plt.figure(figsize=(10,5))
# mais de uma série em um gráfico
notas_provas = {"Cida": [5, 8, 1], "Amanda": [7, 6, 8], "José": [4, 2, 3],
                "Carlos": [10, 9, 9], "Joyce": [6, 5, 10], "Alexandre": [3, 2, 8]}

# lista com todas as notas separadas por listas
notas = [nota for nota in notas_provas.values()]

# função para gerar aleatoriamente uma HEX COLOR (#FFFFFF)
def random_color():
    import random
    r = lambda: random.randint(0,255) # 255 = FF
    hex_number = f"#{r():02x}{r():02x}{r():02x}"
    return hex_number

i = 0
# lista de notas de cada aluno
for aluno_nota in notas:
    # cada nota de um aluno
    cor = random_color()
    for nota in aluno_nota:
        plt.bar(i, nota, align="center", width=0.8, alpha=0.5, color=cor)
        # parâmetros = (x, y, valor)
        plt.text(i, nota+0.1, nota)
    i += 1

# grid no eixo y
plt.grid(True, axis="y")

# Título
plt.title("Notas da Turma A")

# comprimento e rótulos
plt.xticks(range(len(notas_provas)), list(notas_provas.keys()))
plt.yticks(range(11))

plt.show()_____no_output_____import matplotlib.pyplot as plt
x = [i for i in range(1, 21)]
y = [5, 5, 5, 5, 5, 5, 5, 6, 4, 10, 0, 6, 4, 5, 5, 5, 5, 5, 5, 5]

# para usar um estilo temporariamente
with plt.style.context(["seaborn-dark", "dark_background"]):
    plt.tick_params(axis="both",
                    bottom=False,
                    left=False,
                    labelbottom=False,
                    labelleft=False)
    plt.plot(x,y, "r-")
    # salvar em disco
    plt.savefig("./grafico_batimentos.png")_____no_output_____

## Já vimos a forma básica de criar gráficos, porém o matplotlib nos fornece uma gama muito maior de opções.

## Para isso, precisamos antes entender como funcionam as suas partes._____no_output_____## Partes de um gráfico

!["Esquema das partes de um gráfico Matplotlib"](https://files.realpython.com/media/fig_map.bc8c7cabd823.png) _____no_output_____##Conceitos Gerais

###Uma figura do Matplotlib pode ser categorizada em várias partes:

* **Figure**: é uma figura inteira que pode conter um ou mais de um eixo (plotagens). Você pode pensar em uma figura como uma tela que contém gráficos. É o nível mais alto.

* **Axes**: é o que adicionamos ao _Figure_. Uma figura pode conter muitos _Axes_. Ele contém dois ou três objetos do tipo _Axis_ (no caso de 3D). Cada _Axes_ tem um title, um label x e um label y.

* **Axis**: são a linha numérica como objetos e cuidam da geração dos limites do gráfico.

* **Artist**: Tudo o que se pode ver na figura é um artista como objetos de texto, objetos Line2D, objetos de coleção. A maioria dos artistas está ligada aos machados._____no_output_____![Esquema das partes de um gráfico Matplotlib 2](https://s3.amazonaws.com/assets.datacamp.com/blog_assets/Matplotlib+Tutorial/content_content_unknown.png)_____no_output_____# Vejamos os detalhes do gráfico:
![Anatomia de um gráfico Matplotlib](https://files.realpython.com/media/anatomy.7d033ebbfbc8.png)_____no_output_____

fig = plt.figure(figsize=(10, 10))

# ax1 será meu objeto a ser plotado dentro de fig
ax1 = fig.add_subplot(221) # (nrows, ncols, index)

# setando parâmetros do objeto ax1
ax1.set_xlim([0.5, 4.5])
ax1.set_ylim([-2, 8])

ax1.set_title("Título do \"EIXOS\" 1")

ax1.set_ylabel("Rótulo do eixo Y")
ax1.set_xlabel("Rótulo do eixo X")

# --------------------------------------
# criando outro objeto, mas no index 4 agora
ax4 = fig.add_subplot(224)

ax4.set_xlim([0.5, 4.5])
ax4.set_ylim([-2, 8])

ax4.set_title("Título do \"EIXOS\" 4")

ax4.set_ylabel("Rótulo do Y, eixos 4")
ax4.set_xlabel("Rótulo do X, eixos 4")

# --------------------------------------
plt.show()_____no_output_____import matplotlib.pyplot as plt

# objeto figura
fig = plt.figure()
# eixos
ax = fig.add_subplot(111)

ax.plot([1, 2, 3, 4], [10, 20, 25, 30], color='lightblue', linewidth=3)
ax.bar([1, 2, 3], [i*i for i in range(2, 5)], color='red', width=0.5)
ax.scatter([0.3, 3.8, 1.2, 2.5], [11, 25, 9, 26], color='darkgreen', marker='^')
# intervalo do eixo x
ax.set_xlim(0.5, 4.5)

plt.show()_____no_output_____import matplotlib.pyplot as plt

plt.plot([1, 2, 3, 4], [10, 20, 25, 30], color='lightblue', linewidth=3)
ax.bar([1, 2, 3], [i*i for i in range(2, 5)], color='red', width=0.5)
plt.scatter([0.3, 3.8, 1.2, 2.5], [11, 25, 9, 26], color='darkgreen', marker='^')
# %matplotlib
plt.xlim(0.5, 4.5)
plt.show()_____no_output_____

## [Clique aqui para visualizar a galeria do Matplotlib](https://matplotlib.org/gallery/index.html)

 Obs: Existem também outras bibliotecas para criação de gráficos com Python, como por exemplo a Seaborn e a Plotly.
_____no_output_____# Desafios:

* Gerar_____no_output_____

_____no_output_____

![Python logo](https://www.python.org/static/community_logos/python-logo-generic.svg)
# Onde aprender mais:
* [Documentação do Python 3](https://docs.python.org/3/)
---
* Coursera - [Introdução à Ciência da Computação com Python - parte 01 - USP](https://pt.coursera.org/learn/ciencia-computacao-python-conceitos)
* Coursera - [Introdução à Ciência da Computação com Python - parte 02 - USP](https://pt.coursera.org/learn/ciencia-computacao-python-conceitos-2)
---
* Youtube - [Curso em Vídeo - Python 3 - Mundo 01](https://www.youtube.com/playlist?list=PLHz_AreHm4dlKP6QQCekuIPky1CiwmdI6)
* Youtube - [Curso em Vídeo - Python 3 - Mundo 02](https://www.youtube.com/playlist?list=PLHz_AreHm4dk_nZHmxxf_J0WRAqy5Czye)
* Youtube - [Curso em Vídeo - Python 3 - Mundo 03](https://www.youtube.com/playlist?list=PLHz_AreHm4dksnH2jVTIVNviIMBVYyFnH)
* Youtube - [Python 300](https://www.youtube.com/playlist?list=PL8eBmR3QtPL0j3QLEjQ6rcx8rVB8Ir893)
* Youtube - [Python para Zumbis](https://www.youtube.com/playlist?list=PLUukMN0DTKCtbzhbYe2jdF4cr8MOWClXc)
* Youtube - [Aulas Python - Ignorância Zero](https://www.youtube.com/playlist?list=PLfCKf0-awunOu2WyLe2pSD2fXUo795xRe)
* Youtube - [Análise de Dados em Python - LabHacker](https://www.youtube.com/playlist?list=PLqiFjCF_dtcymXtdjwAP4s7tRoW4CYwnH)
---
* Telegram - [Live de Python](https://t.me/livepython)
---
* [GeeksForGeeks](https://www.geeksforgeeks.org/python-programming-language/)
* [W3Schools](https://www.w3schools.com/python/)
* [LearnPython](https://www.learnpython.org/)
* [DataCamp](https://www.datacamp.com/courses/tech:python)
* [TutorialsPoint](https://www.tutorialspoint.com/python)
* [Apostila Python e Orientação à Objetos - Caelum](https://www.caelum.com.br/download/caelum-python-objetos-py14.pdf)
* [PythonAcademy](https://pythonacademy.com.br)
_____no_output_____


# Notebook from MagriLab/API-ESN
Path: source/.ipynb_checkpoints/ESN_bias_drdt-checkpoint.ipynb

<code>
# In this file the process to compute the reservoir state derivative of an Echo State Netowrk is
# defined as a cell for a tensorflow Recurrent Neural Netowrk

# to pass to py_func
def np_eigenvals(x):
    return np.linalg.eigvals(x).astype('complex128')

# Definition of the EchoState NN as a child of RNNCell of tensorflow
# It generates the states of the reservoir; there is no output matrix
class EchoStateRNNCelldrdt(tf.keras.layers.AbstractRNNCell):

    def __init__(self, num_units, num_inputs=1, decay=0.1, rho=0.6,
                 sparseness=0.0,
                 sigma_in=1.0,
                 b_in=1.0,
                 rng=None,
                 activation=None,
                 reuse = False,
                 win=None,
                 wecho=None):
        """
        Args:
            num_units: int, Number of units in the ESN cell.
            num_inputs: int, The number of inputs to the RNN cell.
            decay: float, Decay/leaking of the ODE of each unit.
            rho: float, Target spectral radius 1.
            sparseness: float [0,1], sparseness of the inner weight matrix.
            rng: np.random.RandomState, random number generator. (to be able to have always the same random matrix...)
            activation: Nonlinearity to use.
            reuse: if reusing existing matrix
            win: input weights matrix
            wecho: echo state matrix
        """

        # Basic RNNCell initialization (see tensorflow documentation)
        super(EchoStateRNNCelldrdt, self).__init__() # use the initializer of RNNCell (i.e. defines a bunch of standard variables)

        # hyperparameters
        self._num_units  = num_units
        self._activation = activation
        self._num_inputs = num_inputs
        self.decay = decay
        self.rho = rho
        self.sparseness = sparseness
        self.sigma_in = sigma_in
        self.b_in  = np.reshape(b_in, [1,1])

        # Random number generator initialization
        self.rng = rng
        if rng is None:
            self.rng = np.random.RandomState()

        # build initializers for tensorflow variables
        if (reuse == False):
            self.win = self.buildInputMatrix()
            self.wecho = self.buildEchoMatrix()
        else: #in case they are passed as an argument
            self.win = win.astype('float64')
            self.wecho = wecho.astype('float64')

        # convert the weight to tf variable
        self.Win   = tf.Variable(self.win, name='Win', trainable=False)
        self.Wecho = tf.Variable(self.wecho, name='Wecho', trainable=False)

        self.setEchoStateProperty()

    @property
    def state_size(self):
        return self._num_units*2

    @property
    def output_size(self):
        return self._num_units*2

    # function that has to be implemented in the tensorflow framework
    # return the output and new state of the RNN for given inputs and current state of the RNN
    # state and output coincide in the reservoir
    def call(self, inputs, state):
        """ Reservoir state and reservoir state evolution:
            x(i+1) = f(W*inp(i) + U*g(x(i))).
            dx_dt(i+1) = dx(i+1)_d(inp(i))*d(inp(i))_dt + dx(i+1)_d(x(i))*d(x(i))_dt
        """

        # separating input and its derivative
        x     = inputs[:,:self._num_inputs]
        x_der = inputs[:,self._num_inputs:]
        # separating state and its derivative
        s     = state[0][:,:self._num_units]
        s_der = state[0][:,self._num_units:]

        #state at next time step
        new_state = self._activation(
                    tf.matmul(tf.concat([x,self.b_in],axis=1),self.Win) +
                    tf.matmul(s, self.Wecho))

        # initialize state derivative at next time step
        deriv     = tf.TensorArray(
                    tf.float64, self._num_units, dynamic_size=False, element_shape=[1,1])

        # Compute the derivative: dr2_dt = dr2_dx * dx_dt + dr2_dr1 * dr1_dt
        # dx_dt is x_der and dr1_dt is s_der
        # need to use a for loop with tf.gradients since tf.jacobian does not work in graph mode for RNN as of now (even with tape.gradient)
        # k = 0
        for j in tf.range(self._num_units):
            drj_dt   = tf.reshape(tf.reduce_sum(tf.multiply(tf.gradients(new_state[0,j],x)[0],
                                                                  x_der)), [1,1]) + \
                       tf.reshape(tf.reduce_sum(tf.multiply(tf.gradients(new_state[0,j],s)[0],
                                                                  s_der)), [1,1])
            deriv    = deriv.write(j,drj_dt)
#             k       += 1

        new_state = tf.concat([new_state, tf.reshape(deriv.stack()[:,0],[1,self._num_units])],
                              axis=1)

        return new_state, new_state

    def setEchoStateProperty(self):
        """ optimize U to obtain alpha-improved echo-state property """
        self.Wecho = self.normalizeEchoStateWeights(self.Wecho)

    # construct the Win matrix (dimension num_inputs x num_units)
    def buildInputMatrix(self):
        """
            Returns:

            Matrix representing the
            input weights to an ESN
        """

        # Each unit is connected randomly to a given input with a weight from a
        # uniform distribution between +- sigma_in (input scaling)
        # +1 for added input bias in the input matrix
        W = np.zeros((self._num_inputs+1,self._num_units))
        for i in range(self._num_units):
            W[self.rng.randint(0,self._num_inputs+1),i] = \
            self.rng.uniform(-self.sigma_in,self.sigma_in)

        return W.astype('float64')

    def getInputMatrix(self):
        return self.win

    def buildEchoMatrix(self):
        """
            Returns:

            Matrix representing the
            inner weights to an ESN
        """

        # Build random matrix from uniform distribution
        W = self.rng.uniform(-1.0,1.0, [self._num_units, self._num_units]).astype("float64") * \
                (self.rng.rand(self._num_units, self._num_units) < (1. - self.sparseness) ) # trick to add zeros to have the sparseness required
        return W

    def normalizeEchoStateWeights(self, W):
        # Sets the spectral radius rho

        eigvals = tf.py_function(np_eigenvals, [W], tf.complex128)
        W = W / tf.reduce_max(tf.abs(eigvals))*self.rho #imposes spectral radius

        return W

    def getEchoMatrix(self):
        return self.wecho_____no_output_____# # NAKD
# import numpy as np
# import tensorflow as tf

# import h5py

# # to pass to py_func
# def np_eigenvals(x):
#     return np.linalg.eigvals(x).astype('complex128')

# # Definition of the EchoState NN as a child of RNNCell of tensorflow
# class EchoStateRNNCelldrdt(tf.keras.layers.AbstractRNNCell):

#     def __init__(self, num_units, num_inputs=1, decay=0.1, rho=0.6,
#                  sparseness=0.0,
#                  sigma_in=1.0,
#                  b_in=1.0,
#                  rng=None,
#                  activation=None,
#                  reuse = False,
#                  win=None,
#                  wecho=None):
#         """
#         Args:
#             num_units: int, Number of units in the ESN cell.
#             num_inputs: int, The number of inputs to the RNN cell.
#             decay: float, Decay/leaking of the ODE of each unit.
#             rho: float, Target spectral radius 1.
#             sparseness: float [0,1], sparseness of the inner weight matrix.
#             rng: np.random.RandomState, random number generator. (to be able to have always the same random matrix...)
#             activation: Nonlinearity to use.
#             reuse: if reusing existing matrix
#             win: input weights matrix
#             wecho: echo state matrix
#         """

#         # Basic RNNCell initialization (see tensorflow documentation)
#         super(EchoStateRNNCelldrdt, self).__init__() # use the initializer of RNNCell (i.e. defines a bunch of standard variables)

#         # fixed variables
#         self._num_units  = num_units
#         self._activation = activation
#         self._num_inputs = num_inputs

#         # variables that potentially can be changed/optimized (for future version)
#         self.decay = decay
#         self.rho = rho # rho has to be <1. to ensure the echo state property (see [2])
#         self.sparseness = sparseness
#         self.sigma_in = sigma_in
#         self.b_in  = np.reshape(b_in, [1,1])

#         # Random number generator initialization
#         self.rng = rng
#         if rng is None:
#             self.rng = np.random.RandomState()

#         # build initializers for tensorflow variables
#         if (reuse == False):
#             self.win = self.buildInputMatrix()
#             self.wecho = self.buildEchoMatrix()
#         else:
#             self.win = win.astype('float64')
#             self.wecho = wecho.astype('float64')

#         # convert the weight to tf variable
#         self.Win   = tf.Variable(self.win, name='Win', trainable=False)
#         self.Wecho = tf.Variable(self.wecho, name='Wecho', trainable=False)

#         self.setEchoStateProperty()

#     @property
#     def state_size(self):
#         return self._num_units*2

#     @property
#     def output_size(self):
#         return self._num_units*2

#     # function that has to be implemented in the tensorflow framework
#     # receives input and previous state and returns state and output (which are the same for the ESN without Wout)
#     def call(self, inputs, state):
#         """ Compute the reservoir state and its derivative """

#         # separating input and its derivative
#         x     = inputs[:,:self._num_inputs]
#         x_der = inputs[:,self._num_inputs:]
#         # separating state and its derivative
#         s     = state[0][:,:self._num_units]
#         s_der = state[0][:,self._num_units:]

#         #state at next time step
#         new_state = self._activation(
#                     tf.matmul(tf.concat([x,self.b_in],axis=1),self.Win) +
#                     tf.matmul(s, self.Wecho))

#         # initialize state derivative at next time step
#         deriv     = tf.TensorArray(
#                     tf.float64, self._num_units, dynamic_size=False, element_shape=[1,1])

#         # Compute the derivative: dr2_dt = dr2_dx * dx_dt + dr2_dr1 * dr1_dt
#         # dx_dt is x_der and dr1_dt is s_der
#         # need to use a for loop with tf.gradients since tf.jacobian does not work in graph mode for RNN as of now (even with tape.gradient)
#         # k = 0
#         for j in tf.range(self._num_units):
#             drj_dt   = tf.reshape(tf.reduce_sum(tf.multiply(tf.gradients(new_state[0,j],x)[0],
#                                                                   x_der)), [1,1]) + \
#                        tf.reshape(tf.reduce_sum(tf.multiply(tf.gradients(new_state[0,j],s)[0],
#                                                                   s_der)), [1,1])
#             deriv    = deriv.write(j,drj_dt)
# #             k       += 1

#         new_state = tf.concat([new_state, tf.reshape(deriv.stack()[:,0],[1,self._num_units])],
#                               axis=1)

#         return new_state, new_state

#     def setEchoStateProperty(self):
#         """ optimize U to obtain alpha-improved echo-state property """
#         # I know it's stupid for the time being but it is a placeholder for future treatment of the matrix
#         # (potential meta-optimization and other)
#         self.Wecho = self.normalizeEchoStateWeights(self.Wecho)

#     # construct the Win matrix (dimension num_inputs x num_units)
#     def buildInputMatrix(self):
#         """
#             Returns:

#             Matrix representing the
#             input weights to an ESN
#         """

#         # Input weight matrix initializer according to [3,4]
#         # Each unit is connected randomly to a given input with a weight from a uniform distribution

#         # without bias at the input
#         #W = np.zeros((self._num_inputs,self._num_units))
#         #for i in range(self._num_units):
#             #W[self.rng.randint(0,self._num_inputs),i] = self.rng.uniform(-self.sigma_in,self.sigma_in)

#         # Added bias in the input matrix
#         W = np.zeros((self._num_inputs+1,self._num_units))
#         for i in range(self._num_units):
#             W[self.rng.randint(0,self._num_inputs+1),i] = \
#             self.rng.uniform(-self.sigma_in,self.sigma_in)

#         # Dense input weigth [input] as in [1,2]
#         # Input weigth matrix [input]
#         #W = self.rng.uniform(-self.sigma_in, self.sigma_in, [self.num_inputs, self._num_units]).astype("float64")

#         # Dense input weigth [bias, input] (as in [1,2])
#         #W = self.rng.uniform(-self.sigma_in, self.sigma_in, [self.num_inputs+1, self._num_units]).astype("float64")

#         return W.astype('float64')

#     def getInputMatrix(self):
#         return self.win

#     def buildEchoMatrix(self):
#         """
#             Returns:

#             A 1-D tensor representing the
#             inner weights to an ESN (to be optimized)
#         """

#         # Inner weight tensor initializer
#         # 1) Build random matrix from normal distribution between [0.,1.]
#         #W = self.rng.randn(self._num_units, self._num_units).astype("float64") * \
#                 #(self.rng.rand(self._num_units, self._num_units) < (1. - self.sparseness) )

#         # 2) Build random matrix from uniform distribution
#         W = self.rng.uniform(-1.0,1.0, [self._num_units, self._num_units]).astype("float64") * \
#                 (self.rng.rand(self._num_units, self._num_units) < (1. - self.sparseness) ) # trick to add zeros to have the sparseness required
#         return W

#     def normalizeEchoStateWeights(self, W):
#         # Normalize to spectral radius rho

#         eigvals = tf.py_function(np_eigenvals, [W], tf.complex128)
#         W = W / tf.reduce_max(tf.abs(eigvals))*self.rho # sufficient conditions to ensure that the spectral radius is rho

#         return W

#     def getEchoMatrix(self):
#         return self.wecho

# #     def save_ESN(self,fln,Wout=None,CurrState=None,lmb=None):

# #         hf = h5py.File(fln,'w')

# #         hf.create_dataset('Win',data=self.getInputMatrix() )
# #         hf.create_dataset('Wecho',data=self.getEchoMatrix() )
# #         hf.create_dataset('Wout',data=Wout)

# #         hf.create_dataset('rho',data=self.rho)
# #         hf.create_dataset('num_inputs',data=self._num_inputs)
# #         hf.create_dataset('num_units',data=self._num_units)
# #         hf.create_dataset('sigma_in',data=self.sigma_in)
# #         hf.create_dataset('sparseness',data=self.sparseness)
# #         hf.create_dataset('decay',data=self.decay)
# #         hf.create_dataset('lmb',data=lmb)
# #         hf.create_dataset('state',data=CurrState)

# #         hf.close()_____no_output_____
</code>


# Notebook from Pankaj-Ra/Coursera-Data-Science-in-Python
Path: 2_Applied Plotting, Charting And Data Representation in Python/Course2/Week2.ipynb

# Basic Plotting with matplotlib_____no_output_____You can show matplotlib figures directly in the notebook by using the `%matplotlib notebook` and `%matplotlib inline` magic commands.

`%matplotlib notebook` provides an interactive environment._____no_output_____

%matplotlib notebook_____no_output_____import matplotlib as mpl
mpl.get_backend()_____no_output_____import matplotlib.pyplot as plt
plt.plot?_____no_output_____# because the default is the line style '-',
# nothing will be shown if we only pass in one point (3,2)
plt.plot(3, 2)_____no_output_____# we can pass in '.' to plt.plot to indicate that we want
# the point (3,2) to be indicated with a marker '.'
plt.plot(3, 2, '.')_____no_output_____

Let's see how to make a plot without using the scripting layer._____no_output_____

# First let's set the backend without using mpl.use() from the scripting layer
from matplotlib.backends.backend_agg import FigureCanvasAgg
from matplotlib.figure import Figure

# create a new figure
fig = Figure()

# associate fig with the backend
canvas = FigureCanvasAgg(fig)

# add a subplot to the fig
ax = fig.add_subplot(111)

# plot the point (3,2)
ax.plot(3, 2, '.')

# save the figure to test.png
# you can see this figure in your Jupyter workspace afterwards by going to
# https://hub.coursera-notebooks.org/
canvas.print_png('test.png')_____no_output_____

We can use html cell magic to display the image._____no_output_____

%%html
_____no_output_____# create a new figure
plt.figure()

# plot the point (3,2) using the circle marker
plt.plot(3, 2, 'o')

# get the current axes
ax = plt.gca()

# Set axis properties [xmin, xmax, ymin, ymax]
ax.axis([0,6,0,10])_____no_output_____# create a new figure
plt.figure()

# plot the point (1.5, 1.5) using the circle marker
plt.plot(1.5, 1.5, 'o')
# plot the point (2, 2) using the circle marker
plt.plot(2, 2, 'o')
# plot the point (2.5, 2.5) using the circle marker
plt.plot(2.5, 2.5, 'o')_____no_output_____# get current axes
ax = plt.gca()
# get all the child objects the axes contains
ax.get_children()_____no_output_____

# Scatterplots_____no_output_____

import numpy as np

x = np.array([1,2,3,4,5,6,7,8])
y = x

plt.figure()
plt.scatter(x, y) # similar to plt.plot(x, y, '.'), but the underlying child objects in the axes are not Line2D_____no_output_____import numpy as np

x = np.array([1,2,3,4,5,6,7,8])
y = x

# create a list of colors for each point to have
# ['green', 'green', 'green', 'green', 'green', 'green', 'green', 'red']
colors = ['green']*(len(x)-1)
colors.append('red')

plt.figure()

# plot the point with size 100 and chosen colors
plt.scatter(x, y, s=100, c=colors)_____no_output_____# convert the two lists into a list of pairwise tuples
zip_generator = zip([1,2,3,4,5], [6,7,8,9,10])

print(list(zip_generator))
# the above prints:
# [(1, 6), (2, 7), (3, 8), (4, 9), (5, 10)]

zip_generator = zip([1,2,3,4,5], [6,7,8,9,10])
# The single star * unpacks a collection into positional arguments
print(*zip_generator)
# the above prints:
# (1, 6) (2, 7) (3, 8) (4, 9) (5, 10)_____no_output_____# use zip to convert 5 tuples with 2 elements each to 2 tuples with 5 elements each
print(list(zip((1, 6), (2, 7), (3, 8), (4, 9), (5, 10))))
# the above prints:
# [(1, 2, 3, 4, 5), (6, 7, 8, 9, 10)]

zip_generator = zip([1,2,3,4,5], [6,7,8,9,10])
# let's turn the data back into 2 lists
x, y = zip(*zip_generator) # This is like calling zip((1, 6), (2, 7), (3, 8), (4, 9), (5, 10))
print(x)
print(y)
# the above prints:
# (1, 2, 3, 4, 5)
# (6, 7, 8, 9, 10)_____no_output_____plt.figure()
# plot a data series 'Tall students' in red using the first two elements of x and y
plt.scatter(x[:2], y[:2], s=100, c='red', label='Tall students')
# plot a second data series 'Short students' in blue using the last three elements of x and y
plt.scatter(x[2:], y[2:], s=100, c='blue', label='Short students')_____no_output_____# add a label to the x axis
plt.xlabel('The number of times the child kicked a ball')
# add a label to the y axis
plt.ylabel('The grade of the student')
# add a title
plt.title('Relationship between ball kicking and grades')_____no_output_____# add a legend (uses the labels from plt.scatter)
plt.legend()_____no_output_____# add the legend to loc=4 (the lower right hand corner), also gets rid of the frame and adds a title
plt.legend(loc=4, frameon=False, title='Legend')_____no_output_____# get children from current axes (the legend is the second to last item in this list)
plt.gca().get_children()_____no_output_____# get the legend from the current axes
legend = plt.gca().get_children()[-2]_____no_output_____# you can use get_children to navigate through the child artists
legend.get_children()[0].get_children()[1].get_children()[0].get_children()_____no_output_____# import the artist class from matplotlib
from matplotlib.artist import Artist

def rec_gc(art, depth=0):
    if isinstance(art, Artist):
        # increase the depth for pretty printing
        print("  " * depth + str(art))
        for child in art.get_children():
            rec_gc(child, depth+2)

# Call this function on the legend artist to see what the legend is made up of
rec_gc(plt.legend())_____no_output_____

# Line Plots_____no_output_____

import numpy as np

linear_data = np.array([1,2,3,4,5,6,7,8])
exponential_data = linear_data**2

plt.figure()
# plot the linear data and the exponential data
plt.plot(linear_data, '-o', exponential_data, '-o')_____no_output_____# plot another series with a dashed red line
plt.plot([22,44,55], '--r')_____no_output_____plt.xlabel('Some data')
plt.ylabel('Some other data')
plt.title('A title')
# add a legend with legend entries (because we didn't have labels when we plotted the data series)
plt.legend(['Baseline', 'Competition', 'Us'])_____no_output_____# fill the area between the linear data and exponential data
plt.gca().fill_between(range(len(linear_data)),
                       linear_data, exponential_data,
                       facecolor='blue',
                       alpha=0.25)_____no_output_____

Let's try working with dates!_____no_output_____

plt.figure()

observation_dates = np.arange('2017-01-01', '2017-01-09', dtype='datetime64[D]')

plt.plot(observation_dates, linear_data, '-o',  observation_dates, exponential_data, '-o')_____no_output_____

Let's try using pandas_____no_output_____

import pandas as pd

plt.figure()
observation_dates = np.arange('2017-01-01', '2017-01-09', dtype='datetime64[D]')
observation_dates = map(pd.to_datetime, observation_dates) # trying to plot a map will result in an error
plt.plot(observation_dates, linear_data, '-o',  observation_dates, exponential_data, '-o')_____no_output_____plt.figure()
observation_dates = np.arange('2017-01-01', '2017-01-09', dtype='datetime64[D]')
observation_dates = list(map(pd.to_datetime, observation_dates)) # convert the map to a list to get rid of the error
plt.plot(observation_dates, linear_data, '-o',  observation_dates, exponential_data, '-o')_____no_output_____x = plt.gca().xaxis

# rotate the tick labels for the x axis
for item in x.get_ticklabels():
    item.set_rotation(45)_____no_output_____# adjust the subplot so the text doesn't run off the image
plt.subplots_adjust(bottom=0.25)_____no_output_____ax = plt.gca()
ax.set_xlabel('Date')
ax.set_ylabel('Units')
ax.set_title('Exponential vs. Linear performance')_____no_output_____# you can add mathematical expressions in any text element
ax.set_title("Exponential ($x^2$) vs. Linear ($x$) performance")_____no_output_____

# Bar Charts_____no_output_____

plt.figure()
xvals = range(len(linear_data))
plt.bar(xvals, linear_data, width = 0.3)_____no_output_____new_xvals = []

# plot another set of bars, adjusting the new xvals to make up for the first set of bars plotted
for item in xvals:
    new_xvals.append(item+0.3)

plt.bar(new_xvals, exponential_data, width = 0.3 ,color='red')_____no_output_____from random import randint
linear_err = [randint(0,15) for x in range(len(linear_data))]

# This will plot a new set of bars with errorbars using the list of random error values
plt.bar(xvals, linear_data, width = 0.3, yerr=linear_err)_____no_output_____# stacked bar charts are also possible
plt.figure()
xvals = range(len(linear_data))
plt.bar(xvals, linear_data, width = 0.3, color='b')
plt.bar(xvals, exponential_data, width = 0.3, bottom=linear_data, color='r')_____no_output_____# or use barh for horizontal bar charts
plt.figure()
xvals = range(len(linear_data))
plt.barh(xvals, linear_data, height = 0.3, color='b')
plt.barh(xvals, exponential_data, height = 0.3, left=linear_data, color='r')_____no_output_____


# Notebook from chenlingantelope/scVI_TSP
Path: tests/notebooks/data_loading.ipynb

# Data Loading Tutorial_____no_output_____

# Below is code that helps us test the notebooks
# when not testing, we make the save_path a directory called data
# in the scVI root directory, feel free to move anywhere_____no_output_____def allow_notebook_for_test():
    print("Testing the data loading notebook")

test_mode = False
save_path = "data/"

# Feel free to move this to any convenient location
if not test_mode:
    save_path = "../../data"_____no_output_____from scvi.dataset import LoomDataset, CsvDataset, Dataset10X, DownloadableAnnDataset
import urllib.request
import os
from scvi.dataset import BrainLargeDataset, CortexDataset, PbmcDataset, RetinaDataset, HematoDataset, CbmcDataset, BrainSmallDataset, SmfishDataset[2019-10-01 09:07:08,503] INFO - scvi._settings | Added StreamHandler with custom formatter to 'scvi' logger.

## Generic Datasets
`scvi` supports dataset loading for the following three generic file formats:
* `.loom` files
* `.csv` files
* `.h5ad` files
* datasets processed with Cell Ranger (or from `10x` website)

Most of the dataset loading instances implemented in scvi use a positional argument `filename` and an optional argument `save_path` (value by default: `data/`). Files will be downloaded or searched for at the location `os.path.join(save_path, filename)`, make sure this path is valid when you specify the arguments.

scVI can now also handle `10x` datasets with CITE-seq protein measurements (shown below)._____no_output_____### Loading a `.loom` file
Any `.loom` file can be loaded with initializing `LoomDataset` with `filename`.

Optional parameters:
* `save_path`: save path (default to be `data/`) of the file
* `url`: url the dataset if the file needs to be downloaded from the web
* `new_n_genes`: the number of subsampling genes - set it to be `False` to turn off subsampling
* `subset_genes`: a list of gene names for subsampling_____no_output_____

# Loading a remote dataset
remote_loom_dataset = LoomDataset("osmFISH_SScortex_mouse_all_cell.loom",
                                  save_path=save_path,
                                  url='http://linnarssonlab.org/osmFISH/osmFISH_SScortex_mouse_all_cells.loom')[2019-10-01 09:07:10,167] INFO - scvi.dataset.dataset | Downloading file at /Users/adamgayoso/Google Drive/Berkeley/Software/scVI/data/osmFISH_SScortex_mouse_all_cell.loom
[2019-10-01 09:07:10,244] INFO - scvi.dataset.loom | Preprocessing dataset
[2019-10-01 09:07:10,271] WARNING - scvi.dataset.loom | Removing non-expressing cells
[2019-10-01 09:07:10,389] INFO - scvi.dataset.loom | Finished preprocessing dataset
[2019-10-01 09:07:10,392] WARNING - scvi.dataset.dataset | X is a protected attribute and cannot be set with this name in initialize_cell_attribute, changing name to X_cell and setting
[2019-10-01 09:07:10,393] INFO - scvi.dataset.dataset | Remapping labels to [0,N]
[2019-10-01 09:07:10,395] INFO - scvi.dataset.dataset | Remapping batch_indices to [0,N]
# Loading a local dataset
local_loom_dataset = LoomDataset("osmFISH_SScortex_mouse_all_cell.loom",
                                 save_path=save_path)[2019-10-01 09:07:11,038] INFO - scvi.dataset.loom | Preprocessing dataset
[2019-10-01 09:07:11,070] WARNING - scvi.dataset.loom | Removing non-expressing cells
[2019-10-01 09:07:11,193] INFO - scvi.dataset.loom | Finished preprocessing dataset
[2019-10-01 09:07:11,197] WARNING - scvi.dataset.dataset | X is a protected attribute and cannot be set with this name in initialize_cell_attribute, changing name to X_cell and setting
[2019-10-01 09:07:11,198] INFO - scvi.dataset.dataset | Remapping labels to [0,N]
[2019-10-01 09:07:11,200] INFO - scvi.dataset.dataset | Remapping batch_indices to [0,N]

### Loading a `.csv` file
Any `.csv` file can be loaded with initializing `CsvDataset` with `filename`.

Optional parameters:
* `save_path`: save path (default to be `data/`) of the file
* `url`: url of the dataset if the file needs to be downloaded from the web
* `compression`: set `compression` as `.gz`, `.bz2`, `.zip`, or `.xz` to load a zipped `csv` file
* `new_n_genes`: the number of subsampling genes - set it to be `None` to turn off subsampling
* `subset_genes`: a list of gene names for subsampling

Note: `CsvDataset` currently only supoorts `.csv` files that are genes by cells. _____no_output_____If the dataset has already been downloaded at the location `save_path`, it will not be downloaded again._____no_output_____

# Loading a remote dataset
remote_csv_dataset = CsvDataset("GSE100866_CBMC_8K_13AB_10X-RNA_umi.csv.gz",
                                save_path=save_path,
                                new_n_genes=600,
                                compression='gzip',
                                url = "https://www.ncbi.nlm.nih.gov/geo/download/?acc=GSE100866&format=file&file=GSE100866%5FCBMC%5F8K%5F13AB%5F10X%2DRNA%5Fumi%2Ecsv%2Egz")[2019-10-01 09:07:39,627] INFO - scvi.dataset.dataset | Downloading file at /Users/adamgayoso/Google Drive/Berkeley/Software/scVI/data/GSE100866_CBMC_8K_13AB_10X-RNA_umi.csv.gz
[2019-10-01 09:07:41,457] INFO - scvi.dataset.csv | Preprocessing dataset
[2019-10-01 09:09:46,750] INFO - scvi.dataset.csv | Finished preprocessing dataset
[2019-10-01 09:09:50,655] INFO - scvi.dataset.dataset | Remapping labels to [0,N]
[2019-10-01 09:09:50,659] INFO - scvi.dataset.dataset | Remapping batch_indices to [0,N]
[2019-10-01 09:09:51,992] INFO - scvi.dataset.dataset | Computing the library size for the new data
[2019-10-01 09:09:53,389] INFO - scvi.dataset.dataset | Downsampled from 8617 to 8617 cells
[2019-10-01 09:10:16,073] INFO - scvi.dataset.dataset | Downsampling from 36280 to 600 genes
[2019-10-01 09:10:16,406] INFO - scvi.dataset.dataset | Computing the library size for the new data
[2019-10-01 09:10:16,453] INFO - scvi.dataset.dataset | Filtering non-expressing cells.
[2019-10-01 09:10:16,469] INFO - scvi.dataset.dataset | Computing the library size for the new data
[2019-10-01 09:10:16,477] INFO - scvi.dataset.dataset | Downsampled from 8617 to 8617 cells
# Loading a local dataset
local_csv_dataset = CsvDataset("GSE100866_CBMC_8K_13AB_10X-RNA_umi.csv.gz",
                               save_path=save_path,
                               new_n_genes=600,
                               compression='gzip') [2019-10-01 09:10:16,547] INFO - scvi.dataset.csv | Preprocessing dataset
[2019-10-01 09:11:43,587] INFO - scvi.dataset.csv | Finished preprocessing dataset
[2019-10-01 09:11:45,660] INFO - scvi.dataset.dataset | Remapping labels to [0,N]
[2019-10-01 09:11:45,662] INFO - scvi.dataset.dataset | Remapping batch_indices to [0,N]
[2019-10-01 09:11:46,563] INFO - scvi.dataset.dataset | Computing the library size for the new data
[2019-10-01 09:11:47,483] INFO - scvi.dataset.dataset | Downsampled from 8617 to 8617 cells
[2019-10-01 09:12:05,327] INFO - scvi.dataset.dataset | Downsampling from 36280 to 600 genes
[2019-10-01 09:12:05,929] INFO - scvi.dataset.dataset | Computing the library size for the new data
[2019-10-01 09:12:05,976] INFO - scvi.dataset.dataset | Filtering non-expressing cells.
[2019-10-01 09:12:05,995] INFO - scvi.dataset.dataset | Computing the library size for the new data
[2019-10-01 09:12:06,005] INFO - scvi.dataset.dataset | Downsampled from 8617 to 8617 cells

### Loading a `.h5ad` file
[AnnData](http://anndata.readthedocs.io/en/latest/) objects can be stored in `.h5ad` format. Any `.h5ad` file can be loaded with initializing `DownloadableAnnDataset` with `filename`.

Optional parameters:
* `save_path`: save path (default to be `data/`) of the file
* `url`: url the dataset if the file needs to be downloaded from the web_____no_output_____

# Loading a remote anndata dataset
remote_ann_dataset = DownloadableAnnDataset(
    "TM_droplet_mat.h5ad",
    save_path=save_path,
    url='https://github.com/YosefLab/scVI/blob/master/tests/data/TM_droplet_mat.h5ad?raw=true'
)

# Loading a local anndata dataset (output not shown)
# import anndata
# anndataset = anndata.read(save_path + "TM_droplet_mat.h5ad")
# dataset = AnnDatasetFromAnnData(ad = anndataset)[2019-10-01 09:12:06,873] INFO - scvi.dataset.dataset | Downloading file at /Users/adamgayoso/Google Drive/Berkeley/Software/scVI/data/TM_droplet_mat.h5ad
[2019-10-01 09:12:06,919] INFO - scvi.dataset.anndataset | Dense size under 1Gb, casting to dense format (np.ndarray).
[2019-10-01 09:12:06,921] INFO - scvi.dataset.dataset | Remapping labels to [0,N]
[2019-10-01 09:12:06,923] INFO - scvi.dataset.dataset | Remapping batch_indices to [0,N]
[2019-10-01 09:12:06,924] INFO - scvi.dataset.dataset | Computing the library size for the new data
[2019-10-01 09:12:06,926] INFO - scvi.dataset.dataset | Downsampled from 47 to 47 cells

### Loading outputs from Cell Ranger (`10x Genomics`)_____no_output_____scVI can download datasets from the `10x` website, or load a local directory with outputs generated by Cell Ranger. _____no_output_____`10x` has published several datasets on their [website](https://www.10xgenomics.com).
Initialize `Dataset10X` by passing in the dataset name of one of the following datasets that `scvi` currently supports. Please check `dataset10x.py` for a full list of supported datasets. If the dataset has already been downloaded at the location `save_path`, it will not be downloaded again.

The batch indices after the 10x barcode (e.g., `AAAAAA-1`) is automatically added to the `batch_indices` dataset attribute.

Optional parameters:
* `save_path`: save path (default to be `data/`) of the file
* `type`: set `type` (default to be `filtered`) to be `filtered` or `raw` to choose one from the two datasets that's available on `10X`
* `dense`: bool of whether to load as a dense matrix (scVI can be faster since it doesn't have to densify minibatches). We recommend setting this to `True` (not default).
* `measurement_names_column` column in which to find measurement names in the corresponding `.tsv` file.
_____no_output_____#### Downloading `10x` data

`pbmc_10k_protein_v3` is the name of a publicly available dataset from `10x`._____no_output_____

tenX_dataset = Dataset10X("pbmc_10k_protein_v3", save_path=os.path.join(save_path, "10X"), measurement_names_column=1)[2019-10-01 09:12:06,970] INFO - scvi.dataset.dataset | Downloading file at /Users/adamgayoso/Google Drive/Berkeley/Software/scVI/data/10X/pbmc_10k_protein_v3/filtered_feature_bc_matrix.tar.gz
[2019-10-01 09:12:08,414] INFO - scvi.dataset.dataset10X | Preprocessing dataset
[2019-10-01 09:12:08,417] INFO - scvi.dataset.dataset10X | Extracting tar file
[2019-10-01 09:12:41,192] INFO - scvi.dataset.dataset10X | Finished preprocessing dataset
[2019-10-01 09:12:41,346] INFO - scvi.dataset.dataset | Remapping labels to [0,N]
[2019-10-01 09:12:41,348] INFO - scvi.dataset.dataset | Remapping batch_indices to [0,N]
[2019-10-01 09:12:41,387] INFO - scvi.dataset.dataset | Computing the library size for the new data
[2019-10-01 09:12:41,484] INFO - scvi.dataset.dataset | Downsampled from 7865 to 7865 cells

#### Loading local `10x` data _____no_output_____It is also possible to create a Dataset object from 10X data saved locally. Here scVI assumes that in the directory defined by the `save_path` there exists another directory with the `matrix`, `features/genes` and `barcodes` files. As long as this directory was directly outputted by Cell Ranger and no other files were added, this function will work. If you are struggling to use this function, you may want to load your data using `scanpy` and save an `AnnData` object.

In the example below, inside the save path a directory named `filtered_feature_bc_matrix` exists containing ONLY the files `barcodes.tsv.gz`, `features.tsv.gz`, `matrix.mtx.gz`. The name and compression of these files may depend on the version of Cell Ranger and scVI will adapt accordingly. _____no_output_____

local_10X_dataset = Dataset10X(
    save_path=os.path.join(save_path, "10X/pbmc_10k_protein_v3/"),
    measurement_names_column=1,
)[2019-10-01 09:12:41,522] INFO - scvi.dataset.dataset10X | Preprocessing dataset
[2019-10-01 09:13:13,262] INFO - scvi.dataset.dataset10X | Finished preprocessing dataset
[2019-10-01 09:13:13,403] INFO - scvi.dataset.dataset | Remapping labels to [0,N]
[2019-10-01 09:13:13,404] INFO - scvi.dataset.dataset | Remapping batch_indices to [0,N]
[2019-10-01 09:13:13,443] INFO - scvi.dataset.dataset | Computing the library size for the new data
[2019-10-01 09:13:13,557] INFO - scvi.dataset.dataset | Downsampled from 7865 to 7865 cells

## Exploring a dataset object

A GeneExpressionDataset has `cell_attributes`, `gene_attributes`, and `cell_categorical_attributes`.

The `pbmc10k_protein_v3` from 10X Genomics also has CITE-seq protein data, which we show how to access._____no_output_____

print(tenX_dataset)GeneExpressionDataset object with n_cells x nb_genes = 7865 x 33538
    gene_attribute_names: 'gene_names'
    cell_attribute_names: 'barcodes', 'local_vars', 'batch_indices', 'labels', 'protein_expression', 'local_means'
    cell_categorical_attribute_names: 'labels', 'batch_indices'
    cell_measurements_columns: {'protein_expression': 'protein_names'}
tenX_dataset.gene_names_____no_output_____tenX_dataset.protein_expression_____no_output_____tenX_dataset.protein_names_____no_output_____tenX_dataset.barcodes_____no_output_____

## Subsetting a dataset object

At the core, we have two methods `GeneExpressionDataset.update_cells(subset)` and `GeneExpressionDataset.update_genes(subset)`.

The `subset` should be defined as an `np.ndarray` of either ``int``'s with arbitrary shape which values are the indexes of the cells/genes to keep or boolean array used as a mask-like index. When subsetting, all gene and cell attributes are also updated.

These methods can be used directly, but we also have helpful wrappers around them. For example:
_____no_output_____

# Take the top 3000 genes by variance across cells
tenX_dataset.subsample_genes(new_n_genes=3000)[2019-10-01 09:15:35,060] INFO - scvi.dataset.dataset | Downsampling from 33538 to 3000 genes
[2019-10-01 09:15:35,172] INFO - scvi.dataset.dataset | Computing the library size for the new data
[2019-10-01 09:15:35,207] INFO - scvi.dataset.dataset | Filtering non-expressing cells.
[2019-10-01 09:15:35,245] INFO - scvi.dataset.dataset | Computing the library size for the new data
[2019-10-01 09:15:35,282] INFO - scvi.dataset.dataset | Downsampled from 7865 to 7865 cells
# Retain cells with >= 1200 UMI counts
tenX_dataset.filter_cells_by_count(1200)[2019-10-01 09:16:05,710] INFO - scvi.dataset.dataset | Computing the library size for the new data
[2019-10-01 09:16:05,746] INFO - scvi.dataset.dataset | Downsampled from 7865 to 7661 cells
# Retain genes that start with letter "A"
retain = []
for i, g in enumerate(tenX_dataset.gene_names):
    if g.startswith("A"):
        retain.append(i)
tenX_dataset.subsample_genes(subset_genes=retain)[2019-10-01 09:19:35,978] INFO - scvi.dataset.dataset | Downsampling from 3000 to 215 genes
[2019-10-01 09:19:36,024] INFO - scvi.dataset.dataset | Computing the library size for the new data
[2019-10-01 09:19:36,032] WARNING - scvi.dataset.dataset | This dataset has some empty cells, this might fail scVI inference.Data should be filtered with `my_dataset.filter_cells_by_count()
[2019-10-01 09:19:36,049] INFO - scvi.dataset.dataset | Filtering non-expressing cells.
[2019-10-01 09:19:36,055] INFO - scvi.dataset.dataset | Computing the library size for the new data
[2019-10-01 09:19:36,059] INFO - scvi.dataset.dataset | Downsampled from 7661 to 7660 cells

## Built-In Datasets _____no_output_____We've also implemented seven built-in datasets to make it easier to reproduce results from the scVI paper.

* **PBMC**: 12,039 human peripheral blood mononuclear cells profiled with 10x;
* **RETINA**: 27,499 mouse retinal bipolar neurons, profiled in two batches using the Drop-Seq technology;
* **HEMATO**: 4,016 cells from two batches that were profiled using in-drop;
* **CBMC**: 8,617 cord blood mononuclear cells profiled using 10x along with, for each cell, 13 well-characterized mononuclear antibodies;
* **BRAIN SMALL**: 9,128 mouse brain cells profiled using 10x.
* **BRAIN LARGE**: 1.3 million mouse brain cells profiled using 10x;
* **CORTEX**: 3,005 mouse Cortex cells profiled using the Smart-seq2 protocol, with the addition of UMI
* **SMFISH**: 4,462 mouse Cortex cells profiled using the osmFISH protocol
* **DROPSEQ**: 71,639 mouse Cortex cells profiled using the Drop-Seq technology
* **STARMAP**: 3,722 mouse Cortex cells profiled using the STARmap technology

We do not show the outputs of running the commands below_____no_output_____### Loading `STARMAP` dataset
`StarmapDataset` consists of 3722 cells profiled in 3 batches. The cells come with spatial coordinates of their location inside the tissue from which they were extracted and cell type labels retrieved by the authors ofthe original publication.

Reference: X.Wang et al., Science10.1126/science.aat5691 (2018)_____no_output_____### Loading `DROPSEQ` dataset
`DropseqDataset` consists of 71,639 mouse Cortex cells profiled using the Drop-Seq technology. To facilitate comparison with other methods we use a random filtered set of 15000 cells and then keep only a filtered set of 6000 highly variable genes. Cells have cell type annotaions and even sub-cell type annotations inferred by the authors of the original publication.

Reference: https://www.biorxiv.org/content/biorxiv/early/2018/04/10/299081.full.pdf_____no_output_____### Loading `SMFISH` dataset
`SmfishDataset` consists of 4,462 mouse cortex cells profiled using the OsmFISH protocol. The cells come with spatial coordinates of their location inside the tissue from which they were extracted and cell type labels retrieved by the authors of the original publication.

Reference: Simone Codeluppi, Lars E Borm, Amit Zeisel, Gioele La Manno, Josina A van Lunteren, Camilla I Svensson, and Sten Linnarsson. Spatial organization of the somatosensory cortex revealed by cyclic smFISH. bioRxiv, 2018._____no_output_____

smfish_dataset = SmfishDataset(save_path=save_path)_____no_output_____

### Loading `BRAIN-LARGE` dataset

Loading BRAIN-LARGE requires at least 32 GB memory!

`BrainLargeDataset` consists of 1.3 million mouse brain cells, spanning the cortex, hippocampus and subventricular zone, and profiled with 10x chromium. We use this dataset to demonstrate the scalability of scVI. It can be used to demonstrate the scalability of scVI.

Reference: 10x genomics (2017). URL https://support.10xgenomics.com/single-cell-gene-expression/datasets. _____no_output_____

brain_large_dataset = BrainLargeDataset(save_path=save_path) _____no_output_____

### Loading `CORTEX` dataset
`CortexDataset` consists of 3,005 mouse cortex cells profiled with the Smart-seq2 protocol, with the addition of UMI. To facilitate com- parison with other methods, we use a filtered set of 558 highly variable genes. The `CortexDataset` exhibits a clear high-level subpopulation struc- ture, which has been inferred by the authors of the original publication using computational tools and annotated by inspection of specific genes or transcriptional programs. Similar levels of annotation are provided with the `PbmcDataset` and `RetinaDataset`.

Reference: Zeisel, A. et al. Cell types in the mouse cortex and hippocampus revealed by single-cell rna-seq. Science 347, 1138–1142 (2015). _____no_output_____

cortex_dataset = CortexDataset(save_path=save_path, total_genes=558) _____no_output_____

### Loading `PBMC` dataset
`PbmcDataset` consists of 12,039 human peripheral blood mononu- clear cells profiled with 10x.

Reference: Zheng, G. X. Y. et al. Massively parallel digital transcriptional profiling of single cells. Nature Communications 8, 14049 (2017). _____no_output_____

pbmc_dataset = PbmcDataset(save_path=save_path, save_path_10X=os.path.join(save_path, "10X"))_____no_output_____

### Loading `RETINA` dataset
`RetinaDataset` includes 27,499 mouse retinal bipolar neu- rons, profiled in two batches using the Drop-Seq technology.

Reference: Shekhar, K. et al. Comprehensive classification of retinal bipolar neurons by single-cell transcriptomics. Cell 166, 1308–1323.e30 (2017). _____no_output_____

retina_dataset = RetinaDataset(save_path=save_path)_____no_output_____

### Loading `HEMATO` dataset
`HematoDataset` includes 4,016 cells from two batches that were profiled using in-drop. This data provides a snapshot of hematopoietic progenitor cells differentiating into various lineages. We use this dataset as an example for cases where gene expression varies in a continuous fashion (along pseudo-temporal axes) rather than forming discrete subpopulations.

Reference: Tusi, B. K. et al. Population snapshots predict early haematopoietic and erythroid hierarchies. Nature 555, 54–60 (2018)._____no_output_____

hemato_dataset = HematoDataset(save_path=os.path.join(save_path, 'HEMATO/')) _____no_output_____

### Loading `CBMC` dataset
`CbmcDataset` includes 8,617 cord blood mononuclear cells pro- filed using 10x along with, for each cell, 13 well-characterized mononuclear antibodies. We used this dataset to analyze how the latent spaces inferred by dimensionality-reduction algorithms summarize protein marker abundance.

Reference: Stoeckius, M. et al. Simultaneous epitope and transcriptome measurement in single cells. Nature Methods 14, 865–868 (2017)._____no_output_____

cbmc_dataset = CbmcDataset(save_path=os.path.join(save_path, "citeSeq/"))_____no_output_____

### Loading `BRAIN-SMALL` dataset
`BrainSmallDataset` consists of 9,128 mouse brain cells profiled using 10x. This dataset is used as a complement to PBMC for our study of zero abundance and quality control metrics correlation with our generative posterior parameters.

Reference: _____no_output_____

brain_small_dataset = BrainSmallDataset(save_path=save_path, save_path_10X=os.path.join(save_path, "10X"))_____no_output_____


# Notebook from vantuyls/covid19-dashboard
Path: _notebooks/2020-03-17-covid19-compare-country-death-trajectories.ipynb

# Compare Death Rate Trajectories
> Comparing different countries' death rate trajectories with those of Italy, South Korea, Japan and Lombardy, as well as US states.

- comments: false
- toc: true
- author: Pratap Vardhan, Vlad Stan
- categories: [growth, compare, death, interactive]
- image: images/covid-compare-country-death-trajectories.png
- permalink: /compare-country-death-trajectories/_____no_output_____

#hide
import pandas as pd
import altair as alt
from IPython.display import HTML

pd.options.mode.chained_assignment = None  # default='warn'_____no_output_____#hide
url = ('https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_covid19_deaths_global.csv')
df = pd.read_csv(url)
# rename countries
df['Country/Region'] = df['Country/Region'].replace({'Korea, South': 'South Korea'})
df = df[~df['Country/Region'].isin(['Cruise Ship'])]   # Remove Ships
dt_cols = df.columns[~df.columns.isin(['Province/State', 'Country/Region', 'Lat', 'Long'])]_____no_output_____#hide
url_italy = ('https://raw.githubusercontent.com/pcm-dpc/COVID-19/master/dati-regioni/dpc-covid19-ita-regioni.csv')
df_italy = pd.read_csv(url_italy)_____no_output_____#hide
dff = (df.groupby('Country/Region')[dt_cols].sum()
       .stack().reset_index(name='Confirmed Cases')
       .rename(columns={'level_1': 'Date', 'Country/Region': 'Country'}))
dff['Date'] = pd.to_datetime(dff['Date'], format='%m/%d/%y')_____no_output_____#hide
MIN_CASES = 10
LAST_DATE = dt_cols[-1]
# sometimes last column may be empty, then go backwards
for c in dt_cols[::-1]:
    if not df[c].fillna(0).eq(0).all():
        LAST_DATE = c
        break
countries = dff[dff['Date'].eq(LAST_DATE) & dff['Confirmed Cases'].ge(MIN_CASES) &
        dff['Country'].ne('China')
       ].sort_values(by='Confirmed Cases', ascending=False)
countries = countries['Country'].values_____no_output_____#hide
# more regions of Italy can be added alongside Lombardia in the next line of code (ex: 'Emilia Romagna', 'Veneto', etc.)
df_reg = df_italy[df_italy['denominazione_regione'].isin(['Lombardia'])]
df_reg = df_reg.sort_values(by=['denominazione_regione', 'deceduti'])
mask = (df_reg['deceduti'] >= MIN_CASES)
df_reg_gt10 = df_reg.loc[mask]
df_simple = df_reg_gt10[['denominazione_regione','data', 'deceduti']]
df_simple.rename(columns={'denominazione_regione':'Country'}, inplace=True)
df_simple.rename(columns={'data':'Date'}, inplace=True)
df_simple.rename(columns={'deceduti':'Confirmed Cases'}, inplace=True)
df_simple['Date'] = pd.to_datetime(df_simple["Date"]).dt.strftime('%Y-%m-%d')
df_simple['Date'] = pd.to_datetime(df_simple["Date"])_____no_output_____#hide
df_simple[f'Days since {MIN_CASES}th death'] = ''
current_region = df_simple.iloc[0]['Country']
days = 0
for index, row in df_simple.iterrows():
  if row['Country'] == current_region:
    df_simple[f'Days since {MIN_CASES}th death'][index] = days
    days += 1
  else:
    days = 0
    current_region = df_simple['Country'][index]
    df_simple[f'Days since {MIN_CASES}th death'][index] = days
    days += 1

df_simple["Country"]= df_simple["Country"].replace("Lombardia", "Lombardy")_____no_output_____#hide
SINCE_CASES_NUM = 10
COL_X = f'Days since {SINCE_CASES_NUM}th death'
dff2 = dff[dff['Country'].isin(countries)].copy()
days_since = (dff2.assign(F=dff2['Confirmed Cases'].ge(SINCE_CASES_NUM))
              .set_index('Date')
              .groupby('Country')['F'].transform('idxmax'))
dff2[COL_X] = (dff2['Date'] - days_since.values).dt.days.values
dff2 = dff2[dff2[COL_X].ge(0)]

# append Italy data to the rest of the data
dff2 = dff2.append(df_simple)_____no_output_____#hide
def get_country_colors(x):
    mapping = {
        'Italy': 'black',
        'Lombardy': '#DC143C',
        'Iran': '#A1BA59',
        'South Korea': '#C71585',
        'Spain': 'green',
        'Germany': '#9D755D',
        'France': 'purple',
        'US': '#0059B3',
        'Switzerland': '#9D755D',
        'Norway': '#C1B7AD',
        'United Kingdom': '#2495D3',
        'Netherlands': 'orange',
        'Sweden': '#FFCD00',
        'Belgium': '#C1B7AD',
        'Denmark': '#C1B7AD',
        'Austria': '#C1B7AD',
        'Romania': '#FF4500',
        'Japan': '#9467bd'}
    return mapping.get(x, '#C1B7AD')_____no_output_____#hide_input
baseline_countries = ['Italy', 'South Korea', 'Japan']
max_date = dff2['Date'].max()
color_domain = list(dff2['Country'].unique())
color_range = list(map(get_country_colors, color_domain))

def make_since_chart(highlight_countries=[], baseline_countries=baseline_countries):
    selection = alt.selection_multi(fields=['Country'], bind='legend',
                                    init=[{'Country': x} for x in highlight_countries + baseline_countries])

    base = alt.Chart(dff2, width=550).encode(
        x=f'{COL_X}:Q',
        y=alt.Y('Confirmed Cases:Q', scale=alt.Scale(type='log'), axis=alt.Axis(title='Cumulative Deaths')),
        color=alt.Color(
            'Country:N',
            scale=alt.Scale(domain=color_domain, range=color_range),
            legend=alt.Legend(columns=2, symbolLimit=100)),
        tooltip=list(dff2),
        opacity=alt.condition(selection, alt.value(1), alt.value(0.05))
    )
    max_day = dff2[COL_X].max()
    ref = pd.DataFrame([[x, SINCE_CASES_NUM*1.33**x] for x in range(max_day+1)], columns=[COL_X, 'Confirmed Cases'])
    base_ref = alt.Chart(ref).encode(x=f'{COL_X}:Q', y='Confirmed Cases:Q')
    return (
        base_ref.mark_line(color='black', opacity=.5, strokeDash=[3,3]) +
        base_ref.transform_filter(
            alt.datum[COL_X] >= max_day
        ).mark_text(dy=-6, align='right', fontSize=10, text='33% Daily Growth') +
        base.mark_line(point=True).add_selection(selection) +
        base.transform_filter(
            alt.datum['Date'] >= int(max_date.timestamp() * 1000)
        ).mark_text(dy=-8, align='right', fontWeight='bold').encode(text='Country:N')
    ).properties(
        title=f"Compare {', '.join(highlight_countries)} death trajectory with {', '.join(baseline_countries)}"
    )_____no_output_____

## Learning from Italy, South Korea, Japan_____no_output_____Italy, South Korea & Japan are three countries which show significantly different total deaths growth rates and evolution over time.

**South Korea** and **Japan** have a lower growth rate since thier 10th death. **Italy** has been growing close to 33% until 3 weeks since their 10th death.

Where does your country stand today?

Click (Shift+ for multiple) on Countries legend to filter the visualization._____no_output_____

#hide_input
HTML(f'Last Updated on {pd.to_datetime(LAST_DATE).strftime("%B, %d %Y")}')_____no_output_____#hide_input
chart = make_since_chart()
chart_____no_output_____#hide_input
chart2 = make_since_chart(['Spain', 'Germany'])
chart2_____no_output_____#hide_input
chart2 = make_since_chart(['Lombardy', 'Netherlands', 'US', 'United Kingdom', 'Spain'])
chart2_____no_output_____#hide_input
chart3 = make_since_chart(['US', 'France'])
chart3_____no_output_____#hide_input
chart4 = make_since_chart(['Germany', 'United Kingdom'])
chart4_____no_output_____

Select a country from the drop down list below to toggle  the visualization._____no_output_____

#hide_input
base = alt.Chart(dff2, width=600).encode(
    x=f'{COL_X}:Q',
    y=alt.Y('Confirmed Cases:Q', scale=alt.Scale(type='log'), axis=alt.Axis(title='Cumulative Deaths')),
    color=alt.Color('Country:N', scale=alt.Scale(domain=color_domain, range=color_range), legend=None),
    tooltip=['Country', 'Date', 'Confirmed Cases', COL_X]
)

country_selection = alt.selection_single(
    name='Select', fields=['Country'],
    bind=alt.binding_select(options=list(sorted(set(countries) - set(baseline_countries)))),
    init={'Country': 'US'})

date_filter = alt.datum['Date'] >= int(max_date.timestamp() * 1000)
base2 = base.transform_filter(alt.FieldOneOfPredicate('Country', baseline_countries))
base3 = base.transform_filter(country_selection)
base4 = base3.transform_filter(date_filter)

max_day = dff2[COL_X].max()
ref = pd.DataFrame([[x, SINCE_CASES_NUM*1.33**x] for x in range(max_day+1)], columns=[COL_X, 'Confirmed Cases'])
base_ref = alt.Chart(ref).encode(x=f'{COL_X}:Q', y='Confirmed Cases:Q')
base_ref_f = base_ref.transform_filter(alt.datum[COL_X] >= max_day)

chart5 = (
 base_ref.mark_line(color='black', opacity=.5, strokeDash=[3,3]) +
 base_ref_f.mark_text(dy=-6, align='right', fontSize=10, text='33% Daily Growth') +
 base2.mark_line(point=True, tooltip=True) +
 base3.mark_line(point={'size':50}, tooltip=True) +
 base2.transform_filter(date_filter).mark_text(dy=-8, align='right').encode(text='Country:N') +
 base4.mark_text(dx=8, align='left', fontWeight='bold').encode(text='Country:N') +
 base4.mark_text(dx=8, dy=12, align='left', fontWeight='bold').encode(text='Confirmed Cases:Q')
).add_selection(country_selection).properties(
    title=f"Country's death trajectory compared to {', '.join(baseline_countries)}"
)
chart5_____no_output_____

## Learning From Lombardy, Italy

Lombardy, a region of Italy with a population of only 10 million people, has reached over 4,000 deaths due to Covid-19. At its highest daily death toll, Lombardy had 546 deaths in one day. This shows that the virus has a high mortality rate even on a small demographic scale, likely due to overwhelmed healthcare services.

Where does your country stand in comparison to Lombardy?

Click (Shift+ for multiple) on Countries legend to filter the visualization._____no_output_____

#hide_input
chart5 = make_since_chart(['United Kingdom', 'US', 'Spain', 'Netherlands', 'France', 'South Korea'], ['Lombardy'])
chart5_____no_output_____

Visualizations are created by [Pratap Vardhan](https://twitter.com/PratapVardhan)[^1]

Lombardy comparison chart by [Vlad Stan](https://twitter.com/vladpke).

[^1]: Source: ["2019 Novel Coronavirus COVID-19 (2019-nCoV) Data Repository by Johns Hopkins CSSE"](https://systems.jhu.edu/research/public-health/ncov/) [GitHub repository](https://github.com/CSSEGISandData/COVID-19). Link to [original notebook](https://github.com/pratapvardhan/notebooks/blob/master/covid19/covid19-compare-country-death-trajectories.ipynb).

[^2]: Source: ["Dati COVID-19 Italia"](http://www.salute.gov.it/portale/home.html) [GitHub repository](https://github.com/pcm-dpc/COVID-19)._____no_output_____