# Notebook from BorisYourich/EurOPDX-Galaxy
Path: workflow_evaluation/jax-data.ipynb

import re
import os
import sys
import pandas as pd
import numpy as np_____no_output_____pipeline_dir = "galaxy_counts"
jax_dir = "jax_zscores"_____no_output_____

input_files = os.popen("ls -1 ./" + pipeline_dir + " | sed 's/_.*//'").read()
files = input_files.split('\n')

#last element is dir name
for filename in files[:-1]:
    #download the relative files
    os.system("curl -o "+ filename + " http://tumor.informatics.jax.org/mtbwi/pdxDetails.do?modelID=" + filename)
    print(filename)
    textfile = open(filename, 'r')
    filetext = textfile.read()
    textfile.close()
    match = re.findall(".expression.*ToDataTable.*", filetext)
    genes = match[0].split('([')[1].split("],[")
    columns = []
    column_names = ['gene']
    #which columns to use, and which z-score values to use
    for i,one in enumerate(genes[0].split('},')):
        if 'stRNA_Seq' in one:
            columns.append(i*2 + 1)
            column_names.append('stRNA_Seq')
        elif 'RNA_Seq' in one:
            columns.append(i*2 + 1)
            column_names.append('RNA_Seq')

    new = []
    for i,item in enumerate(genes[1:]):
        new.append(item.split(","))
        new[i][0] = new[i][0].strip("'")
    df = pd.DataFrame(new)
    df = df[[0,*columns]]
    df = df.set_axis(column_names, axis=1, inplace=False)
    df.to_csv(jax_dir+"/"+filename)
    print(df)
    os.remove(filename)  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100 57871    0 57871    0     0  52099      0 --:--:--  0:00:01 --:--:-- 52136
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0

input_files = os.popen("ls -1 ./"+ pipeline_dir).read()
files = input_files.split('\n')
normalized_dir="galaxy_normalized_by_sample"

for filename in files[:-1]:
    df = pd.read_table(pipeline_dir+"/"+filename)
    df = df.set_axis(['gene','id','count'], axis=1, inplace=False)
    df = df.drop(labels='id',axis=1)
    # remove zero valued genes
    df = df.loc[df['count'] != 0]
    df['count'] = df['count'].div(df['count'].quantile(q=0.75))
    print(df)
    df.to_csv(normalized_dir+"/"+filename.split('_')[0])             gene     count
0          TSPAN6  1.644079
2            DPM1  1.382453
3           SCYL3  1.187054
4        C1orf112  1.698664
5             FGR  0.002039
...           ...       ...
60669  AC114778.2  0.001142
60671    Z97205.3  0.001020
60672  AC010184.1  0.006688
60673  AL357874.3  0.014273
60674  AL645933.5  0.116611

[27436 rows x 2 columns]
             gene     count
0          TSPAN6  1.063986
2            DPM1  1.972314
3           SCYL3  0.860223
4        C1orf112  0.658895
5             FGR  0.007447
...           ...       ...
60670  AL162231.5  0.001064
60671    Z97205.3  0.003351
60672  AC010184.1  0.004585
60673  AL357874.3  0.017021
60674  AL645933.5  0.007245

[29920 rows x 2 columns]
             gene     count
0          TSPAN6  1.650407
2            DPM1  2.548829
3           SCYL3  0.441454
4        C1orf112  0.865824
5             FGR  0.006596
...           ...       ...
60667  AC023296.1  0.001319
60669  AC114778.2  0.002850
60672  AC010184.1  0.010581
60673  AL357874.3  0.005277
60674  AL645933.5  0.081834

[26083 rows x 2 columns]
             gene     count
0          TSPAN6  2.327273
1            TNMD  0.127273
2            DPM1  1.409790
3           SCYL3  0.572308
4        C1orf112  0.608671
...           ...       ...
60669  AC114778.2  0.008392
60671    Z97205.3  0.034126
60672  AC010184.1  0.022559
60673  AL357874.3  0.050350
60674  AL645933.5  0.012741

[31809 rows x 2 columns]
             gene     count
0          TSPAN6  0.858981
2            DPM1  1.509224
3           SCYL3  1.048753
4        C1orf112  1.258182
5             FGR  0.016519
...           ...       ...
60667  AC023296.1  0.009010
60669  AC114778.2  0.017976
60672  AC010184.1  0.004775
60673  AL357874.3  0.034539
60674  AL645933.5  0.416951

[30283 rows x 2 columns]
             gene     count
0          TSPAN6  1.304300
1            TNMD  0.028241
2            DPM1  1.966624
3           SCYL3  1.199949
4        C1orf112  2.113119
...           ...       ...
60669  AC114778.2  0.005481
60671    Z97205.3  0.002747
60672  AC010184.1  0.002850
60673  AL357874.3  0.039795
60674  AL645933.5  0.005443

[31309 rows x 2 columns]
             gene     count
0          TSPAN6  1.536210
1            TNMD  0.023865
2            DPM1  3.096784
3           SCYL3  1.399247
4        C1orf112  2.499281
...           ...       ...
60669  AC114778.2  0.003409
60671    Z97205.3  0.006557
60672  AC010184.1  0.011978
60673  AL357874.3  0.036366
60674  AL645933.5  0.002125

[33543 rows x 2 columns]
             gene     count
0          TSPAN6  3.277298
1            TNMD  0.002175
2            DPM1  2.665115
3           SCYL3  1.942707
4        C1orf112  2.541907
...           ...       ...
60667  AC023296.1  0.002175
60669  AC114778.2  0.001174
60671    Z97205.3  0.007883
60672  AC010184.1  0.017974
60673  AL357874.3  0.027184

[27996 rows x 2 columns]
             gene     count
0          TSPAN6  4.429018
1            TNMD  0.004739
2            DPM1  1.977578
3           SCYL3  1.072489
4        C1orf112  2.181038
...           ...       ...
60667  AC023296.1  0.014216
60671    Z97205.3  0.017691
60672  AC010184.1  0.013552
60673  AL357874.3  0.025273
60674  AL645933.5  0.179435

[31518 rows x 2 columns]
             gene     count
0          TSPAN6  3.024552
1            TNMD  0.008313
2            DPM1  0.821626
3           SCYL3  0.856444
4        C1orf112  0.377407
...           ...       ...
60667  AC023296.1  0.009699
60669  AC114778.2  0.004863
60672  AC010184.1  0.021240
60673  AL357874.3  0.027711
60674  AL645933.5  0.033281

[27061 rows x 2 columns]
             gene     count
0          TSPAN6  8.793711
1            TNMD  0.339737
2            DPM1  4.427705
3           SCYL3  1.455319
4        C1orf112  1.408332
...           ...       ...
60669  AC114778.2  0.008615
60671    Z97205.3  0.018443
60672  AC010184.1  0.022053
60673  AL357874.3  0.026289
60674  AL645933.5  0.171446

[31593 rows x 2 columns]
             gene     count
0          TSPAN6  2.114804
2            DPM1  1.577589
3           SCYL3  1.579775
4        C1orf112  1.020643
5             FGR  0.002197
...           ...       ...
60667  AC023296.1  0.003296
60669  AC114778.2  0.031925
60671    Z97205.3  0.003307
60673  AL357874.3  0.016479
60674  AL645933.5  0.041846

[30880 rows x 2 columns]
             gene     count
0          TSPAN6  2.452381
2            DPM1  4.040179
3           SCYL3  0.683170
4        C1orf112  0.705699
5             FGR  0.038690
...           ...       ...
60670  AL162231.5  0.001488
60671    Z97205.3  0.005818
60672  AC010184.1  0.010268
60673  AL357874.3  0.031250
60674  AL645933.5  0.009911

[29241 rows x 2 columns]
             gene     count
0          TSPAN6  2.668478
2            DPM1  1.156250
3           SCYL3  0.856630
4        C1orf112  0.710217
5             FGR  0.005435
...           ...       ...
60670  AL162231.5  0.002717
60671    Z97205.3  0.001698
60672  AC010184.1  0.005149
60673  AL357874.3  0.008152
60674  AL645933.5  0.037948

[30100 rows x 2 columns]
             gene     count
0          TSPAN6  4.969697
2            DPM1  1.487013
3           SCYL3  0.398896
4        C1orf112  0.482056
5             FGR  0.002165
...           ...       ...
60667  AC023296.1  0.006494
60669  AC114778.2  0.020411
60672  AC010184.1  0.002165
60673  AL357874.3  0.004329
60674  AL645933.5  0.167446

[24605 rows x 2 columns]
             gene     count
0          TSPAN6  1.388350
2            DPM1  1.264563
3           SCYL3  1.238228
4        C1orf112  0.417573
5             FGR  0.009709
...           ...       ...
60669  AC114778.2  0.004951
60671    Z97205.3  0.002233
60672  AC010184.1  0.058301
60673  AL357874.3  0.004854
60674  AL645933.5  0.108374

[26760 rows x 2 columns]
             gene     count
0          TSPAN6  1.820117
1            TNMD  0.010361
2            DPM1  3.615956
3           SCYL3  1.267070
4        C1orf112  0.833189
...           ...       ...
60669  AC114778.2  0.002020
60671    Z97205.3  0.025332
60672  AC010184.1  0.050060
60673  AL357874.3  0.050078
60674  AL645933.5  0.074253

[32828 rows x 2 columns]
             gene     count
0          TSPAN6  1.026832
2            DPM1  2.024849
3           SCYL3  1.271464
4        C1orf112  1.828277
5             FGR  0.015371
...           ...       ...
60670  AL162231.5  0.001025
60671    Z97205.3  0.004089
60672  AC010184.1  0.002070
60673  AL357874.3  0.019470
60674  AL645933.5  0.230829

[29638 rows x 2 columns]
             gene     count
0          TSPAN6  0.961938
2            DPM1  1.686857
3           SCYL3  0.234774
4        C1orf112  0.251712
5             FGR  0.001386
...           ...       ...
60667  AC023296.1  0.001386
60669  AC114778.2  0.008802
60671    Z97205.3  0.001358
60672  AC010184.1  0.004449
60673  AL357874.3  0.011089

[27679 rows x 2 columns]

input_files = os.popen("ls -1 ./"+ jax_dir).read()
files = input_files.split('\n')
pipeline_norm_dir = "galaxy_normalized_by_sample"
jax_rna = "jax_normalized_rna"
jax_strna = "jax_normalized_strna"

rna = pd.read_csv("rnaseq_expn.csv", delimiter=';')
rna = rna.drop(labels=['ensembl_gene_id', 'platform'],axis=1)
rna = rna.rename(columns={"gene_symbol_hint": "gene"})
rna['mean'] = (rna['mean'].str.split()).apply(lambda x: float(x[0].replace(',', '.'))).astype(float)
rna['sd'] = (rna['sd'].str.split()).apply(lambda x: float(x[0].replace(',', '.'))).astype(float)

strna = pd.read_csv("strnaseq_expn.csv", delimiter=';')
strna = strna.drop(labels=['ensembl_gene_id', 'platform'],axis=1)
strna = strna.rename(columns={"gene_symbol_hint": "gene"})
strna['mean'] = (strna['mean'].str.split()).apply(lambda x: float(x[0].replace(',', '.'))).astype(float)
strna['sd'] = (strna['sd'].str.split()).apply(lambda x: float(x[0].replace(',', '.'))).astype(float)

for filename in files[:-1]:
    print(filename)

    z_score = pd.read_csv(jax_dir+"/"+filename)
    normalized = pd.read_csv(pipeline_norm_dir+"/"+filename)

    #somehow this ugly extra column appeared in the csv
    z_score = z_score.drop(labels='Unnamed: 0',axis=1)
    normalized = normalized.drop(labels='Unnamed: 0',axis=1)

    #print(z_score)
    #print(rna)
    #print(normalized)
    #print(normalized.loc[normalized['gene'] == 'A1CF'].iloc[0])
    norm_factors = []
    stgene = []
    gene = []
    counts_strna = []
    counts_rna = []
    counts = 0
    i = 0

    for index, row in z_score.iterrows():
        if 'stRNA_Seq' in z_score:
            if strna.isin([row['gene']]).any().any():
                if normalized.isin([row['gene']]).any().any():
                    mean = strna.loc[strna['gene'] == row['gene']].iloc[0]['mean']
                    sd = strna.loc[strna['gene'] == row['gene']].iloc[0]['sd']
                    count = (float(row['stRNA_Seq'])*sd)+mean
                    stgene.append(row['gene'])
                    counts_strna.append(count)
        if 'RNA_Seq' in z_score:
            if rna.isin([row['gene']]).any().any():
                if normalized.isin([row['gene']]).any().any():
                    mean = rna.loc[rna['gene'] == row['gene']].iloc[0]['mean']
                    sd = rna.loc[rna['gene'] == row['gene']].iloc[0]['sd']
                    count = (float(row['RNA_Seq'])*sd)+mean
                    stgene.append(row['gene'])
                    counts_strna.append(count)
                    #norm_factor = float(normalized[normalized['gene'] == row['gene']].iloc[0]['count']) / count
                    #norm_factors.append(norm_factor)
                    #counts += norm_factor
                    #i += 1

    if 'stRNA_Seq' in z_score:
        dictt = {'gene': stgene, 'count': counts_strna}
        df = pd.DataFrame(dictt)
        print(df)
        df.to_csv(jax_strna+"/"+filename)
    if 'RNA_Seq' in z_score:
        dictt = {'gene': stgene, 'count': counts_strna}
        df = pd.DataFrame(dictt)
        print(df)
        df.to_csv(jax_rna+"/"+filename)
    #print(counts/i)
    #print(max(norm_factors), " ", min(norm_factors))J000092712
        gene      count
0       ABL1  12.603812
1     ACVR1B   9.209382
2       ADAR  14.254944
3     ADARB1  12.878800
4       AGO1  10.725500
..       ...        ...
322    XRCC3   9.934656
323     YAP1  12.341800
324     YES1  12.135435
325  ZFP36L1  13.252550
326  ZMYND19   9.720186

[327 rows x 2 columns]
J000106560
        gene      count
0       ABL1  12.656488
1     ACVR1B   9.711981
2       ADAR  14.185948
3     ADARB1   9.081463
4       AGO1  11.464985
..       ...        ...
322    XRCC3   8.419992
323     YAP1  12.707090
324     YES1  11.909295
325  ZFP36L1  12.331412
326  ZMYND19   8.996276

[327 rows x 2 columns]
TM00016
        gene      count
0       A1CF   5.205280
1       ABL1  11.360984
2     ACVR1B  10.176140
3       ADAR  14.007412
4     ADARB1   9.343120
..       ...        ...
312    XRCC3  10.651512
313     YAP1  12.341800
314     YES1  12.418095
315  ZFP36L1  12.737288
316  ZMYND19   9.927297

[317 rows x 2 columns]
TM00065
        gene      count
0       ABL1  12.557285
1     ACVR1B   9.099201
2       ADAR  13.653724
3     ADARB1  11.321428
4       AFF2   5.793952
..       ...        ...
330    XRCC3   8.020440
331     YAP1  12.350306
332     YES1  12.474480
333  ZFP36L1  12.745488
334  ZMYND19   8.817604

[335 rows x 2 columns]
TM00089
        gene      count
0       ABL1  11.132870
1     ACVR1B   9.453418
2       ADAR  14.196318
3     ADARB1   9.688560
4       AFF2   6.750680
..       ...        ...
333    XRCC3   8.775036
334     YAP1  10.656640
335     YES1  12.397905
336  ZFP36L1  12.792992
337  ZMYND19   9.412860

[338 rows x 2 columns]
TM00090
        gene      count
0       A1CF   3.154644
1       A1CF   4.329160
2       ABL1  11.847602
3       ABL1  11.864408
4     ACVR1B   9.742620
..       ...        ...
669     YES1  11.640780
670  ZFP36L1  11.821780
671  ZFP36L1  11.539652
672  ZMYND19   8.707652
673  ZMYND19   9.105534

[674 rows x 2 columns]
        gene      count
0       A1CF   3.154644
1       A1CF   4.329160
2       ABL1  11.847602
3       ABL1  11.864408
4     ACVR1B   9.742620
..       ...        ...
669     YES1  11.640780
670  ZFP36L1  11.821780
671  ZFP36L1  11.539652
672  ZMYND19   8.707652
673  ZMYND19   9.105534

[674 rows x 2 columns]
TM00096
        gene      count
0       ABL1  10.165352
1     ACVR1B  11.340004
2       ADAR  14.039982
3     ADARB1   9.830800
4       AGO1  10.024628
..       ...        ...
316    XRCC3   9.681648
317     YAP1  12.631720
318     YES1  12.428190
319  ZFP36L1  11.651060
320  ZMYND19   9.072129

[321 rows x 2 columns]
TM00107
        gene      count
0       A1CF   2.790820
1       ABL1  12.099425
2     ACVR1B   9.191118
3       ADAR  14.331100
4     ADARB1   8.910799
..       ...        ...
333    XRCC3   8.994348
334     YAP1  11.547542
335     YES1  12.681255
336  ZFP36L1  12.920674
337  ZMYND19   9.250540

[338 rows x 2 columns]
TM00137
        gene      count
0       A1CF   8.167400
1       ABL1  11.636294
2     ACVR1B  11.452636
3       ADAR  12.033670
4     ADARB1   7.392400
..       ...        ...
320    XRCC3   9.744900
321     YAP1  12.450520
322     YES1  10.681755
323  ZFP36L1  13.001882
324  ZMYND19  10.194537

[325 rows x 2 columns]
TM00170
        gene      count
0       A1CF   8.748438
1       ABL1  11.488945
2     ACVR1B  11.856711
3       ADAR  13.715932
4     ADARB1   7.908148
..       ...        ...
326    XRCC3   9.219096
327     YAP1  12.573296
328     YES1  11.261400
329  ZFP36L1  12.618080
330  ZMYND19   9.229924

[331 rows x 2 columns]
TM00226
        gene      count
0       A1CF   2.618640
1       ABL1  11.400314
2     ACVR1B  12.081498
3       ADAR  13.955300
4     ADARB1   9.505680
..       ...        ...
341    XRCC3   9.049128
342     YAP1  12.831040
343     YES1  12.397905
344  ZFP36L1  11.330762
345  ZMYND19   8.878380

[346 rows x 2 columns]
TM00231
        gene      count
0       A1CF   3.973248
1       ABL1  10.565594
2     ACVR1B  10.314548
3       ADAR  13.432540
4     ADARB1   9.230794
..       ...        ...
326    XRCC3   9.710212
327     YAP1  12.528698
328     YES1  12.805320
329  ZFP36L1  12.952526
330  ZMYND19   9.010020

[331 rows x 2 columns]
TM00256
        gene      count
0       A1CF   3.953680
1       ABL1  12.068924
2     ACVR1B  11.321232
3       ADAR  13.929244
4     ADARB1   7.229840
..       ...        ...
332    XRCC3   9.231856
333     YAP1  12.214960
334     YES1  11.620590
335  ZFP36L1  13.001882
336  ZMYND19   9.733548

[337 rows x 2 columns]
TM00279
        gene      count
0       A1CF   4.291594
1       ABL1  12.603071
2     ACVR1B   9.701768
3       ADAR  12.119260
4     ADARB1  12.409411
..       ...        ...
310    XRCC3   8.852840
311     YAP1  12.528698
312     YES1  12.267705
313  ZFP36L1  12.809192
314  ZMYND19   9.078740

[315 rows x 2 columns]
TM00314
        gene      count
0       A1CF   8.702960
1       ABL1  11.389742
2     ACVR1B  11.560534
3       ADAR  13.584604
4     ADARB1   6.841498
..       ...        ...
309    XRCC3   7.620888
310     YAP1  12.038120
311     YES1  12.143640
312  ZFP36L1  12.586228
313  ZMYND19   9.607884

[314 rows x 2 columns]
TM00335
        gene      count
0       A1CF   2.426996
1       ABL1  10.840310
2     ACVR1B  11.437978
3       ADAR  14.282716
4     ADARB1  10.660105
..       ...        ...
338    XRCC3   9.094236
339     YAP1  12.528698
340     YES1  10.255095
341  ZFP36L1  12.681784
342  ZMYND19   9.882764

[343 rows x 2 columns]
TM00999
        gene      count
0       ABL1  12.202646
1     ACVR1B   9.387716
2       ADAR  14.378710
3     ADARB1  12.472400
4       AFF2   5.606042
..       ...        ...
331    XRCC3   9.456752
332     YAP1  12.106240
333     YES1  12.912750
334  ZFP36L1  10.718018
335  ZMYND19  10.328157

[336 rows x 2 columns]
TM01571
        gene      count
0       A1CF   2.785520
1       ABL1  12.816194
2     ACVR1B   9.753770
3       ADAR  12.463594
4     ADARB1   9.647920
..       ...        ...
324    XRCC3   9.456752
325     YAP1  12.251200
326     YES1  12.044580
327  ZFP36L1  12.946178
328  ZMYND19  10.642164

[329 rows x 2 columns]
#sample='TM00089'

fig = plt.figure(figsize=(12,15))
axes = fig.subplots(6,3)

cr = {}
i=0
for r in axes:
    for p in r:
        sample=files[i]
        try:
            jax=pd.read_csv(f"{jax_rna}/{sample}",usecols=['gene','count'],index_col='gene')
        except FileNotFoundError:
            jax=pd.read_csv(f"{jax_strna}/{sample}",usecols=['gene','count'],index_col='gene')

        our=pd.read_csv(f"{pipeline_norm_dir}/{sample}",usecols=['gene','count'],index_col='gene')
        our = np.log(our.loc[jax.index])
 #       p.set_yscale('log')
        p.scatter(jax['count'],our['count'])
        cr[sample] = our['count'].corr(jax['count'])
        p.set_title(f"{sample} (R2 = {cr[sample]:.3f})")
        i += 1

fig.tight_layout()
fig.savefig('scatter.pdf')
fig.show()

print(min(cr.values()),max(cr.values()))0.9522565214929398 0.9858204959616322
import numpy as np
#jax /= jax.max()
ourl = np.log(our)
#ourl /= ourl.max()_____no_output_____ourl['count'].corr(jax['count'])_____no_output_____our = our.loc[jax.index]_____no_output_____import matplotlib.pyplot as plt

plt.scatter(jax['count'],our['count'])_____no_output_____files_____no_output_____len(files)_____no_output_____len(jax.index)_____no_output_____jax.index_____no_output_____


# Notebook from YannSia/tutorials
Path: session2/sigmoid_neuron_TODO.ipynb

This is the second part of the second AI session!
Here you will code a **sigmoid neuron**._____no_output_____

#Some useful imports
import math
from pylab import *
import matplotlib.pyplot as plt
from random import *_____no_output_____

**Quick reminder**

The **sigmoid activation function** that we will note $\sigma$ is:
$$\sigma(x) = \frac{1}{1+e^{-x}}$$
And we have :
$$\sigma'(x) = \sigma(x)(1-\sigma(x))$$
$$\\[0.2 cm]$$

Let's consider:
$~~~~~$-$\hat{y_i}$ the value to predict for the i-th training.

$~~~~~$-$y_i$ The predicted value by the model for the i-th training.

$~~~~~$-n the number of training.

The **loss function** is the following:
$$L = \frac{1}{n}\sum_{i =1}^n(\hat{y_i} - y_i)^2$$
$$\\[0.4 cm]$$

*$\rightarrow$How to compute $\frac{\partial L}{\partial \omega_i}$ for the k-th input?*

Let's consider:

$$
h_k = \sum_{i=1}^{n} a_{i}^{k}*w_{i} + b \\
y_k = \sigma(h_k)
$$

Using the *chain rule*,
$$\frac{\partial L}{\partial \omega_i} = \frac{\partial L}{\partial y_k} \frac{\partial y_k}{\partial h_k}\frac{\partial h_k}{\partial w_i}$$

we have:
$$\frac{\partial L}{\partial \omega_i} = -2(\hat{y_i} - y_i) \sigma'(h_k)a_{i}^k$$

To simplify the calculations during the execution of the algorithm we will note:

$$\delta  = -2(\hat{y_i} - y_i)\sigma'(h_k)$$

_____no_output_____

"""
The sigmoid function
"""
def sigmoid(valeur):
  #TODO
  ..._____no_output_____class Neuron:

  """
  Build of a perceptron
  weights : List of weights
  bias : The bias.
  """
  def __init__(self, weights, bias):
    self.weights = weights
    self.bias = bias

  """
  Function called when you want to get the output from the input
  input : List of input values.
  """
  def forward(self, input):
    assert(len(input) == len(self.weights))
    #TODO
    ...

  """
  Function to compute the delta value
  """
  def delta(self, predicted_output, expected_output, input):
    #TODO
    ...

  """
  Updates the weights and bias
  """
  def backward(self, input, delta):
    learning_rate = 0.1
    #TODO
    ...

  """
  Computes the accuracy of our neuron.
  inputs: List of List of inputs
  outputs: list of the expected output values.
  """
  def accuracy(self, inputs, outputs):
    #TODO
    ...

  """
  Computes the output from the input using our neuron.
  """
  def predict(self, input):
    return int(round(self.forward(input)))_____no_output_____

Now that you have programmed a **sigmoid neuron**, you can use it! Let's try to allow this neuron to **learn** to reproduce the behavior of an OR gate.
This time, you don't have to look for the right weights and biases yourself. They will be learned automatically._____no_output_____

# Example for a OR gate:

or_neuron = Neuron([random(),random()], random()) # Build a neuron with random weights and bias
or_input = [[1,1], [1,0], [0,1], [0,0]] # List of possible inputs for the neuron
or_output = [1, 1, 1, 0] # List of the expected outputs_____no_output_____

Now that we have created the neuron, we will have to train it!
So we have to make a loop that trains the neuron enough to predict all the outputs correctly. We call **epochs** the number of times we will go through this loop. One **epoch** corresponds to one training on all the data. So, to train on 10 epochs is training the neuron 10 times on the 4 inputs data.
For each epoch we need:

1.   Run the data through the neural network
2.   Compute the loss
3.   Backpropagate the loss

In order to be able to follow the evolution of our neuron, calculate, at each end of loop, the accuracy and save it somewhere. You can then draw a graph to see if the accuracy of your neuron is increasing over time._____no_output_____

# Training
accuracy = []
nb_epochs = 2000
for j in range(nb_epochs):
  #TODO_____no_output_____

If you are here, it means that you have trained a neuron to reproduce an OR gate! Check by yourself the outputs_____no_output_____

for i in range(4):
  print("Predicted : "+str(or_neuron.predict(or_input[i]))+" Expected ouput : "+str(or_output[i]))_____no_output_____

Using plt.plot, check how the accuracy has evolved_____no_output_____

plt.plot(accuracy)
plt.xlabel("nb of epochs")
plt.ylabel("accuracy")
plt.show()_____no_output_____

To understand what exactly our little neuron does, we will draw its decision line.
This line corresponds to the case $y = \frac{1}{2}$ where $y$ represents the output for a ($x_2$,$x_1$).

$ y = \frac{1}{2} \iff x_2w_2 + x_1w_1 + b = 0 \iff x_2 = \frac{-x_1w_1 - b}{w_2}$

To draw the decision line, we have to draw $f$ where $f(x) = \frac{-w_1x - b}{w_2}$
_____no_output_____

x = linspace(-0.1, 1.1, 30)

y = -or_neuron.weights[0] / or_neuron.weights[1] * x - or_neuron.bias / or_neuron.weights[1]

plt.figure(figsize=(8,8))
plt.plot(x, y)
plt.xlim(-0.1, 1.1)
plt.ylim(-0.1, 1.1)
plt.scatter([0,1,1], [1,0,1], c = 'red')
plt.scatter([0], [0], c = 'blue')
plt.title("OR gate decision boundary")
plt.show()_____no_output_____

And if you now try to train a sigmoid neuron to reproduce the behavior of an XOR gate...?_____no_output_____

xor_neuron = Neuron([randint(0,20),randint(0,20)], randint(-20,-15))
xor_input = [[1,1], [1,0], [0,1], [0,0]]
xor_output = [0, 1, 1, 0]_____no_output_____# Training
#TODO
_____no_output_____

Let's see what we have after the training_____no_output_____

# Decision line
x = linspace(-0.1, 1.1, 30)
y = - xor_neuron.weights[0] / xor_neuron.weights[1] * x - xor_neuron.bias / xor_neuron.weights[1]

plt.figure(figsize=(8, 8))
plt.plot(x, y)
plt.xlim(-0.1, 1.1)
plt.ylim(-0.1, 1.1)
plt.scatter([0,1], [0,1], c = 'red')
plt.scatter([0,1], [1,0], c = 'blue')
plt.title("XOR gate decision boundary")
plt.show()_____no_output_____# TODO : Plot the accuracy over the number of epochs_____no_output_____


# Notebook from NicoSerranoP/PySyft
Path: examples/tutorials/translations/chinese/Part 07 - Federated Learning with Federated Dataset.ipynb

epochs = 50_____no_output_____

# Part 7 - 使用FederatedDataset进行联邦学习

在这里，我们介绍了一种使用联合数据集的新工具。 我们创建了一个`FederatedDataset`类，该类旨在像PyTorch Dataset类一样使用，并提供给联合数据加载器`FederatedDataLoader`，它将以联合方式对其进行迭代。

作者:
- Andrew Trask - Twitter: [@iamtrask](https://twitter.com/iamtrask)
- Théo Ryffel - GitHub: [@LaRiffle](https://github.com/LaRiffle)

中文版译者：
- Hou Wei - github：[@dljgs1](https://github.com/dljgs1)_____no_output_____我们使用上上节课提到的沙箱_____no_output_____

import torch as th
import syft as sy
sy.create_sandbox(globals(), verbose=False)_____no_output_____

找一个数据集_____no_output_____

boston_data = grid.search("#boston", "#data")
boston_target = grid.search("#boston", "#target")_____no_output_____

加载模型和优化器_____no_output_____

n_features = boston_data['alice'][0].shape[1]
n_targets = 1

model = th.nn.Linear(n_features, n_targets)_____no_output_____

在这里，我们将获取的数据转换为`FederatedDataset`中的数据。看看拥有部分数据的工作机。
Here we cast the data fetched in a `FederatedDataset`. See the workers which hold part of the data._____no_output_____

# 将结果转换为BaseDatasets
datasets = []
for worker in boston_data.keys():
    dataset = sy.BaseDataset(boston_data[worker][0], boston_target[worker][0])
    datasets.append(dataset)

# 建立FederatedDataset对象
dataset = sy.FederatedDataset(datasets)
print(dataset.workers)
optimizers = {}
for worker in dataset.workers:
    optimizers[worker] = th.optim.Adam(params=model.parameters(),lr=1e-2)_____no_output_____

放入一个`FederatedDataLoader`并进行设置_____no_output_____

train_loader = sy.FederatedDataLoader(dataset, batch_size=32, shuffle=False, drop_last=False)_____no_output_____

最后，我们在各个epoch进行迭代。 您会发现这与纯本地PyTorch训练相比有多相似！_____no_output_____

for epoch in range(1, epochs + 1):
    loss_accum = 0
    for batch_idx, (data, target) in enumerate(train_loader):
        model.send(data.location)

        optimizer = optimizers[data.location.id]
        optimizer.zero_grad()
        pred = model(data)
        loss = ((pred.view(-1) - target)**2).mean()
        loss.backward()
        optimizer.step()

        model.get()
        loss = loss.get()

        loss_accum += float(loss)

        if batch_idx % 8 == 0:
            print('Train Epoch: {} [{}/{} ({:.0f}%)]\tBatch loss: {:.6f}'.format(
                epoch, batch_idx, len(train_loader),
                       100. * batch_idx / len(train_loader), loss.item()))

    print('Total loss', loss_accum)
_____no_output_____

# 恭喜!!! 是时候加入社区了!

祝贺您完成本笔记本教程！ 如果您喜欢此方法，并希望加入保护隐私、去中心化AI和AI供应链（数据）所有权的运动，则可以通过以下方式做到这一点！

### 给 PySyft 加星

帮助我们的社区的最简单方法是仅通过给GitHub存储库加注星标！ 这有助于提高人们对我们正在构建的出色工具的认识。

- [Star PySyft](https://github.com/OpenMined/PySyft)

### 加入我们的 Slack!

保持最新进展的最佳方法是加入我们的社区！ 您可以通过填写以下表格来做到这一点[http://slack.openmined.org](http://slack.openmined.org)

### 加入代码项目!

对我们的社区做出贡献的最好方法是成为代码贡献者！ 您随时可以转到PySyft GitHub的Issue页面并过滤“projects”。这将向您显示所有概述，选择您可以加入的项目！如果您不想加入项目，但是想做一些编码，则还可以通过搜索标记为“good first issue”的GitHub问题来寻找更多的“一次性”微型项目。

- [PySyft Projects](https://github.com/OpenMined/PySyft/issues?q=is%3Aopen+is%3Aissue+label%3AProject)
- [Good First Issue Tickets](https://github.com/OpenMined/PySyft/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22)

### 捐赠

如果您没有时间为我们的代码库做贡献，但仍想提供支持，那么您也可以成为Open Collective的支持者。所有捐款都将用于我们的网络托管和其他社区支出，例如黑客马拉松和聚会！

[OpenMined's Open Collective Page](https://opencollective.com/openmined)_____no_output_____


# Notebook from lgpreston75/collie_recs
Path: tutorials/03_advanced_matrix_factorization.ipynb

     Run in Google Colab

     View source on GitHub

     Download notebook

_____no_output_____

# for Collab notebooks, we will start by installing the ``collie_recs`` library
!pip install collie_recs --quiet_____no_output_____%reload_ext autoreload
%autoreload 2
%matplotlib inline

%env DATA_PATH data/env: DATA_PATH=data/
import os

import joblib
from pytorch_lightning.utilities.seed import seed_everything

from collie_recs.interactions import ApproximateNegativeSamplingInteractionsDataLoader
from collie_recs.metrics import auc, evaluate_in_batches, mapk, mrr
from collie_recs.model import CollieTrainer, MatrixFactorizationModel
from collie_recs.movielens import read_movielens_df_item_____no_output_____

## Load Data From ``01_prepare_data`` Notebook
If you're running this locally on Jupyter, you should be able to run the next cell quickly without a problem! If you are running this on Colab, you'll need to regenerate the data by running the cell below that, which should only take a few extra seconds to complete. _____no_output_____

try:
    # let's grab the ``Interactions`` objects we saved in the last notebook
    train_interactions = joblib.load(os.path.join(os.environ.get('DATA_PATH', 'data/'),
                                                  'train_interactions.pkl'))
    val_interactions = joblib.load(os.path.join(os.environ.get('DATA_PATH', 'data/'),
                                                'val_interactions.pkl'))
except FileNotFoundError:
    # we're running this notebook on Colab where results from the first notebook are not saved
    # regenerate this data below
    from collie_recs.cross_validation import stratified_split
    from collie_recs.interactions import Interactions
    from collie_recs.movielens import read_movielens_df
    from collie_recs.utils import convert_to_implicit, remove_users_with_fewer_than_n_interactions

    df = read_movielens_df(decrement_ids=True)
    implicit_df = convert_to_implicit(df, min_rating_to_keep=4)
    implicit_df = remove_users_with_fewer_than_n_interactions(implicit_df, min_num_of_interactions=3)

    interactions = Interactions(
        users=implicit_df['user_id'],
        items=implicit_df['item_id'],
        ratings=implicit_df['rating'],
        allow_missing_ids=True,
    )

    train_interactions, val_interactions = stratified_split(interactions, test_p=0.1, seed=42)

print('Train:', train_interactions)
print('Val:  ', val_interactions)Train: Interactions object with 49426 interactions between 943 users and 1674 items, returning 10 negative samples per interaction.
Val:   Interactions object with 5949 interactions between 943 users and 1674 items, returning 10 negative samples per interaction.

# More Advanced Ways to Train a Matrix Factorization Model
Thus far in the tutorials, we have only walked through the Collie necessities: preparing data, data splits, training a model, and evaluating a model.

This optional notebook will be an extension of the last notebook, diving into some of the more advanced features of Collie we can use without the use of metadata. _____no_output_____

# this handy PyTorch Lightning function fixes random seeds across all the libraries used here
seed_everything(22)Global seed set to 22

## Faster Data Loading Through Approximate Negative Sampling _____no_output_____With sufficiently large enough data, verifying that each negative sample is one a user has *not* interacted with becomes expensive. With many items, this can soon become a bottleneck in the training process.

Yet, when we have many items, the chances a user has interacted with most is increasingly rare. Say we have ``1,000,000`` items and we want to sample ``10`` negative items for a user that has positively interacted with ``200`` items. The chance that we accidentally select a positive item in a random sample of ``10`` items is just ``0.2%``. At that point, it might be worth it to forgo the expensive check to assert our negative sample is true, and instead just randomly sample negative items with the hope that most of the time, they will happen to be negative.

This is the theory behind the ``ApproximateNegativeSamplingInteractionsDataLoader``, an alternate DataLoader built into Collie. Let's train a model with this below, noting how similar this procedure looks to that in the previous tutorial. _____no_output_____

train_loader = ApproximateNegativeSamplingInteractionsDataLoader(train_interactions, batch_size=1024, shuffle=True)
val_loader = ApproximateNegativeSamplingInteractionsDataLoader(val_interactions, batch_size=1024, shuffle=False)_____no_output_____model = MatrixFactorizationModel(
    train=train_loader,
    val=val_loader,
    embedding_dim=10,
    lr=1e-2,
)_____no_output_____trainer = CollieTrainer(model, max_epochs=10, deterministic=True)

trainer.fit(model)
model.eval()GPU available: False, used: False
TPU available: False, using: 0 TPU cores

  | Name            | Type            | Params
----------------------------------------------------
0 | user_biases     | ZeroEmbedding   | 943
1 | item_biases     | ZeroEmbedding   | 1.7 K
2 | user_embeddings | ScaledEmbedding | 9.4 K
3 | item_embeddings | ScaledEmbedding | 16.7 K
4 | dropout         | Dropout         | 0
----------------------------------------------------
28.8 K    Trainable params
0         Non-trainable params
28.8 K    Total params
0.115     Total estimated model params size (MB)
mapk_score, mrr_score, auc_score = evaluate_in_batches([mapk, mrr, auc], val_interactions, model)

print(f'MAP@10 Score: {mapk_score}')
print(f'MRR Score:    {mrr_score}')
print(f'AUC Score:    {auc_score}')_____no_output_____

We're seeing a small hit on performance and only a marginal improvement in training time compared to the standard ``MatrixFactorizationModel`` we trained in Tutorial ``02`` because MovieLens 100K has so few items. ``ApproximateNegativeSamplingInteractionsDataLoader`` is especially recommended for when we have more items in our data and training times need to be optimized.

For more details on this and other DataLoaders in Collie (including those for out-of-memory datasets), check out the [docs](https://collie.readthedocs.io/en/latest/index.html)! _____no_output_____## Multiple Optimizers _____no_output_____Training recommendation models at ShopRunner, we have encountered something we call "the curse of popularity."

This is best thought of in the viewpoint of a model optimizer - say we have a user, a positive item, and several negative items that we hope have recommendation scores that score lower than the positive item. As an optimizer, you can either optimize every single embedding dimension (hundreds of parameters) to achieve this, or instead choose to score a quick win by optimizing the bias terms for the items (just add a positive constant to the positive item and a negative constant to each negative item).

While we clearly want to have varied embedding layers that reflect each user and item's taste profiles, some models learn to settle for popularity as a recommendation score proxy by over-optimizing the bias terms, essentially just returning the same set of recommendations for every user. Worst of all, since popular items are... well, popular, **the loss of this model will actually be decent, solidifying the model getting stuck in a local loss minima**.

To counteract this, Collie supports multiple optimizers in a ``MatrixFactorizationModel``. With this, we can have a faster optimizer work to optimize the embedding layers for users and items, and a slower optimizer work to optimize the bias terms. With this, we impel the model to do the work actually coming up with varied, personalized recommendations for users while still taking into account the necessity of the bias (popularity) terms on recommendations.

At ShopRunner, we have seen significantly better metrics and results from this type of model. With Collie, this is simple to do, as shown below. _____no_output_____

model = MatrixFactorizationModel(
    train=train_interactions,
    val=val_interactions,
    embedding_dim=10,
    lr=1e-2,
    bias_lr=1e-1,
    optimizer='adam',
    bias_optimizer='sgd',
)_____no_output_____trainer = CollieTrainer(model, max_epochs=10, deterministic=True)

trainer.fit(model)
model.eval()GPU available: False, used: False
TPU available: False, using: 0 TPU cores

  | Name            | Type            | Params
----------------------------------------------------
0 | user_biases     | ZeroEmbedding   | 943
1 | item_biases     | ZeroEmbedding   | 1.7 K
2 | user_embeddings | ScaledEmbedding | 9.4 K
3 | item_embeddings | ScaledEmbedding | 16.7 K
4 | dropout         | Dropout         | 0
----------------------------------------------------
28.8 K    Trainable params
0         Non-trainable params
28.8 K    Total params
0.115     Total estimated model params size (MB)
mapk_score, mrr_score, auc_score = evaluate_in_batches([mapk, mrr, auc], val_interactions, model)

print(f'MAP@10 Score: {mapk_score}')
print(f'MRR Score:    {mrr_score}')
print(f'AUC Score:    {auc_score}')_____no_output_____

Again, we're not seeing as much performance increase here compared to the model in Tutorial ``02`` because MovieLens 100K has so few items. For a more dramatic difference, try training this model on a larger dataset, such as MovieLens 10M, adjusting the architecture-specific hyperparameters, or train longer. _____no_output_____## Item-Item Similarity _____no_output_____While we've trained every model thus far to work for member-item recommendations (given a *member*, recommend *items* - think of this best as "Personalized recommendations for you"), we also have access to item-item recommendations for free (given a seed *item*, recommend similar *items* - think of this more like "People who interacted with this item also interacted with...").

With Collie, accessing this is simple! _____no_output_____

# start by reading in the item information with MovieLens data
df_item = read_movielens_df_item()

# all IDs must start at 0 in Collie, so we'll make that adjustment here
df_item['item_id'] -= 1_____no_output_____df_item.loc[df_item['movie_title'] == 'Star Wars (1977)']_____no_output_____# let's start by finding movies similar to Star Wars (1977)
item_similarities = model.item_item_similarity(item_id=49)

item_similarities_____no_output_____df_item.iloc[item_similarities.index][:5]_____no_output_____

These films seem similar to Star Wars 1!

Best of all, the ``item_item_similarity`` method is available in all Collie models, not just ``MatrixFactorizationModel``!

In the next two tutorials, we will incorporate item metadata into recommendations for even better results.

1 Note that the author of this tutorial has never seen Star Wars, and thus cannot accurately say that these films are similar. Sorry to Star Wars fans._____no_output_____----- _____no_output_____


# Notebook from hossainlab/dsnotes
Path: book/pandas/02-Reading Data.ipynb

Table of Contents

1  Reading Data into Pandas

1.1  Read CSV

1.2  Read Excel Sheet

1.3  Read Multiple Excel Sheets

1.4  From URL

1.5  Modify Dataset

1.6  Read Biological Data(.txt)

1.7  Read Biological Data(.tsv)

1.8  Read HTML Data

1.8.1  About the Author

_____no_output_____# Reading Data into Pandas_____no_output_____

# conventional way to import pandas
import pandas as pd _____no_output_____# Set options for displaying rows and columns _____no_output_____

## Read CSV_____no_output_____

# read data from csv file
corona = pd.read_csv("../data/covid-19_cleaned_data.csv")_____no_output_____# Examine first few rows
corona.head() _____no_output_____

## Read Excel Sheet_____no_output_____

# read data from excel file
movies = pd.read_excel("../data/movies.xls")_____no_output_____# examine first few rows
movies.head() _____no_output_____

## Read Multiple Excel Sheets _____no_output_____

import xlrd
# Import xlsx file and store each sheet in to a df list
xl_file = pd.ExcelFile("../data/data.xls",)
# Dictionary comprehension
dfs = {sheet_name: xl_file.parse(sheet_name) for sheet_name in xl_file.sheet_names}_____no_output_____# Data from each sheet can be accessed via key
keylist = list(dfs.keys())_____no_output_____# Examine the sheet name
keylist[1:10]_____no_output_____# Accessing first sheet
dfs[keylist[0]]_____no_output_____

## From URL_____no_output_____

# read a dataset of Chipotle orders directly from a URL and store the results in a DataFrame
orders = pd.read_table('http://bit.ly/chiporders')_____no_output_____# examine the first 5 rows
orders.head()_____no_output_____# examine the last 5 rows
orders.tail()_____no_output_____# examine the first `n` number of rows
orders.head(10)_____no_output_____# examine the last `n` number of rows
orders.tail(10)_____no_output_____

## Modify Dataset_____no_output_____

# read a dataset of movie reviewers (modifying the default parameter values for read_table)
users = pd.read_table('http://bit.ly//movieusers')_____no_output_____# examine the first 5 rows
users.head()_____no_output_____# DataFrame looks ugly. let's modify the default parameter for read_table
user_cols = ['user_id', 'age', 'gender', 'occupation', 'zip_code']
users = pd.read_table('http://bit.ly//movieusers', sep='|' , header=None, names=user_cols)_____no_output_____# now take a look at modified dataset
users.head()_____no_output_____

## Read Biological Data(.txt)_____no_output_____

# read text/csv data into pandas
chrom = pd.read_csv("../data/Encode_HMM_data.txt", delimiter="\t", header=None)_____no_output_____# Examine first few rows
chrom.head()_____no_output_____# it's not much better to see. so we have to modify this dataset
cols_name = ['chrom', 'start', 'stop', 'type']
chrom = pd.read_csv("../data/Encode_HMM_data.txt", delimiter="\t", header=None, names=cols_name)_____no_output_____# now examine first few rows
chrom.head()_____no_output_____

## Read Biological Data(.tsv)_____no_output_____

pokemon = pd.read_csv("../data/pokemon.tsv", sep="\t")_____no_output_____pokemon.head() _____no_output_____

## Read HTML Data_____no_output_____

# Read HTML data from web
url = 'https://www.fdic.gov/bank/individual/failed/banklist.html'
data = pd.io.html.read_html(url)_____no_output_____# Check type
type(data)_____no_output_____# access data
data[0]_____no_output_____

About the Author

This repo was created by Jubayer Hossain

Jubayer Hossain is a student of Microbiology at Jagannath University and the founder of Health Data Research Organization. He is also a team member of a bioinformatics research group known as Bio-Bio-1.

This work is licensed under a Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International License.mm_____no_output_____


# Notebook from jenna-jordan/Advent_of_Code
Path: 2021/Day_12.ipynb

# AoC Day 12

Jenna Jordan

13 December -

## Prompt

--- Day 12: Passage Pathing ---

With your submarine's subterranean subsystems subsisting suboptimally, the only way you're getting out of this cave anytime soon is by finding a path yourself. Not just a path - the only way to know if you've found the best path is to find all of them.

Fortunately, the sensors are still mostly working, and so you build a rough map of the remaining caves (your puzzle input). For example:

```
start-A
start-b
A-c
A-b
b-d
A-end
b-end
```

This is a list of how all of the caves are connected. You start in the cave named start, and your destination is the cave named end. An entry like b-d means that cave b is connected to cave d - that is, you can move between them.

So, the above cave system looks roughly like this:

```
    start
    /   \
c--A-----b--d
    \   /
     end
```

Your goal is to find the number of distinct paths that start at start, end at end, and don't visit small caves more than once. There are two types of caves: big caves (written in uppercase, like A) and small caves (written in lowercase, like b). It would be a waste of time to visit any small cave more than once, but big caves are large enough that it might be worth visiting them multiple times. So, all paths you find should visit small caves at most once, and can visit big caves any number of times.

Given these rules, there are 10 paths through this example cave system:

```
start,A,b,A,c,A,end
start,A,b,A,end
start,A,b,end
start,A,c,A,b,A,end
start,A,c,A,b,end
start,A,c,A,end
start,A,end
start,b,A,c,A,end
start,b,A,end
start,b,end
```
(Each line in the above list corresponds to a single path; the caves visited by that path are listed in the order they are visited and separated by commas.)

Note that in this cave system, cave d is never visited by any path: to do so, cave b would need to be visited twice (once on the way to cave d and a second time when returning from cave d), and since cave b is small, this is not allowed.

Here is a slightly larger example:

```
dc-end
HN-start
start-kj
dc-start
dc-HN
LN-dc
HN-end
kj-sa
kj-HN
kj-dc
```
The 19 paths through it are as follows:

```
start,HN,dc,HN,end
start,HN,dc,HN,kj,HN,end
start,HN,dc,end
start,HN,dc,kj,HN,end
start,HN,end
start,HN,kj,HN,dc,HN,end
start,HN,kj,HN,dc,end
start,HN,kj,HN,end
start,HN,kj,dc,HN,end
start,HN,kj,dc,end
start,dc,HN,end
start,dc,HN,kj,HN,end
start,dc,end
start,dc,kj,HN,end
start,kj,HN,dc,HN,end
start,kj,HN,dc,end
start,kj,HN,end
start,kj,dc,HN,end
start,kj,dc,end
```
Finally, this even larger example has 226 paths through it:

```
fs-end
he-DX
fs-he
start-DX
pj-DX
end-zg
zg-sl
zg-pj
pj-he
RW-he
fs-DX
pj-RW
zg-RW
start-pj
he-WI
zg-he
pj-fs
start-RW
```

How many paths through this cave system are there that visit small caves at most once?_____no_output_____

# get input

with open("inputs/day12.txt") as file:
    inputs = file.read().splitlines()

test1_inputs = ["start-A", "start-b", "A-c", "A-b", "b-d", "A-end", "b-end"]

test2_inputs = ["dc-end", "HN-start", "start-kj", "dc-start", "dc-HN", "LN-dc", "HN-end", "kj-sa", "kj-HN", "kj-dc"]

test3_inputs = ["fs-end", "he-DX", "fs-he", "start-DX", "pj-DX", "end-zg", "zg-sl", "zg-pj", "pj-he", "RW-he", "fs-DX", "pj-RW", "zg-RW", "start-pj", "he-WI", "zg-he", "pj-fs", "start-RW"]_____no_output_____

### Resources

- [A* Search Algorithm](https://towardsdatascience.com/a-star-a-search-algorithm-eb495fb156bb)_____no_output_____

def get_connections(inputs):
    conn_list = []
    for i in inputs:
        connection = i.split("-")
        conn_list.append((connection[0],connection[1]))
    return conn_list_____no_output_____test1_connections = get_connections(test1_inputs)
test1_connections_____no_output_____def get_nodes(connections):
    nodes = set()
    for conn in connections:
        for n in conn:
            nodes.add(n)
    return list(nodes)_____no_output_____test1_nodes = get_nodes(test1_connections)
test1_nodes_____no_output_____def get_edges(connections):
    nodes = get_nodes(connections)
    # initialize edges dict: mapping of node to list of connected nodes
    edges = {}
    for n in nodes:
        edges[n] = []
    # add nodes edge is actually connected to
    for c in connections:
        n1, n2 = c
        edges[n1].append(n2)
        edges[n2].append(n1)

    return edges_____no_output_____test1_edges = get_edges(test1_connections)
test1_edges_____no_output_____small_caves1 = [x for x in test1_nodes if x.islower() and (x not in ["start", "end"])]
small_caves1_____no_output_____def find_a_path(nodes, edges, paths = None):
    if not paths:
        paths = []
    small_caves = [x for x in nodes if x.islower() and (x not in ["start", "end"])]
    path = ['start']
    end_reached = False
    while not end_reached:

    next_node = edges[path[-1]][0]
    path.append(next_node)
    if path[-1] == 'end':
        return path
    next_node = edges[path[-1]][0]
    if (next_node in small_caves) and (next_node in path):
        return path
    else:
        path.append(next_node)
        _____no_output_____def find_next_cave(edges, path = None, small_caves = None):
    if not path:
        path = ['start']
    current_cave = path[-1]
    if current_cave == 'end':

    for next_cave in edges[current_cave]:
        if next_cave == 'end':
            path.append('end')
            return_____no_output_____def find_all_paths(connections):
    nodes = get_nodes(connections)
    edges = get_edges(connections)
    paths = []
    small_caves = [x for x in nodes if x.islower() and (x not in ["start", "end"])]
    for next_node in edges['start']:
        path = ['start', next_node]
        if next_node == 'end':
            path.append('end')
            paths.append(path)
        elif (next_node in small_caves) and (next_node in path):
            break
        else:
            for next_node2 in edges[next_node]:

                _____no_output_____def fizzbuzz(num):
    # fizz on %3, buzz on %5, fizzbuzz on %15
    appendStr = ""
    if num == 1:
        appendStr = "(1)"
    else:
        if num %3 == 0:
            appendStr += f"fizz"
        if num % 5 == 0:
            appendStr += f"buzz"
        appendStr += f" ({num})\n" + fizzbuzz(num-1)
    return appendStr

print(fizzbuzz(21))fizz (21)
buzz (20)
 (19)
fizz (18)
 (17)
 (16)
fizzbuzz (15)
 (14)
 (13)
fizz (12)
 (11)
buzz (10)
fizz (9)
 (8)
 (7)
fizz (6)
buzz (5)
 (4)
fizz (3)
 (2)
(1)


# Notebook from thashmadech/is445_spring2022
Path: week15/.ipynb_checkpoints/inClass_week14-checkpoint.ipynb

# In Class Notebook, Week 14_____no_output_____You can always paste the URL of this notebook (https://github.com/UIUC-iSchool-DataViz/is445_spring2021/blob/master/week14/inClass_week14.ipynb ) into the nbviewer interface for a plain-text rendering:

https://kokes.github.io/nbviewer.js/viewer.html_____no_output_____Let me write something!_____no_output_____Before yt, let's check out ipyvolume

## ipyvolume_____no_output_____

# import our usual things
import numpy as np
import pandas as pd
import ipyvolume_____no_output_____x, y, z = np.random.random((3, 100)) # 3 dimensions, 100 points total_____no_output_____ipyvolume.quickscatter?_____no_output_____# quick 3d scatter plot
ipyvolume.quickscatter(x,y,z, size=5, marker='diamond')_____no_output_____

Now using "real" data -- data from a VERY low resolution simulation of 2 galaxies colliding. (Galaxies are collections of billions of stars, we live in the Milky Way galaxy, our star (the Sun!) is about 1/2 out on a disk of stars in the Milky Way)._____no_output_____

galaxy_file = '/Users/jillnaiman/Downloads/outarrsnap_001_fac1n3.txt'_____no_output_____

We note: the data is super confusing to look at!  Luckily there is a data reader -- solverlibs.py -- included in the galaxyFiles.zip!_____no_output_____

# to load solver -- can copy to current directory

# or update path:
from sys import path
path.append('/Users/jillnaiman/Downloads/')
#path.append('C:\\Users\\jillnaiman\\Downloads\\') # maybe windows?!_____no_output_____path_____no_output_____from solverlibs import read_in_galaxy_data_____no_output_____read_in_galaxy_data?_____no_output_____# Note: a terrible person has not included doc strings!!!_____no_output_____t_h, r_h, v_h, e_h, N, part_type = read_in_galaxy_data(galaxy_file)_____no_output_____r_h.shape # 82 particles, 3 positions -- x/y/z, 500 time steps_____no_output_____N_____no_output_____# 3D plot of my initial time step:
nT = 0 # first time step

x, y, z = r_h[:,0, nT], r_h[:,1,nT], r_h[:,2,nT]_____no_output_____ipyvolume.quickscatter(x,y,z, size=3, marker="sphere") # plots one time step_____no_output_____# all timesteps by "unwrapping"

x = r_h[:,0,:].ravel() # 1D
y = r_h[:,1,:].ravel()
z = r_h[:,2,:].ravel()_____no_output_____x_____no_output_____ipyvolume.quickscatter(x,y,z, size=3, marker='sphere')_____no_output_____

This is not super useful, but we can do better using a quiver plot:_____no_output_____

ipyvolume.figure()
# use quiver plot -- x,y,z positions and vx,vy,vz velocity vector components

for i in range(r_h.shape[0]): # loop through every particle
    ipyvolume.quiver(r_h[i,0,:], # ith particle, x position, all time steps
                    r_h[i,1,:], # ith particle, y postion, all time steps
                    r_h[i,2,:], # ith particle, z postion, all timesteps
                    )_____no_output_____


# Notebook from binzhihao/py-ai-notebook
Path: Pattern-Recognition-and-Machine-Learning/Chap-01-Introduction/01-06-Information-Theory.ipynb

# 1.6 信息论_____no_output_____### 熵_____no_output_____考虑一个离散变量 $x$。

对于该变量的一个观测值，我们需要一个量来衡量从这个观测量中得到了多少关于 $x$ 的信息。
从主观意义上来说，一个不常见的观测值所提供的新信息要比常见的观测值要多。

我们的信息量 $h$ 依赖于一个概率分布 $p(x)$，$h(x)$ 需要满足这样的性质：
- 关于 $p(x)$ 是单调（递减）的，高概率对应的信息量低，低概率对应的信息量高；
- 假设两个事件 $x， y$ 是不相关的，即 $p(x, y)=p(x)p(y)$，那么从事件 $(x, y)$ 中获得的信息量应该等于单独从 $x$ 和 $y$ 事件中获得的信息量之和，即 $h(x, y)=h(x) + h(y)$；

我们将 $h$ 和 $p$ 之间的关系表示为 $h(p)$，从上面的例子中，我们可以看到

$$
h(pq) = h(p) h(q)
$$

归纳法不难给出 $h(p^n) = nh(p)$，所以

$$
h(p^{n/m}) = nh(p^{1/m}) = \frac n m h(p^{m/m}) = \frac n m h(p)
$$

因此由连续性，我们有 $h(p^x) = x h(p)$。

对任意正数 $p,q$，总有 $q^x = p$ 成立，则我们有

$$
\frac{h(p)}{\ln (p)} = \frac{h(q^x)}{\ln (p^x)} = \frac{xh(q)}{x\ln (q)} = \frac{h(q)}{\ln (q)}
$$

从而 $h(p) \propto \ln p$，即 $h$ 与 $p$ 的对数成正比。

一个可能的定义为

$$
h(x) = - \log_2 p(x)
$$

负号保证信息量为正且满足单调递减的要求，在这里，我们选取了 2 作为对数的底，因此 $h(x)$ 的单位为比特（`bit`）。

有了对于单一的 $x$ 的信息量，更一般的想法是衡量一组 $x$ 的信息量，因此，我们考虑对 x 求期望，得到熵（`entropy`）：

$$
H[x]=-\sum_x p(x) \log_2 p(x)
$$

因为 $\lim_{p\to 0} plnp = 0$，所以如果 $p(x)=0$，那么 $p(x)\ln p(x)=0$_____no_output_____### 编码长度和熵_____no_output_____现在假设我们要传送状态变量给一个接收器。

考虑一个有 8 个状态的变量 $x$，每个状态都是等概率的。为了表示这个状态，我们至少需要 3 比特的信息来表示状态，此时熵值为

$$
H[x] = - 8 \times \frac 1 8 \log_2 \frac 1 8 = 3~\text{bits}
$$

再考虑不等概率的情况，假设状态和概率为

|a|b|c|d|e|f|g|h|
|---|---|---|---|---|---|---|---|
|$\frac{1}{2}$|$\frac{1}{4}$|$\frac{1}{8}$|$\frac{1}{16}$|$\frac{1}{64}$|$\frac{1}{64}$|$\frac{1}{64}$|$\frac{1}{64}$|

熵为

$$
H[x] = - \frac{1}{2} \log_2 \frac{1}{2} - \frac{1}{4}\log_2 \frac{1}{4} - \frac{1}{8} \log_2 \frac{1}{8} - \frac{1}{16} \log_2 \frac{1}{16} - \frac{4}{64} \log_2 \frac{1}{64} = 2~\text{bits}
$$

不等概率的熵要比等概率的熵要小。

考虑字符的霍夫曼编码：

|a|b|c|d|e|f|g|h|
|---|---|---|---|---|---|---|---|
|0|10|110|1110|111100|111101|111110|111111|

其平均字符长度为：

$$
\frac{1}{2} \times 1 + \frac{1}{4} \times 2 + \frac{1}{8} \times 3 + \frac{1}{16} \times 4 + \frac{4}{64} \times 6 = 2~\text{bits}
$$

因此，熵和霍夫曼编码长度之间存在一定的关系。

`Shannon` 在 1948 年的 `noiseless coding theorem` 说明，这样的熵是编码长度的下界。_____no_output_____### 自然对数熵_____no_output_____现在考虑自然对数表示的熵。此时熵的单位是 `nat` 而不是 `bit`。

我们之前用平均信息量的方式引入了熵，事实上熵的概念是从物理上引入的。

假设我们将 $N$ 个相同的小球放入一些容器中，使得第 $i$ 个容器中有 $n_i$ 个球，$\sum_i n_i = N$。

我们按顺序从第 1 个容器开始往里面放入小球，第 1 个有 $N$ 种选择，第 2 个有 $N-1$ 种选择……总共有 $N!$ 种选择。

但是放入每个容器中的 $n_i$ 个球是不区分顺序的，有 $n_i!$ 种可能性放置这些小球，因此总的可能方法有

$$
W = \frac{N!}{\prod_i n_i!}
$$

定义熵为

$$
H = \frac{1}{N} \ln W = \frac 1 N\ln N! - \frac 1 N \sum_{i} \ln n_i !
$$

当 $N \to \infty$ 时，由 `Stirling` 近似公式：

$$
\ln N! \approx N \ln N - N
$$

我们有

$$
H = \lim_{N\to \infty} \frac 1 N (N\ln N - N) - \frac 1 N \sum_{i} (n_i \ln n_i - n_i)
= -\lim_{N\to \infty} \sum_i \left(\frac{n_i}{N}\right) \ln \left(\frac{n_i}{N}\right)
= -\sum_i p_i \ln p_i
$$

$p_i = \lim_{N\to\infty} \left(\frac{n_i}{N}\right)$ 是小球放入第 i 个容器的概率。

在物理上，容器中小球的分布叫做微观状态，各个容器中的小球数目叫做宏观状态。

对于一个离散随机变量 $X$ 的取值 $x_i$ 当作一个容器，即有 $p(X=x_i)=p_i$，则它的熵为：

$$
H[p] = - \sum_i p(x_i) \ln p(x_i)
$$

一般来说，一个离散分布的点越多，分布越均匀，那么对应的熵也越大。

如果其中有一个 $p_i=1$，那么熵取到最小值 0，最大值可以用 Lagrange 乘子求解，Lagrange 函数为

$$
\tilde H = - \sum_i p(x_i) \ln p(x_i) + \lambda \left(\sum_i p(x_i) - 1\right)
$$

最大化这个函数得到：

$$
\frac{\partial \tilde H}{\partial p(x_i)} = -1 -\ln p(x_i) + \lambda = 0
$$

得到最大熵应该满足一个均匀离散分布 $p(x_i)=p(x_j), \forall i,j$。

若状态总数为 $M$，则 $p(x_i) = \frac 1 M$，此时，我们有 $H = \ln M$。

为了验证的确是个最大值，我们可以计算它的二阶导数来验证。_____no_output_____### 连续变量的熵_____no_output_____对于连续变量，我们将 $x$ 划分为长度为 $\Delta$ 的小段，积分中值定理告诉我们，在第 $i$ 小段 $(i\Delta, (i+1)\Delta)$ 中存在 $x_i$，使得：

$$
\int_{i\Delta}^{i+1\Delta} p(x) dx = p(x_i) \Delta
$$

我们可以将落在第 $i$ 小段的 $x$ 对应为 $x_i$，则观测到 $x_i$ 的概率为 $p(x_i) \Delta$，这给出了一个关于 $x_i$ 的离散分布，其熵为：

$$
H_{\Delta} = - \sum_i p(x_i) \Delta \ln(p(x_i) \Delta) = - \sum_i p(x_i) \Delta \ln(p(x_i)) - \ln\Delta
$$

其中 $p(x_i) \Delta= 1$，现在忽略最后的常数项，然后考虑 $\Delta \to0$ 的情况，我们有

$$
-\lim_{\Delta \to 0} \left\{ \sum_i p(x_i) \Delta \ln(p(x_i) \Delta) \right\} = -\int p(x) \ln p(x) dx
$$

这一项叫做微分熵（`differential entropy`），它与离散熵的差别在于一个 $\ln \Delta$，在 $\Delta \to0$ 时会发散的部分。

对于多元向量 $\mathbf x$，微分熵定义为

$$
H[\mathbf x]=-\int p(\mathbf x) \ln p(\mathbf x) d\mathbf x
$$

在离散分布的例子中，我们看到最大化熵的分布是均匀离散分布。

为了最大化连续分布的熵，我们需要加上 $p(x)$ 的一阶矩和二阶矩都固定的条件。

考虑一维情况，我们的问题为

$$
\begin{align}
\max_{p(x)}~& -\int p(x) \ln p(x) dx \\
s.t.~& 1 = \int_{-\infty}^{\infty} p(x) dx  \\
& \mu = \int_{-\infty}^{\infty} xp(x) dx\\
& \sigma^2 = \int_{-\infty}^{\infty} (x-\mu)^2p(x) dx\\
\end{align}
$$

`Lagrange` 函数为

$$
-\int p(x) \ln p(x) dx + \lambda_1 \left(\int_{-\infty}^{\infty} p(x) dx - 1\right) + \lambda_2 \left(\int_{-\infty}^{\infty} xp(x) dx - \mu\right) + \lambda_3 \left(\int_{-\infty}^{\infty} (x-\mu)^2p(x) dx - \sigma^2) \right)
$$

令这个泛函对 $p(x)$ 的导数为 `0`，得到

$$
1-\ln p(x)+\lambda_1 + \lambda_2 x + \lambda_3 (x-\mu)^2 = 0
$$

从而

$$
p(x) = \exp\left\{1+\lambda_1 + \lambda_2 x + \lambda_3 (x-\mu)^2\right\}
$$

将这个形式带入约束条件，不难得到

$$
p(x) = \frac{1}{(2\pi\sigma^2)^{1/2}} \exp\left\{-\frac{(x-\mu)^2}{2\sigma^2}\right\}
$$

即最大熵的分布是高斯分布。

注意到，我们并没有对 $p(x)$ 做非负的约束，但是最大化给出的结果是一个非负的结果。

高斯分布的熵为：

$$
\frac{1}{2} \left\{1+\ln(2\pi\sigma^2)\right\}
$$

从这个表达式我们可以看出，与离散熵不同，连续熵可以为负值：$\sigma^2 < \frac{1}{2\pi e}, H(x) < 0$，原因是概率密度函数 p(x) 没有小于 1 的限制，因此 $-p(x)\ln p(x)$ 不能保证非负。_____no_output_____### 条件熵_____no_output_____假设我们有 $\mathbf{x,y}$ 的联合分布，假设 $\mathbf x$ 已知，那么 $\mathbf y$ 的包含的信息量应当由 $-\ln p(\mathbf{y|x})$ 来衡量，因此我们可以定义其条件期望为给定 $\mathbf x$ 下 $\mathbf y$ 的条件熵（`conditional entropy`）：

$$
H[\mathbf{y|x}] = -\iint p(\mathbf{x,y}) \ln(\mathbf{y|x}) d\mathbf xd\mathbf y
$$

可以计算出，我们有：

$$
H[\mathbf{x, y}] = H[\mathbf{y|x}] + H[\mathbf{x}]
$$

因此，用来描述 $\mathbf{x,y}$ 的信息量等于描述 $\mathbf x$ 的信息量加上给定 $\mathbf x$ 的条件下描述 $\mathbf y$ 所需的信息量。_____no_output_____## 1.6.1 相对熵和互信息_____no_output_____假设我们有一个未知分布 $p(\mathbf x)$，然后对它建模得到了一个概率分布 $q(\mathbf x)$。

用 $q(\mathbf x)$ 的分布来传送 $x$ 所用的信息量（$q(\mathbf x)$），比用真实的分布 $p(\mathbf x)$ 传送所用的信息量（$p(\mathbf x)$）要多一些，多出来的部分为：

$$
\begin{align}
KL(p\|q)
&= - \int p(\mathbf x)\ln(q(\mathbf x)) dx - \left(-\int p(\mathbf x)\ln p(\mathbf x)\right)\\
&= - \int p(\mathbf x)\ln \left\{\frac{q(\mathbf x)}{p(\mathbf x)}\right\}
\end{align}
$$

这叫做相对熵（`relative entropy`）或者 `Kullback-Leibler` 散度（`Kullback-Leibler divergence`）。

`K-L` 散度不是对称的，即 $KL(p\|q) \neq KL(q\|p)$。

`K-L` 散度满足 $KL(p\|q) \geq 0$，当且仅当 $p(\mathbf x)=q(\mathbf x)$ 时取等号。_____no_output_____### 凸函数_____no_output_____如果一个函数的弦（`chord`）始终在函数上或者函数上方，那么这个函数就是凸函数。

一个凸函数的例子如图所示：_____no_output_____

<code>
import matplotlib.pyplot as plt
import numpy as np
import scipy as sp

%matplotlib inline

fig, ax = plt.subplots()

xx = np.linspace(-3, 4, 100)
yy = xx ** 2

ax.plot(xx, yy, 'r')
ax.set_xlim(-4, 5)
ax.set_yticks([])
ax.set_ylim(-10, 20)

ax.set_xticks([-2, 0.5, 3])
ax.set_xticklabels([r"$a$", r"$x_\lambda$", r"$b$"], fontsize="xx-large")

for i in [-2, 3]:
    ax.plot([i, i], [-10, i ** 2], 'g--')

ax.plot([-2, 3], [4, 9], 'b')
ax.plot([0.5, 0.5], [-10, 6.5], 'g')
ax.annotate(
    "chord",
    fontsize="xx-large",
    xytext=(-3, 12),
    xy=(-0.2, 6),
    arrowprops=dict(arrowstyle="->",
                    connectionstyle="arc3")
           )

plt.show()_____no_output_____
</code>

在 $x=a$ 和 $x=b$ 的区间内的任意点 $x$ 都可以写成 $\lambda a +(1-\lambda)b, 0\leq\lambda \leq 1$，对应的弦上的值为 $\lambda f(a) +(1-\lambda) f(b)$，函数值为 $f(\lambda a +(1-\lambda)b)$。凸性等价于

$$
f(\lambda a +(1-\lambda)b) \leq \lambda f(a) +(1-\lambda) f(b)
$$

这也等价于函数的二阶导数非负（如果二阶导数存在）。

证明如下：

- 必要性

假设凸性的式子成立，令 $\lambda = \frac 1 2, b = a + 2\epsilon$，则有：

$$
\begin{align}
\frac 1 2f(a)+\frac 1 2f(b) & \geq f\left(\frac 1 2a + \frac 1 2b\right) \\
& = \frac 1 2 f\left((\frac 1 2a + \frac 1 2b\right) + \frac 1 2 f\left((\frac 1 2a + \frac 1 2b\right) \\
& = \frac 1 2 f(a+\epsilon) + \frac{1}{2} f(b-\epsilon)
\end{align}
$$

从而

$$
f(b) - f(b-\epsilon) \geq  f(a+\epsilon) - f(a)
$$

取 $\epsilon\to 0$，我们有

$$
f'(b) \geq f'(a)
$$

从而

$$
f''(x) \geq 0
$$

- 充分性

假设二阶导数非负：$f''(x) \geq 0$，那么由泰勒展开，存在一个 $x^\star$ 使得

$$
f(x) = f(x_0) + f'(x_0)(x-x_0) + \frac 1 2 f''(x^\star) (x-x_0)^2 \geq f(x_0) + f'(x_0)(x-x_0)
$$

令 $x_0 = \lambda a +(1-\lambda b), x = a$，则有

$$
f(a) \geq f(x_0) + f'(x_0)((1-\lambda)(a-b))
$$

令 $x_0 = \lambda a +(1-\lambda b), x = b$，则有

$$
f(b) \geq f(x_0) + f'(x_0)(\lambda(a-b))
$$

合并两个不等式，去掉 $f'(x_0)$ 的项：

$$
\lambda f(a) +(1-\lambda) \geq f(b) f(\lambda a +(1-\lambda)b)
$$

凸函数的例子：

- $x\ln x$
- $x^2$

如果等式只在 $\lambda = 1$ 和 $\lambda = 0$ 时成立，则称 $f(x)$ 是严格凸函数。

如果 $f(x)$ 是凸函数，那么 $-f(x)$ 是凹函数。_____no_output_____### Jensen 不等式 _____no_output_____使用数学归纳法不难证明，对于任意凸函数 $f(x)$，`Jensen` 不等式成立：

$$
f\left(\sum_{i=1}^M \lambda _i x_i\right) \leq \sum_{i=1}^M\lambda_i f(x_i)
$$

其中 $\lambda_i \geq 0, \sum_i \lambda_i = 1$。等式成立的条件为 $f$ 是线性，或者所有的 $x_i$ 都相等。

证明如下：

$M = 1$ 时显然成立；假设对于某个 $M$ 不等式成立，那么对于 $M+1$ 来说：

$$
f\left(\sum_{i=1}^{M+1} \lambda_i x_i\right) = f\left(\lambda_{M+1} x_{M+1} + \sum_{i=1}^M \lambda _i x_i\right) = f\left(\lambda_{M+1} x_{M+1} + (1-\lambda_{M+1})\sum_{i=1}^M \eta_i x_i\right)
$$

其中

$$
\eta_i = \frac{\lambda_i}{1-\lambda_{M+1}}
$$

考虑凸函数性质，我们有

$$
f\left(\sum_{i=1}^{M+1} \lambda_i x_i\right) = f\left(\lambda_{M+1} x_{M+1} + (1-\lambda_{M+1})\sum_{i=1}^M \eta_i x_i\right) \leq \lambda_{M+1} f(x_{M+1}) + (1-\lambda_{M+1}) f\left(\sum_{i=1}^M \eta_i x_i\right)
$$

再应用假设条件，我们有
$$
f\left(\sum_{i=1}^{M+1} \lambda_i x_i\right) \leq \lambda_{M+1} f(x_{M+1}) + (1-\lambda_{M+1}) f\left(\sum_{i=1}^M \eta_i x_i\right) \leq \lambda_{M+1} f(x_{M+1}) + \sum_{i=1}^M (1-\lambda_{M+1})\eta_i f\left(x_i\right) = \sum_{i=1}^{M+1} \lambda_i f(x_i)
$$

如果我们认为 $\lambda_i$ 是某个离散分布取 $x_i$ 的概率，那 `Jenson` 不等式可以表示为

$$
f\left(\mathbb E[\mathbf x]\right) \leq \mathbb E[f(\mathbf x)]
$$

不等式成立的条件为 $x$ 为常数，或者 $f$ 是线性的。

更一般的，我们有：

$$
f\left(\mathbb E[y(\mathbf x)]\right) \leq \mathbb E[f(y(\mathbf x))]
$$

对于连续形式，我们有

$$
f\left(\int y(\mathbf x)p(\mathbf x)d\mathbf x\right) \leq \int f(y(\mathbf x))p(\mathbf x)d\mathbf x
$$_____no_output_____### 应用 Jensen 不等式_____no_output_____将 `Jensen` 不等式应用到 `K-L` 散度（$-\ln x$ 是凸函数）上：

$$
KL(p||q)=-\int p(\mathbf x)\ln\left\{\frac{q(\mathbf x)}{p(\mathbf x)}\right\} d\mathbf x
\geq -\ln \int \left\{p(\mathbf x) \frac{q(\mathbf x)}{p(\mathbf x)}\right\}
= -\ln \int q(\mathbf x)d\mathbf x = 0
$$

因为 $-\ln x$ 是严格凸的，因此等式只能在 $\frac{q(\mathbf x)}{p(\mathbf x)}$ 为常数时成立，考虑到两者都是概率分布（积分都为 1），所以只能有 $p(\mathbf x) = q(\mathbf x)$ 成立。

因此，我们可以将 `K-L` 散度来衡量两个分布的差异。

另一方面，我们看到，当我们使用真实的数据分布传送数据，所用信息量的期望是最小的。_____no_output_____### 最小化相对熵_____no_output_____假设我们有一组从未知分布 $p(\mathbf x)$ 中产生的数据，我们的模型为 $q(\mathbf{x|\theta})$，一个决定参数 $\mathbf \theta$ 的方法就是最小化 $p(\mathbf x)$ 和 $q(\mathbf{x|\theta})$ 的 `K-L` 散度。

然而我们并不知道 $p(x)$ 的真实分布，一个近似的方法是使用已观测的数据进行模拟：

$$
KL(p|q) \approx \sum_{n=1}^N (-\ln q(\mathbf{x}_n | \mathbf{\theta}) + \ln p(\mathbf x_n))
$$

由于右边的部分与 $\mathbf\theta$ 无关，因此，最小化 `K-L` 散度就相当于最大化似然函数。_____no_output_____### 互信息_____no_output_____考虑两个变量 $\mathbf{x,y}$ 的分布，如果它们独立，则 $p(\mathbf{x,y}) = p(\mathbf{x})p(\mathbf{y})$，如果他们不独立，则我们用 $p(\mathbf{x,y})$ 和 $p(\mathbf{x})p(\mathbf{y})$ 的 `K-L` 散度来衡量它们接近独立的程度：

$$
I[\mathbf{x,y}] \equiv KL(p(\mathbf{x,y})||p(\mathbf{x})p(\mathbf{y}))
= - \iint p(\mathbf{x,y}) \ln \frac{p(\mathbf{x})p(\mathbf{y})}{p(\mathbf{x,y})} d\mathbf xd\mathbf y
$$

这叫做 $\mathbf x$ 和 $\mathbf y$ 的互信息（`mutual information`）。从 `K-L` 散度的性质中我们可以看出，$I[\mathbf{x,y}] \geq 0$，当且仅当 $\mathbf{x,y}$ 是独立的时候等号成立。

另外，我们从定义式中可以得到：

$$
I[\mathbf{x,y}] = H[\mathbf x] - H[\mathbf{x|y}] = H[\mathbf y] - H[\mathbf{y|x}]
$$

所以我们可以将互信息看成是给定 $\mathbf y$ 后 $\mathbf x$ 信息量的减少值。_____no_output_____


# Notebook from jckantor/CBE30338-Sp2020
Path: notebooks/06.03-TCLab PCR UI Development.ipynb

# PCR Thermal Cycler UI Development

Information from [ThermoFisher Scientific](https://www.thermofisher.com/us/en/home/life-science/cloning/cloning-learning-center/invitrogen-school-of-molecular-biology/pcr-education/pcr-reagents-enzymes/pcr-cycling-considerations.html)._____no_output_____

%matplotlib inline

import ipywidgets as widgets
from IPython.display import display
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd

def PCR_thermal_cycle(filename):

    filename_text = widgets.Text(value=filename, description="Filename")

    heat_rate_label = widgets.Label('Heat Rate [deg C/sec]', layout=widgets.Layout(width='140px'))
    heat_rate = widgets.FloatSlider(value = 0.5, min=0.1, max=2, step=0.01)

    cool_rate_label = widgets.Label('Cool Rate [deg C/sec]', layout=widgets.Layout(width='140px'))
    cool_rate = widgets.FloatSlider(value = 0.5, min=0.1, max=2, step=0.01)

    pcr_cycles_label = widgets.Label('PCR Cycles', layout=widgets.Layout(width='140px'))
    pcr_cycles = widgets.IntSlider(value=12, min=1, max=30)

    denaturation_label = widgets.Label('Denaturation', layout=widgets.Layout(width='80px'))
    denaturation_temp = widgets.FloatSlider(value=94, min=90, max=98, step=0.1, description='Temperature', readout_format='0.1f')
    denaturation_dur = widgets.BoundedIntText(value=60, min=30, max=120, step=1, description='Duration')
    denaturation = widgets.HBox([denaturation_label, denaturation_temp, denaturation_dur])

    annealing_label = widgets.Label('Annealing', layout=widgets.Layout(width='80px'))
    annealing_temp = widgets.FloatSlider(value=64, min=55, max=70, step=0.1, description='Temperature', readout_format='0.1f')
    annealing_dur = widgets.BoundedIntText(value=60, min=20, max=120, step=1, description='Duration')
    annealing = widgets.HBox([annealing_label, annealing_temp, annealing_dur])

    extension_label = widgets.Label('Extension', layout=widgets.Layout(width='80px'))
    extension_temp = widgets.FloatSlider(value=70, min=68, max=72, step=0.1, description='Temperature', readout_format='0.1f')
    extension_dur = widgets.BoundedIntText(value=20, min=10, max=120, step=1, description='Duration')
    extension = widgets.HBox([extension_label, extension_temp, extension_dur])

    def create_setpoint_trajectory(filename_text, heat_rate, cool_rate, pcr_cycles, denaturation_temp,
            denaturation_dur, annealing_temp, annealing_dur, extension_temp, extension_dur):
        Ta = 22
        tp = [0]
        Tp = [Ta]

        for k in range(0, pcr_cycles):
            # denaturation
            tp.append(tp[-1] + (denaturation_temp - Tp[-1]) / heat_rate)
            Tp.append(denaturation_temp)
            tp.append(tp[-1] + denaturation_dur)
            Tp.append(denaturation_temp)

            # annealing
            tp.append(tp[-1] - (annealing_temp - denaturation_temp) / cool_rate)
            Tp.append(annealing_temp)
            tp.append(tp[-1] + annealing_dur)
            Tp.append(annealing_temp)

            # extension
            tp.append(tp[-1] + (extension_temp - annealing_temp) / heat_rate)
            Tp.append(extension_temp)
            tp.append(tp[-1] + extension_dur)
            Tp.append(extension_temp)

        tp.append(tp[-1] - (Ta - Tp[-1]) / cool_rate)
        Tp.append(Ta)
        fig, ax = plt.subplots(1, 1, figsize=(0.75 + max(tp)/600, 4))
        ax.plot(tp, Tp)
        ax.set_xlabel('time / sec')
        ax.set_ylabel('temperature / deg C')
        ax.set_title('PCR Temperature Program')
        ax.plot(tp, Tp, 'r.', ms=3)
        ax.grid(True)
        df = pd.DataFrame()
        df['tp'] = tp
        df['Tp'] = Tp
        df.to_csv(filename_text)

    display(filename_text)
    display(widgets.HBox([heat_rate_label, heat_rate]))
    display(widgets.HBox([cool_rate_label, cool_rate]))
    display(widgets.HBox([pcr_cycles_label, pcr_cycles]))
    display(denaturation)
    display(annealing)
    display(extension)

    out = widgets.interactive_output(create_setpoint_trajectory, {
        'filename_text': filename_text,
        'heat_rate': heat_rate,
        'cool_rate': cool_rate,
        'pcr_cycles': pcr_cycles,
        'denaturation_temp': denaturation_temp,
        'denaturation_dur': denaturation_dur,
        'annealing_temp': annealing_temp,
        'annealing_dur': annealing_dur,
        'extension_temp': extension_temp,
        'extension_dur': extension_dur,
        })

    display(out)_____no_output_____

## PCR Thermal Profile for Heater 1_____no_output_____

PCR_thermal_cycle('pcr/pcr1.csv')_____no_output_____

## PCR Thermal Profile for Heater 2_____no_output_____

PCR_thermal_cycle('pcr/pcr2.csv')_____no_output_____

## Parallel Operation_____no_output_____

import glob
csv_files = glob.glob("pcr/*.csv")

def get_data(file1, file2):
    fig, ax = plt.subplots(1,2, figsize=(15,5))
    if file1:
        df1 = pd.read_csv(file1)
        ax[0].plot(df1['tp'], df1['Tp'], 'b')
    if file2:
        df2 = pd.read_csv(file2)
        ax[0].plot(df2['tp'], df2['Tp'], 'r')
    ax[0].grid(True)
    if file1 and file2:
        tmax = max(df1['tp'].max(), df2['tp'].max())
        t = np.linspace(0, tmax, int(tmax+1))
        u1 = np.interp(t, df1['tp'].to_numpy(), df1['Tp'].to_numpy())
        u2 = np.interp(t, df2['tp'].to_numpy(), df2['Tp'].to_numpy())
        ax[1].plot(u1, u2)
        ax[1].axis('square')

widgets.interact(get_data,
    file1 = widgets.Combobox(value="pcr/pcr1.csv", options=csv_files, ensure_option=True),
    file2 =  widgets.Combobox(value="pcr/pcr2.csv", options=csv_files, ensure_option=True)
)_____no_output_____


# Notebook from bonilab/malariaibm-interrupted-feeding-and-recombination
Path: iif_analysis/3Fig-triple_double_trend_across_therapy-0p05.ipynb

# Fig.2 - Evolution of MDR Types (w/ 5x3 subpanels)_____no_output_____Change plot's default size and font_____no_output_____

import matplotlib.pyplot as plt
plt.rcParams['figure.figsize'] = [20, 12]
rc = {"font.family" : "sans-serif",
      "font.style" : "normal",
      "mathtext.fontset" : "dejavusans"}
plt.rcParams.update(rc)
plt.rcParams["font.sans-serif"] = ["Myriad Pro"] + plt.rcParams["font.sans-serif"]_____no_output_____

You need to define these variables each run to reflect certain drug coverage settings._____no_output_____

file_path_cyc = 'raw_data/0p05/set7_c/monthly/set7cf_%smonthly_data_0.txt'
file_path_mft = 'raw_data/0p05/set7_m/monthly/set7mf_%smonthly_data_0.txt'
file_path_adpcyc = 'raw_data/0p05/set7_ac/monthly/set7acf_%smonthly_data_0.txt'
plot_savepath = ''_____no_output_____IQR_only=False_____no_output_____

Load Computed NTF IQR Data_____no_output_____

# Compute Median Value of NTF
from plot_helper import NTF_IQR_compute
MFT_ntf = NTF_IQR_compute('raw_data/0p05/set7_m/summary/set7mf_%ssummary_0.txt')
Cyc_ntf = NTF_IQR_compute('raw_data/0p05/set7_c/summary/set7cf_%ssummary_0.txt')
AC_ntf = NTF_IQR_compute('raw_data/0p05/set7_ac/summary/set7acf_%ssummary_0.txt')_____no_output_____

Load output files to prepare `dflist`_____no_output_____

from constant import REPORTDAYS, HEADER_NAME, COLUMNS_TO_DROP
import pandas as pd_____no_output_____dflist_cyc = []
dflist_mft = []
dflist_adpcyc = []
for i in range(1,101):
  dflist_cyc.append(pd.read_csv(file_path_cyc % i, index_col=False, \
                                names=HEADER_NAME, sep='\t').drop(columns=COLUMNS_TO_DROP))
  dflist_mft.append(pd.read_csv(file_path_mft % i, index_col=False, \
                                names=HEADER_NAME, sep='\t').drop(columns=COLUMNS_TO_DROP))
  dflist_adpcyc.append(pd.read_csv(file_path_adpcyc % i, index_col=False, \
                                names=HEADER_NAME, sep='\t').drop(columns=COLUMNS_TO_DROP))_____no_output_____

Test plot to get y-limit of each row (NO ANNOTATION)_____no_output_____

# prepare plt for 5x3 subpanels
from plot_helper import xaxis_label_ticker
import matplotlib.ticker as ticker
xlocator = 5*365
ticks_x = xaxis_label_ticker()

fig, axs = plt.subplots(5, 3, sharex='col', sharey='row',
                        gridspec_kw={'hspace': 0, 'wspace': 0})
fig.patch.set_facecolor('white')
fig.suptitle('Evolution of Multiple-Drug-Resistant Types', y=0.92, fontweight='bold')
(ax1, ax2, ax3), (ax4, ax5, ax6), (ax7, ax8, ax9), \
      (ax10, ax11, ax12), (ax13, ax14, ax15) = axs

# plot trends for most-dangerous-triple (2 types)
from plotter import fig2_dangerous_triple, \
                    fig2_dangerous_double

#  MFT
fig2_dangerous_triple(ax1, dflist_mft, 'TYY..Y2.', IQR_only)
fig2_dangerous_triple(ax4, dflist_mft, 'KNF..Y2.', IQR_only)
fig2_dangerous_double(ax7, dflist_mft, 'DHA-PPQ', IQR_only)
fig2_dangerous_double(ax10, dflist_mft, 'ASAQ', IQR_only)
fig2_dangerous_double(ax13, dflist_mft, 'AL', IQR_only)

#  Cycling
fig2_dangerous_triple(ax2, dflist_cyc, 'TYY..Y2.', IQR_only)
fig2_dangerous_triple(ax5, dflist_cyc, 'KNF..Y2.', IQR_only)
fig2_dangerous_double(ax8, dflist_cyc, 'DHA-PPQ', IQR_only)
fig2_dangerous_double(ax11, dflist_cyc, 'ASAQ', IQR_only)
fig2_dangerous_double(ax14, dflist_cyc, 'AL', IQR_only)

#  Adaptive Cycling
fig2_dangerous_triple(ax3, dflist_adpcyc, 'TYY..Y2.', IQR_only)
fig2_dangerous_triple(ax6, dflist_adpcyc, 'KNF..Y2.', IQR_only)
fig2_dangerous_double(ax9, dflist_adpcyc, 'DHA-PPQ', IQR_only)
fig2_dangerous_double(ax12, dflist_adpcyc, 'ASAQ', IQR_only)
fig2_dangerous_double(ax15, dflist_adpcyc, 'AL', IQR_only)

# Labeling Strategies
ax1.set_title('MFT')
ax2.set_title('5-Year Cycling')
ax3.set_title('Adaptive Cycling')

ax13.xaxis.set_major_locator(ticker.MultipleLocator(xlocator))
ax13.xaxis.set_major_formatter(ticks_x)
ax14.xaxis.set_major_locator(ticker.MultipleLocator(xlocator))
ax14.xaxis.set_major_formatter(ticks_x)
ax15.xaxis.set_major_locator(ticker.MultipleLocator(xlocator))
ax15.xaxis.set_major_formatter(ticks_x)

ax13.set_xlabel('Year')
ax14.set_xlabel('Year')
ax15.set_xlabel('Year')

(_, row1ylim) = ax1.get_ylim()
(_, row2ylim) = ax4.get_ylim()
(_, row3ylim) = ax7.get_ylim()
(_, row4ylim) = ax10.get_ylim()
(_, row5ylim) = ax13.get_ylim()_____no_output_____

Final Version of Plot_____no_output_____

#upper_row_lim = max(row1ylim, row2ylim)
lower_row_lim = max(row3ylim, row4ylim, row5ylim)
TITLE_FONTSIZE = 20
XLABEL_FONTSIZE = 20
YLABEL_PADDING = 290
lower_row_lowlim = 0 - lower_row_lim * 0.05_____no_output_____fig, axs = plt.subplots(5, 3, sharex='col', sharey='row',
                        gridspec_kw={'hspace': 0, 'wspace': 0})
fig.patch.set_facecolor('white')

fig.suptitle('Evolution of Multiple-Drug-Resistant Types', y=0.95, fontweight='bold', fontsize=TITLE_FONTSIZE)
(ax1, ax2, ax3), (ax4, ax5, ax6), (ax7, ax8, ax9), \
      (ax10, ax11, ax12), (ax13, ax14, ax15) = axs

ax7.set_ylim(lower_row_lowlim, lower_row_lim)
ax8.set_ylim(lower_row_lowlim, lower_row_lim)
ax9.set_ylim(lower_row_lowlim, lower_row_lim)
ax10.set_ylim(lower_row_lowlim, lower_row_lim)
ax11.set_ylim(lower_row_lowlim, lower_row_lim)
ax12.set_ylim(lower_row_lowlim, lower_row_lim)
ax13.set_ylim(lower_row_lowlim, lower_row_lim)
ax14.set_ylim(lower_row_lowlim, lower_row_lim)
ax15.set_ylim(lower_row_lowlim, lower_row_lim)

# plot trends for most-dangerous-triple (2 types combined)
from plotter import fig2_dangerous_triple, \
                    fig2_dangerous_double

#  MFT
fig2_dangerous_triple(ax1, dflist_mft, 'TYY..Y2.', IQR_only, annoty=row1ylim, ntf=MFT_ntf)
fig2_dangerous_triple(ax4, dflist_mft, 'KNF..Y2.', IQR_only, annoty=row2ylim)
fig2_dangerous_double(ax7, dflist_mft, 'DHA-PPQ', IQR_only, annoty=lower_row_lim)
fig2_dangerous_double(ax10, dflist_mft, 'ASAQ', IQR_only, annoty=lower_row_lim)
fig2_dangerous_double(ax13, dflist_mft, 'AL', IQR_only, annoty=lower_row_lim)

#  Cycling
fig2_dangerous_triple(ax2, dflist_cyc, 'TYY..Y2.', IQR_only, annoty=row1ylim, ntf=Cyc_ntf)
fig2_dangerous_triple(ax5, dflist_cyc, 'KNF..Y2.', IQR_only, annoty=row2ylim)
fig2_dangerous_double(ax8, dflist_cyc, 'DHA-PPQ', IQR_only, annoty=lower_row_lim)
fig2_dangerous_double(ax11, dflist_cyc, 'ASAQ', IQR_only, annoty=lower_row_lim)
fig2_dangerous_double(ax14, dflist_cyc, 'AL', IQR_only, annoty=lower_row_lim)

#  Adaptive Cycling
fig2_dangerous_triple(ax3, dflist_adpcyc, 'TYY..Y2.', IQR_only, annoty=row1ylim, ntf=AC_ntf)
fig2_dangerous_triple(ax6, dflist_adpcyc, 'KNF..Y2.', IQR_only, annoty=row2ylim)
fig2_dangerous_double(ax9, dflist_adpcyc, 'DHA-PPQ', IQR_only, annoty=lower_row_lim)
fig2_dangerous_double(ax12, dflist_adpcyc, 'ASAQ', IQR_only, annoty=lower_row_lim)
fig2_dangerous_double(ax15, dflist_adpcyc, 'AL', IQR_only, annoty=lower_row_lim)

# Labeling Strategies
ax1.set_title('MFT', fontsize=TITLE_FONTSIZE)
ax2.set_title('5-Year Cycling', fontsize=TITLE_FONTSIZE)
ax3.set_title('Adaptive Cycling', fontsize=TITLE_FONTSIZE)
ax13.set_xlabel('Year', fontsize=TITLE_FONTSIZE)
ax14.set_xlabel('Year', fontsize=TITLE_FONTSIZE)
ax15.set_xlabel('Year', fontsize=TITLE_FONTSIZE)

ax13.xaxis.set_major_locator(ticker.MultipleLocator(xlocator))
ax13.xaxis.set_major_formatter(ticks_x)
ax14.xaxis.set_major_locator(ticker.MultipleLocator(xlocator))
ax14.xaxis.set_major_formatter(ticks_x)
ax15.xaxis.set_major_locator(ticker.MultipleLocator(xlocator))
ax15.xaxis.set_major_formatter(ticks_x)

# ax1.set_ylabel('Triple-resistant \ngenotypes carrying \n76T, 86Y, Y184, 580Y, \nand double-copy $pfpm2/3$',
#                multialignment='left',
#                horizontalalignment='left', rotation=0,
#                fontsize=XLABEL_FONTSIZE, labelpad=YLABEL_PADDING)
# ax4.set_ylabel('Triple-resistant \ngenotypes carrying \nK76, N86, 184F, 580Y, \nand double-copy $pfpm2/3$',
#                multialignment='left',
#                horizontalalignment='left', rotation=0,
#                fontsize=XLABEL_FONTSIZE, labelpad=YLABEL_PADDING)
# ax7.set_ylabel('Double-resistant genotypes \nto DHA-PPQ', multialignment='left',
#                horizontalalignment='left', rotation=0,
#                fontsize=XLABEL_FONTSIZE, labelpad=YLABEL_PADDING)
# ax10.set_ylabel('Double-resistant genotypes \nto ASAQ', multialignment='left',
#                 horizontalalignment='left', rotation=0,
#                 fontsize=XLABEL_FONTSIZE, labelpad=YLABEL_PADDING)
# ax13.set_ylabel('Double-resistant genotypes \nto AL', multialignment='left',
#                 horizontalalignment='left', rotation=0,
#                 fontsize=XLABEL_FONTSIZE, labelpad=YLABEL_PADDING)

fig.add_subplot(111, frameon=False)
# hide tick and tick label of the big axis
plt.tick_params(labelcolor='none', top=False, bottom=False, left=False, right=False)
# add common x- and y-labels
plt.ylabel('Genotype Frequency', fontsize=XLABEL_FONTSIZE)

if plot_savepath != '':
  plt.savefig(fname=plot_savepath, format='svg')_____no_output_____


# Notebook from desi-bgs/bgs-cmxsv
Path: doc/nb/blanc_nexp1_zsuccess_revisited.ipynb

!pip install git+https://github.com/desi-bgs/bgs-cmxsv.git --upgrade --userCollecting git+https://github.com/desi-bgs/bgs-cmxsv.git
  Cloning https://github.com/desi-bgs/bgs-cmxsv.git to /tmp/pip-req-build-037e_1gk
  Running command git clone -q https://github.com/desi-bgs/bgs-cmxsv.git /tmp/pip-req-build-037e_1gk
Requirement already satisfied, skipping upgrade: numpy in /global/common/software/desi/cori/desiconda/20200801-1.4.0-spec/conda/lib/python3.8/site-packages (from bgs-sv==0.0) (1.19.1)
Requirement already satisfied, skipping upgrade: matplotlib in /global/common/software/desi/cori/desiconda/20200801-1.4.0-spec/conda/lib/python3.8/site-packages (from bgs-sv==0.0) (3.2.1)
Requirement already satisfied, skipping upgrade: scipy in /global/common/software/desi/cori/desiconda/20200801-1.4.0-spec/conda/lib/python3.8/site-packages (from bgs-sv==0.0) (1.5.0)
Requirement already satisfied, skipping upgrade: fitsio in /global/common/software/desi/cori/desiconda/20200801-1.4.0-spec/conda/lib/python3.8/site-packages (from bgs-sv==0.0) (1.1.2)
Requirement already satisfied, skipping upgrade: python-dateutil>=2.1 in /global/common/software/desi/cori/desiconda/20200801-1.4.0-spec/conda/lib/python3.8/site-packages (from matplotlib->bgs-sv==0.0) (2.8.1)
Requirement already satisfied, skipping upgrade: cycler>=0.10 in /global/common/software/desi/cori/desiconda/20200801-1.4.0-spec/conda/lib/python3.8/site-packages (from matplotlib->bgs-sv==0.0) (0.10.0)
Requirement already satisfied, skipping upgrade: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /global/common/software/desi/cori/desiconda/20200801-1.4.0-spec/conda/lib/python3.8/site-packages (from matplotlib->bgs-sv==0.0) (2.4.7)
Requirement already satisfied, skipping upgrade: kiwisolver>=1.0.1 in /global/common/software/desi/cori/desiconda/20200801-1.4.0-spec/conda/lib/python3.8/site-packages (from matplotlib->bgs-sv==0.0) (1.2.0)
Requirement already satisfied, skipping upgrade: six>=1.5 in /global/common/software/desi/cori/desiconda/20200801-1.4.0-spec/conda/lib/python3.8/site-packages (from python-dateutil>=2.1->matplotlib->bgs-sv==0.0) (1.14.0)
Building wheels for collected packages: bgs-sv
  Building wheel for bgs-sv (setup.py) ... done
  Created wheel for bgs-sv: filename=bgs_sv-0.0-py3-none-any.whl size=1087771 sha256=105fe2720af22f96a8e7fc2210806be7e6cb7e6622aaae17fe693f6a43e4bd3c
  Stored in directory: /tmp/pip-ephem-wheel-cache-z6845bqw/wheels/c5/7b/33/3fa8f7338087a37e7ad7a1378e08d97458e5dbab577a10268f
Successfully built bgs-sv
Installing collected packages: bgs-sv
  Attempting uninstall: bgs-sv
    Found existing installation: bgs-sv 0.0
    Uninstalling bgs-sv-0.0:
      Successfully uninstalled bgs-sv-0.0
Successfully installed bgs-sv-0.0
import numpy as np
import astropy.table as atable
import matplotlib.pyplot as plt

from bgs_sv import sv1_____no_output_____

# read single exposures from Blanc reduction with similar to nominal BGS conditions_____no_output_____

exps = sv1.blanc_nexp1_exposures()_____no_output_____# lets only keep the nightly exposures where we have a corresponding
# deep exposure that we can use as the truth table
deep_exp = sv1.blanc_deep_exposures()

keep = np.zeros(len(exps)).astype(bool)
for i in range(len(exps)):
    if exps[i]['TILEID'] in deep_exp['TILEID']:
        keep[i] = True
print('%i exposures with corresponding deep exposures' % np.sum(keep))175 exposures with corresponding deep exposures
r_depth_ebvair = 180
d_r_depth_ebvair = 20
close_to_nominal = np.abs(exps['R_DEPTH_EBVAIR'] - r_depth_ebvair) < d_r_depth_ebvair
isbgs = (exps['TARGETS'] == 'BGS+MWS')

bgs_exps = exps[isbgs & keep & close_to_nominal]
print(bgs_exps['TILEID', 'NIGHT', 'EXPID'])
print('%i exposures that have deep and are close to nominal' % len(bgs_exps))TILEID  NIGHT   EXPID
------ -------- -----
 80614 20201218 68686
 80616 20201219 68790
 80617 20201220 69024
 80617 20201220 69026
 80612 20201221 69223
 80613 20201221 69225
 80613 20201221 69227
 80613 20201221 69228
 80613 20201221 69230
9 exposures that have deep and are close to nominal

Now lets get sky surface brightness measurements for these exposures_____no_output_____

# get sky surface brightnesses
Iskies = []
for i in np.arange(len(bgs_exps)):
    wave, Isky = sv1.get_obs_sky(int(bgs_exps[i]['NIGHT']), int(bgs_exps[i]['EXPID']), 'model', redux="daily");
    Iskies.append(Isky * 1e17)_____no_output_____# read in BGS reference sky brightness
f_reference = '/global/cfs/cdirs/desi/users/chahah/BGS_reference_sky_brightness.20210210.npy'
w_ref, Isky_ref = np.load(f_reference)_____no_output_____fig = plt.figure(figsize=(10,5))
sub = fig.add_subplot(111)

for Isky in Iskies:
    sub.plot(wave, Isky)

sub.plot(w_ref, Isky_ref, c='k', ls='--', lw=1, label='BGS reference sky')
sub.set_xlabel('wavelength', fontsize=25)
sub.set_xlim(3e3, 1e4)
sub.set_ylabel('sky brightness [$erg/s/cm^2/A/arcsec^2$]', fontsize=25)
sub.set_ylim(0., 10.)
sub.set_title('BGS ref. sky vs Blanc skies', fontsize=25)_____no_output_____

Blanc BGS exposures with R_DEPTH_EBVAIR ~ 180, have darker skies than the reference BGS sky... _____no_output_____# caclulate $z$ success rate for nexp=1 expsoures using the deep exposures as the truth table_____no_output_____

def get_exp_zsuccess(tileid, night, expid):
    ''' get redshift success rate for given exposure
    '''
    # get redrock output for DEEP exposure for BGS_ANY targets
    _zbest_deep = sv1.get_zbest(tileid, 'deep', targetclass='all')

    crit_zwarn = (_zbest_deep['ZWARN'] == 0)
    crit_dchi2 = (_zbest_deep['DELTACHI2']  > 40.)
    crit_stype = (_zbest_deep['SPECTYPE'] != "STAR") # only galaxy spectra
    crit_z_lim = (_zbest_deep['Z'] > 0.0) & (_zbest_deep['Z'] < 0.6) # rough BGS redshift limit
    crit_z_err = (_zbest_deep['ZERR'] < (0.0005 * (1. + _zbest_deep['Z'])))

    truth_table = crit_dchi2 & crit_stype & crit_z_lim & crit_z_err
    zbest_deep = _zbest_deep[truth_table]['TARGETID', 'Z']
    # redrock redshifts from DEEP exposure will be used as true redshifts
    zbest_deep.rename_column('Z', 'Z_TRUE')

    # get redrock output for SINGLE exposure for BGS_ANY targets
    zbest_exp = sv1.get_zbest(tileid, night, expid=expid, targetclass='all')

    # only keep targets that are in the deep
    zbest_exp = atable.join(zbest_deep, zbest_exp, keys='TARGETID', join_type='inner')

    crit_zwarn = (zbest_exp['ZWARN'] == 0)
    crit_dchi2 = (zbest_exp['DELTACHI2']  > 40.)
    crit_stype = (zbest_exp['SPECTYPE'] != "STAR") # only galaxy spectra
    crit_z_lim = (zbest_exp['Z'] > 0.0) & (zbest_exp['Z'] < 0.6) # rough BGS redshift limit
    crit_z_err = (zbest_exp['ZERR'] < (0.0005 * (1. + zbest_exp['Z'])))

    dz_1pz = np.abs(zbest_exp['Z_TRUE'] - zbest_exp['Z']) / (1. + zbest_exp['Z_TRUE'])
    crit_ztrue = (dz_1pz < 0.003)

    # combine all criteria
    zsuccess = crit_zwarn & crit_dchi2 & crit_stype & crit_z_lim & crit_z_err & crit_ztrue

    r_mag = 22.5 - 2.5 * np.log10(zbest_exp['FLUX_R'])
    return r_mag, zsuccess_____no_output_____r_mags, zs_exps = [], []
for tid, n, eid in zip(bgs_exps['TILEID'], bgs_exps['NIGHT'], bgs_exps['EXPID']):
    r_mag, zs_exp = get_exp_zsuccess(tid, n, eid)
    r_mags.append(r_mag)
    zs_exps.append(zs_exp)_____no_output_____# first lets install the python package `feasibgs`, a python package for the BGS spectral simulations
!pip install git+https://github.com/desi-bgs/feasiBGS.git --upgrade --userCollecting git+https://github.com/desi-bgs/feasiBGS.git
  Cloning https://github.com/desi-bgs/feasiBGS.git to /tmp/pip-req-build-y9lckeql
  Running command git clone -q https://github.com/desi-bgs/feasiBGS.git /tmp/pip-req-build-y9lckeql
Building wheels for collected packages: feasibgs
  Building wheel for feasibgs (setup.py) ... done
  Created wheel for feasibgs: filename=feasibgs-0.1-py3-none-any.whl size=47422 sha256=5f002cbbf25788cf1e19e33b38770b2a15e840715d1e243333fd51644db6108e
  Stored in directory: /tmp/pip-ephem-wheel-cache-f42ewzyk/wheels/f7/d7/9d/824c3b88b5bd8c28b8c576f20984e3cf31a80def7e3deae10a
Successfully built feasibgs
Installing collected packages: feasibgs
  Attempting uninstall: feasibgs
    Found existing installation: feasibgs 0.1
    Uninstalling feasibgs-0.1:
      Successfully uninstalled feasibgs-0.1
Successfully installed feasibgs-0.1
from feasibgs import spectral_sims as BGS_spec_sim
from feasibgs import util as UT
import fitsio_____no_output_____# read in source wavelength, flux, and galaxy properties.
wave_s, flux_s, prop = BGS_spec_sim.simulated_GAMA_source_spectra(emlines=True)

# extract true redshift and r-band magnitude for the simulated galaxies
ztrue_sim = prop['zred'] # redshift
r_mag_sim = prop['r_mag'] # Legacy Survey r mag

# read redrock output
frr = '/global/cfs/cdirs/desi/users/chahah/redrock.bgs_spectral_sim.new_bgs_ref_sky.texp300s.fits'
rr = fitsio.read(frr)

# redshift success defined as |z_redrock - z_true|/(1+z_true) < 0.003 and ZWARN flag = 0
zs_ref = UT.zsuccess(rr['Z'], ztrue_sim, rr['ZWARN'], deltachi2=rr['DELTACHI2'], min_deltachi2=40)
#zs_ref[r_mag_sim < 18.2] = True # this is to account for the unusual bright z failures, which we think will eventually be ironed out_____no_output_____fig = plt.figure(figsize=(8,6))
sub = fig.add_subplot(111)

for r_mag, zs_exp in zip(r_mags, zs_exps):
    r_mid, zs, zs_err = UT.zsuccess_rate(r_mag, zs_exp, range=(14, 22), nbins=20)
    sub.plot(r_mid, zs, alpha=0.5)
r_mid, zs, zs_err = UT.zsuccess_rate(r_mag_sim, zs_ref, range=(14, 22), nbins=20)
sub.plot(r_mid, zs, c='k', ls='--', label='BGS spectral sim w/ ref. sky 300s')
sub.legend(loc='lower left', fontsize=20)
sub.axvline(19.5, c='r', ls='--')
sub.set_xlabel('$r$ mag', fontsize=25)
sub.set_xlim(15, 20)
sub.set_ylabel('$z$ success rate', fontsize=25)_____no_output_____zs_19p5_ref = np.sum(zs_ref[r_mag_sim <= 19.5])/np.sum(r_mag_sim <= 19.5)
zs_19p5, zs_20p0 = [], []
for r_mag, zs_exp in zip(r_mags, zs_exps):
    zs_19p5.append(np.sum(zs_exp[r_mag <= 19.5])/np.sum(r_mag <= 19.5))
    zs_20p0.append(np.sum(zs_exp[r_mag <= 20.0])/np.sum(r_mag <= 20.0))_____no_output_____print('BGS spectral sim w/ reference')
print('redshift success rate r < 19.5 = %.3f' % zs_19p5_ref)
print('Blanc BGS exposures')
print('redshift success rate r < 19.5 = %.3f' % np.median(zs_19p5))
print('redshift success rate r < 20.0 = %.3f' % np.median(zs_20p0))BGS spectral sim w/ reference
redshift success rate r < 19.5 = 0.964
Blanc BGS exposures
redshift success rate r < 19.5 = 0.967
redshift success rate r < 20.0 = 0.934

The spectral simulations are able to closely reproduce the SV1 exposures that are close to BGS. _____no_output_____


# Notebook from edur409/ASTROBIOLOGY200
Path: content/01/3/.ipynb_checkpoints/from_ptolemy_to_kepler-checkpoint.ipynb

from IPython.display import HTML

HTML('''
The raw code for this IPython notebook is by default hidden for easier reading.
To toggle on/off the raw code, click here.''')_____no_output_____

# From Ptolemy to Kepler

Ptolemy, Copernicus, Brahe, Kepler

_____no_output_____
# Cosmic Calculations 2.1: Kepler’s Third Law

First, let's review the three laws discovered by Kepler from the careful measurements of [Tycho Brache](https://physicsworld.com/a/kepler-and-tycho-brahe-the-odd-couple/).  This is one of the most ineteresting stories of scientific collaboration that transformed years of observations into laws about the universe.  I recommend to read/watch [***The Character of Physical Law***](https://www.youtube.com/watch?v=j3mhkYbznBk) by Richard Feynman if you want to indulge in the details.

By Both Előd at Hungarian Wikipedia, CC BY-SA 2.5, Link

## Kepler's laws:

***1. The orbit of every planet is an ellipse with the Sun at one of the two foci.***

In the figure below, you can imagine the yellow dot as the sun, and a planet would be moving on the blue curve a certain distance away from it.  The elliptical orbit can be described by the semi-major and semi-minor axes, which define the eccentricity of the orbit.

Look for the *perihelion* and *aphelion* of Earth's orbit. From those values, what's the flattening of Earth's orbit and its eccentricity?_____no_output_____

import numpy as np
import matplotlib.pyplot as plt
from __future__ import print_function
from ipywidgets import interact, interactive, fixed, interact_manual
import ipywidgets as widgets

# Set default font size for plots:
font = {'size'   : 18}
plt.rc('font',**font)

def elliptic_orbit(a,b,t):
    '''Plot an elliptical orbit and see the radial distance from one focal point
    a= semi-major axis
    b=semi-minor axis
    t=location at an angle between 0 and 360'''
    p=np.linspace(0,2*np.pi,360)
    x = a*np.cos(p)
    y = b*np.sin(p)
    plt.figure('Ellipse2',figsize=(10,5))
    plt.plot(x,y,'-')
    plt.axis('equal')
    plt.grid(True)
    #t is the angle varying from 0 to 360 degrees
    X = a*np.cos(t*np.pi/180)
    Y = b*np.sin(t*np.pi/180)
    #Conditionals in case of changing length of largest semi-major axis
    if a>=b:
        c=np.sqrt(a**2-b**2)
        plt.scatter(c,0,s=200,c='y')
        plt.scatter(-c, 0, s=200, facecolors='none', edgecolors='y')
        plt.arrow(c, 0, X-c, Y, head_width=0.1, head_length=0.1, fc='red', ec='red')
        plt.scatter(X,Y,s=50,c='b')
        f=(a-b)/a
        e=c/a
        #print('Orbital flattening : ',f)
        print('Orbital eccentricity : ',e)
        #plt.show()
    else:
        c=np.sqrt(b**2-a**2)
        plt.scatter(0,c,s=200,c='y')
        plt.scatter(0, -c, s=200, facecolors='none', edgecolors='y')
        plt.arrow(0, c, X, Y-c, head_width=0.1, head_length=0.1, fc='red', ec='red')
        plt.scatter(X,Y,s=50,c='b')
        f=(b-a)/b
        e=c/b
        #print('Orbital flattening : ',f)
        print('Orbital eccentricity : ',e)
    plt.show()
    return

interactive(elliptic_orbit, a = (0,20,1),b=(0,20,1),t=(0,360,20),continuous_update=False)
        _____no_output_____

The reason the orbits are ellipses, and not circles, would not be understood until the arrival of Newton's equations on Gravitation.

***2. A line joining a planet and the Sun sweeps out equal areas during equal intervals of time.***

By Gonfer (talk) - Gonfer, CC BY-SA 3.0, Link

When Kepler discovered his third law ($p^2 = a^3$), he knew only that it applied to the orbits of planets about the Sun. In fact, it applies to any orbiting object as long as the following two conditions are met:

1. The object orbits the Sun or another star of precisely the same mass.
2. We use units of years for the orbital period and AU for the orbital distance. (Newton extended the law to all orbiting objects; see [Cosmic Calculations 7.1](#CC-7.1).)

In other words, these two conditions make the relationship a perfect equality.

**Example 1:** The largest asteroid, Ceres, orbits the Sun at an average distance (semimajor axis) of 2.77 AU. What is its orbital period?

***Solution:*** Both conditions are met, so we solve Kepler’s third law for the orbital period $p$ and substitute the given orbital distance, $a = 2.77~AU$.

$$p^2 = a^3$$

$$ p = \sqrt{a^3} = \sqrt{2.77^3} \approx 4.6~y$$

Ceres has an orbital period of 4.6 years.

**Example 2:** A planet is discovered orbiting every three months around a star of the same mass as our Sun. What is the planet’s average orbital distance?

***Solution:*** The ﬁrst condition is met, and we can satisfy the second by converting the orbital period from months to years: $p = 3$ months = 0.25 year. We now solve Kepler’s third law for the average distance a:

$a = \sqrt[3]{p^2}$

$a = \sqrt[3]{0.25^2} \approx 0.40~AU$

The planet orbits its star at an average distance of $0.40~AU$, which is nearly the same as Mercury’s average distance from the Sun.

These observations offered clear proof that Earth is not the center of everything.* Although we now recognize that Galileo won the day, the story was more complex in his own time, when Catholic Church doctrine still held Earth to be the center of the universe. On June 22, 1633, Galileo was brought before a Church inquisition in Rome and ordered to recant his claim that Earth orbits the Sun. Nearly 70 years old and fearing for his life, Galileo did as ordered and his life was spared. However, legend has it that as he rose from his knees, he whispered under his breath, Eppur si muove— Italian for “And yet it moves.” (Given the likely consequences if Church ofﬁcials had heard him say this, most historians doubt the legend.) The Church did not formally vindicate Galileo until 1992, but the Church had given up the argument long before that. Today, Catholic scientists are at the forefront of much astronomical research, and ofﬁcial Church teachings are compatible not only with Earth’s planetary status but also with the theories of the Big Bang and the subsequent evolution of the cosmos and of life.

<object classid="clsid:D27CDB6E-AE6D-11cf-96B8-444553540000" width="750" height="400"><param name="movie" value="KeplerFirstLaw.swf" /><object type="application/x-shockwave-flash" data="KeplerFirstLaw.swf" width="750" height="400">
flash animation
</object></object>

[Go back to the top of the page](#top-title)_____no_output_____