# Notebook from ismms-himc/clustergrammer_widget2
Path: examples/10X_Genomics_3K_Dataset_binder.ipynb

# 5.0 10X Genomics PBMC 3K Dataset_____no_output_____

from clustergrammer import *
net = Network()
df = {}

import clustergrammer_widget2
import clustergrammer_groupby as cby
import gene_exp_10x_____no_output_____from sklearn.metrics import f1_score
import pandas as pd
import numpy as np
from copy import deepcopy

import matplotlib.pyplot as plt
%matplotlib inline _____no_output_____

### Load Data_____no_output_____

df = gene_exp_10x.load_gene_exp_to_df('../data/pbmc3k_filtered_gene_bc_matrices/hg19/')
df.shape_____no_output_____

### Remove Ribosomal and Mitochondrial Genes_____no_output_____

all_genes = df.index.tolist()
print(len(all_genes))
keep_genes = [x for x in all_genes if 'RPL' not in x]
keep_genes = [x for x in keep_genes if 'RPS' not in x]
print(len(keep_genes))

df = df.loc[keep_genes]
df.shape

# Removing Mitochondrial Genes
list_mito_genes = ['MTRNR2L11', 'MTRF1', 'MTRNR2L12', 'MTRNR2L13', 'MTRF1L', 'MTRNR2L6', 'MTRNR2L7',
                'MTRNR2L10', 'MTRNR2L8', 'MTRNR2L5', 'MTRNR2L1', 'MTRNR2L3', 'MTRNR2L4']

all_genes = df.index.tolist()
mito_genes = [x for x in all_genes if 'MT-' == x[:3] or
             x.split('_')[0] in list_mito_genes]
print(mito_genes)

keep_genes = [x for x in all_genes if x not in mito_genes]
df = df.loc[keep_genes]

# # normalize by UMI count
# barcode_umi_sum = df['ge'].sum()
# df['ge'] = df['ge'].div(barcode_umi_sum)32738
32546
['MTRNR2L11', 'MTRNR2L12', 'MTRNR2L13', 'MTRF1L', 'MTRNR2L6', 'MTRNR2L10', 'MTRNR2L7', 'MTRNR2L5', 'MTRNR2L8', 'MTRF1', 'MTRNR2L4', 'MTRNR2L1', 'MTRNR2L3', 'MT-ND1', 'MT-ND2', 'MT-CO1', 'MT-CO2', 'MT-ATP8', 'MT-ATP6', 'MT-CO3', 'MT-ND3', 'MT-ND4L', 'MT-ND4', 'MT-ND5', 'MT-ND6', 'MT-CYB']
def calc_mean_var_disp(df_inst):
    mean_arr = []
    var_arr = []
    mean_names = []
    for inst_gene in df_inst.index.tolist():
        mean_arr.append( df_inst.loc[inst_gene].mean() )
        var_arr.append(df_inst.loc[inst_gene].var())
        mean_names.append(inst_gene)

    ser_mean = pd.Series(data=mean_arr, index=mean_names)
    ser_var = pd.Series(data=var_arr, index=mean_names)
    return ser_mean, ser_var_____no_output_____

### Keep top 5000 expressing genes_____no_output_____

ser_mean, ser_var = calc_mean_var_disp(df)

keep_genes = ser_mean.sort_values(ascending=False)[:5000].index.tolist()

df = df.loc[keep_genes]
print(df.shape)(5000, 2700)

### Keep top 1000 Genes by Dispersion and Randomly Sample Cells_____no_output_____

num_genes = 1000
num_cells = 1000

# keep top genes based on dispersion
ser_mean, ser_var = calc_mean_var_disp(df)
ser_disp = ser_var.divide(ser_mean).sort_values(ascending=False)
keep_genes = ser_disp[:num_genes].index.tolist()
df = df.loc[keep_genes]

# randomly sample cells
df = df.sample(axis=1, n=num_cells, random_state=99)_____no_output_____

### Arcsin Transform_____no_output_____

df.shape
df = np.arcsinh(df/5)_____no_output_____

### Z-score and Subset of Data_____no_output_____

net.load_df(df)
net.normalize(axis='row', norm_type='zscore')
df = net.export_df()
df.shape_____no_output_____

# Visualize Data in Clustergrammer-Widget2_____no_output_____

net.load_df(df)
net.clip(lower=-5, upper=5)
net.cluster()
net_json = net.export_viz_to_widget()
w = clustergrammer_widget2.ExampleWidget(network=net_json)
w_____no_output_____

### Load CIBERSORT gene sigantures_____no_output_____

net.load_file('../data/cell_type_signatures/nm3337_broad_cell_type_sigs.txt')
df_sig = net.export_df()
print(df_sig.shape)(523, 9)
net.load_df(df_sig)
net.clip(lower=-5, upper=5)
net.cluster()
net_json = net.export_viz_to_widget()
w = clustergrammer_widget2.ExampleWidget(network=net_json)
w_____no_output_____

# Predict Cell Types using NM3337 Signatures_____no_output_____

rows = df_sig.index.tolist()
new_rows = [x.split('_')[0] for x in rows]
df_sig_clean = deepcopy(df_sig)
df_sig_clean.index = new_rows_____no_output_____df_pred_cat, df_sig_sim, y_info = cby.predict_cats_from_sigs(df, df_sig_clean,
                                                                   predict_level='Cell Type', unknown_thresh=0.05)_____no_output_____

### Cell Clustering Based on CIBERSORT Signature Genes with CIBERSORT Signature Categories_____no_output_____

# net.load_df(df_pred_cat)
# net.clip(lower=-5, upper=5)
# net.cluster()
# net_json = net.export_viz_to_widget()
# w = clustergrammer_widget2.ExampleWidget(network=net_json)
# w_____no_output_____df.columns = df_pred_cat.columns.tolist()_____no_output_____

### Cell Clustering Based on top 1000 Genes by Disperson with CIBERSORT Signature Categories_____no_output_____

net.load_df(df)
net.clip(lower=-5, upper=5)
net.cluster()
net_json = net.export_viz_to_widget()
w = clustergrammer_widget2.ExampleWidget(network=net_json)
w_____no_output_____


# Notebook from deklund76/Amazon_Vine_Analysis
Path: Vine_Review_Analysis.ipynb

<code>
import os
# Find the latest version of spark 3.0 from http://www.apache.org/dist/spark/ and enter as the spark version
# For example:
# spark_version = 'spark-3.0.3'
spark_version = 'spark-3.0.3'
os.environ['SPARK_VERSION']=spark_version

# Install Spark and Java
!apt-get update
!apt-get install openjdk-11-jdk-headless -qq > /dev/null
!wget -q http://www.apache.org/dist/spark/$SPARK_VERSION/$SPARK_VERSION-bin-hadoop2.7.tgz
!tar xf $SPARK_VERSION-bin-hadoop2.7.tgz
!pip install -q findspark

# Set Environment Variables
import os
os.environ["JAVA_HOME"] = "/usr/lib/jvm/java-11-openjdk-amd64"
os.environ["SPARK_HOME"] = f"/content/{spark_version}-bin-hadoop2.7"

# Start a SparkSession
import findspark
findspark.init()
0% [Working]

Get:1 http://security.ubuntu.com/ubuntu bionic-security InRelease [88.7 kB]

0% [Connecting to archive.ubuntu.com (91.189.88.152)] [1 InRelease 0 B/88.7 kB
0% [Waiting for headers] [Connected to cloud.r-project.org (65.8.185.60)] [Wait
0% [1 InRelease gpgv 88.7 kB] [Waiting for headers] [Connected to cloud.r-proje

Get:2 https://cloud.r-project.org/bin/linux/ubuntu bionic-cran40/ InRelease [3,626 B]

0% [1 InRelease gpgv 88.7 kB] [Waiting for headers] [2 InRelease 3,626 B/3,626
0% [1 InRelease gpgv 88.7 kB] [Waiting for headers] [Waiting for headers] [Wait

Ign:3 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  InRelease
Hit:4 http://ppa.launchpad.net/c2d4u.team/c2d4u4.0+/ubuntu bionic InRelease
Hit:5 http://archive.ubuntu.com/ubuntu bionic InRelease
Get:6 http://archive.ubuntu.com/ubuntu bionic-updates InRelease [88.7 kB]
Hit:7 http://ppa.launchpad.net/cran/libgit2/ubuntu bionic InRelease
Ign:8 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64  InRelease
Hit:9 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  Release
Hit:10 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64  Release
Hit:11 http://ppa.launchpad.net/deadsnakes/ppa/ubuntu bionic InRelease
Get:12 http://security.ubuntu.com/ubuntu bionic-security/universe amd64 Packages [1,431 kB]
Get:13 http://archive.ubuntu.com/ubuntu bionic-backports InRelease [74.6 kB]
Hit:14 http://ppa.launchpad.net/graphics-drivers/ppa/ubuntu bionic InRelease
Get:17 http://archive.ubuntu.com/ubuntu bionic-updates/restricted amd64 Packages [640 kB]
Get:18 http://archive.ubuntu.com/ubuntu bionic-updates/universe amd64 Packages [2,210 kB]
Get:19 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 Packages [2,803 kB]
Fetched 7,341 kB in 4s (1,678 kB/s)
Reading package lists... Done
# Start Spark session
from pyspark.sql import SparkSession
spark = SparkSession.builder.appName("Vine_Review_Analysis").getOrCreate()_____no_output_____from pyspark import SparkFiles
url = "https://s3.amazonaws.com/amazon-reviews-pds/tsv/amazon_reviews_us_Video_Games_v1_00.tsv.gz"
spark.sparkContext.addFile(url)
df = spark.read.option("encoding", "UTF-8").csv(SparkFiles.get(""), sep="\t", header=True, inferSchema=True)
df.show()+-----------+-----------+--------------+----------+--------------+--------------------+----------------+-----------+-------------+-----------+----+-----------------+--------------------+--------------------+-----------+
|marketplace|customer_id|     review_id|product_id|product_parent|       product_title|product_category|star_rating|helpful_votes|total_votes|vine|verified_purchase|     review_headline|         review_body|review_date|
+-----------+-----------+--------------+----------+--------------+--------------------+----------------+-----------+-------------+-----------+----+-----------------+--------------------+--------------------+-----------+
|         US|   12039526| RTIS3L2M1F5SM|B001CXYMFS|     737716809|Thrustmaster T-Fl...|     Video Games|          5|            0|          0|   N|                Y|an amazing joysti...|Used this for Eli...| 2015-08-31|
|         US|    9636577| R1ZV7R40OLHKD|B00M920ND6|     569686175|Tonsee 6 buttons ...|     Video Games|          5|            0|          0|   N|                Y|Definitely a sile...|Loved it,  I didn...| 2015-08-31|
|         US|    2331478|R3BH071QLH8QMC|B0029CSOD2|      98937668|Hidden Mysteries:...|     Video Games|          1|            0|          1|   N|                Y|            One Star|poor quality work...| 2015-08-31|
|         US|   52495923|R127K9NTSXA2YH|B00GOOSV98|      23143350|GelTabz Performan...|     Video Games|          3|            0|          0|   N|                Y|good, but could b...|nice, but tend to...| 2015-08-31|
|         US|   14533949|R32ZWUXDJPW27Q|B00Y074JOM|     821342511|Zero Suit Samus a...|     Video Games|          4|            0|          0|   N|                Y|   Great but flawed.|Great amiibo, gre...| 2015-08-31|
|         US|    2377552|R3AQQ4YUKJWBA6|B002UBI6W6|     328764615|Psyclone Recharge...|     Video Games|          1|            0|          0|   N|                Y|            One Star|The remote consta...| 2015-08-31|
|         US|   17521011|R2F0POU5K6F73F|B008XHCLFO|      24234603|Protection for yo...|     Video Games|          5|            0|          0|   N|                Y|              A Must|I have a 2012-201...| 2015-08-31|
|         US|   19676307|R3VNR804HYSMR6|B00BRA9R6A|     682267517|   Nerf 3DS XL Armor|     Video Games|          5|            0|          0|   N|                Y|          Five Stars|Perfect, kids lov...| 2015-08-31|
|         US|     224068| R3GZTM72WA2QH|B009EPWJLA|     435241890|One Piece: Pirate...|     Video Games|          5|            0|          0|   N|                Y|          Five Stars|            Excelent| 2015-08-31|
|         US|   48467989| RNQOY62705W1K|B0000AV7GB|     256572651|Playstation 2 Dan...|     Video Games|          4|            0|          0|   N|                Y|          Four Stars|Slippery but expe...| 2015-08-31|
|         US|     106569|R1VTIA3JTYBY02|B00008KTNN|     384411423|Metal Arms: Glitc...|     Video Games|          5|            0|          0|   N|                N|          Five Stars|Love the game. Se...| 2015-08-31|
|         US|   48269642|R29DOU8791QZL8|B000A3IA0Y|     472622859|72 Pin Connector ...|     Video Games|          1|            0|          0|   N|                Y| Game will get stuck|Does not fit prop...| 2015-08-31|
|         US|   52738710|R15DUT1VIJ9RJZ|B0053BQN34|     577628462|uDraw Gametablet ...|     Video Games|          2|            0|          0|   N|                Y|We have tried it ...|This was way too ...| 2015-08-31|
|         US|   10556786|R3IMF2MQ3OU9ZM|B002I0HIMI|     988218515|NBA 2K12(Covers M...|     Video Games|          4|            0|          0|   N|                Y|          Four Stars|Works great good ...| 2015-08-31|
|         US|    2963837|R23H79DHOZTYAU|B0081EH12M|     770100932|New Trigger Grips...|     Video Games|          1|            1|          1|   N|                Y|Now i have to buy...|It did not fit th...| 2015-08-31|
|         US|   23092109| RIV24EQAIXA4O|B005FMLZQQ|      24647669|Xbox 360 Media Re...|     Video Games|          5|            0|          0|   N|                Y|          Five Stars|perfect lightweig...| 2015-08-31|
|         US|   23091728|R3UCNGYDVN24YB|B002BSA388|      33706205|Super Mario Galaxy 2|     Video Games|          5|            0|          0|   N|                Y|          Five Stars|               great| 2015-08-31|
|         US|   10712640| RUL4H4XTTN2DY|B00BUSLSAC|     829667834|Nintendo 3DS XL -...|     Video Games|          5|            0|          0|   N|                Y|          Five Stars|Works beautifully...| 2015-08-31|
|         US|   17455376|R20JF7Z4DHTNX5|B00KWF38AW|     110680188|Captain Toad:  Tr...|     Video Games|          5|            0|          0|   N|                Y|          Five Stars|Kids loved the ga...| 2015-08-31|
|         US|   14754850|R2T1AJ5MFI2260|B00BRQJYA8|     616463426|Lego Batman 2: DC...|     Video Games|          4|            0|          0|   N|                Y|          Four Stars|           Goodngame| 2015-08-31|
+-----------+-----------+--------------+----------+--------------+--------------------+----------------+-----------+-------------+-----------+----+-----------------+--------------------+--------------------+-----------+
only showing top 20 rows

# Create the vine_table. DataFrame
vine_df = df.select(["review_id","star_rating","helpful_votes","total_votes","vine","verified_purchase"])
vine_df.show()+--------------+-----------+-------------+-----------+----+-----------------+
|     review_id|star_rating|helpful_votes|total_votes|vine|verified_purchase|
+--------------+-----------+-------------+-----------+----+-----------------+
| RTIS3L2M1F5SM|          5|            0|          0|   N|                Y|
| R1ZV7R40OLHKD|          5|            0|          0|   N|                Y|
|R3BH071QLH8QMC|          1|            0|          1|   N|                Y|
|R127K9NTSXA2YH|          3|            0|          0|   N|                Y|
|R32ZWUXDJPW27Q|          4|            0|          0|   N|                Y|
|R3AQQ4YUKJWBA6|          1|            0|          0|   N|                Y|
|R2F0POU5K6F73F|          5|            0|          0|   N|                Y|
|R3VNR804HYSMR6|          5|            0|          0|   N|                Y|
| R3GZTM72WA2QH|          5|            0|          0|   N|                Y|
| RNQOY62705W1K|          4|            0|          0|   N|                Y|
|R1VTIA3JTYBY02|          5|            0|          0|   N|                N|
|R29DOU8791QZL8|          1|            0|          0|   N|                Y|
|R15DUT1VIJ9RJZ|          2|            0|          0|   N|                Y|
|R3IMF2MQ3OU9ZM|          4|            0|          0|   N|                Y|
|R23H79DHOZTYAU|          1|            1|          1|   N|                Y|
| RIV24EQAIXA4O|          5|            0|          0|   N|                Y|
|R3UCNGYDVN24YB|          5|            0|          0|   N|                Y|
| RUL4H4XTTN2DY|          5|            0|          0|   N|                Y|
|R20JF7Z4DHTNX5|          5|            0|          0|   N|                Y|
|R2T1AJ5MFI2260|          4|            0|          0|   N|                Y|
+--------------+-----------+-------------+-----------+----+-----------------+
only showing top 20 rows

voted_df = vine_df.filter("total_votes>=20")
voted_df.show()+--------------+-----------+-------------+-----------+----+-----------------+
|     review_id|star_rating|helpful_votes|total_votes|vine|verified_purchase|
+--------------+-----------+-------------+-----------+----+-----------------+
| R4PKAZRQJJX14|          1|           21|         34|   N|                N|
|R2CI0Y288CC7E2|          1|           21|         35|   N|                Y|
|R127WEQY2FM1T3|          1|          147|        175|   N|                Y|
|R3EZ0EPYLDA34S|          1|           14|         31|   N|                Y|
|R2FJ94555FZH32|          2|           55|         60|   N|                N|
|R1U3AR67RE273L|          1|           51|         65|   N|                Y|
|R3PZOXA5X1U8KW|          4|           31|         36|   N|                N|
| R6KTC1OPIOIIG|          2|           19|         34|   N|                Y|
|R36O341WWXXKNP|          5|           28|         31|   N|                N|
|R3GSK9MM8DNOYI|          1|            4|         32|   N|                N|
|R10LZVBLQHBVJ0|          2|          151|        198|   N|                N|
|R1VR5GLGY1GE7N|          1|           49|         51|   N|                Y|
|R3GUUC6NT4QMCL|          1|            5|         36|   N|                N|
|R1ZSD020RT18I5|          1|            7|        139|   N|                N|
|R113X48B575ASY|          1|           15|         45|   N|                N|
| RJLYLGRYH7SFQ|          1|           34|        116|   N|                N|
|R2OWDAR0EDEPTF|          1|           23|         28|   N|                N|
|R3CICXKAHN6NUD|          1|            0|         23|   N|                Y|
|R29BOS5HMAY1LO|          5|           88|        110|   N|                N|
| R8WD0136CETA7|          2|           99|        119|   N|                N|
+--------------+-----------+-------------+-----------+----+-----------------+
only showing top 20 rows

helpful_df = voted_df.filter("helpful_votes/total_votes>=0.5")
helpful_df.show()+--------------+-----------+-------------+-----------+----+-----------------+
|     review_id|star_rating|helpful_votes|total_votes|vine|verified_purchase|
+--------------+-----------+-------------+-----------+----+-----------------+
| R4PKAZRQJJX14|          1|           21|         34|   N|                N|
|R2CI0Y288CC7E2|          1|           21|         35|   N|                Y|
|R127WEQY2FM1T3|          1|          147|        175|   N|                Y|
|R2FJ94555FZH32|          2|           55|         60|   N|                N|
|R1U3AR67RE273L|          1|           51|         65|   N|                Y|
|R3PZOXA5X1U8KW|          4|           31|         36|   N|                N|
| R6KTC1OPIOIIG|          2|           19|         34|   N|                Y|
|R36O341WWXXKNP|          5|           28|         31|   N|                N|
|R10LZVBLQHBVJ0|          2|          151|        198|   N|                N|
|R1VR5GLGY1GE7N|          1|           49|         51|   N|                Y|
|R2OWDAR0EDEPTF|          1|           23|         28|   N|                N|
|R29BOS5HMAY1LO|          5|           88|        110|   N|                N|
| R8WD0136CETA7|          2|           99|        119|   N|                N|
|R16621F16PQN5A|          3|           24|         31|   N|                Y|
|R1LXBPG1M9DDOC|          1|          102|        106|   N|                N|
| RNEMGSUKJ4535|          2|           21|         25|   N|                Y|
|R3R2UKBFKP1IQ4|          1|           19|         23|   N|                Y|
|R2T90XM5G2DBZA|          3|           15|         22|   N|                Y|
| R5ZMK0Y0KRBRZ|          1|           79|         91|   N|                N|
|R12E2W0XOV3I2S|          1|           24|         30|   N|                Y|
+--------------+-----------+-------------+-----------+----+-----------------+
only showing top 20 rows

paid_df = helpful_df.filter("vine == 'Y'")
paid_df.show()+--------------+-----------+-------------+-----------+----+-----------------+
|     review_id|star_rating|helpful_votes|total_votes|vine|verified_purchase|
+--------------+-----------+-------------+-----------+----+-----------------+
|R3KKUSGFZWSUIY|          5|           56|         63|   Y|                N|
|R10FO5UKKVZBK2|          3|           23|         23|   Y|                N|
| RM4KSGEOR7MU1|          5|           19|         24|   Y|                N|
| RG7VRMYLEXD23|          4|           22|         26|   Y|                N|
|R11O4YSCPSNL6L|          3|           20|         26|   Y|                N|
|R286MFBAJ8NPD6|          5|           46|         51|   Y|                N|
|R1JRR530H4COA2|          5|           22|         28|   Y|                N|
| RQ5WD90PUNBU9|          5|           21|         24|   Y|                N|
|R12648VHCQWUV9|          4|           21|         28|   Y|                N|
|R3KAW29CJ8L6DQ|          5|           17|         20|   Y|                N|
|R1OWK33OPI45KT|          5|           55|         58|   Y|                N|
|R2UP7VTED8O425|          5|           35|         41|   Y|                N|
|R3N4QAXTQCA1ED|          5|           18|         20|   Y|                N|
|R3EPHK0JGQGE91|          5|           21|         28|   Y|                N|
| RCP3KRO47S0GG|          4|           51|         58|   Y|                N|
|R3JT907UUA1WUI|          4|           85|        103|   Y|                N|
| RY0555ILXIYXW|          5|           38|         50|   Y|                N|
|R24PS9N276OXF4|          5|          241|        262|   Y|                N|
|R3R0F8ETAGTIOT|          5|           14|         20|   Y|                N|
| ROXZOMHBY8Y9S|          4|           21|         23|   Y|                N|
+--------------+-----------+-------------+-----------+----+-----------------+
only showing top 20 rows

unpaid_df = helpful_df.filter("vine == 'N'")
unpaid_df.show()+--------------+-----------+-------------+-----------+----+-----------------+
|     review_id|star_rating|helpful_votes|total_votes|vine|verified_purchase|
+--------------+-----------+-------------+-----------+----+-----------------+
| R4PKAZRQJJX14|          1|           21|         34|   N|                N|
|R2CI0Y288CC7E2|          1|           21|         35|   N|                Y|
|R127WEQY2FM1T3|          1|          147|        175|   N|                Y|
|R2FJ94555FZH32|          2|           55|         60|   N|                N|
|R1U3AR67RE273L|          1|           51|         65|   N|                Y|
|R3PZOXA5X1U8KW|          4|           31|         36|   N|                N|
| R6KTC1OPIOIIG|          2|           19|         34|   N|                Y|
|R36O341WWXXKNP|          5|           28|         31|   N|                N|
|R10LZVBLQHBVJ0|          2|          151|        198|   N|                N|
|R1VR5GLGY1GE7N|          1|           49|         51|   N|                Y|
|R2OWDAR0EDEPTF|          1|           23|         28|   N|                N|
|R29BOS5HMAY1LO|          5|           88|        110|   N|                N|
| R8WD0136CETA7|          2|           99|        119|   N|                N|
|R16621F16PQN5A|          3|           24|         31|   N|                Y|
|R1LXBPG1M9DDOC|          1|          102|        106|   N|                N|
| RNEMGSUKJ4535|          2|           21|         25|   N|                Y|
|R3R2UKBFKP1IQ4|          1|           19|         23|   N|                Y|
|R2T90XM5G2DBZA|          3|           15|         22|   N|                Y|
| R5ZMK0Y0KRBRZ|          1|           79|         91|   N|                N|
|R12E2W0XOV3I2S|          1|           24|         30|   N|                Y|
+--------------+-----------+-------------+-----------+----+-----------------+
only showing top 20 rows

# Paid Review Summary
paid_reviews = paid_df.count()
five_star_paid_reviews = paid_df.filter("star_rating == 5").count()
percent_paid_five_stars = five_star_paid_reviews / paid_reviews * 100_____no_output_____# Unpaid Review Summary
unpaid_reviews = unpaid_df.count()
five_star_unpaid_reviews = unpaid_df.filter("star_rating == 5").count()
percent_unpaid_five_stars = five_star_unpaid_reviews / unpaid_reviews * 100_____no_output_____# Create schema and data for summary of analysis
columns = ['review_type', 'reviews', 'five_star_reviews', 'five_star_review_percentage']
data = [('paid', paid_reviews, five_star_paid_reviews, percent_paid_five_stars),
        ('unpaid', unpaid_reviews, five_star_unpaid_reviews, percent_unpaid_five_stars)]

# Create rdd object for summary dataframe
rdd = spark.sparkContext.parallelize(data)

analysis_summary = rdd.toDF(columns)

analysis_summary.show()+-----------+-------+-----------------+---------------------------+
|review_type|reviews|five_star_reviews|five_star_review_percentage|
+-----------+-------+-----------------+---------------------------+
|       paid|     94|               48|          51.06382978723404|
|     unpaid|  40471|            15663|         38.701786464381904|
+-----------+-------+-----------------+---------------------------+

</code>


# Notebook from mayou36/2021-06-14_ESCAPESummerSchool
Path: 01_HEPandBigData.ipynb

Introduction to the Scikit-HEP
Big Data Python ecosystem for analysis in Particle Physics

Eduardo Rodrigues
University of Liverpool

ESCAPE Summer School 2021
Data Science for Astronomers, Astroparticle and Particle Physics, 7-18 June 2021

_____no_output_____---

    The School is organized in the framework and with the support of the European Science Cluster of Astronomy
    & Particle physics ESFRI research infrastructures (ESCAPE),
    funded by the European Union's Horizon 2020 - Grant N. 824064.

---_____no_output_____### Abstract

Data analysis in High Energy Physics (HEP) has evolved considerably in recent years. In particular, the role of Python has gained
much momentum, sharing at present the show with C++ as a language of choice.
Several (community) domain-specific projects have seen the day, providing (HEP) data analysis packages that profit from, and talk to well with,
the huge Python scientific ecosystem, which navigates around NumPy and friends.
This lecture introduces the Scikit-HEP project, which I started in late 2016 with a few colleagues from various backgrounds
and domains of expertise.
Scikit-HEP is a community-driven and community-oriented project with the aim of providing Particle Physics at large
with a Big Data ecosystem for analysis in Python. It has developed considerably in the past couple of years,
and is now part of the official software stack of the experiments ATLAS, Belle II, CMS and KM3NeT.
In this lecture ample time will be provided to "play around" with the material, in Jupyter notebooks._____no_output_____### Lecture material

- Notebooks GitHub repository https://github.com/eduardo-rodrigues/2021-06-14_ESCAPESummerSchool.
- Can be nicely viewed [with nbviewer](https://nbviewer.jupyter.org/github/eduardo-rodrigues/2021-06-14_ESCAPESummerSchool/tree/master/).

(Launching with Binder from the Github repository? Please be patient, the repository may take of order 5 minutes to load completely on Binder.)_____no_output_____## **1.1 Particle Physics and Big Data**

A lot of what has happened in the HEP Python ecosystem can be thought of as trying to **bridge the Particle Physics and Big Data worlds** and **profit from what the Data Science scientific software stack has to offer**.

We will come back to software, but what about the data itself? Is that "Big Data"?_____no_output_____**The CERN [ROOT team](https://root.cern/) advertises that of the order of 1 EB - 1 million TB! - of data exists right now in the `.root` format.**

Impressive. **We are already in the exascale era!**_____no_output_____For the record, the CERN Data Centre had accumulated more than 200 PB of data back in 2017 already!
[[CERN news, July 6, 2017](https://home.cern/news/news/computing/cern-data-centre-passes-200-petabyte-milestone)]:

_____no_output_____And in just an extra 1.5 years, 50% more data got saved in CERN's Data Centre.
Citing the [CERN news, March 1, 2019](https://home.cern/news/news/computing/lhc-pushing-computing-limits),

> "*The CERN Advanced Storage system (CASTOR), which relies on a tape-based backend for permanent data archiving, reached **330 PB of data (equivalent to 330 million gigabytes) stored on tape, an equivalent of over 2000 years of 24/7 HD video recording**. In November 2018 alone, a record-breaking 15.8 PB of data were recorded on tape, a remarkable achievement given that it corresponds to more than what was recorded during the first year of the LHC’s Run 1.*"

In fact, in 2018, over 115 PB of data in total (including about 88 PB of LHC data) were recorded on tape._____no_output_____The accummulation of data generated by the [LHC](https://home.cern/science/accelerators/large-hadron-collider) experiments alone, over a decade-ish, speaks for itself, as seen by this graph on CERN computing:

data (in terabytes) recorded on tapes at CERN month-by-month (2010–2018) [[ref](https://cds.cern.ch/images/CERN-HOMEWEB-PHO-2019-003-1)]:_____no_output__________no_output_____**And what about the outside world?**

Let's look at Amazon for the sake of argument [[ref](https://aws.amazon.com/snowmobile/)]:
>"*AWS Snowmobile is an Exabyte-scale data transfer service used to move extremely large amounts of data to AWS. You can transfer up to 100PB per Snowmobile, a 45-foot long ruggedized shipping container, pulled by a semi-trailer truck.*"
_____no_output_____> "*Each Snowmobile includes a network cable connected to a high-speed switch capable of supporting 1Tbps of data transfer spread across multiple 40Gbps connections.*"

This is to be compared with the throughput of 1-2 Tbps the [LHCb experiment](http://lhcb-public.web.cern.ch/)'s first High-Level Trigger (HLT1, partial reconstruction on GPUs) will put to buffer while the real-time calibration and alignment is run, which is needed to digest the data in the second stage, HLT2 (full reconstruction) ... next year!_____no_output_____### **Is it relevant and useful to learn non-HEP tools?**

We've just quickly recalled that data requirements for Particle Physics match those of the Big Data world. Huge amounts of data are in fact used by companies worldwide for just about any business, see for example the report "State of Data Science and Analytics, IDC InfoBrief, April 2019":
_____no_output_____International surveys indicate **over 50M data workers worldwide! Can we really compete in terms of tools for data analysis?
... Or should we rather try and profit from, and even contribute to, the huge ecosystem available to do Data Science?**

Anyway, what are data scientists, data engineers, etc., using for their daily work? That largely involves to some (larger and larger) extent Machine Learning, statistics, and even AI. International surveys give a hint ... the **tools are dominated by Python tools ...!**_____no_output_____From the 2019 Figure Eight report "The State of AI and Machine Learning",
> "*Some popular frameworks and tools technical practitioners prefer in different stages of the ML pipeline are: Numpy and Pandas for loading data; Matplotlib for visualization; Scikit-learn and TensorFlow (including Keras) for ML models.*":

_____no_output_____### **The reign of Python**

One may say that 2019 was the year for Python. Now it simply is more popular than anything else:

The trend is only more clear when checking the popularity of programming languages for Machine Learning, BTW._____no_output_____Particle Physics is clearly following this same trend - Python ever more popular for/in analysis. Here is an example study from CMS:

_____no_output_____

Take aways

- Particle Physics and Data Science deal with Big Data and share requirements.
- The Data Science world has over the years built an extensive, powerful, well maintained and documented software ecosystem.
- It would be real shame for Particle Physics not to profit from it, as user but potentially also as a contributor.
- Python is the programming language of choice.

_____no_output_____## **1.2 The "Python in HEP" Big Data ecosystem**_____no_output_____So, we concluded that:
- The Data Science world relies on an extensive, powerful, well maintained and documented software ecosystem in Python.
- Particle Physics has much to gain if engaging and contributing.

It's a lot about making it possible, natural and efficient to perform HEP analysis in the NumPy ecosystem ...!_____no_output_____### **The central role of NumPy**

The (vast) scientific Python ecosystem can be organised, schematically, as a layered set of libraries and packages ever more specialised, atop a series of building blocks. The [NumPy library](https://numpy.org/) is a central pillar._____no_output__________no_output_____This fact has been very recently acknowledged by Nature. Quoting [ZDNet (25/09/2020)](https://www.zdnet.com/article/numpys-contribution-to-python-is-remarkable-but-where-it-goes-next-could-be-even-more-so/),
> "Something fascinating happened in the world of scientific publishing last week: The prestigious journal Nature featured an overview of a 15-year-old programming library for the language Python. The widely popular library, called NumPy, gives Python the ability to perform scientific computing functions. "_____no_output_____The Nature article, [Harris, C.R., Millman, K.J., van der Walt, S.J. *et al.*, Array programming with NumPy, Nature 585, 357–362 (2020)](https://www.nature.com/articles/s41586-020-2649-2), features in particular the following diagram,
_____no_output_____, which provides a view of the contemporary Python scientific ecosystem, from foundational libraries to domain-specific and application-specific libraries.

BTW, if you look carefully, you will see above a single HEP "library", namely Scikit-HEP!
Aside the fact that the authors misinterpreted the scope of the [Scikit-HEP project](https://scikit-hep.org/), which rather is a *domain-specific project*, this is excellent news for us (HEP), to show up in such a publication, since the appearance acknowledges our position out there! Scikit-HEP will be the focus of the next notebooks ..._____no_output_____### **Intermezzo - The HEP Software Foundation**

The goal of the [HEP Software Foundation (HSF)](https://hepsoftwarefoundation.org/) is to *facilitate coordination and common efforts in software and computing across HEP in general*.
It was founded in 2014, explicitly to address current and future computing & software challenges in common._____no_output__________no_output_____### **PyHEP - "Python in HEP"**

The HSF comprises a series of Working Groups and (less formal) Activity Areas ... among which the [**“Python in HEP” WG**](https://hepsoftwarefoundation.org/workinggroups/pyhep.html).

_____no_output_____As the WG homepage says,

> *The PyHEP working group brings together a community of developers and users of Python in Particle Physics, with the aim of improving the sharing of knowledge and expertise. It embraces the broad community, from HEP to the Astroparticle and Intensity Frontier communities.*

Several of the [HSF Gitter channels](https://gitter.im/HSF/) relate to PyHEP topics, and are the means by which everyone communicates and exchanges informations, experiences, and asks questions ...

Feel free to join. It's trivial!_____no_output_____#### **The PyHEP workshops**

These are major events for the WG. Check out the forthcoming [PyHEP 2021 workshop](https://indico.cern.ch/e/PyHEP2021) ...

_____no_output_____### **"PyHEP community projects"**

A series of Python projects have seen the light in recent years, where by *projects* I select endeavours that provide one or more Python libraries *with a community around it*. ROOT and SWAN (both are CERN projects) presently share the arena with a few community projects that build around the general Python scientific ecosystem instead of on ROOT:

Coffea

FAST-HEP

ROOT

Scikit-HEP

SWAN

zfit

_____no_output_____**Scikit-HEP** is:
- The one I co-founded in late 2016 with a few colleagues, hence intimately involved with.
- The oldest of such projects.
- The one with more libraries provided.
- The project on which all of the others (i.e. non-CERN) depend on (they depend on at least one of Scikit-HEP libraries).

For these reasons the next few notebooks will focus on Scikit-HEP packages. The last notebook will work with the 2 main packages of the `zfit` project._____no_output_____

On these PyHEP community projects and libraries

The ecosystem that is going to be showcased in these tutorial notebooks is already advanced enough to perform analysis in it. All the packages are written and maintained by other physicists or scientists in general. As any other library, these packages are neither perfect (bug free) nor do they have all the features that you may want today. They are, however, built for simple contributions: if there are missing features or bugs, please report or directly contribute with code and become part of the ecosystem!

_____no_output_____### **For reference - some useful links collected in one place:**

#### HSF PyHEP Working Group
Information on the [HEP Software Foundation](hepsoftwarefoundation.org/) PyHEP ("Python in HEP") Working Group.
- Webpage: https://hepsoftwarefoundation.org/workinggroups/pyhep.html
- Gitter channel: https://gitter.im/HSF/PyHEP

#### Scikit-HEP project
- Website: https://scikit-hep.org/
- GitHub: https://github.com/scikit-hep/_____no_output_____### **How to explore these lectures**

- **The notebooks are topical and self-consistent for you to run though them at your own pace and leisure.
Run what topics sound appealing to you ...**

- You will also find some corresponding exercises notebooks. As the names indicate, they suggest exercises beyond the introduction to the various topics and related packages. Again, feel free to decide what to try out.

- Be warned, I am no expert on all packages! But I will try and help as much as possible.

- You liked these tutorials? Consider dropping a line and/or giving the GitHub repository a star, as that's a trivial way to convey positive feedback._____no_output_____


# Notebook from nate-stein/cs207_nate_stein
Path: lectures/L13/L13.ipynb

# Lecture 13

### Monday, October 23rd 2017_____no_output_____## Last time:
* Data structures motivation
* Abstract data types
* Sequences
* Linked lists

## This time:
* Iterators and Iterables
* Trees, B-trees, and BSTs_____no_output_____# From pointers to iterators

One can simply follow the `next` pointers to the next **position** in a linked list.

This suggests an abstraction of the **position** to an **iterator**.

Such an abstraction allows us to treat arrays and linked lists with an identical interface.

The salient points of this abstraction are:
- The notion of a `next`
- The notion of a `first` to a `last`

We already implemented the sequence protocol.

Now we suggest an additional abstraction that is more fundamental than the notion of a sequence: the **iterable**._____no_output_____# Iterators and Iterables in `Python`

Just as a sequence is something implementing `__getitem__` and `__len__`, an **iterable** is something implementing `__iter__`.

`__len__` is not needed and indeed may not make sense.
```python
len(open('fname.txt')) # File iterator has no length
```

Example `14-1` in `Fluent Python`: The `Sentence` sequence and shows how it can be iterated upon._____no_output_____

import reprlib
class Sentence:
    def __init__(self, text):
        self.text = text
        self.words = text.split()

    def __getitem__(self, index):
        return self.words[index]

    def __len__(self):
        #completes sequence protocol, but not needed for iterable
        return len(self.words)

    def __repr__(self):
        return 'Sentence(%s)' % reprlib.repr(self.text)_____no_output_____# Sequence'
s = Sentence("Dogs will save the world.")
print(len(s), "   ", s[3], "   ",  s)5     the     Sentence('Dogs will save the world.')
min(s), max(s)_____no_output_____list(s)_____no_output_____

To iterate over an object `x`, `Python` automatically calls `iter(x)` (i.e. `x.__iter__`).

An **iterable** is something which, when `iter` is called on it, returns an **iterator**.

(1) If `__iter__` is defined, it is called to implement an iterator.

(2) If not, `__getitem__` is called starting from index `0`.

(3) If no `__iter__` and no `__getitem__` then raise a `TypeError`.

Any `Python` sequence is iterable because sequences implement `__getitem__`. The standard sequences also implement `__iter__`; for future proofing you should too because  (2) might be deprecated in a future version of `Python`._____no_output_____We know that `for` operates on iterables:_____no_output_____

for i in s:
    print(i)Dogs
will
save
the
world.

What's actually going on here?_____no_output_____

it = iter(s) # Build an iterator from an iterable
while True:
    try:
        nextval = next(it) # Get the next item in the iterator
        print(nextval)
    except StopIteration:
        del it # Iterator is exhausted.  Release reference and discard.
        breakDogs
will
save
the
world.

We can completely abstract away a sequence in favor of an iterable (i.e. we don't need to support indexing anymore)

Example `14-4` in `Fluent Python`:_____no_output_____

class SentenceIterator: # has __next__ and __iter__
    def __init__(self, words):
        self.words = words
        self.index = 0

    def __next__(self):
        try:
            word = self.words[self.index]
        except IndexError:
            raise StopIteration()
        self.index += 1
        return word

    def __iter__(self):
        return self

class Sentence: # An iterable b/c it has __iter__
    def __init__(self, text):
        self.text = text
        self.words = text.split()

    def __iter__(self):
        return SentenceIterator(self.words) # Returns an instance of the iterator

    def __repr__(self):
        return 'Sentence(%s)' % reprlib.repr(self.text)_____no_output_____s2 = Sentence("What is data science?")_____no_output_____for i in s2:
    print(i)What
is
data
science?
s2it=iter(s2) # Make the iterable an iterator
print(next(s2it), "\n\n") # Get the next entry
s2it2=iter(s2) # Reset the iterator
print(next(s2it), "   ", next(s2it2)) # Get the next entry of s2it and s2it2What

is     What

While we could have implemented `__next__` in Sentence itself, making it an iterator, we will run into the problem of "exhausting an iterator".

The iterator above keeps state in `self.index` and we must be able to start anew by creating a new instance if we want to re-iterate. Thus the `__iter__` in the iterable simply returns the `SentenceIterator`.

From `Fluent Python` ("Sentence Take #2:  A Classic Iterator"):
> A common cause of errors in building iterables and iterators is to confuse the two. To be clear: iterables have an `__iter__` method that instantiates a new iterator every time. Iterators implement a `__next__` method that returns individual items, and an `__iter__` method that returns self._____no_output_____`min()` and `max()` also work even though we no longer satisfy the sequence protocol.

`min` and `max` are pairwise comparisons and can be handled via iteration.

The take home message is that in programming with these iterators we don't need either the length or indexing to work to implement many algorithms: we have abstracted these away._____no_output_____

min(s2), max(s2)_____no_output_____

# Trees

- A tree is a hierarchical data structure that has a bunch of items.
- Each item may have a value.
- Items may point to other items or not (leaf nodes).
- Each item is pointed to by exactly one other item with the sole exception of the root item.  Nothing points to the root item.

There are many types of trees used to express data structures.  For example,
* Binary Tree
* B-Tree
* Heap
* Multi-way tree

Each of these has many subtypes._____no_output_____#### Trees arise everywhere:

- In parsing of code
- Evolutionary trees in biology
- Language origin trees
- Unix file system
- HTML tags

Just like with lists, one can consider looking at a tree in two ways: a collection of nodes or a tree with a root and multiple sub-trees.

Today, and in your homework, the focus is on Binary trees._____no_output_____Once again, one can represent trees using the recursive data structures we used to represent linked lists (from cs61a):

![](http://wla.berkeley.edu/~cs61a/fa11/lectures/img/tree.png)

_____no_output_____You could also use a tree in which the nodes all themselves have data. This is often used to represent a binary tree._____no_output_____## Some Terminology (Binary Tree)
* The root node is the only node that has nothing pointing to it.
* A node can point to other nodes called children.
* The node doing the pointing is called the parent node.
* In a binary tree, a given node only points to two other nodes (maximum of two children).
* If a child node does not contain a value, then it is said to be empty (or null).
* A node with pointers to two empty values is called a leaf._____no_output_____## Binary Search Trees (BSTs)
These are binary trees with the requirement that all children to the left of a particular node have lower values and all children to the right have higher values.

There are ways of dealing with repeated values (e.g. storing a counter), but we will just use BSTs that do not have repeated values.

A major benefit of BSTs:
* Efficient searching:  Can be $O(\log(n))$ for a balanced tree where $n$ is the height of the tree.

An algorithm for the deletion of a node is as follows:
1. If the node to be deleted has no children then just remove it.
2. If the node to be deleted has only one child, remove the node and replace it with its child.
3. If the node to be deleted has two children, replace the node to be deleted with the maximum value in the left subtree. Finally, delete the node with the maximum value in the left-subtree._____no_output_____## Practice

In your homework, you will implement a BST.  The implementation should be able to insert new values and optionally be able to delete values from the tree.

Consider the array $$\left[13, 7, 19, 17, 3, 29, 5, 31, 2, 11\right].$$  With your group, at your whiteboard, do the following:
1. Draw a BST for the array
2. Delete the node with value $13$_____no_output_____


# Notebook from oliverphilcox/ChempyMulti
Path: Chempy_tutorials/3-Chempy_inputs.ipynb

## Input to Chempy
*Chempy* was developed with the intention to flexibly test all the assumptions going into chemical evolution modeling. Here we give a range of input parameters that can be played around with in *Chempy*_____no_output_____

%pylab inlinePopulating the interactive namespace from numpy and matplotlib
from Chempy.parameter import ModelParameters
a = ModelParameters()_____no_output_____

## Star formation rate (SFR)
Determines how many stars a formed at a specific time of the simulation_____no_output_____

# We load the SFR class

from Chempy.sfr import SFR

# Its initialised with t_0, t_end, and number of time-steps

basic_sfr = SFR(0,13.5,136)

# Then we load it with its default parameters

getattr(basic_sfr, a.basic_sfr_name)(S0 = a.S_0 * a.mass_factor,a_parameter = a.a_parameter, loc = a.sfr_beginning, scale = a.sfr_scale)_____no_output_____# Here a list of already implemented SFR functions

print('these SFR functions are already implemented, see source/sfr.py')
print(a.basic_sfr_name_list)

# The SFR should always sum to 1 if you want to make your own SFR function

print('the sfr sums to 1')

# Plot the default SFR function (gamma function) with different SFR_peak parameters

for sfr_scale in [2.0,3.5,5.0]:
    basic_sfr = SFR(a.start,a.end,136)
    getattr(basic_sfr, 'gamma_function')(S0 = a.S_0 * a.mass_factor,a_parameter = a.a_parameter, loc = a.sfr_beginning, scale = sfr_scale)
    plt.plot(basic_sfr.t,basic_sfr.sfr, label = 'SFR peak at %.1f' %(sfr_scale))
    print(sum(basic_sfr.sfr)*basic_sfr.dt)
    plt.legend()
plt.xlabel('time in Gyr')these SFR functions are already implemented, see source/sfr.py
['model_A', 'gamma_function', 'prescribed', 'doubly_peaked']
the sfr sums to 1
1.0
1.0
1.0

## Infall
infall of gas from the corona into the ISM (usually diluting the ISM gas).
In *Chempy* the infall can also be prescribed (you will need to make sure that always enough gas for star formation is available), or you can relate the infall to the star formation, e.g. via the Kennicut-Schmidt law._____no_output_____

# Initialising the infall

from Chempy.infall import INFALL
basic_infall = INFALL(np.copy(basic_sfr.t),np.copy(basic_sfr.sfr))
getattr(basic_infall, 'sfr_related')()_____no_output_____# SFR_related infall will be calculated during the Chempy run according to the needed gas mass

basic_infall.infall_____no_output_____# But the infall can also be prescribed here we use the exponential infall:

basic_infall = INFALL(np.copy(basic_sfr.t),np.copy(basic_sfr.sfr))
getattr(basic_infall, 'exponential')((-0.15,0.,0.9))

# And compare it to the SFR

print(sum(basic_infall.sfr)*basic_sfr.dt)
print(sum(basic_infall.infall)*basic_sfr.dt)
plt.plot(basic_infall.t,basic_infall.sfr, label = 'SFR')
plt.plot(basic_infall.t,basic_infall.infall, label = 'Infall')
plt.legend()1.0
0.9

## stellar initial mass function (IMF)
defines the distribution of stellar masses formed in a starburst._____no_output_____

# We initialise the IMF class and plot it for different IMFs

from Chempy.imf import IMF
basic_imf = IMF(a.mmin,a.mmax,a.mass_steps)
getattr(basic_imf, a.imf_type_name)((a.chabrier_para1,a.chabrier_para2,a.chabrier_para3,a.high_mass_slope))
plt.plot(basic_imf.x,basic_imf.dn, label ='Chabrier')
basic_imf = IMF(a.mmin,a.mmax,a.mass_steps)
getattr(basic_imf, 'salpeter')((2.35))
plt.plot(basic_imf.x,basic_imf.dn, label = 'Salpeter')
basic_imf = IMF(a.mmin,a.mmax,a.mass_steps)
getattr(basic_imf, 'normed_3slope')((-1.3,-2.2,-2.7,0.5,1.0))
plt.plot(basic_imf.x,basic_imf.dn, label = 'Kroupa')
plt.yscale('log')
plt.xscale('log')
plt.xlim((0.07,110))
plt.legend()_____no_output_____

### Mass range of exploding CC-SNe
The Kroupa IMF only has 8% of stars exploding as CC-SNe whereas Salpeter has 12%.
_____no_output_____

# We calculate the Kroupa IMF mass fractions

print(basic_imf.imf_mass_fraction(8.,100.))
print(basic_imf.imf_mass_fraction(1.,8.))
print(basic_imf.imf_mass_fraction(0.08,1.))0.0826531531475
0.325021531915
0.59232293366
# And compare the mass fraction exploding as CC-SN to the Salpeter IMF

basic_imf = IMF(a.mmin,a.mmax,a.mass_steps)
getattr(basic_imf, 'salpeter')((2.35))
print(basic_imf.imf_mass_fraction(8.,100.))0.12145269368

IMF sampling
---------------
The IMF can also be realized *stochastically*. Albeit each reaslization is new and it takes more time than the analytic version._____no_output_____

# We sample the Chabrier IMF for 3 different masses:

for item in [1e1,1e3,1e5]:
    basic_imf = IMF(a.mmin,a.mmax,a.mass_steps)
    getattr(basic_imf, a.imf_type_name)((a.chabrier_para1,a.chabrier_para2,a.chabrier_para3,a.high_mass_slope))
    basic_imf.stochastic_sampling(item)
    plt.plot(basic_imf.x,basic_imf.dn, label ='SSP with %dMsun' %(item))
plt.title('Chabrier IMF realized for different SSP masses')
plt.yscale('log')
plt.xscale('log')
plt.xlim((0.07,200))
plt.ylabel('dn/dm')
plt.xlabel('Mass in Msun')
plt.legend()_____no_output_____

## Stellar lifetimes
in order to calculate when a star is dying and returns its newly produced elements into the ISM. The lifetimes are to zeroth order *mass* dependent and to first order *metallicity* dependent_____no_output_____

# Here we show the difference of Lifetime calculations

from Chempy.weighted_yield import lifetime_Argast, lifetime_Raiteri
metallicity = 0.015
plt.plot(basic_imf.x,lifetime_Argast(basic_imf.x,metallicity), label = 'Argast')
plt.plot(basic_imf.x,lifetime_Raiteri(basic_imf.x,metallicity), label = 'Raiteri')
plt.yscale('log')
plt.xscale('log')
plt.ylabel('Age in Gyr')
plt.xlabel('Mass in Msun')
plt.xlim((0.07,200))
plt.legend()_____no_output_____# And here the difference of the lifetimes with metallicity

for metallicity in [0.00015,0.0015,0.015]:
    plt.plot(basic_imf.x,lifetime_Argast(basic_imf.x,metallicity), label = 'Argast Z=%f' %(metallicity))
plt.yscale('log')
plt.xscale('log')
plt.ylabel('Age in Gyr')
plt.xlabel('Mass in Msun')
plt.xlim((0.07,200))
plt.legend()_____no_output_____


# Notebook from 1106601034/sit742
Path: Jupyter/M04-DataManipulation/M04B-EDA.ipynb

![Cloud-First](../figures/CloudFirst.png)

# SIT742: Modern Data Science
**(Module: Data Manipulation)**

---
- Materials in this module include resources collected from various open-source online repositories.
- You are free to use, change and distribute this package.
- If you found any issue/bug for this document, please submit an issue at [tulip-lab/sit742](https://github.com/tulip-lab/sit742/issues)

Prepared by **SIT742 Teaching Team**

---

## Session 4B - Exploratory Data Analysis

This practical session will show you how to use packages for data exploration.

### Content

#### Part 1 Matplotlib Module

#### Part 2 Plotting a Histogram

2.1 [Dataset](#ds)

2.2 [Histogram](#hist)

2.3 [Boxplot](#boxplot)

#### Part 3: Data Understanding

1.1 [Pie Chart](#pie)

1.2 [Bar Chart](#bar)

1.3 [Word Cloud](#wordcloud)

1.4 [Step Plot](#stepplot)

1.5 [Histogram](#histogram)

1.6 [Box Plot](#box)

1.7 [Scatter Plot](#scatter)

#### Part 4: Exercise

_____no_output_____## 1. Matplotlib
_____no_output_____matplotlib is a python plotting library which produces publication quality figures in a variety of hardcopy formats and interactive environments across platforms. You can generate plots, histograms, power spectra, bar charts, errorcharts, scatterplots, etc, with just a few lines of code.

For simple plotting the pyplot interface provides a MATLAB-like interface, particularly when combined with IPython. You have full control of line styles, font properties, axes properties, etc, via an object oriented interface or via a set of functions familiar to MATLAB users._____no_output_____### 1.1 Get started
_____no_output_____To get started with `'matplotlib'` you can either execute:_____no_output_____

from pylab import *_____no_output_____

or_____no_output_____

import matplotlib.pyplot_____no_output_____

In fact it is a convention to import it under the name of `'plt'`:_____no_output_____

import matplotlib.pyplot as plt_____no_output_____

**note: The method, 'import matplotlib.pyplot as plt', is preferred.**_____no_output_____

import matplotlib.pyplot as plt
import numpy as np_____no_output_____

Regardless of the method you use, it is better to configure matplotlib to embed figures in the notebook instead of opening them in a new window for each figure. To do this use the magic function:_____no_output_____

%matplotlib inline_____no_output_____

### 1.2 `plot`
_____no_output_____By using `'subplots()'` you have access to both figure and axes objects. _____no_output_____

#To define the x-Axis and y-Axis
x = np.linspace(0, 10)
y = np.sin(x)_____no_output_____#fig, ax = plt.subplots() is more consice the below code
#If you use it,you unpack this tuple into the variables fig and ax
#Actually, it equals the below 2 lines code.
#fig = plt.figure()
#ax = fig.add_subplot(111) #the 111 menas 1x1 grid, first subplot

fig, ax = plt.subplots()
ax.plot(x, y)_____no_output_____

### 1.3 title and labels
_____no_output_____

#To set the title, xlabel and ylabel
ax.set_title('title here!')
ax.set_xlabel('x')
ax.set_ylabel('sin(x)')
fig_____no_output_____

You can also use [$\LaTeX$](https://www.latex-project.org/about/) in title or labels, or change the font size or font family._____no_output_____

#To set the domain of X-Axis from -10 to 10
x = np.linspace(-10, 10)

#Parameter figsize : (float, float), optional, default: None
#width, height in inches. If not provided, defaults to rcParams["figure.figsize"] = [6.4, 4.8].

#Parameter dpi : integer, optional, default: None
#resolution of the figure. If not provided, defaults to rcParams["figure.dpi"] = 100.
fig, ax = plt.subplots(figsize=(6, 4), dpi=100)

#x, x**3-x**2 are used to define the horizontal / vertical coordinates of the data points. x values are optional. If not given, they default to [0, ..., N-1].
#'bo-' means blue circle with the solid line style. Its name pattern follow the '[color][marker][line]' structure.
#linewidth defines the line width
#markersize defines the circle size
ax.plot(x, x**3-x**2, 'bo-', linewidth=1, markersize=5)

#To set title, xlabel, ylabel and their fontsize
ax.set_title('$x^3-x^2$', fontsize=18)
ax.set_xlabel('$x$', fontsize=18)
ax.set_ylabel('$y$', fontsize=18)
ax_____no_output_____

### 1.4 Subplots

You can pass the number of subplots to `'subplots()'`. In this case, `'axes'` will be an array that each of its elements associates with one of the subgraphs. You can set properties of each `'ax'` object separately like the cell below.

Obviously you caould use a loop to iterate over `'axes'`._____no_output_____

#nrows, ncols : int, optional, default: 1
#They are used to define the number of rows/columns of the subplot grid.
#You can try to comment the below code out and run the 'fig, axes = plt.subplots(nrows=1, ncols=2)', then you will find the difference.
fig, axes = plt.subplots(nrows=2, ncols=1)

#To define the domain of the x-Axis from 0 to 10
x = np.linspace(0, 10)

#x, np.sin(x) are used to define the horizontal / vertical coordinates of the data points. x values are optional. If not given, they default to [0, ..., N-1].
axes[0].plot(x, np.sin(x))
axes[0].set_xlabel('x for sin(x)')
axes[0].set_ylabel('sin(x)')

#x, np.cos(x) are used to define the horizontal / vertical coordinates of the data points. x values are optional. If not given, they default to [0, ..., N-1].
axes[1].plot(x, np.cos(x))
axes[1].set_xlabel('x for cos(x)')
axes[1].set_ylabel('cos(x)')

axes_____no_output_____

`'cos(x)'` label is overlapping with the `'sin'` graph. You can adjust the size of the graph or space between the subplots to fix it._____no_output_____

#nrows, ncols : int, optional, default: 1
#They are used to define the number of rows/columns of the subplot grid.
#You can try to comment the below code out and run the 'fig, axes = plt.subplots(nrows=1, ncols=2)', then you will find the difference.
#You also can use the fig, axes = plt.subplots(nrows=1, ncols=2, figsize=(10,5)) to resize the graph
fig, axes = plt.subplots(nrows=2, ncols=1)

#To adjust the space between the subpots
fig.subplots_adjust(hspace=1)

#To define the domain of the x-Axis from 0 to 10
x = np.linspace(0, 10)

#x, np.sin(x) are used to define the horizontal / vertical coordinates of the data points. x values are optional. If not given, they default to [0, ..., N-1].
axes[0].plot(x, np.sin(x))
axes[0].set_xlabel('x for sin(x)')
axes[0].set_ylabel('sin(x)')

#x, np.cos(x) are used to define the horizontal / vertical coordinates of the data points. x values are optional. If not given, they default to [0, ..., N-1].
axes[1].plot(x, np.cos(x))
axes[1].set_xlabel('x for cos(x)')
axes[1].set_ylabel('cos(x)')

axes_____no_output_____

### 1.5 Legend
_____no_output_____

##To define the domain of the x-Axis from 0 to 10
x = np.linspace(0, 10)

#To define the figsize
fig, ax = plt.subplots(figsize=(7, 5))

#To define the x-Axis and y-Axis and label
ax.plot(x, np.sin(x), label='$sin(x)$')
ax.plot(x, np.cos(x), label='$cos(x)$')

#To place the legend for the two plot and define their fontsize and the location of the legend.
#Possible codes for the 'loc' are from 0 to 10:
ax.legend(fontsize=16, loc=3)
_____no_output_____

### 1.6 Customizing ticks
_____no_output_____In many cases you want to customize the ticks and their labels on x or y axis. First draw a simple graph and look at the ticks on x-axis. _____no_output_____

x = np.linspace(0, 10, num=100)

fig, ax = plt.subplots(figsize=(10, 5))
ax.plot(x, np.sin(x), x, np.cos(x), linewidth=2)_____no_output_____

You can change the ticks easily with passing a list (or array) to `'set_xticks()'` or `'set_yticks()'`:_____no_output_____

xticks = [0, 1, 2, 5, 8, 8.5, 10]
ax.set_xticks(xticks)
fig_____no_output_____

Or even you can change the labels:_____no_output_____

xticklabels = ['$\gamma$', '$\delta$', 'apple', 'b', '', 'c']
ax.set_xticklabels(xticklabels, fontsize=18)
fig_____no_output_____

### 1.7 Saving figures
_____no_output_____

x = np.linspace(0, 10)
fig, ax = plt.subplots(figsize=(7, 5))
ax.plot(x, np.sin(x), label='$sin(x)$')
ax.plot(x, np.cos(x), label='$cos(x)$')
ax.legend(fontsize=16, loc=3)
fig.savefig('P03Saved.pdf', format='PDF', dpi=300)_____no_output_____

### 1.8 Other plot styles

There are many other plot types in addition to simple `'plot'` supported by `'matplotlib'`. You will find a complete list of them on [matplotlib gallery](http://matplotlib.org/gallery.html)._____no_output_____#### 1.8.1 Scatter plot

_____no_output_____

fig, ax = plt.subplots(figsize=(10,5))

x = np.linspace(-0.75, 1., 100)

#s is the marker size in points**2. Default is rcParams['lines.markersize'] ** 2.
#alpha is the alpha blending value, between 0 (transparent) and 1 (opaque).

#edgecolor it the edge color of the marker. Possible values:
# 'face': The edge color will always be the same as the face color.
# 'none': No patch boundary will be drawn.
# A matplotib color.
ax.scatter(x, np.random.randn(x.shape[0]),
                   s = 250*np.abs(np.random.randn(x.shape[0])),
                   alpha=0.5,
                   facecolor='green',
                   edgecolor='face')
ax.set_title('scatter')
ax_____no_output_____

#### 1.8.2 Bar plot
_____no_output_____

fig, ax = plt.subplots()

x = np.arange(1, 6)
ax.bar(x, x**2, align="center")
ax.set_title('bar')_____no_output_____

---
##  2. Plotting a histogram
_____no_output_____

### 2.1 Dataset
_____no_output_____You are provided with a dataset of percentage of body fat and 10 simple body measurements recoreded for 252 men (courtesy of Journal of Statistics Education - JSE). You can read about this and other [JSE datasets here](http://www.amstat.org/publications/jse/jse_data_archive.htm).

First load the data set into an array:_____no_output_____

!pip install wget
_____no_output_____import numpy as np
import wget

link_to_data = 'https://github.com/tulip-lab/sit742/raw/master/Jupyter/data/fat.dat.txt'
DataSet = wget.download(link_to_data)_____no_output_____data = np.genfromtxt("fat.dat.txt")
data.shape_____no_output_____

Based on the [dataset description](http://www.amstat.org/publications/jse/datasets/fat.txt), 5th column represents the weight in lbs. Index the weight column and call it `'weights'`:_____no_output_____

weights = data[:, 5]
weights_____no_output_____

Use array operators to convert the weigts into kg. 1 lb equals to 0.453592 kg._____no_output_____

# weights *= 0.453592 is equivalent to weights = weights * 0.453592
#It multiplies right operand with the left operand and assign the result to left operand
weights *= 0.453592

#Round the converted weights to only two decimals:
weights = weights.round(2)

weights_____no_output_____

### 2.2 Histogram
_____no_output_____A histogtram is a bar plot that shows you the statistical distribution of the data over a variable. The bars represent the frequency of occurenve by classess of data. We use the package `'matplotlib'` and the function `'hist()'` for plotting the histogram. To learn more about `'matplotlib'` make sure you have read tutorial.

The first line of the cell below if for showing the figure in the notebook and not opening it in a separate window._____no_output_____

%matplotlib inline
import matplotlib.pyplot as plt_____no_output_____fig, ax = plt.subplots()
ax.hist(weights)
ax_____no_output_____

The `'hist()'` functions automatically group the data over 10 bins. Usually you need to tweek the number of bins to obtain a more expressive histogram._____no_output_____

fig, ax = plt.subplots(figsize=(7, 5))
ax.hist(weights, bins=20)

# title
ax.set_title('Weight Distribution Diagram ')

# label
ax.set_xlabel('Weight (kg)  ')
ax.set_ylabel('The number of people')

ax_____no_output_____

### 2.3 Boxplot

A `Boxplot` is a convenient way to graphically display numerical data. _____no_output_____

import matplotlib

fig, ax = matplotlib.pyplot.subplots(figsize=(7, 5))

matplotlib.rcParams.update({'font.size': 14})

ax.boxplot(weights, 0, labels=['group1'])

ax.set_ylabel('weight (kg)', fontsize=16)
ax.set_title('Weights BoxPlot', fontsize=16)_____no_output_____

You have already been thought about different sorts of plots, how they help to get a better understanding of the data, and when to use which. In this practical session we will work with `matplotlib` package to learn more about plotting in Python._____no_output_____---
## 3. Data Understanding

You have already been thought about different sorts of plots, how they help to get a better understanding of the data, and when to use which. In this practical session we will work with `matplotlib` package to learn more about plotting in Python._____no_output_____

import numpy as np
import csv

import matplotlib.pyplot as plt
%matplotlib inline_____no_output_____

### 3.1 Pie Chart

Suppose you have the frequency count of a variable (e.g. hair_colour). Draw a pie chart to explain it._____no_output_____

labels = 'Black', 'Red', 'Brown'

# frequency count
hair_colour_freq = [5, 3, 2]  # Black, Red, Brown

# colors
colors = ['yellowgreen', 'gold', 'lightskyblue']

# explode the third one
explode = (0, 0, 0.1)

fig, ax = plt.subplots(figsize=(5, 5))
ax.pie(hair_colour_freq, labels=labels, explode=explode, colors=colors,
       autopct='%1.1f', shadow=True, startangle=90);_____no_output_____

What if we have too many tags and sectors?_____no_output_____

# Excellence in Reasearch Australia
labels = ['HEALTH', 'ENGINEERING', 'COMPUTER SCIENCES', 'HUMAN SOCIETY',
          'TOURISM SERVICES', 'EDUCATION', 'CHEMISTRY', 'BIOLOGY', 'PSYCHOLOGY',
          'CREATIVE ARTS', 'LINGUISTICS', 'BUILT ENVIRONMENT', 'HISTORY',
          'ECONOMICS', 'PHILOSOPHY', 'AGRICULTURE', 'ENVIRONMENT', 'TECHNOLOGY',
          'LAW', 'MATHS', 'EARTH SCIENCES', 'PHYSICS']

# frequency count
xx = [2625.179999, 1306.259999, 1187.039999, 1166.04, 980.8599997, 810.5999998,
      725.6399996, 678.7899998, 436.5999997, 404.3299999, 348.01, 304.33, 294.19,
      293.02, 282.31, 228.21, 197.3399999, 164.0599998, 157, 50.49999998, 49.60999999, 48.08000005]

fig, ax = plt.subplots(figsize=(10, 10))
ax.pie(xx, labels=labels, autopct="%1.1f");_____no_output_____

### 3.2 Bar Chart

Use the hair colour data to draw a bar chart._____no_output_____

labels = ['Black', 'Red', 'Brown']
hair_colour_freq = [5, 3, 2]

fig, ax = plt.subplots(figsize=(7, 5), dpi=100)

x_pos = np.arange(len(hair_colour_freq))
colors = ['black', 'red', 'brown']

ax.bar(x_pos, hair_colour_freq, align='center', color=colors)

ax.set_xlabel("Hair Colour")
ax.set_ylabel("Number of participants")
ax.set_title("Hair Colour Distribution")

ax.set_xticks(x_pos)
ax.set_xticklabels(labels)
ax_____no_output_____

Now suppose we have the hair colour distribution across genders, so we can plot grouped bar charts. Plot a grouped bar chart to show the distribution of colours acros genders._____no_output_____

"""
        black  red  brown
Male      4     1     3
Female    1     2     2

"""

data = np.array([[4, 1, 3],
                 [1, 2, 3]])

x_pos = np.arange(2)
width = 0.2

fig, ax = plt.subplots(figsize=(7, 5), dpi=100)
ax.bar(x_pos, data[:, 0], width=width, color='black', label='Black', align='center')
ax.bar(x_pos+width, data[:, 1], width=width, color='red', label='Red', align='center')
ax.bar(x_pos+2*width, data[:, 2], width=width, color='brown', label='Brown', align='center')

ax.legend()

ax.set_xlabel("Gender")
ax.set_ylabel("Frequency")
ax.set_title("Distribution of hair colour amongst genders")

ax.set_xticks(x_pos+width)
ax.set_xticklabels(['Male', 'Female'])
ax_____no_output_____

Can we plot it more intelligently? We are doing the same thing multiple times! Is it a good idea to use a loop?_____no_output_____

"""
        black  red  brown
Male      4     1     3
Female    1     2     2

"""

data = np.array([[4, 1, 3],
                 [1, 2, 3]])

n_groups, n_colours = data.shape

x_pos = np.arange(n_groups)
width = 0.2
fig, ax = plt.subplots(figsize=(7, 5), dpi=100)

colours = ['black', 'red', 'brown']
labels = ['Black', 'Red', 'Brown']
for i in range(n_colours):
    ax.bar(x_pos + i*width, data[:, i], width=width, color=colours[i], label=labels[i], align='center')

ax.legend()

ax.set_xlabel("Gender")
ax.set_ylabel("Frequency")
ax.set_title("Distribution of hair colour amongst genders")

ax.set_xticks(x_pos+width)
ax.set_xticklabels(['Male', 'Female'])
ax_____no_output_____

What if we want to group the bar charts based on the hair colour?_____no_output_____

"""
        black  red  brown
Male      4     1     3
Female    1     2     2

"""

labels1 = ['Black', 'Red', 'Brown']
labels2 = ['Male','Female']
colours = ['r', 'y']
data = np.array([[4, 1, 3],
                 [1, 2, 3]])

n_groups, n_colours = data.shape
width = 0.2
x_pos = np.arange(n_colours)

fig, ax = plt.subplots(figsize=(7, 5), dpi=100)
for i in range(n_groups):
    ax.bar(x_pos + i*width, data[i, :], width, align='center', label=labels2[i], color=colours[i])  #出错在这里
ax.set_xlabel("Hair Colour")
ax.set_ylabel("Frequency")
ax.set_title("Distribution of gender amongst hair colours")

ax.set_xticks(x_pos+width/2)
ax.set_xticklabels(labels1)

ax.legend()
ax_____no_output_____

#### Stacked bar chart

The other type of bar chart is stacked bar chart. draw a stacked bar plot of the hair colour data grouped on hair colours._____no_output_____

"""
        black  red  brown
Male      4     1     3
Female    1     2     2

"""

labels = ['Black', 'Red', 'Brown']
data = np.array([[4, 1, 3],
                 [1, 2, 3]])

male_freq = data[0,:]

width = 0.4
x_pos = np.arange(n_colours)

fig, ax = plt.subplots(figsize=(7, 5), dpi=100)
ax.bar(x_pos, data[0, :], width, align='center', label='Male', color='r')
ax.bar(x_pos, data[1, :], width, bottom=male_freq, align='center', label='Female', color='y')

ax.set_xlabel("Hair Colour")
ax.set_ylabel("Frequency")
ax.set_title("Distribution of gender amongst hair colours")

ax.set_xticks(x_pos)
ax.set_xticklabels(labels)

ax.legend(loc=0)_____no_output_____

draw a stacked bar plot grouped on the gender._____no_output_____

"""
        black  red  brown
Male      4     1     3
Female    1     2     2

"""

labels = ['Black', 'Red', 'Brown']
data = np.array([[4, 1, 3],
                 [1, 2, 3]])

black = data[:,0]
red = data[:,1]
brown = data[:,2]

x_pos = np.arange(2)
width = 0.4
fig, ax = plt.subplots(figsize=(7, 5), dpi=100)
ax.bar(x_pos, data[:, 0], width=width, color='black', label='Black', align='center')
ax.bar(x_pos, data[:, 1], width=width, bottom=black, color='red', label='Red', align='center')
ax.bar(x_pos, data[:, 2], width=width, color='brown', bottom=black+red, label='Brown', align='center')

ax.legend(loc=0)

ax.set_xlabel("Gender")
ax.set_ylabel("Frequency")
ax.set_title("Distribution of hair colour amongst genders")

ax.set_xticks(x_pos)
ax.set_xticklabels(['Male', 'Female'])_____no_output_____labels = ['Black', 'Red', 'Brown']
male_freq = [4, 1, 3]
female_freq = [1, 2, 2]

x_pos = np.arange(3)

fig, ax = plt.subplots(nrows=1, ncols=2, figsize=(14, 5), dpi=100)

w1 = 0.2
w2 = 0.4

ax[0].bar(x_pos, male_freq, width=w1, align='center', label='Male', color='r')
ax[0].bar(x_pos+width, female_freq, width=w1, align='center', label='Female', color='y')
ax[1].bar(x_pos, male_freq, width=w2, align='center', label='Male', color='r')
ax[1].bar(x_pos, female_freq, width=w2, bottom=male_freq, align='center', label='Female', color='y')

ax[0].set_xlabel("Hair Colour")
ax[0].set_ylabel("Frequency")
ax[0].set_title("Distribution of gender amongst hair colours")
ax[1].set_xlabel("Hair Colour")
ax[1].set_ylabel("Frequency")
ax[1].set_title("Distribution of gender amongst hair colours")

ax[0].set_xticks(x_pos+width/2)
ax[0].set_xticklabels(labels)
ax[1].set_xticks(x_pos)
ax[1].set_xticklabels(labels)

ax[0].legend()
ax[1].legend(loc=0)_____no_output_____

What if we have too many groups? Draw a bar chart for the Excellence in Research Australia data. _____no_output_____

import matplotlib.pyplot as plt
import numpy as np

# Excellence in Research Australia
labels = ['HEALTH', 'ENGINEERING', 'COMPUTER SCIENCES', 'HUMAN SOCIETY',
          'TOURISM SERVICES', 'EDUCATION', 'CHEMISTRY', 'BIOLOGY', 'PSYCHOLOGY',
          'CREATIVE ARTS', 'LINGUISTICS', 'BUILT ENVIRONMENT', 'HISTORY',
          'ECONOMICS', 'PHILOSOPHY', 'AGRICULTURE', 'ENVIRONMENT', 'TECHNOLOGY',
          'LAW', 'MATHS', 'EARTH SCIENCES', 'PHYSICS']

# frequency count
xx = [2625.179999, 1306.259999, 1187.039999, 1166.04, 980.8599997, 810.5999998,
      725.6399996, 678.7899998, 436.5999997, 404.3299999, 348.01, 304.33, 294.19,
      293.02, 282.31, 228.21, 197.3399999, 164.0599998, 157, 50.49999998, 49.60999999, 48.08000005]

xx_pos = np.arange(len(xx))

fig, ax = plt.subplots(figsize=(15, 5))
ax.bar(xx_pos, xx, align='center')
ax.set_xlabel("research subject")
ax.set_ylabel("score")
ax.set_xticks(xx_pos)
ax.set_xticklabels(labels, rotation=90)
ax.set_xlim(-1, len(xx))_____no_output_____

You can also refer to one online [post](http://queirozf.com/entries/add-labels-and-text-to-matplotlib-plots-annotation-examples) for different customization of those plots._____no_output_____

### 3.3 Wordcloud

As you saw, pie-chart is not very helpful when we have too many sectors. It is hard to read and visually ugly. Instead we can use wordcloud representation. A useful tool is [wordle.net](http://wordle.net). Go to [wordle.net](http://wordle.net) and use it to create a wordcloud for the previous data._____no_output_____

for i in range(len(labels)):
    print("{}:{}".format(labels[i], xx[i]))_____no_output_____!pip install wordcloud_____no_output_____import wget

link_to_data = 'https://github.com/tulip-lab/sit742/raw/master/Jupyter/data/constitution.txt'
DataSet = wget.download(link_to_data)_____no_output_____from os import path
from wordcloud import WordCloud

# Read the whole text.
text = open('constitution.txt').read()

# Generate a word cloud image
wordcloud = WordCloud().generate(text)

# Display the generated image:
# the matplotlib way:
import matplotlib.pyplot as plt
plt.imshow(wordcloud, interpolation='bilinear')
plt.axis("off")
plt.show()

# lower max_font_size
wordcloud = WordCloud(max_font_size=30).generate(text)
plt.figure()
plt.imshow(wordcloud, interpolation="bilinear")
plt.axis("off")
plt.show()

# The PIL way (if you don't have matplotlib)
#image = wordcloud.to_image()
#image.show()_____no_output_____

### 3.4 Step plot

Draw a step plot for the seatbelt data._____no_output_____

freq = np.array([0, 2, 1, 5, 7])
labels = ['Never', 'Rarely', 'Sometimes', 'Most-times', 'Always']
freq_cumsum = np.cumsum(freq)
x_pos = np.arange(len(freq))

fig, ax = plt.subplots()
ax.step(x_pos, freq_cumsum, where='mid')
ax.set_xlabel("Fastening seatbelt behaviour")
ax.set_ylabel("Cumulative frequency")
ax.set_xticks(x_pos)
ax.set_xticklabels(labels)
ax_____no_output_____

### 3.5 Histogram

Google for this paper:

``Johnson, Roger W. "Fitting percentage of body fat to simple body measurements." Journal of Statistics Education 4.1 (1996): 265-266.``

Download the dataset and read the dataset description. Draw a histogram of male weights and female weights._____no_output_____

import wget

link_to_data = 'https://github.com/tulip-lab/sit742/raw/master/Jupyter/data/body.dat.txt'
DataSet = wget.download(link_to_data)_____no_output_____data = np.genfromtxt('body.dat.txt')
m_w = data[data[:, -1] == 1][:, -3]
f_w = data[data[:, -1] == 0][:, -3]

fig, ax = plt.subplots(figsize=(7, 5), dpi=100)
ax.hist(m_w, bins=15, alpha=0.6, label='male')
ax.hist(f_w, bins=15, alpha=0.6, label='female')
ax.set_xlabel("weight (kg)")
ax.set_title("Weight Distribution amongst gXenders")
ax.legend()_____no_output_____

### 3.6 Boxplot
Draw a box plot for male and female weights of the previous dataset._____no_output_____

fig, ax = plt.subplots(figsize=(7, 5), dpi=100)

#To define the style of the fliers.
#red_square = dict(markerfacecolor='r', marker='s') is another example, you can try define you own style.
green_diamond = dict(markerfacecolor='g', marker='D')

#Set the value of showfliers with True to show the outliers beyond the caps.
ax.boxplot([m_w, f_w], labels=['male', 'female'],showfliers=True, flierprops=green_diamond)

ax.set_title("weight distribution amongst genders")
ax_____no_output_____

### 3.7 Scatter plot

Draw a scatter plot of the car weights and their fuel consumption as displayed in the lecture._____no_output_____

import wget

link_to_data = 'https://github.com/tulip-lab/sit742/raw/master/Jupyter/data/Auto.csv'
DataSet = wget.download(link_to_data)_____no_output_____datafile = 'Auto.csv'
data = np.genfromtxt(datafile, delimiter=',')
data = []
with open(datafile, 'r') as fp:
    reader = csv.reader(fp, delimiter=',')
    for row in reader:
        data.append(row)
miles = [dd[1] for dd in data[1:]]
weights = [dd[5] for dd in data[1:]]_____no_output_____fig, ax = plt.subplots(figsize=(15, 5), dpi=100)
ax.scatter(weights,miles, alpha=0.6, edgecolor='none', s=100)
ax.set_xlabel('Car Weight (tons)')
ax.set_ylabel('Miles Per Gallon')
_____no_output_____

Can I also show the number of cylinders on this graph? In other words use the scatter plot to show three variable?_____no_output_____

cylinder = 75 * np.array([int(dd[2]) for dd in data[1:]])_____no_output_____fig, ax = plt.subplots(figsize=(15, 5), dpi=100)
ax.scatter(weights,miles, alpha=0.6, edgecolor='none', s=cylinder)
ax.set_xlabel('Car Weight (tons)')
ax.set_ylabel('Miles Per Gallon')
ax_____no_output_____

---
## 4. Mini Exercise

In 1970, US Congress instituted a random selection process for the military draft. All 366 possible birth dates were placed in plastic capsules in a rotating drum and were selected one by one. The first date drawn from the drum received draft number one and eligible men born on that date were drafted first. The data is provided in a text file with a structure like:

```
Day    Month    MO.NUMBER    DAY_OF_YEAR    DRAFT_NO.
1      JAN      1            1              305
2      JAN      1            2              159
.
31     JAN      1            31             221
1      FEB      2            32             86
.
31     Dec      12           366            100
```
_____no_output_____Using what you have learnt by now, can you tell if it was a fair lottary or not?_____no_output_____Read the data file and save the values in a 2D array._____no_output_____

import wget

link_to_data = 'https://github.com/tulip-lab/sit742/raw/master/Jupyter/data/DraftLottery.txt'
DataSet = wget.download(link_to_data)_____no_output_____data = []
with open('DraftLottery.txt', 'r') as fp:
    reader = csv.reader(fp, delimiter='\t')
    for row in reader:
        data.append(row)

birthdays = np.array([int(row[3]) for row in data[1:]])
draft_no = np.array([int(row[4]) for row in data[1:]])
months = np.array([int(row[2]) for row in data[1:]])_____no_output_____

Plot a `'scatter plot'` of the draft priority vs birthdays._____no_output_____

fig, ax = plt.subplots(figsize=(10, 7), dpi=100)
ax.scatter(birthdays, draft_no, alpha=0.7, s = 100, edgecolor='none')
ax.set_xlabel("Birthday (day of the year)", fontsize=12)
ax.set_ylabel("Draft priority value", fontsize=12)
ax.set_title("USA Draft Lottery Data", fontsize=14)
ax_____no_output_____

In a truly random lottery there should be no relationship between the date and the draft number. To investigate this further we draw boxplots by months and compare them together._____no_output_____

fig, ax = plt.subplots(figsize=(10, 7), dpi=100)
months_range = range(1, 13)

# boxplot data
boxplot_data = [draft_no[months == mm] for mm in months_range]
ax.boxplot(boxplot_data)

# medians
medians = [np.median(dd) for dd in boxplot_data]
ax.plot(months_range, medians, "g--", lw=2)

# means
means = [dd.mean() for dd in boxplot_data]
ax.plot(months_range, means, "k--", lw=2)

month_labels = ["Jan", "Feb", "Mar", "Apr", "May", "Jun",
                "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"]
ax.set_xlabel("Month", fontsize=12)
ax.set_xticklabels(month_labels)
ax.set_ylabel("Draft priority value", fontsize=12)
ax.set_title("USA Draft Lottery Data", fontsize=14)
ax_____no_output_____

While it is impossible to view this trend in a scatterplot of draft number vs. birth date, a series of side-by-side boxplots by month illustrate it clearly. A further investigation of the lottery revealed that the birthdates were placed in the drum by month and were not thoroughly mixed._____no_output_____


# Notebook from YosefLab/scvi_tutorials
Path: python_in_R.ipynb

# Using Python in R with `reticulate`

In this tutorial, we will demonstrate how to perform basic Python operations in R using the library `reticulate`. This includes converting between R and Python dataframe objects and running python functions. Since `scvi-tools` is written in Python, such an interface is necessary to take advantage of these models within the R environment._____no_output_____### Import Libraries_____no_output_____

library(reticulate)
library(anndata)
library(sceasy)

library(Seurat)
library(SeuratData)Attaching SeuratObject

Registered S3 method overwritten by 'cli':
  method     from
  print.boxx spatstat.geom

Warning message in if (is.na(desc)) {:
“the condition has length > 1 and only the first element will be used”
Warning message in if (is.na(desc)) {:
“the condition has length > 1 and only the first element will be used”
Warning message in if (is.na(desc)) {:
“the condition has length > 1 and only the first element will be used”
Warning message in if (is.na(desc)) {:
“the condition has length > 1 and only the first element will be used”
Warning message in if (is.na(desc)) {:
“the condition has length > 1 and only the first element will be used”
Warning message in if (is.na(desc)) {:
“the condition has length > 1 and only the first element will be used”
Warning message in if (is.na(desc)) {:
“the condition has length > 1 and only the first element will be used”
Warning message in if (is.na(desc)) {:
“the condition has length > 1 and only the first element will be used”
Warning message in if (is.na(desc)) {:
“the condition has length > 1 and only the first element will be used”
Warning message in if (is.na(desc)) {:
“the condition has length > 1 and only the first element will be used”
Warning message in if (is.na(desc)) {:
“the condition has length > 1 and only the first element will be used”
Warning message in if (is.na(desc)) {:
“the condition has length > 1 and only the first element will be used”
── Installed datasets ────────────────────────────────────────────────────────────────────────────────────────── SeuratData v0.2.1 ──

✔ ifnb     3.0.0                                                  ✔ stxBrain 0.1.1
✔ pbmc3k   3.1.4

──────────────────────────────────────────────────────────────── Key ────────────────────────────────────────────────────────────────

✔ Dataset loaded successfully
❯ Dataset built with a newer version of Seurat than installed
❓ Unknown version of Seurat installed

## Operating between Python and R_____no_output_____First, we will create a dummy list, and convert between R and Python. Note that R is 1-indexed while Python is 0-indexed, so when retrieiving elements the user should be conscious of what kind of object they are operating on._____no_output_____

lst <- list(1, 2, 3)
print(lst)
print(typeof(lst))[[1]]
[1] 1

[[2]]
[1] 2

[[3]]
[1] 3

[1] "list"

We will convert this R list to a Python list via a function provided by the `reticulate` library called `r_to_py()`. This works for various fundamental R types like vectors, lists, arrays, data frames, functions, and primitives. Any python object will have `typeof(obj)` as `environment`. To see the Python type, we can call `class(obj)` instead._____no_output_____

py_lst <- r_to_py(lst)
print(py_lst)
print(typeof(py_lst))
print(class(py_lst))[1.0, 2.0, 3.0]
[1] "environment"
[1] "python.builtin.list"   "python.builtin.object"

We can call instance functions of a Python object by replacing the usual dot notation with `$` instead. So something like `lst.append(5)` would become `lst$append(5)`._____no_output_____

py_lst$append(5)
print(py_lst)_____no_output_____

Note, arguments passed into these functions can either be Python or R objects. R objects passed in as arguments will automatically converted to the corresponding Python type via the `r_to_py()` function. However, this can sometimes result in unexpected results. For example, `0` in R will be automatically inferred as a float, which can result in an error when trying to pop an element below. We workaround this by explictly casting the R term to an integer with `as.integer(0)` or using the `0L` syntax, which results in the proper type conversion._____no_output_____

# This will fail.
py_lst$pop(0)_____no_output_____py_lst$pop(0L)
print(py_lst)_____no_output_____

Finally, we will convert back into an R list with the function `py_to_r()` which executes the inverse of `r_to_py()`._____no_output_____

lst <- py_to_r(py_lst)
print(lst)[1] 2 3 5

## Import Python libraries_____no_output_____Now, we load the scanpy library via reticulate using the `import()` function. The `convert` boolean argument determines whether the output of Python functions is automatically converted to an R object equivalent via the `py_to_r()` function. Here, we set it to `FALSE` intentionally since often times we would like to retain the Python format for further manipulation in Python (e.g. with scanpy). Additionally, this keeps data type conversion more explicit, avoiding type confusion._____no_output_____

sc <- import('scanpy', convert = FALSE)_____no_output_____

## Load Dataset with SeuratData_____no_output_____

data("pbmc3k")
pbmc <- pbmc3k_____no_output_____pbmc_____no_output_____

In order to make use of `scvi-tools`, we use a third-party library called `sceasy` to convert the SeuratObject into an AnnData object, the primary format used by `scanpy` and `scvi-tools`._____no_output_____

adata <- convertFormat(pbmc, from="seurat", to="anndata", main_layer="counts", drop_single_values=FALSE)
adata_____no_output_____

We can access the AnnData fields in the same way we call instance functions, with the `$` syntax._____no_output_____

adata$obs$head()_____no_output_____class(adata$obs)_____no_output_____head(py_to_r(adata$obs))_____no_output_____

Above, we loaded the `anndata` R library. It is important to know when dealing with a Python AnnData object and an R AnnDataR6 Object. We can distinguish these by using the `class()` method, then using the `py_to_r(), r_to_py()` functions to interoperate between the two. Generally, it is recommended to use the R AnnDataR6 object to manipulate fields._____no_output_____

class(adata)_____no_output_____class(py_to_r(adata))_____no_output_____# Convert adata object to R AnnDataR6 object.
adata <- py_to_r(adata)_____no_output_____

We can set fields in the AnnData object using the `$` syntax. Here, we run CPM normalization using scanpy and save it to a new layer in the AnnData object. For the sake of demonstration, we do not use the inplace update option that scanpy provides. Note, this only works well if using the AnnDataR6 object._____no_output_____

X_norm <- sc$pp$normalize_total(adata, target_sum = 1e+09, inplace = FALSE)["X"]
adata$layers["X_norm"] <- X_norm_____no_output_____head(as.data.frame(adata$layers["X_norm"]))_____no_output_____

Now you should be comfortable interoperating between R and Python. Once you configure your AnnData object to contain all the necessary fields for your model of choice, you can intialize and train with the AnnData object. Visit our tutorials page for examples of running `scvi-tools` in R._____no_output_____## Session Info_____no_output_____

sI <- sessionInfo()
sI$loadedOnly <- NULL
print(sI, locale=FALSE)R version 4.0.3 (2020-10-10)
Platform: x86_64-conda-linux-gnu (64-bit)
Running under: Ubuntu 16.04.6 LTS

Matrix products: default
BLAS/LAPACK: /data/yosef2/users/jhong/miniconda3/envs/r_tutorial/lib/libopenblasp-r0.3.12.so

attached base packages:
[1] stats     graphics  grDevices utils     datasets  methods   base

other attached packages:
[1] stxBrain.SeuratData_0.1.1 pbmc3k.SeuratData_3.1.4
[3] ifnb.SeuratData_3.0.0     SeuratData_0.2.1
[5] SeuratObject_4.0.2        Seurat_4.0.4
[7] sceasy_0.0.6              anndata_0.7.5.3
[9] reticulate_1.22


# Notebook from mustaric/MASTTools
Path: tesscut/begin_tesscut_request-devel.ipynb

# Beginner: Cutout of the TESS Full Frame Image Time Series using Python Requests

This notebook shows the user how to use the MAST programmatic interface to create a cutout of the small section of the TESS FFIs. For this example we will determine the RA and Dec for TOI 109.01, which is on TIC 29344935. We then perform a query to determine which sectors contain this RA and Dec, peform a cutout of the FFI time series, open the resulting target pixel files and plot the first image of each file with the WCS overlayed on the image.

This tutorial shows the users how to do the following: use astroquery.catalogs to query the TIC, use the python requests module to query the available sectors, and python requests to obtain a ffi cutout with tesscut.

### Table of Contents
  [Imports](#imports_ID)

  [Astroquery Search of the TIC](#catalog_ID)

  [Use Tesscut to get FFI Cutout](#tesscut_ID)

  [Additional Resources](#resources_ID)

  [About this Notebook](#about_ID) _____no_output_____## Import statements for this notebook.
_____no_output_____

import numpy as np
from astropy.io import fits
import requests
from astroquery.mast import Catalogs
import zipfile
from astropy.wcs import WCS
import matplotlib.pyplot as plt

%matplotlib inline
#Next line puts your mast api token in a useful variable for later.
#See https://auth.mast.stsci.edu/info
masttoken=%env MAST_DV_TOKEN
urlbase="https://mastdev.stsci.edu/tesscut/api/v0.1"_____no_output_____

## Get RA and Dec of your target using Astroquery Catalogs

Here we use the astroquery.mast Catalogs query_criteria function to request the RA and Dec of for the object with TIC ID = 29344935. We collect the RA and Declination and store them in their own variables.  For good measure, we print out some of the resulting values from the catalog search in a table.

Note, that the catalog name is "Tic". This specific capitalization is required for this to work._____no_output_____

ticid=25155310
ticid=307210830  #Mdwarf
#ticid=261136679
starName="TIC "+str(ticid)
rad_search=10/60 #radius in degrees
#catalogData=Catalogs.query_criteria(catalog="Tic",ID=str(ticid))
catalogData = Catalogs.query_object(starName, radius=rad_search, catalog="TIC")
Ra=catalogData[0]['ra']
Dec=catalogData[0]['dec']
print(catalogData[0]['ID','Jmag','ra','dec','objType'])

bright=catalogData['Jmag']<12    ID     Jmag    ra      dec   objType
--------- ----- -------- ------- -------
307210830 7.933 124.5319 -68.313    STAR

## Use sector query to determine which TESS sectors have FFI data

Using the tess sector information service, we make a request to determine which sectors/cameras/CCDs contain data for this target. Remember that there is a set of FFIs for each Sector and those are broken up into 4 cameras which each have 4 CCDs.  We will do this with a radius=0 cone search to find only those FFI sets that contain the star of interest. You can make the query using a larger radius, which may matter if the star is near the edge of one of the CCDs.

Note, the request is returned in a json format. The 'results' key contains an array of dictionaries with the information we are looking for.  _____no_output_____

myparams={"ra":Ra, "dec":Dec, "radius":"0m"}
url=urlbase+"/sector"

requestData=requests.get(url=url,params=myparams)
print(requestData.headers.get('content-type'))application/json; charset=UTF-8
sectors=requestData.json()['results']
sectors_____no_output_____

Each element of the array shows one set of sector, camer and CCD that has data for the requested region of sky. _____no_output_____
## Request a 10 x 10 pixel timeseries of FFI cutouts around specified RA and Dec.
This delivers a zipped file containing a cutout for each set of FFIs as listed above. It is also possible to request only one sector using the "sector" parameter._____no_output_____

myparams={"ra":Ra, "dec":Dec, "x":60, "y":120,
           "units":"px","sector":"All"}
header={'Authorization': "token "+masttoken}

url=urlbase+"/astrocut"

r=requests.get(url=url,params=myparams,headers=header)
#Check the content type.
print(r)
print(r.headers.get('content-type'))<Response [200]>
application/zip

Write the resulting zip file.s_____no_output_____

open('astrocut.zip', 'wb').write(r.content)_____no_output_____

### Open the zip file so we can see what is in the header of the two files.
We use python's zipfile to unzip the file, but this could also be done using unzip from the command line.  _____no_output_____

zipRef=zipfile.ZipFile('astrocut.zip','r')
zipRef.extractall('.')
zipRef.close()
cutoutnames = zipRef.namelist()
print(cutoutnames)['tess-s0001-4-2_84.291198_-80.469143_120x60_astrocut.fits']

### Inspect the contents of both files.
Use the fits.info function to see the contents of the file. Notice that it has three extensions just like a target pixel file. Most of the interesting information is in the PIXELS extension._____no_output_____

file1=cutoutnames[0]
file2=cutoutnames[1]
fits.info(file1)_____no_output_____fits.info(file1)Filename: tess-s0001-4-3_124.5319_-68.313_120x60_astrocut.fits
No.    Name      Ver    Type      Cards   Dimensions   Format
  0  PRIMARY       1 PrimaryHDU      54   ()
  1  PIXELS        1 BinTableHDU    244   1282R x 12C   [D, E, J, 7200J, 7200E, 7200E, 7200E, 7200E, J, E, E, 38A]
  2  APERTURE      1 ImageHDU       143   (120, 60)   float64
hdu1=fits.open(file1)
#hdu1[1].header_____no_output_____

### Compare the time for the start of observation for both files.
Since one is from Sector 1 and the other is from Sector 2, these are approximately a month a part. _____no_output_____

hdu2=fits.open(file2)
print("File 1 Start Time: " + hdu1[0].header['DATE-OBS'])
print("File 2 Start Time: " + hdu2[0].header['DATE-OBS'])
print(np.shape(hdu1[1].data[0]['FLUX']))
print(np.shape(hdu1[2].data))File 1 Start Time: 2018-07-25T19:29:42.708Z
File 2 Start Time: 2018-08-23T14:29:41.724Z
(60, 120)
(60, 120)

## Plot the First Image of the First Sector of Data

We use the WCS in the header to place a red dot on the image for the catalog position of the star on the figure as a demonstration of the WCS. Remember that the WCS is based on the WCS stored in the FFI file for the central part of the time series and there can be some motion during a sector of observation. _____no_output_____

first_image=hdu1[1].data['FLUX'][0]
wcs=WCS(hdu1[2].header)

fig = plt.figure(figsize=(10,10))
fig.clf()
#fig.add_subplot(111, projection=wcs)
plt.imshow(first_image, origin='lower', cmap=plt.cm.viridis,vmin=-1,vmax=10100)
#plt.imshow(first_image.reshape(np.shape(first_image)[::-1]), origin='lower', cmap=plt.cm.viridis)
plt.xlabel('CCD Column',fontsize=14)
plt.ylabel('CCD Row',fontsize=14)
plt.grid(axis='both',color='white', ls='solid')

coord=list(map(lambda x,y:[x,y], catalogData['ra'],catalogData['dec']))
starloc=wcs.all_world2pix([[Ra,Dec]],0)  #Second is origin
nearbyloc=wcs.all_world2pix(coord,0)
plt.scatter(starloc[0,0],starloc[0,1],s=45,color='red')
plt.scatter(nearbyloc[bright,0],nearbyloc[bright,1],s=15,color='orange')_____no_output_____

## Additional Resources
[TESScut API Documentation](https://mast.stsci.edu/tesscut/)

[Astrocut Documentation](https://astrocut.readthedocs.io/en/latest/)

[TESS Homepage](https://archive.stsci.edu/tess)
_____no_output_____
## About this Notebook
**Author:** Susan E. Mullally, STScI Archive Scientist

**Updated On:** 2018-11-13_____no_output_____[Top of Page](#title_ID)
 _____no_output_____