# Notebook from plrlhb12/compare_variants
Path: compare variants-logistic regression.ipynb

<code>
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
import matplotlib as mpl

%matplotlib inline
#mpl.rcParams.update(mpl.rcParamsDefault)
%config InlineBackend.figure_format = 'retina'

#plt.style.use("seaborn-white")
print(plt.style.available)

#import matplotlib as mpl
#mpl.rcParams['figure.dpi'] = 80
#mpl.rcParams['figure.dpi'] = 300['Solarize_Light2', '_classic_test_patch', 'bmh', 'classic', 'dark_background', 'fast', 'fivethirtyeight', 'ggplot', 'grayscale', 'seaborn', 'seaborn-bright', 'seaborn-colorblind', 'seaborn-dark', 'seaborn-dark-palette', 'seaborn-darkgrid', 'seaborn-deep', 'seaborn-muted', 'seaborn-notebook', 'seaborn-paper', 'seaborn-pastel', 'seaborn-poster', 'seaborn-talk', 'seaborn-ticks', 'seaborn-white', 'seaborn-whitegrid', 'tableau-colorblind10']
pd.set_option('display.max_columns', None)
pd.set_option('display.max_rows', None)
# pd.options.display.max_columns = None
# pd.options.display.max_rows = None_____no_output_____!pwd_____no_output_____%ls -lthtotal 3527344
-rw-r--r--   1 pengl7  NIH\Domain Users   300K Sep 15 11:56 venn_diagram_after_fliter_2.ipynb
-rw-r--r--@  1 pengl7  NIH\Domain Users   162B Sep 15 11:29 ~$ader of the long read sequence vcf file from 10X Genomics.docx
-rw-r--r--@  1 pengl7  NIH\Domain Users    16K Sep 15 11:29 header of the long read sequence vcf file from 10X Genomics.docx
-rw-r--r--   1 pengl7  NIH\Domain Users    13K Sep 15 10:58 compare variants-logistic regression.ipynb
-rw-r-----   1 pengl7  NIH\Domain Users   115M Sep 15 09:38 UNHS_GT19-38445_bcfc1_2.annotated_qual30_pass.chr1.vcf
-rw-r-----   1 pengl7  NIH\Domain Users   110M Sep 15 09:37 Jax_KOLF2-ARID2-A2_bcfc1_2.annotated_qual30_pass.chr1.vcf
-rw-r-----   1 pengl7  NIH\Domain Users   268M Sep 14 21:25 long_psomagen_KOLF2.1_variants.annotated_qual30_pass.chr1.vcf
-rw-r-----   1 pengl7  NIH\Domain Users   301M Sep 14 21:24 long_Jax_KOLF2C1_variants.annotated_qual30_pass.chr1.vcf
-rw-r--r--@  1 pengl7  NIH\Domain Users   1.4M Sep 10 13:00 check variants and genotype.pptx
-rw-r--r--@  1 pengl7  NIH\Domain Users   229K Sep 10 11:54 check variants from different platforms.pptx
-rw-r--r--   1 pengl7  NIH\Domain Users   317K Sep 10 11:01 venn_diagram_after_fliter.ipynb
-rw-r--r--   1 pengl7  NIH\Domain Users    14K Sep 10 10:58 KOLF2.1_Jax_short_psomagen_long_filter.png
-rw-r--r--@  1 pengl7  NIH\Domain Users    14K Sep 10 10:58 KOLF2.1_Jax_short_psomagen_long_filter.pdf
-rw-r--r--   1 pengl7  NIH\Domain Users    15K Sep 10 10:51 Psomagen_KOLF2.1_filter.png
-rw-r--r--@  1 pengl7  NIH\Domain Users    14K Sep 10 10:51 Psomagen_KOLF2.1_filter.pdf
-rw-r--r--   1 pengl7  NIH\Domain Users    16K Sep 10 10:14 KOLF2.1_vs_C1_filter.png
-rw-r--r--@  1 pengl7  NIH\Domain Users    14K Sep 10 10:14 KOLF2.1_vs_C1_filter.pdf
-rw-r--r--   1 pengl7  NIH\Domain Users    22K Sep 10 10:12 3_kolf2.1_filter.png
-rw-r--r--   1 pengl7  NIH\Domain Users    14K Sep 10 10:12 3_kolf2.1_filter.pdf
-rw-r--r--   1 pengl7  NIH\Domain Users    23K Sep 10 09:55 compare variants-caluclate qual.ipynb
drwxr-xr-x  19 pengl7  NIH\Domain Users   608B Sep  7 23:45 metrics/
drwxr-xr-x  29 pengl7  NIH\Domain Users   928B Sep  7 23:24 output_figs/
-rw-r--r--   1 pengl7  NIH\Domain Users   453K Sep  4 17:23 compare_variants_venn_diagram.ipynb
-rw-r-----   1 pengl7  NIH\Domain Users    80M Sep  4 17:15 long_psomagen_KOLF2.1_variants_filt30_merge.vcf
-rw-r-----   1 pengl7  NIH\Domain Users    80M Sep  4 17:14 long_Jax_KOLF2C1_variants_filt30_merge.vcf
-rw-r-----   1 pengl7  NIH\Domain Users    95M Sep  4 17:02 long_Jax_KOLF2C1_variants_filt30.vcf
-rw-r-----   1 pengl7  NIH\Domain Users    94M Sep  4 17:00 long_psomagen_KOLF2.1_variants_filt30.vcf
-rw-r--r--@  1 pengl7  NIH\Domain Users   4.1K Sep  4 14:04 subset_annot_calc_variant_types.html
-rw-r--r--   1 pengl7  NIH\Domain Users    12K Sep  4 12:08 compare-variants.ipynb
-rw-r--r--@  1 pengl7  NIH\Domain Users   6.4K Sep  4 10:07 30Aug2020-diff-vcf.txt
-rw-r--r--   1 pengl7  NIH\Domain Users    25K Sep  1 16:13 calculate karyotype resutls.ipynb
-rw-r-----   1 pengl7  NIH\Domain Users    36M Sep  1 15:00 Jax_KOLF2-ARID2-A2_bcfc1_qaul
-rw-r-----   1 pengl7  NIH\Domain Users    34M Sep  1 14:59 UNHS_GT19-38445_bcfc1_qaul
-rw-r-----   1 pengl7  NIH\Domain Users    38M Sep  1 14:48 long_psomagen_KOLF2.1_variants_qual
-rw-r-----   1 pengl7  NIH\Domain Users    37M Sep  1 14:48 long_Jax_KOLF2C1_variants_qual
-rw-r--r--@  1 pengl7  NIH\Domain Users    93K Aug 31 14:15 How_to_Venn.ipynb
-rw-r-----   1 pengl7  NIH\Domain Users    91M Aug 31 11:07 long_Jax_KOLF2C1_variants_cut_merage.vcf
-rw-r-----   1 pengl7  NIH\Domain Users    93M Aug 31 11:07 long_psomagen_KOLF2.1_variants_cut_merage.vcf
-rw-r-----   1 pengl7  NIH\Domain Users    81M Aug 31 11:06 UNHS_GT19-38445_bcfc1_cut_merage.vcf
-rw-r-----   1 pengl7  NIH\Domain Users    84M Aug 31 11:05 Jax_KOLF2-ARID2-A2_bcfc1_cut_merage.vcf
long_jax = pd.read_csv("long_Jax_KOLF2C1_variants.annotated_qual30_pass.chr1.vcf", sep="\t", header=None)
long_jax.head(2)_____no_output_____long_jax.shape_____no_output_____pd.set_option('display.max_colwidth', None)
long_jax.iloc[:, 7][:10]_____no_output_____long_jax_sub = long_jax.iloc[:, [2,5,7]]
long_jax_sub.columns = ["varaint_ID", "Qual", "INFO"]
long_jax_sub.head()_____no_output_____# this method doesn't work well since there are too many ; inside the column of INFP
# long_jax["AC"], long_jax["AF"], long_jax["BaseQRankSum"], long_jax["ClippingR"] = long_jax_sub["INFO"].str.split(";")[:5]_____no_output_____df_info = pd.DataFrame(long_jax_sub["INFO"].str.split(";"))_____no_output_____df_info.head()_____no_output_____type(info)_____no_output_____info[:5]_____no_output_____long_jax.describe()_____no_output_____len(long_jax)_____no_output_____long_pso = pd.read_csv("long_psomagen_KOLF2.1_variants.annotated_qual30_pass.chr1.vcf", sep="\t", header=None)
long_pso.columns = ["A"]
long_pso.head()_____no_output_____ls20 = long_jax[long_jax["A"] < 20]
print('total variants are: ', len(long_pso))
print('varints less than QUAL 20 are: ', len(long_jax[long_jax["A"] < 20]))
print('varints less than QUAL 30 are: ', len(long_jax[long_jax["A"] < 30]))_____no_output_____long_pso.describe()_____no_output_____print('total variants are: ', len(long_pso))
print('varints less than QUAL 20 are: ', len(long_pso[long_pso["A"] <= 20]))
print('varints less than QUAL 30 are: ', len(long_pso[long_pso["A"] <= 30]))_____no_output_____short_jax = pd.read_csv("Jax_KOLF2-ARID2-A2_bcfc1_qaul", sep="\t", header=None)
short_jax.columns = ["A"]
short_jax.head()_____no_output_____short_jax.describe()_____no_output_____print('total variants are: ', len(short_jax))
print('varints less than QUAL 20 are: ', len(short_jax[short_jax["A"] <= 30]))_____no_output_____short_pso = pd.read_csv("UNHS_GT19-38445_bcfc1_qaul", sep="\t", header=None)
short_pso.columns = ["A"]
short_pso.head()_____no_output_____short_pso.describe()_____no_output_____print('total variants are: ', len(short_pso))
print('varints less than QUAL 20 are: ', len(short_pso[short_pso["A"] <= 30]))_____no_output_____
</code>


# Notebook from vishalbelsare/stellargraph
Path: demos/interpretability/gcn-node-link-importance.ipynb

# Interpreting nodes and edges with saliency maps in GCN
_____no_output_____

Run the latest release of this notebook:

_____no_output_____This demo shows how to use integrated gradients in graph convolutional networks to obtain accurate importance estimations for both the nodes and edges. The notebook consists of three parts:
- setting up the node classification problem for Cora citation network
- training and evaluating a GCN model for node classification
- calculating node and edge importances for model's predictions of query ("target") nodes

**References**

[1] Axiomatic Attribution for Deep Networks. M. Sundararajan, A. Taly, and Q. Yan.
    Proceedings of the 34th International Conference on Machine Learning, Sydney, Australia, PMLR 70, 2017
    ([link](https://arxiv.org/pdf/1703.01365.pdf)).

[2] Adversarial Examples on Graph Data: Deep Insights into Attack and Defense. H. Wu, C. Wang, Y. Tyshetskiy, A. Docherty, K. Lu, and L. Zhu. arXiv: 1903.01610 ([link](https://arxiv.org/abs/1903.01610))._____no_output_____

# install StellarGraph if running on Google Colab
import sys
if 'google.colab' in sys.modules:
  %pip install -q stellargraph[demos]==1.3.0b_____no_output_____# verify that we're using the correct version of StellarGraph for this notebook
import stellargraph as sg

try:
    sg.utils.validate_notebook_version("1.3.0b")
except AttributeError:
    raise ValueError(
        f"This notebook requires StellarGraph version 1.3.0b, but a different version {sg.__version__} is installed.  Please see <https://github.com/stellargraph/stellargraph/issues/1172>."
    ) from None_____no_output_____import networkx as nx
import pandas as pd
import numpy as np
from scipy import stats
import os
import time
import stellargraph as sg
from stellargraph.mapper import FullBatchNodeGenerator
from stellargraph.layer import GCN
from tensorflow import keras
from tensorflow.keras import layers, optimizers, losses, metrics, Model, regularizers
from sklearn import preprocessing, feature_extraction, model_selection
from copy import deepcopy
import matplotlib.pyplot as plt
from stellargraph import datasets
from IPython.display import display, HTML
%matplotlib inline_____no_output_____

## Loading the CORA network_____no_output_____(See [the "Loading from Pandas" demo](../basics/loading-pandas.ipynb) for details on how data can be loaded.)_____no_output_____

dataset = datasets.Cora()
display(HTML(dataset.description))
G, subjects = dataset.load()_____no_output_____

### Splitting the data_____no_output_____For machine learning we want to take a subset of the nodes for training, and use the rest for validation and testing. We'll use scikit-learn again to do this.

Here we're taking 140 node labels for training, 500 for validation, and the rest for testing._____no_output_____

train_subjects, test_subjects = model_selection.train_test_split(
    subjects, train_size=140, test_size=None, stratify=subjects
)
val_subjects, test_subjects = model_selection.train_test_split(
    test_subjects, train_size=500, test_size=None, stratify=test_subjects
)_____no_output_____

### Converting to numeric arrays_____no_output_____For our categorical target, we will use one-hot vectors that will be fed into a soft-max Keras layer during training. To do this conversion ..._____no_output_____

target_encoding = preprocessing.LabelBinarizer()

train_targets = target_encoding.fit_transform(train_subjects)
val_targets = target_encoding.transform(val_subjects)
test_targets = target_encoding.transform(test_subjects)

all_targets = target_encoding.transform(subjects)_____no_output_____

### Creating the GCN model in Keras_____no_output_____To feed data from the graph to the Keras model we need a generator. Since GCN is a full-batch model, we use the `FullBatchNodeGenerator` class.

Note: For interpretability we require a dense matrix so we set `sparse=False` in the `FullBatchNodeGenerator`._____no_output_____

generator = FullBatchNodeGenerator(G, sparse=False)Using GCN (local pooling) filters...

For training we map only the training nodes returned from our splitter and the target values._____no_output_____

train_gen = generator.flow(train_subjects.index, train_targets)_____no_output_____

Now we can specify our machine learning model: tn this example we use two GCN layers with 16-dimensional hidden node features at each layer with ELU activation functions._____no_output_____

layer_sizes = [16, 16]
gcn = GCN(
    layer_sizes=layer_sizes,
    activations=["elu", "elu"],
    generator=generator,
    dropout=0.3,
    kernel_regularizer=regularizers.l2(5e-4),
)_____no_output_____# Expose the input and output tensors of the GCN model for node prediction, via GCN.in_out_tensors() method:
x_inp, x_out = gcn.in_out_tensors()
# Snap the final estimator layer to x_out
x_out = layers.Dense(units=train_targets.shape[1], activation="softmax")(x_out)_____no_output_____

### Training the model_____no_output_____Now let's create the actual Keras model with the input tensors `x_inp` and output tensors being the predictions `x_out` from the final dense layer_____no_output_____

model = keras.Model(inputs=x_inp, outputs=x_out)

model.compile(
    optimizer=optimizers.Adam(lr=0.01),  # decay=0.001),
    loss=losses.categorical_crossentropy,
    metrics=[metrics.categorical_accuracy],
)_____no_output_____

Train the model, keeping track of its loss and accuracy on the training set, and its generalisation performance on the validation set (we need to create another generator over the validation data for this)_____no_output_____

val_gen = generator.flow(val_subjects.index, val_targets)_____no_output_____

Train the model_____no_output_____

history = model.fit(
    train_gen, shuffle=False, epochs=20, verbose=2, validation_data=val_gen
)Epoch 1/20
1/1 - 0s - loss: 1.9493 - categorical_accuracy: 0.1429 - val_loss: 1.8287 - val_categorical_accuracy: 0.3040
Epoch 2/20
1/1 - 0s - loss: 1.7875 - categorical_accuracy: 0.3214 - val_loss: 1.7499 - val_categorical_accuracy: 0.3040
Epoch 3/20
1/1 - 0s - loss: 1.6701 - categorical_accuracy: 0.3357 - val_loss: 1.6854 - val_categorical_accuracy: 0.3040
Epoch 4/20
1/1 - 0s - loss: 1.5617 - categorical_accuracy: 0.3286 - val_loss: 1.6155 - val_categorical_accuracy: 0.3160
Epoch 5/20
1/1 - 0s - loss: 1.4499 - categorical_accuracy: 0.3786 - val_loss: 1.5322 - val_categorical_accuracy: 0.3680
Epoch 6/20
1/1 - 0s - loss: 1.3278 - categorical_accuracy: 0.5071 - val_loss: 1.4399 - val_categorical_accuracy: 0.4700
Epoch 7/20
1/1 - 0s - loss: 1.1788 - categorical_accuracy: 0.6214 - val_loss: 1.3484 - val_categorical_accuracy: 0.5820
Epoch 8/20
1/1 - 0s - loss: 1.0673 - categorical_accuracy: 0.7571 - val_loss: 1.2649 - val_categorical_accuracy: 0.6440
Epoch 9/20
1/1 - 0s - loss: 0.9381 - categorical_accuracy: 0.8357 - val_loss: 1.1877 - val_categorical_accuracy: 0.6900
Epoch 10/20
1/1 - 0s - loss: 0.8570 - categorical_accuracy: 0.8571 - val_loss: 1.1152 - val_categorical_accuracy: 0.7280
Epoch 11/20
1/1 - 0s - loss: 0.7681 - categorical_accuracy: 0.9286 - val_loss: 1.0464 - val_categorical_accuracy: 0.7540
Epoch 12/20
1/1 - 0s - loss: 0.6665 - categorical_accuracy: 0.9429 - val_loss: 0.9840 - val_categorical_accuracy: 0.7740
Epoch 13/20
1/1 - 0s - loss: 0.5994 - categorical_accuracy: 0.9500 - val_loss: 0.9309 - val_categorical_accuracy: 0.7820
Epoch 14/20
1/1 - 0s - loss: 0.5016 - categorical_accuracy: 0.9643 - val_loss: 0.8893 - val_categorical_accuracy: 0.7880
Epoch 15/20
1/1 - 0s - loss: 0.4481 - categorical_accuracy: 0.9786 - val_loss: 0.8585 - val_categorical_accuracy: 0.7860
Epoch 16/20
1/1 - 0s - loss: 0.3930 - categorical_accuracy: 0.9786 - val_loss: 0.8370 - val_categorical_accuracy: 0.7840
Epoch 17/20
1/1 - 0s - loss: 0.3617 - categorical_accuracy: 0.9714 - val_loss: 0.8221 - val_categorical_accuracy: 0.7860
Epoch 18/20
1/1 - 0s - loss: 0.3515 - categorical_accuracy: 0.9714 - val_loss: 0.8109 - val_categorical_accuracy: 0.7880
Epoch 19/20
1/1 - 0s - loss: 0.3070 - categorical_accuracy: 0.9857 - val_loss: 0.8035 - val_categorical_accuracy: 0.7880
Epoch 20/20
1/1 - 0s - loss: 0.2896 - categorical_accuracy: 0.9786 - val_loss: 0.7987 - val_categorical_accuracy: 0.7920
sg.utils.plot_history(history)_____no_output_____

Evaluate the trained model on the test set_____no_output_____

test_gen = generator.flow(test_subjects.index, test_targets)
test_metrics = model.evaluate(test_gen)
print("\nTest Set Metrics:")
for name, val in zip(model.metrics_names, test_metrics):
    print("\t{}: {:0.4f}".format(name, val))
Test Set Metrics:
	loss: 0.7585
	categorical_accuracy: 0.8037

## Node and link importance via saliency maps_____no_output_____In order to understand why a selected node is predicted as a certain class we want to find the node feature importance, total node importance, and link importance for nodes and edges in the selected node's neighbourhood (ego-net). These importances give information about the effect of changes in the node's features and its neighbourhood on the prediction of the node, specifically:

- **Node feature importance**: Given the selected node $t$ and the model's prediction $s(c)$ for class $c$. The feature importance can be calculated for each node $v$ in the selected node's ego-net where the importance of feature $f$ for node $v$ is the change predicted score $s(c)$ for the selected node when the feature $f$ of node $v$ is perturbed.
- **Total node importance**: This is defined as the sum of the feature importances for node $v$ for all features. Nodes with high importance (positive or negative) affect the prediction for the selected node more than links with low importance.
- **Link importance**: This is defined as the change in the selected node's predicted score $s(c)$ if the link $e=(u, v)$ is removed from the graph. Links with high importance (positive or negative) affect the prediction for the selected node more than links with low importance.

Node and link importances can be used to assess the role of nodes and links in model's predictions for the node(s) of interest (the selected node). For datasets like CORA-ML, the features and edges are binary, vanilla gradients may not perform well so we use integrated gradients [[1]](#refs) to compute them.

Another interesting application of node and link importances is to identify model vulnerabilities to attacks via perturbing node features and graph structure (see [[2]](#refs))._____no_output_____To investigate these importances we use the StellarGraph `saliency_maps` routines:_____no_output_____

from stellargraph.interpretability.saliency_maps import IntegratedGradients_____no_output_____

Select the target node whose prediction is to be interpreted_____no_output_____

graph_nodes = list(G.nodes())
target_nid = 1109199
target_idx = graph_nodes.index(target_nid)
y_true = all_targets[target_idx]  # true class of the target node_____no_output_____all_gen = generator.flow(graph_nodes)
y_pred = model.predict(all_gen)[0, target_idx]
class_of_interest = np.argmax(y_pred)

print(
    "Selected node id: {}, \nTrue label: {}, \nPredicted scores: {}".format(
        target_nid, y_true, y_pred.round(2)
    )
)Selected node id: 1109199,
True label: [0 1 0 0 0 0 0],
Predicted scores: [0.06 0.57 0.15 0.06 0.05 0.04 0.07]

Get the node feature importance by using integrated gradients_____no_output_____

int_grad_saliency = IntegratedGradients(model, train_gen)_____no_output_____

For the parameters of `get_node_importance` method, `X` and `A` are the feature and adjacency matrices, respectively. `target_idx` is the node of interest, and `class_of_interest` is set as the predicted label of the node. `steps` indicates the number of steps used to approximate the integration in integrated gradients calculation. A larger value of `steps` gives better approximation, at the cost of higher computational overhead._____no_output_____

integrated_node_importance = int_grad_saliency.get_node_importance(
    target_idx, class_of_interest, steps=50
)
To change all layers to have dtype float64 by default, call `tf.keras.backend.set_floatx('float64')`. To change just this layer, pass dtype='float64' to the layer constructor. If you are the author of this layer, you can disable autocasting by passing autocast=False to the base Layer constructor.

To change all layers to have dtype float64 by default, call `tf.keras.backend.set_floatx('float64')`. To change just this layer, pass dtype='float64' to the layer constructor. If you are the author of this layer, you can disable autocasting by passing autocast=False to the base Layer constructor.

integrated_node_importance.shape_____no_output_____print("\nintegrated_node_importance", integrated_node_importance.round(2))
print("integrate_node_importance.shape = {}".format(integrated_node_importance.shape))
print(
    "integrated self-importance of target node {}: {}".format(
        target_nid, integrated_node_importance[target_idx].round(2)
    )
)
integrated_node_importance [0. 0. 0. ... 0. 0. 0.]
integrate_node_importance.shape = (2708,)
integrated self-importance of target node 1109199: 3.61

Check that number of non-zero node importance values is less or equal the number of nodes in target node's K-hop ego net (where K is the number of GCN layers in the model)_____no_output_____

G_ego = nx.ego_graph(G.to_networkx(), target_nid, radius=len(gcn.activations))_____no_output_____print("Number of nodes in the ego graph: {}".format(len(G_ego.nodes())))
print(
    "Number of non-zero elements in integrated_node_importance: {}".format(
        np.count_nonzero(integrated_node_importance)
    )
)Number of nodes in the ego graph: 202
Number of non-zero elements in integrated_node_importance: 202

We now compute the link importance using integrated gradients [1]. Integrated gradients are obtained by accumulating the gradients along the path between the baseline (all-zero graph) and the state of the graph. They provide better sensitivity for the graphs with binary features and edges compared with the vanilla gradients._____no_output_____

integrate_link_importance = int_grad_saliency.get_integrated_link_masks(
    target_idx, class_of_interest, steps=50
)
print("integrate_link_importance.shape = {}".format(integrate_link_importance.shape))integrate_link_importance.shape = (2708, 2708)

Some sanity checks:
We expect the number of non-zero elements in the integrated link importance be same or less than the number of real edges in the ego graph._____no_output_____

(X, _, A), _ = train_gen[0]_____no_output_____# The built-in number_of_edges function for the ego graph does not count the self-loops and some reversed edges
# in the non-directed graph so we do the sanity check as the following.
G_ego_edges = set()
for i in np.nonzero(A[0, target_idx])[0]:
    G_ego_edges.add((graph_nodes[target_idx], graph_nodes[i]))
    for j in np.nonzero(A[0, i])[0]:
        G_ego_edges.add((graph_nodes[i], graph_nodes[j]))
print("Number of edges in the ego graph: {}".format(len(G_ego_edges)))
print(
    "Number of non-zero elements in integrate_link_importance: {}".format(
        np.count_nonzero(integrate_link_importance)
    )
)Number of edges in the ego graph: 210
Number of non-zero elements in integrate_link_importance: 210

We can now find the nodes that have the highest importance to the prediction of the selected node:_____no_output_____

sorted_indices = np.argsort(integrate_link_importance.flatten())
N = len(graph_nodes)
integrated_link_importance_rank = [
    (graph_nodes[k // N], graph_nodes[k % N]) for k in sorted_indices[::-1]
]
topk = 10
print(
    "Top {} most important links by integrated gradients are:\n {}".format(
        topk, integrated_link_importance_rank[-topk:]
    )
)Top 10 most important links by integrated gradients are:
 [(6214, 1105764), (6214, 13960), (6214, 345340), (6214, 6378), (6214, 399339), (6214, 124064), (6214, 95589), (6214, 1106172), (148170, 1128997), (1109199, 6214)]
# Set the labels as an attribute for the nodes in the graph. The labels are used to color the nodes in different classes.
nx.set_node_attributes(G_ego, values={x[0]: {"subject": x[1]} for x in subjects.items()})_____no_output_____

In the following, we plot the link and node importance (computed by integrated gradients) of the nodes within the ego graph of the target node.

For nodes, the shape of the node indicates the positive/negative importance the node has. 'round' nodes have positive importance while 'diamond' nodes have negative importance. The size of the node indicates the value of the importance, e.g., a large diamond node has higher negative importance.

For links, the color of the link indicates the positive/negative importance the link has. 'red' links have positive importance while 'blue' links have negative importance. The width of the link indicates the value of the importance, e.g., a thicker blue link has higher negative importance._____no_output_____

integrated_node_importance.max()_____no_output_____integrate_link_importance.max()_____no_output_____node_size_factor = 1e2
link_width_factor = 2

nodes = list(G_ego.nodes())
colors = pd.DataFrame(
    [v[1]["subject"] for v in G_ego.nodes(data=True)], index=nodes, columns=["subject"]
)
colors = np.argmax(target_encoding.transform(colors), axis=1) + 1

fig, ax = plt.subplots(1, 1, figsize=(15, 10))
pos = nx.spring_layout(G_ego)

# Draw ego as large and red
node_sizes = [integrated_node_importance[graph_nodes.index(k)] for k in nodes]
node_shapes = ["o" if w > 0 else "d" for w in node_sizes]

positive_colors, negative_colors = [], []
positive_node_sizes, negative_node_sizes = [], []
positive_nodes, negative_nodes = [], []
node_size_scale = node_size_factor / np.max(node_sizes)
for k in range(len(nodes)):
    if nodes[k] == target_idx:
        continue
    if node_shapes[k] == "o":
        positive_colors.append(colors[k])
        positive_nodes.append(nodes[k])
        positive_node_sizes.append(node_size_scale * node_sizes[k])

    else:
        negative_colors.append(colors[k])
        negative_nodes.append(nodes[k])
        negative_node_sizes.append(node_size_scale * abs(node_sizes[k]))

# Plot the ego network with the node importances
cmap = plt.get_cmap("jet", np.max(colors) - np.min(colors) + 1)
nc = nx.draw_networkx_nodes(
    G_ego,
    pos,
    nodelist=positive_nodes,
    node_color=positive_colors,
    cmap=cmap,
    node_size=positive_node_sizes,
    vmin=np.min(colors) - 0.5,
    vmax=np.max(colors) + 0.5,
    node_shape="o",
)
nc = nx.draw_networkx_nodes(
    G_ego,
    pos,
    nodelist=negative_nodes,
    node_color=negative_colors,
    cmap=cmap,
    node_size=negative_node_sizes,
    vmin=np.min(colors) - 0.5,
    vmax=np.max(colors) + 0.5,
    node_shape="d",
)
# Draw the target node as a large star colored by its true subject
nx.draw_networkx_nodes(
    G_ego,
    pos,
    nodelist=[target_nid],
    node_size=50 * abs(node_sizes[nodes.index(target_nid)]),
    node_shape="*",
    node_color=[colors[nodes.index(target_nid)]],
    cmap=cmap,
    vmin=np.min(colors) - 0.5,
    vmax=np.max(colors) + 0.5,
    label="Target",
)

# Draw the edges with the edge importances
edges = G_ego.edges()
weights = [
    integrate_link_importance[graph_nodes.index(u), graph_nodes.index(v)]
    for u, v in edges
]
edge_colors = ["red" if w > 0 else "blue" for w in weights]
weights = link_width_factor * np.abs(weights) / np.max(weights)

ec = nx.draw_networkx_edges(G_ego, pos, edge_color=edge_colors, width=weights)
plt.legend()
plt.colorbar(nc, ticks=np.arange(np.min(colors), np.max(colors) + 1))
plt.axis("off")
plt.show()_____no_output_____

We then remove the node or edge in the ego graph one by one and check how the prediction changes. By doing so, we can obtain the ground truth importance of the nodes and edges. Comparing the following figure and the above one can show the effectiveness of integrated gradients as the importance approximations are relatively consistent with the ground truth._____no_output_____

X_bk = deepcopy(X)
A_bk = deepcopy(A)
selected_nodes = np.array([[target_idx]], dtype="int32")
nodes = [graph_nodes.index(v) for v in G_ego.nodes()]
edges = [(graph_nodes.index(u), graph_nodes.index(v)) for u, v in G_ego.edges()]

clean_prediction = model.predict([X, selected_nodes, A]).squeeze()
predict_label = np.argmax(clean_prediction)

groud_truth_node_importance = np.zeros((N,))
for node in nodes:
    # we set all the features of the node to zero to check the ground truth node importance.
    X_perturb = deepcopy(X_bk)
    X_perturb[:, node, :] = 0
    predict_after_perturb = model.predict([X_perturb, selected_nodes, A]).squeeze()
    groud_truth_node_importance[node] = (
        clean_prediction[predict_label] - predict_after_perturb[predict_label]
    )

node_shapes = [
    "o" if groud_truth_node_importance[k] > 0 else "d" for k in range(len(nodes))
]
positive_colors, negative_colors = [], []
positive_node_sizes, negative_node_sizes = [], []
positive_nodes, negative_nodes = [], []
# node_size_scale is used for better visulization of nodes
node_size_scale = node_size_factor / max(groud_truth_node_importance)

for k in range(len(node_shapes)):
    if nodes[k] == target_idx:
        continue
    if node_shapes[k] == "o":
        positive_colors.append(colors[k])
        positive_nodes.append(graph_nodes[nodes[k]])
        positive_node_sizes.append(
            node_size_scale * groud_truth_node_importance[nodes[k]]
        )
    else:
        negative_colors.append(colors[k])
        negative_nodes.append(graph_nodes[nodes[k]])
        negative_node_sizes.append(
            node_size_scale * abs(groud_truth_node_importance[nodes[k]])
        )
X = deepcopy(X_bk)
groud_truth_edge_importance = np.zeros((N, N))
for edge in edges:
    A = deepcopy(A_bk)
    if A[0, edge[0], edge[1]] == 0:
        continue
    # we set the weight of a given edge to zero to check the ground truth link importance
    A[:, edge[0], edge[1]] = 0
    predict_after_perturb = model.predict([X, selected_nodes, A]).squeeze()
    groud_truth_edge_importance[edge[0], edge[1]] = (
        predict_after_perturb[predict_label] - clean_prediction[predict_label]
    ) / (0 - 1)
    A[:, edge[0], edge[1]] = 1

fig, ax = plt.subplots(1, 1, figsize=(15, 10))
cmap = plt.get_cmap("jet", np.max(colors) - np.min(colors) + 1)
# Draw the target node as a large star colored by its true subject
nx.draw_networkx_nodes(
    G_ego,
    pos,
    nodelist=[target_nid],
    node_size=50 * abs(node_sizes[nodes.index(target_idx)]),
    node_color=[colors[nodes.index(target_idx)]],
    cmap=cmap,
    node_shape="*",
    vmin=np.min(colors) - 0.5,
    vmax=np.max(colors) + 0.5,
    label="Target",
)
# Draw the ego net
nc = nx.draw_networkx_nodes(
    G_ego,
    pos,
    nodelist=positive_nodes,
    node_color=positive_colors,
    cmap=cmap,
    node_size=positive_node_sizes,
    vmin=np.min(colors) - 0.5,
    vmax=np.max(colors) + 0.5,
    node_shape="o",
)
nc = nx.draw_networkx_nodes(
    G_ego,
    pos,
    nodelist=negative_nodes,
    node_color=negative_colors,
    cmap=cmap,
    node_size=negative_node_sizes,
    vmin=np.min(colors) - 0.5,
    vmax=np.max(colors) + 0.5,
    node_shape="d",
)
edges = G_ego.edges()
weights = [
    groud_truth_edge_importance[graph_nodes.index(u), graph_nodes.index(v)]
    for u, v in edges
]
edge_colors = ["red" if w > 0 else "blue" for w in weights]
weights = link_width_factor * np.abs(weights) / np.max(weights)

ec = nx.draw_networkx_edges(G_ego, pos, edge_color=edge_colors, width=weights)
plt.legend()
plt.colorbar(nc, ticks=np.arange(np.min(colors), np.max(colors) + 1))
plt.axis("off")
plt.show()_____no_output_____

By comparing the above two figures, one can see that the integrated gradients are quite consistent with the brute-force approach. The main benefit of using integrated gradients is scalability. The gradient operations are very efficient to compute on deep learning frameworks with the parallelism provided by GPUs. Also, integrated gradients can give the importance of individual node features, for all nodes in the graph. Achieving this by brute-force approach is often non-trivial. _____no_output_____

Run the latest release of this notebook:

_____no_output_____


# Notebook from LDSSA/ds-prep-course-2022
Path: Week 06/SLU10 - OOP Inheritance/Solutions notebook.ipynb

# SLU10 | Object Oriented Programming - Inheritance: Exercise Notebook

***_____no_output_____## Start by importing these packages_____no_output_____

# for evaluation purposes
import hashlib
def _hash(s):
    return hashlib.blake2b(
        bytes(str(s), encoding='utf8'),
        digest_size=5
    ).hexdigest()

import math
import inspect

from utils import get_random_number
from utils import Spell_____no_output_____

## PART 1

### Exercise 1

Create a class named `Worker()` and define the `__init__()` method of the created class to have three arguments:

1. name
2. age
3. department

Now, do the following:

- Assign these variables to attributes by using the `self` keyword;

- Define an extra attribute in the `__init__()` method, **without user input**, called `income` and with default value of **`1200`**;

- Define another one attribute in the `__init__()` method, called `company` and with default value  **`"Dunder Mifflin Paper Company"`**;

- Create a `presentation()` method that will **`return`** a string `speech` with the message:

    **_"Hi, my name is `name`! I'm work at `company` on the `department` department, nice to meet you."_**

- Create another method called `show_income()` that will **`return`** a string with the message:

    **_"My salary is € `income` per month"_**

In the end, create the `employee` object for a **"Jim Halpert"**, **25** from **"Sales"** department.

Please make sure the output is exactly the same as the above messages as the assert will look and compare the full sentence.
_____no_output_____

class Worker():
    """
    Abstract base class representing a Worker
    """

    # def __init__(...)
    ### BEGIN SOLUTION
    def __init__(self, name, age, department):
        """
        Initialize the Worker object
        """
        self.name = name
        self.age = age
        self.company = "Dunder Mifflin Paper Company"
        self.department = department
        self.income = 1200
    ### END SOLUTION

    # def presentation(...)
    ### BEGIN SOLUTION
    def presentation(self):
        speech = "Hi, my name is {name}! I'm work at {company} on the {department} department, nice to meet you.".format(
            name=self.name,
            company=self.company,
            department=self.department)

        return speech
    ### END SOLUTION

    # def show_income(...)
    ### BEGIN SOLUTION
    def show_income(self):
        return "My salary is € {income} per month".format(income=self.income)
    ### END SOLUTION_____no_output_____# Creating our first Worker!

employee = Worker(name='Jim Halpert',
                  age=25,
                  department='Sales')_____no_output_____assert _hash(employee.name) == 'aeb6411c48', 'Ops!! Did you set the name?'
assert _hash(employee.age) == 'f4073f1323', 'Something is wrong with the age...'
assert _hash(employee.company) == '656a034445', 'Ops!! Did you set the company as we said to?'
assert _hash(employee.department) == 'f0f668e3a2', 'Where are the department?'
assert _hash(employee.income) == 'f9ef548c29', 'Ops!! We are talking about money, right?'

assert _hash(type(employee.age)) == 'd461979a1d', 'Are you sure that `age` is a numerical value?'
assert _hash(type(employee.income)) == 'd461979a1d', 'Are you sure that `income` is a numerical value?'

assert _hash(employee.presentation()) == 'e75d412926', "Something going wrong with the presentation() method"
assert _hash(employee.show_income()) == '5f5cc6ca00', "Something going wrong with the show_income() method"

print("---- Yay! All asserts passed ---- ")---- Yay! All asserts passed ----
# Please dear employee, present yourself:
print(employee.presentation())

# And sorry for the indiscretion, but how much do they pay to you?
print(employee.show_income())

# Oh, ok, Thanks!Hi, my name is Jim Halpert! I'm work at Dunder Mifflin Paper Company on the Sales department, nice to meet you.
My salary is € 1200 per month

_____no_output_____### Exercise 2

One of the mainly benefits of **Inheritance** is that we can create more complicated classes that inherit variables or methods from their **parent** classes. This saves us time and helps us build more complex objects, since these **child** classes can also include additional variables or methods.

- Create another class, **which inherits `Worker`**, and call it `Manager`;
- Define a new method `promote()` that will multiply the `income` **by** a (pseudo)random integer from 2 to 10; ❗
- Then, implement an object called `boss` for a **"Michael Scott"**, **35** from **"Management"**;

So, make sure that `boss` belongs to `Manager` class.

❗ _use the **`get_random_number()`** function, already imported in the first line, for generating pseudo-random integers._ ❗_____no_output_____

# class Manager(...)
### BEGIN SOLUTION
class Manager(Worker):
    """
    A new class representing a Manager
    """

    def promote(self):
        self.income *= get_random_number(2, 10)
### END SOLUTION_____no_output_____# Now let create our Manager, and call him a Boss!

boss = Manager(name='Michael Scott',
                  age=35,
                  department='Management')_____no_output_____assert _hash(boss.name) == '7d84d32222', 'Ops!! Did you changed something with the name?'
assert _hash(boss.age) == '7d8e4de993', 'Ops!! Did you changed something with the age?'
assert _hash(boss.company) == '656a034445', 'Ops! The company should not change'
assert _hash(boss.department) == 'c24f31960b', 'Ops!! Did you changed something with the department?'

# Now all our boss need is a promotion!
# Let's promote him...
boss.promote()
assert _hash((boss.income >= (1200*2)) & (boss.income <= (1200*10))) == 'dd931fabaa', 'Ops! The promotion did not work'

assert _hash(boss.presentation()) == '11be1d4ed9', 'Ops!! Did you changed something with the presentation() method?'
assert _hash((isinstance(boss, Manager))) == 'dd931fabaa', 'boss must belong to the Manager Child Class'
assert _hash((isinstance(boss, Worker))) == 'dd931fabaa', 'boss must belong to the Worker Parent Class'
assert _hash((isinstance(employee, Worker))) == 'dd931fabaa', 'Did you changed something with our employer?'
assert _hash((isinstance(employee, Manager))) == '3ab05b19da', 'Did you changed something with our employer?'

# Well, take a look at these details!
print("Instance test:", isinstance(boss, Manager),'| boss belongs to the Manager Child Class')
print("Instance test:", isinstance(boss, Worker),'| boss belongs to the Worker Parent Class')
print('')
print("Type test:", type(boss),'| boss is the type of Manager Class')
print("Type test:", type(employee),'| employer is the type of Worker Class')
print('')
print("---- Well Done, all asserts passed ---- ")Instance test: True | boss belongs to the Manager Child Class
Instance test: True | boss belongs to the Worker Parent Class

Type test: <class '__main__.Manager'> | boss is the type of Manager Class
Type test: <class '__main__.Worker'> | employer is the type of Worker Class

---- Well Done, all asserts passed ----
# Hello, amazing boss, can you also present yourself?
boss.presentation()

# I heard you were promoted, so tell me the news!
boss.show_income()_____no_output_____

***_____no_output_____## Part 2

### Exercise 3

Now, let's take a look at these enchanted charms:

What's the **`parent`** class in this code above?

Options:

* a. Spell.
* b. Hocus_Pocus.
* c. Tontus_Talontus.
* d. put_spell.

Write the letter with the **correct answer** to a variable called **`answer_3`** as a string, for example:

    answer_3 = 'c'

⚠ **Note:** _try to think before writing anything. If you don't know the right answer, check the learning material._ ⚠

But please try not to guess the answer by testing the function itself (**don't call it**).

_You're learning it for yourself, not for correct answers or some grades!_ 🙂

⚠ **Another Note:** _Make sure you don't answer with capital letters or the assert will not pass._ ⚠_____no_output_____

# answer_3 = ''

### BEGIN SOLUTION
# the answer is going to be 'a', the parent is Spell class
# because the Hocus_Pocus and Tontus_Talontus are classes
# created from Spell's attributes (they inherit its traits)

answer_3 = 'a'
### END SOLUTION_____no_output_____

### Exercise 4

Which option is correct about the **`child`** class in this code above?

Options:

* a. Spell.
* d. Hocus_Pocus.
* c. Tontus_Talontus.
* d. Spell and Hocus_Pocus.
* e. Hocus_Pocus and Tontus_Talontus.
* f. Tontus_Talontus and Spell.

Save the answer in a variable called **`answer_4`**. Example:

    answer_4 = 'c'_____no_output_____

# answer_4 = ''

### BEGIN SOLUTION
# the answer is going to be 'e',
# the child classes are Hocus_Pocus and Tontus_Talontus
# which are created inheriting Spell class attributes

answer_4 = 'e'
### END SOLUTION_____no_output_____

### Exercise 5

What's the output of the following line?

**`put_spell(Tontus_Talontus())`**

Options:

* a. `'Pocus Charm | Hocus Pocus | Causes the victim to become confused and dizzy'` will be printed.
* b. `'Pocus Charm | Hocus Pocus | No description'` will be printed.
* c. Nothing. The code is not valid.
* d. `'Tontus Charm | Tontus_Talontus | Causes the victim to become confused and dizzy'` will be printed.
* e. `'Tontus Charm | Tontus_Talontus | No description'` will be printed.

Save the answer in a variable called **`answer_5`**. Example:

    answer_5 = 'c'_____no_output_____

# answer_5 = ''

### BEGIN SOLUTION
# the answer is going to be 'd',
# the method `put_spell` get an Spell as input
# and just print the Spell detail, which returns
# a string concatenation for Spell's name, charm and description.

answer_5 = 'd'
### END SOLUTION_____no_output_____

### Exercise 6

What's the output of the following lines?

**`spell = Hocus_Pocus()
put_spell(spell)`**

Options:

* a. `'Pocus Charm | Hocus Pocus | Causes the victim to become confused and dizzy'` will be printed.
* b. `'Pocus Charm | Hocus Pocus | No description'` will be printed.
* c. Nothing. The code is not valid.
* d. `'Tontus Charm | Tontus_Talontus | Causes the victim to become confused and dizzy'` will be printed.
* e. `'Tontus Charm | Tontus_Talontus | No description'` will be printed.

Save the answer in a variable called **`answer_6`**. Example:

    answer_6 = 'c'_____no_output_____

# answer_6 = ''

### BEGIN SOLUTION
# the answer is going to be 'b',
# because the Hocus_Pocus class does not have
# its own get_description() implemented,
# so it'll use the method from parent class

answer_6 = 'b'
### END SOLUTION_____no_output_____assert isinstance(answer_3, str), 'The answer should be a string'
assert isinstance(answer_4, str), 'The answer should be a string'
assert isinstance(answer_5, str), 'The answer should be a string'
assert isinstance(answer_6, str), 'The answer should be a string'

answers = [answer_3, answer_4, answer_5, answer_6]
assert _hash(answers) == 'a437db6d5b', 'Oops. One or more answers are not correct! Try it again! :)'

print("---- all asserts passed ---- ")---- all asserts passed ----

_____no_output_____Do you know another better way to get more practice with **OOP Inheritance** besides biology?

_Yeap... Exactly with Math (and geometry)!_ 📐

### Exercise 7

Start by creating a new class `Rectangle` with the following:
+ Attributes
    - **length**: float
    - **width**: float

+ Methods
    - `get_area()`: returns the product of the rectangle **length** with **width**
    - `get_perimeter()`: returns the sum of double the **lenght** and double the **width**_____no_output_____

# class Rectangle():

### BEGIN SOLUTION
class Rectangle():
    def __init__(self, length, width):
        self.length = length
        self.width = width

    def get_area(self):
        return self.length * self.width

    def get_perimeter(self):
        return 2 * self.length + 2 * self.width
### END SOLUTION_____no_output_____# considering a 28,04 m by 17,11 m rectangle:
r = Rectangle(length=28.04, width=17.11)

assert math.isclose(r.get_area(), 479.765, abs_tol=0.001), 'Area computation is not correct!'
assert math.isclose(r.get_perimeter(), 90.300, abs_tol=0.001), 'Perimeter computation is not correct!'
assert _hash(type(r)) == 'f533583264', 'You should implement a Rectangle Class!'

print("Retangle Area:", r.get_area(),'| The Area is correct!')
print("Retangle Perimeter:", r.get_perimeter(),'| The Perimeter is correct!')
print('')
print("---- UhuuuuL! all asserts passed ---- ")Retangle Area: 479.76439999999997 | The Area is correct!
Retangle Perimeter: 90.3 | The Perimeter is correct!

---- UhuuuuL! all asserts passed ----

### Exercise 8

Now, let's implement another object:

- Create a `Square` class, as a **child** of the `Rectangle` class.

- Change **only** the `__init__()` method in a way that now it **only** accepts one argument **`side`**.

- You can call the `__init__()` method from the **parent** class and assign both **`length`** and **`width`** to be equal to **`side`**_____no_output_____

# class Square(...)
### BEGIN SOLUTION
class Square(Rectangle):
    def __init__(self, side):
        super().__init__(length=side, width=side)
### END SOLUTION_____no_output_____# considering a 28,04 m by 28,04 m square:
s = Square(side=28.04)

assert math.isclose(s.get_area(), 786.241, abs_tol=0.001), 'Area computation is not correct!'
assert math.isclose(s.get_perimeter(), 112.16, abs_tol=0.001), 'Perimeter computation is not correct!'

error_message = 'Did you use the super() function?'
assert 'self.width' not in inspect.getsource(Square.__init__), error_message
assert 'self.length' not in inspect.getsource(Square.__init__), error_message
assert "super()." in inspect.getsource(Square.__init__), error_message

error_message =  'Do not re-implement the methods, use the ones from the Rectangle :)'
assert 'self.length' in inspect.getsource(Square.get_area), error_message
assert 'self.width' in inspect.getsource(Square.get_area),  error_message
assert 'self.length' in inspect.getsource(Square.get_perimeter), error_message
assert 'self.width' in inspect.getsource(Square.get_perimeter), error_message

print("Square Area:", s.get_area(),'| The Area is correct!')
print("Square Perimeter:", s.get_perimeter(),'| The Perimeter is correct!')
print('')
print("---- all asserts passed ---- ")Square Area: 786.2416 | The Area is correct!
Square Perimeter: 112.16 | The Perimeter is correct!

---- all asserts passed ----

### Exercise 9

Create a new class `Triangle`, independent from the other classes:

+ Attributes
    - **base**: float
    - **height**: float

+ Methods
    - `get_tri_area()` which calculates the area of the triangle (do you remember the formula? 🤓)_____no_output_____

# class Triangle():

### BEGIN SOLUTION
class Triangle():
    def __init__(self, base, height):
        self.base = base
        self.height = height

    def get_tri_area(self):
        return 0.5 * self.base * self.height
### END SOLUTION_____no_output_____# considering a 28 m by 17 m triangle:
t = Triangle(base=28, height=17)

assert math.isclose(t.get_tri_area(), 238, abs_tol=0.001), 'The area of the triangle is miscalculated!'

print("Triangle Area:", t.get_tri_area(),'| The Area is correct!')
print('')
print("---- all asserts passed ---- ")Triangle Area: 238.0 | The Area is correct!

---- all asserts passed ----

### Exercise 10

Now for the `RightPyramid` class, create methods that compute the area of this geometric solid using both the `Square` and `Triangle` classes as **parents**.

**Override** the `get_area()` method in order to compute the equation of the **RightPyramid area**.

🌟 **Hint:** _use the `super()` function to  make everything easier._ 🌟

So, you're in luck, geometry is not the subject for this SLU! Here are the formulas that you need:

* The area of a Right Pyramid is equal to the sum of the base_area with its lateral_area.
* The Lateral area can be computed as the next image shows (1/2 * perimeter * slant_height)

Other options do exist to compute the lateral area, like using the **triangle area**. Feel free to use what is easier for you as **this is about python, not geometry!**

+ Attributes:
    - **square_side** (this is equivalent to one side of the square base, just like the previous exercises. So, base is also equal to 1/4 of the perimeter `p` of the square base of the pyramid)
    - **slant_height**

+ Methods:
    - `get_area()`_____no_output_____

# class RightPyramid(...)
### BEGIN SOLUTION
class RightPyramid(Square, Triangle):
    def __init__(self, square_side, slant_height):
        self.square_side = square_side
        self.slant_height = slant_height
        self.height = slant_height
        self.length = square_side
        super().__init__(self.square_side)

    def get_area(self):
        base_area = super().get_area()
        perimeter = super().get_perimeter()
        return 0.5 * perimeter * self.slant_height + base_area

# alternative solution:
#     def get_area(self):
#         base_area = super().get_area()
#         triangle_area = super().get_tri_area()
#         return triangle_area * 4 + base_area
### END SOLUTION_____no_output_____# Creating one Right Pyramid, right now!
rp = RightPyramid(square_side=17, slant_height=28)

assert isinstance(rp, RightPyramid), 'ops, are you sure it is a right pyramid object?'
assert isinstance(rp, Square), 'Probably your class is not properly defined, should be a child class of the Square class'
assert isinstance(rp, Triangle), 'Probably your class is not properly defined, should be a child class of the Triangle class'

error_message = 'Did you use the super approach?'
assert 'super()' in inspect.getsource(rp.__init__), error_message
assert 'super()' in inspect.getsource(rp.get_area), error_message

assert math.isclose(rp.get_area(), 1241, abs_tol=0.001), 'The area of the right pyramid is miscalculated!'

print("Right Pyramid Area:", rp.get_area(),'| The Area is correct!')
print('')
print("---- all asserts passed ---- ")Right Pyramid Area: 1241.0 | The Area is correct!

---- all asserts passed ----

Feeling of the day:

***_____no_output_____## Part 3

Uhuuul! 🎉 This is the last part of this notebook exercise! And you're rocking it! 🤘🤘🏻🤘🏼🤘🏽🤘🏾🤘🏿

In the following exercice the goal it is to create a **BankAccount program**, composed by **one grand-parent class**, **one parent class** and **one child class**, in order to manage the clients accounts. This is the general idea:

### Exercise 11

First things first, the `BankAccount` class:

+ Attributes
    - **customer**
    - **number**
    - **balance** with default value of 0 (zero) but passed as an argument anyway (you'll need it for **Ex.12** and **Ex.13**)
    - **currency** variable is always € (no need to pass it as an argument)

+ Methods
    - `deposit()`:
        * Deposit **amount** into the bank account. (increase the balance)
        * If the **amount** is invalid (< 0) it should `return` an alert message to the user

      ⚠ **Warning**: it should not deduce the amount and `return`, such as **`Invalid deposit amount`** message.

    - `withdraw()`:
        * Withdraw **amount** from the bank account. (decrease the balance)
        * This method must ensure that the account has sufficient funds for the asked **amount**.

      ⚠ **Warning**: Again, it should be > 0 and also `return` an alert if the value is less than zero or if it's bigger than the current balance

      (a.k.a **`Invalid withdraw amount`** message)

    - `check_balance()`: `print()` a statement of the account balance, example **`"The balance of account number 48522888 is €1000"`**_____no_output_____

class BankAccount:
    """
    Abstract base class representing a bank account
    """
    ### BEGIN SOLUTION
    def __init__(self, customer, number, balance=0):
        """
        Initialize the BankAccount class with a customer,
        number and opening balance (default value of 0)
        """
        self.customer = customer
        self.number = number
        self.balance = balance
        self.currency = '€'

    def deposit(self, amount):
        """
        Deposit amount into the bank account
        """
        if amount > 0:
            self.balance += amount
        else:
            return 'Invalid deposit amount:' + str(amount)

    def withdraw(self, amount):
        """
        Withdraw amount from the bank account,
        ensuring that are sufficient funds
        """
        if amount > 0:
            if amount > self.balance:
                return 'Insufficient funds!'
            else:
                self.balance -= amount
        else:
            return 'Invalid withdrawal amount:' + str(amount)

    def check_balance(self):
        """
        Print a statement of the account balance
        """
        return 'The balance of account number ' + str(self.number) + ' is ' + str(self.currency) + ' ' + str(self.balance)
    ### END SOLUTION_____no_output_____# Now, let's create our account!
account = BankAccount(customer='Dwight Schrute', number=21457666)

assert isinstance(account, BankAccount), 'Your class is not properly defined'
assert account.number == 21457666, 'You have to define an "number" attribute for your class!'
assert _hash(account.customer) == 'dd1e4dfba6', 'You have to define a "customer" attribute for your class!'
assert account.balance == 0, 'The default value of "balance" should be 0!'
assert 'balance=0' in inspect.getsource(account.__init__), 'balance=0 must be passed as an argument anyway'
assert account.currency == '€', 'Currency symbol is not working... Did you change the existing line?'

print("The account for %s was successfully created. \nHis account number is %s and for now his account is empty."
     % (account.customer, account.number))
print('')
print("---- Horay! All asserts passed ---- ")The account for Dwight Schrute was successfully created.
His account number is 21457666 and for now his account is empty.

---- Horay! All asserts passed ----
# Let's deposit that big, fat check!
account.deposit(amount=25000)

assert account.balance == 25000, 'Your balance, after making a deposit should change!'

print("%s just made a deposit of €25'000. \nHis new account balance is: %s%s"
      %(account.customer, account.currency, account.balance))
print('')
print("---- Yeah! all asserts passed ---- ")Dwight Schrute just made a deposit of €25'000.
His new account balance is: €25000

---- Yeah! all asserts passed ----
# And then just go shopping
# online... better be online, right?!

account.withdraw(amount=2250)

assert account.balance == 22750, 'Your balance, after making a withdrawal should change or it is not correct!'

print("%s bought the best fertilizers for his beet production. \nHis new account balance is: %s%s"
      %(account.customer, account.currency, account.balance))
print('')
print("---- Yoooo! all asserts passed ---- ")Dwight Schrute bought the best fertilizers for his beet production.
His new account balance is: €22750

---- Yoooo! all asserts passed ----
# checking the returned message for invalid deposit/withdraw:
assert type(account.deposit(-10)) == str, 'Are you returning a string message for invalid deposit?'
assert type(account.withdraw(-10)) == str, 'Are you returning a string message for invalid withdraw?'

# Finally let's ensure that your program does not accept invalid operations:
account.deposit(amount=-10)
assert account.balance == 22750, 'Your balance, after making an invalid deposit should not change!'
account.withdraw(amount=-10)
assert account.balance == 22750, 'Your balance, after making an invalid withdraw should not change!'

print("---- Amaaazing! all asserts passed ---- ")---- Amaaazing! all asserts passed ----

### Exercise 12

Now we need to implement the `CheckingAccount` class, as a **child class** of `BankAccount`, that has a similar structure of the previous exercise:

+ Attributes:
    - **annual_fee**
    - **withdraw_limit**
    - All the BankAccount attributes

+ Methods:
    - `withdraw()`:
        **override** the **parent class** method and create an `if` clause with an extra condition, in order to limit the possible amount that a person can withdraw daily.

    - `charge_fee()`:
        Deduct the annual fee from the account balance, making sure that the account **never goes negative**! If the balance is not enough, make sure you deduct the maximum possible.

       🌟 **Hint:** use python's `min()` function 🌟_____no_output_____

# class CheckingAccount(...)
### BEGIN SOLUTION
class CheckingAccount(BankAccount):
    """
    A class representing a checking account
    """
    def __init__(self, customer, number, annual_fee,
                withdraw_limit, balance=0):
        """
        Initialize checking account
        """
        self.annual_fee = annual_fee
        self.withdraw_limit = withdraw_limit
        super().__init__(customer, number, balance)

    def withdraw(self, amount):
        """
        Withdraw amount if sufficient funds exist and amount is less
        than the single withdraw limit
        """
        if amount > self.balance:
            return 'Insufficient funds'

        elif amount > self.withdraw_limit:
            return 'Exceeds the single withdraw limit!'

        elif amount < 0:
            return 'Invalid withdrawal amount:' + str(amount)

        else: self.balance -= amount

    def charge_fee(self):
        """
        Deduct the annual fee from the account balance
        """
        self.balance -= min(self.annual_fee, self.balance)

### END SOLUTION_____no_output_____# Updating our account to be a Checking Account
cheking_account = CheckingAccount(customer='Dwight Schrute', number=21457666, annual_fee=20., withdraw_limit=200)

assert isinstance(cheking_account, CheckingAccount) == True, 'Your class is not properly defined'
assert isinstance(cheking_account, BankAccount) == True, 'Your class has no relation with the BankAccount class'

assert cheking_account.number == 21457666, 'You have to define an "number" attribute for your class!'
assert _hash(cheking_account.customer) == 'dd1e4dfba6', 'You have to define a "customer" attribute for your class!'
assert cheking_account.balance == 0, 'The default value of "balance" should be 0!'
assert cheking_account.currency == '€', 'Currency symbol is not working... Did you changed the existing line?'
assert cheking_account.annual_fee == 20, 'You have to define a "annual fee" attribute for your class!'

print("---- Weee! all asserts passed ---- ")
print('')
print("Now, the new cheking account for %s was successfully created. \nHis account number is %s and for now his account is empty."
     % (cheking_account.customer, cheking_account.number))---- Weee! all asserts passed ----

Now, the new cheking account for Dwight Schrute was successfully created.
His account number is 21457666 and for now his account is empty.
#Let's do something crazy! Witdhraw more than balance...
cheking_account.withdraw(amount=220)

assert cheking_account.balance == 0, "The created current account must not authorize withdraws when the amount is higher than the account balance"

# Making a new deposit and checking balance:
cheking_account.deposit(amount=750)
cheking_account.check_balance()
assert cheking_account.balance == 750, 'Ops! Did you change something on deposit() method?'

# Another withdraw that should be blocked by withdraw limit:
cheking_account.withdraw(amount=220)
assert cheking_account.balance == 750

# Charge current account annual fee:
cheking_account.charge_fee()
assert cheking_account.balance == 730

# Some withdraws to reach a balance of 10:
cheking_account.withdraw(amount=200)
cheking_account.withdraw(amount=200)
cheking_account.withdraw(amount=200)
cheking_account.withdraw(amount=120)
assert cheking_account.balance == 10, 'Ops, Are you withdrawing correctly?'

# Charge current account annual fee:
cheking_account.charge_fee()
assert cheking_account.balance == 0, 'Ops! Are you charging the annual fee as much as you can?'

# Making a new deposit and checking balance:
cheking_account.deposit(amount=750)
cheking_account.check_balance()
assert cheking_account.balance == 750, 'Ops! Did you change something on deposit() method?'

print("---- Great!! all asserts passed ---- ")
print('')
print(cheking_account.check_balance())
print('Applying annual fee and...')
cheking_account.charge_fee()
print(cheking_account.check_balance())---- Great!! all asserts passed ----

The balance of account number 21457666 is € 750.0
Applying annual fee and...
The balance of account number 21457666 is € 730.0

### Exercise 13

Finally, let's create the `InvestimentAccount` class, as a **child class** of `CheckingAccount` (🧩 _Yup! It's right, we have a multilevel inheritance here_ 🧩)

+ Attributes:
    - **ticker** 🔍 _a code that represents a company in the stock market_ 🔎
    - **investment_fee**
    - All the CheckingAccount attributes ⚠ (the **`withdraw_limit`** needs always to be the same as the balance)

+ Methods:
    - `withdraw()`:
        **override** the method **again**, and now, deduce from the account balance **`amount + investment_fee`**.

       🌟 **Hint:** remember that the account **never goes negative**!   🌟

    - `bet_on_market()`:
        will multiply the balance **by** a (pseudo)random integer from 1 to 10; ❗

       ❗ **Hint:** use **`get_random_number()`** function again, remember?! ❗
    _____no_output_____

# class InvestimentAccount(...)
### BEGIN SOLUTION
class InvestimentAccount(CheckingAccount):
    """
    A class representing a investment account
    """
    def __init__(self, customer, number, annual_fee, ticker, investment_fee, balance=0):
        """
        Initialize investment account
        """
        self.ticker = ticker
        self.investment_fee = investment_fee
        super().__init__(customer, number, annual_fee, withdraw_limit=balance, balance=balance)

    def withdraw(self, amount):
        """
        Withdraw amount if sufficient funds exist (amount + investment_fee)
        and amount is less than the withdraw limit (balance)
        """
        super().withdraw(amount + self.investment_fee)

    def bet_on_market(self):
        """
        multiply the balance by a (pseudo)random integer from 1 to 10
        """
        self.balance = self.balance * get_random_number(1, 10)

### END SOLUTION_____no_output_____# Let's open the our new investment account:
investment_account = InvestimentAccount(
    customer='Dwight Schrute',number=21457666,annual_fee=20,ticker='DMP',investment_fee=10, balance = 1
)

assert investment_account.number == 21457666, 'You have to define an "number" attribute for your class!'
assert _hash(investment_account.customer) == 'dd1e4dfba6', 'You have to define a "customer" attribute for your class!'
assert investment_account.balance == 1, 'The default value of "balance" should be 1!'
assert investment_account.currency == '€', 'Currency symbol is not working... Did you changed the existing line?'
assert investment_account.annual_fee == 20, 'You have to define a "annual fee" attribute for your class!'
assert investment_account.investment_fee == 10, 'Ops! Where is the "investment fee" attribute of your class?!'

# Let's open the our new investment account #2:
investment_account = InvestimentAccount(
    customer='Dwight Schrute',number=21457666,annual_fee=20,ticker='DMP',investment_fee=10
)

assert isinstance(investment_account, InvestimentAccount) == True, 'Your class is not properly defined'
assert isinstance(investment_account, CheckingAccount) == True, 'Your class has no relation with the CheckingAccount class'
assert isinstance(investment_account, BankAccount) == True, 'Your class has no relation with the BankAccount class'

assert investment_account.number == 21457666, 'You have to define an "number" attribute for your class!'
assert _hash(investment_account.customer) == 'dd1e4dfba6', 'You have to define a "customer" attribute for your class!'
assert investment_account.balance == 0, 'The default value of "balance" should be 0!'
assert investment_account.currency == '€', 'Currency symbol is not working... Did you changed the existing line?'
assert investment_account.annual_fee == 20, 'You have to define a "annual fee" attribute for your class!'
assert investment_account.investment_fee == 10, 'Ops! Where is the "investment fee" attribute of your class?!'

assert _hash(investment_account.ticker) == '941024a0fd', 'Ops! what ticker are we investing?'
assert investment_account.withdraw_limit == 0, 'Investiment account does not have a properly withdraw limit...'

investment_account.deposit(amount=1570)
assert investment_account.balance == 1570, 'Ops! Did you change something on deposit() method?'

investment_account.bet_on_market()
assert _hash((investment_account.balance >= (1570*1)) & (investment_account.balance <= (1570*10))) == 'dd931fabaa', 'Ops! The bet_on_market() did not work'

investment_account.withdraw(250)
assert _hash((investment_account.balance >= ((1570*1)-10) & (investment_account.balance <= ((1570*10)-10)))) == 'dd931fabaa', 'Did you deduce "investment fee" on withdraw() method?'

investment_account.withdraw(investment_account.balance+5)
assert _hash((investment_account.balance >= ((1570*1)-10) & (investment_account.balance <= ((1570*10)-10)))) == 'dd931fabaa','You cannot withdraw more funds than you have...'

print("---- Everything is awesome!! all asserts passed ---- ")
print('')
print(investment_account.check_balance())
print('{} is playing on market again...'.format(investment_account.customer))
print('And now...')
investment_account.bet_on_market()
print(investment_account.check_balance())---- Everything is awesome!! all asserts passed ----

The balance of account number 21457666 is € 15700
Dwight Schrute is playing on market again...
And now...
The balance of account number 21457666 is € 15700

***

### And now... Mission Accomplished!!! 🎉🎉🎉

Now you can rest proudly,
because your work was excellent!

**But don't forget...**

***_____no_output_____# Submit your work!

To submit your work, [follow the steps here, in the step "Grading the Exercise Notebook"!](https://github.com/LDSSA/ds-prep-course-2022#22---working-on-the-learning-units)_____no_output_____


# Notebook from techthiyanes/haystack
Path: tutorials/Tutorial7_RAG_Generator.ipynb

# Generative QA with "Retrieval-Augmented Generation"

[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/deepset-ai/haystack/blob/master/tutorials/Tutorial7_RAG_Generator.ipynb)

While extractive QA highlights the span of text that answers a query,
generative QA can return a novel text answer that it has composed.
In this tutorial, you will learn how to set up a generative system using the
[RAG model](https://arxiv.org/abs/2005.11401) which conditions the
answer generator on a set of retrieved documents._____no_output_____### Prepare environment

#### Colab: Enable the GPU runtime
Make sure you enable the GPU runtime to experience decent speed in this tutorial.
**Runtime -> Change Runtime type -> Hardware accelerator -> GPU**

_____no_output_____

# Make sure you have a GPU running
!nvidia-smi_____no_output_____

Here are the packages and imports that we'll need:_____no_output_____

# Install the latest release of Haystack in your own environment
#! pip install farm-haystack

# Install the latest master of Haystack
!pip install --upgrade pip
!pip install git+https://github.com/deepset-ai/haystack.git#egg=farm-haystack[colab,faiss]_____no_output_____from typing import List
import requests
import pandas as pd
from haystack import Document
from haystack.document_stores import FAISSDocumentStore
from haystack.nodes import RAGenerator, DensePassageRetriever
from haystack.utils import fetch_archive_from_http_____no_output_____

Let's download a csv containing some sample text and preprocess the data.
_____no_output_____

# Download sample
doc_dir = "data/tutorial7/"
s3_url = "https://s3.eu-central-1.amazonaws.com/deepset.ai-farm-qa/datasets/small_generator_dataset.csv.zip"
fetch_archive_from_http(url=s3_url, output_dir=doc_dir)

# Create dataframe with columns "title" and "text"
df = pd.read_csv("small_generator_dataset.csv", sep=",")
# Minimal cleaning
df.fillna(value="", inplace=True)

print(df.head())_____no_output_____

We can cast our data into Haystack Document objects.
Alternatively, we can also just use dictionaries with "text" and "meta" fields_____no_output_____

# Use data to initialize Document objects
titles = list(df["title"].values)
texts = list(df["text"].values)
documents: List[Document] = []
for title, text in zip(titles, texts):
    documents.append(Document(content=text, meta={"name": title or ""}))_____no_output_____

Here we initialize the FAISSDocumentStore, DensePassageRetriever and RAGenerator.
FAISS is chosen here since it is optimized vector storage._____no_output_____

# Initialize FAISS document store.
# Set `return_embedding` to `True`, so generator doesn't have to perform re-embedding
document_store = FAISSDocumentStore(faiss_index_factory_str="Flat", return_embedding=True)

# Initialize DPR Retriever to encode documents, encode question and query documents
retriever = DensePassageRetriever(
    document_store=document_store,
    query_embedding_model="facebook/dpr-question_encoder-single-nq-base",
    passage_embedding_model="facebook/dpr-ctx_encoder-single-nq-base",
    use_gpu=True,
    embed_title=True,
)

# Initialize RAG Generator
generator = RAGenerator(
    model_name_or_path="facebook/rag-token-nq",
    use_gpu=True,
    top_k=1,
    max_length=200,
    min_length=2,
    embed_title=True,
    num_beams=2,
)_____no_output_____

We write documents to the DocumentStore, first by deleting any remaining documents then calling `write_documents()`.
The `update_embeddings()` method uses the retriever to create an embedding for each document.
_____no_output_____

# Delete existing documents in documents store
document_store.delete_documents()

# Write documents to document store
document_store.write_documents(documents)

# Add documents embeddings to index
document_store.update_embeddings(retriever=retriever)_____no_output_____

Here are our questions:_____no_output_____

QUESTIONS = [
    "who got the first nobel prize in physics",
    "when is the next deadpool movie being released",
    "which mode is used for short wave broadcast service",
    "who is the owner of reading football club",
    "when is the next scandal episode coming out",
    "when is the last time the philadelphia won the superbowl",
    "what is the most current adobe flash player version",
    "how many episodes are there in dragon ball z",
    "what is the first step in the evolution of the eye",
    "where is gall bladder situated in human body",
    "what is the main mineral in lithium batteries",
    "who is the president of usa right now",
    "where do the greasers live in the outsiders",
    "panda is a national animal of which country",
    "what is the name of manchester united stadium",
]_____no_output_____

Now let's run our system!
The retriever will pick out a small subset of documents that it finds relevant.
These are used to condition the generator as it generates the answer.
What it should return then are novel text spans that form and answer to your question!_____no_output_____

# Or alternatively use the Pipeline class
from haystack.pipelines import GenerativeQAPipeline
from haystack.utils import print_answers

pipe = GenerativeQAPipeline(generator=generator, retriever=retriever)
for question in QUESTIONS:
    res = pipe.run(query=question, params={"Generator": {"top_k": 1}, "Retriever": {"top_k": 5}})
    print_answers(res, details="minimum")_____no_output_____

## About us

This [Haystack](https://github.com/deepset-ai/haystack/) notebook was made with love by [deepset](https://deepset.ai/) in Berlin, Germany

We bring NLP to the industry via open source!
Our focus: Industry specific language models & large scale QA systems.

Some of our other work:
- [German BERT](https://deepset.ai/german-bert)
- [GermanQuAD and GermanDPR](https://deepset.ai/germanquad)
- [FARM](https://github.com/deepset-ai/FARM)

Get in touch:
[Twitter](https://twitter.com/deepset_ai) | [LinkedIn](https://www.linkedin.com/company/deepset-ai/) | [Slack](https://haystack.deepset.ai/community/join) | [GitHub Discussions](https://github.com/deepset-ai/haystack/discussions) | [Website](https://deepset.ai)

By the way: [we're hiring!](https://www.deepset.ai/jobs)_____no_output_____


# Notebook from DARK-art108/Ritesh_Yadav
Path: _notebooks/2022-01-13-Question_Generation_with_GPT3_+_Creative_Prompting.ipynb

# Question Generation with GPT3 + Creative Prompting

> Performing Some Task using GPT-3

- toc:true
- branch: master
- badges: false
- image: images/gpt-3.png
- comments: true
- author: Ritesh Yadav
- permalink: /GPT3/
- categories: [nlp, gpt3, question generation, prompt engineering]_____no_output_____## Install OpenAI Python SDK_____no_output_____

!pip install openaiCollecting openai
  Downloading openai-0.11.5.tar.gz (466 kB)

     |▊                               | 10 kB 29.4 MB/s eta 0:00:01
     |█▍                              | 20 kB 32.2 MB/s eta 0:00:01
     |██                              | 30 kB 20.2 MB/s eta 0:00:01
     |██▉                             | 40 kB 17.3 MB/s eta 0:00:01
     |███▌                            | 51 kB 14.3 MB/s eta 0:00:01
     |████▏                           | 61 kB 12.9 MB/s eta 0:00:01
     |█████                           | 71 kB 12.0 MB/s eta 0:00:01
     |█████▋                          | 81 kB 13.1 MB/s eta 0:00:01
     |██████▎                         | 92 kB 13.5 MB/s eta 0:00:01
     |███████                         | 102 kB 11.7 MB/s eta 0:00:01
     |███████▊                        | 112 kB 11.7 MB/s eta 0:00:01
     |████████▍                       | 122 kB 11.7 MB/s eta 0:00:01
     |█████████▏                      | 133 kB 11.7 MB/s eta 0:00:01
     |█████████▉                      | 143 kB 11.7 MB/s eta 0:00:01
     |██████████▌                     | 153 kB 11.7 MB/s eta 0:00:01
     |███████████▎                    | 163 kB 11.7 MB/s eta 0:00:01
     |████████████                    | 174 kB 11.7 MB/s eta 0:00:01
     |████████████▋                   | 184 kB 11.7 MB/s eta 0:00:01
     |█████████████▍                  | 194 kB 11.7 MB/s eta 0:00:01
     |██████████████                  | 204 kB 11.7 MB/s eta 0:00:01
     |██████████████▊                 | 215 kB 11.7 MB/s eta 0:00:01
     |███████████████▍                | 225 kB 11.7 MB/s eta 0:00:01
     |████████████████▏               | 235 kB 11.7 MB/s eta 0:00:01
     |████████████████▉               | 245 kB 11.7 MB/s eta 0:00:01
     |█████████████████▌              | 256 kB 11.7 MB/s eta 0:00:01
     |██████████████████▎             | 266 kB 11.7 MB/s eta 0:00:01
     |███████████████████             | 276 kB 11.7 MB/s eta 0:00:01
     |███████████████████▋            | 286 kB 11.7 MB/s eta 0:00:01
     |████████████████████▍           | 296 kB 11.7 MB/s eta 0:00:01
     |█████████████████████           | 307 kB 11.7 MB/s eta 0:00:01
     |█████████████████████▊          | 317 kB 11.7 MB/s eta 0:00:01
     |██████████████████████▌         | 327 kB 11.7 MB/s eta 0:00:01
     |███████████████████████▏        | 337 kB 11.7 MB/s eta 0:00:01
     |███████████████████████▉        | 348 kB 11.7 MB/s eta 0:00:01
     |████████████████████████▋       | 358 kB 11.7 MB/s eta 0:00:01
     |█████████████████████████▎      | 368 kB 11.7 MB/s eta 0:00:01
     |██████████████████████████      | 378 kB 11.7 MB/s eta 0:00:01
     |██████████████████████████▊     | 389 kB 11.7 MB/s eta 0:00:01
     |███████████████████████████▍    | 399 kB 11.7 MB/s eta 0:00:01
     |████████████████████████████    | 409 kB 11.7 MB/s eta 0:00:01
     |████████████████████████████▉   | 419 kB 11.7 MB/s eta 0:00:01
     |█████████████████████████████▌  | 430 kB 11.7 MB/s eta 0:00:01
     |██████████████████████████████▏ | 440 kB 11.7 MB/s eta 0:00:01
     |██████████████████████████████▉ | 450 kB 11.7 MB/s eta 0:00:01
     |███████████████████████████████▋| 460 kB 11.7 MB/s eta 0:00:01
     |████████████████████████████████| 466 kB 11.7 MB/s
Requirement already satisfied: requests>=2.20 in /usr/local/lib/python3.7/dist-packages (from openai) (2.23.0)
Requirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from openai) (4.62.3)
Collecting pandas>=1.2.3
  Downloading pandas-1.3.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (11.3 MB)
     |████████████████████████████████| 11.3 MB 42.7 MB/s
Collecting pandas-stubs>=1.1.0.11
  Downloading pandas_stubs-1.2.0.40-py3-none-any.whl (161 kB)
     |████████████████████████████████| 161 kB 48.3 MB/s
Collecting openpyxl>=3.0.7
  Downloading openpyxl-3.0.9-py2.py3-none-any.whl (242 kB)
     |████████████████████████████████| 242 kB 52.7 MB/s
Requirement already satisfied: et-xmlfile in /usr/local/lib/python3.7/dist-packages (from openpyxl>=3.0.7->openai) (1.1.0)
Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas>=1.2.3->openai) (2.8.2)
Requirement already satisfied: numpy>=1.17.3 in /usr/local/lib/python3.7/dist-packages (from pandas>=1.2.3->openai) (1.19.5)
Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.7/dist-packages (from pandas>=1.2.3->openai) (2018.9)
Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from pandas-stubs>=1.1.0.11->openai) (3.10.0.2)
Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.7.3->pandas>=1.2.3->openai) (1.15.0)
Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests>=2.20->openai) (2.10)
Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests>=2.20->openai) (3.0.4)
Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests>=2.20->openai) (2021.10.8)
Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests>=2.20->openai) (1.24.3)
Building wheels for collected packages: openai
  Building wheel for openai (setup.py) ... done
  Created wheel for openai: filename=openai-0.11.5-py3-none-any.whl size=161464 sha256=604ae3e818b3dc26bade5925151a4404156543dd2fee70dd437372e651bb86ee
  Stored in directory: /root/.cache/pip/wheels/5a/52/66/06c234d9748d81b9622d0b9765c3bd10e99d059ed93f13b0b3
Successfully built openai
Installing collected packages: pandas-stubs, pandas, openpyxl, openai
  Attempting uninstall: pandas
    Found existing installation: pandas 1.1.5
    Uninstalling pandas-1.1.5:
      Successfully uninstalled pandas-1.1.5
  Attempting uninstall: openpyxl
    Found existing installation: openpyxl 2.5.9
    Uninstalling openpyxl-2.5.9:
      Successfully uninstalled openpyxl-2.5.9
ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.
google-colab 1.0.0 requires pandas~=1.1.0; python_version >= "3.0", but you have pandas 1.3.5 which is incompatible.
Successfully installed openai-0.11.5 openpyxl-3.0.9 pandas-1.3.5 pandas-stubs-1.2.0.40

## Few-shot prompting with question templates
-----
⚡ **The Idea**

For each question type (see below), Create nice reusable question templates with few examples.

- Fill in the blanks
- MCQs
- True or False / Boolean
- Multi-choice/Multiple Fill in the blanks

Question templates are a combination of prompt tokens and question phrases. For the question generation task "Few" in the phrase "few examples" depends on the question type. For instance you will witness for Fill in the lanks and MCQs 2 examples works just fine. For True or False you could just go with 1 example and May be in your experiments you may discover Zero-short can be put to best use for your needs.

**Note:**
- Here examples simply mean string templates of some static knowledge put together to match the desired Question (and answer) style.
- Having categories of templates can help, because the closer your templates are with the actual paragraphs you are using to generate question in terms of domains the stronger will be the generation results.

**Disclaimer:**
- The strength and quality of your generations are a function of your Prompt Tokens and Example phrases. Choose creative yet relevant prompt tokens.
- PROMPT PROGRAMMING is a SKILL you can improve purely by TRIAL AND ERROR with your ENGLISH knowledge.

_____no_output_____#### 1. Fill in the blanks
------

**Below is a sample prompt template for Fill in the blank questions** As said before the examples are based on the some static knowledge. Here we have 2 ``` <Paragraph/Ques-Ans> ``` pairs. You can have as many example as you want. In this example ```Paragraph:, Question:, Answer: ``` are the 3 prompt tokens.

* Line 1. Paragraph: Jesus, according to some biblical sources, was born in this town some two millennia ago in Bethlehem.
* Line 2. Question: Where was Jesus born ______ ? Answer: Bethlehem
* Line 3. Paragraph: Sachin Tendulkar was born in India. He debuted for Indian cricket in 1988.
* Line 4. Question: In ______ Sachin started playing cricket? Answer: 1988_____no_output_____

import openai
openai.api_key = "<your_openai_api_key>"_____no_output_____prompt_template= """
Paragraph: Jesus, according to some biblical sources, was born in this town some two millennia ago in Bethlehem.
Question: Where was Jesus born ______ ? Answer: Bethlehem
Paragraph: Sachin Tendulkar was born in India. He debuted for Indian cricket in 1988.
Question: In ______ Sachin started playing cricket. Answer: 1988
"""

custom_prompt="""
Paragraph: Elon Musk was a born in South Africa in 1971 and he joined Tesla in 2004.
"""

prompt = prompt_template + custom_prompt
completion = openai.Completion.create(engine="davinci", prompt=prompt, max_tokens=32, temperature=0.7)

print("Generated Question..")
generated = completion.choices[0].text
if "Paragraph" in generated:
   ind = generated.index("Paragraph")
   print(generated[:ind])
Generated Question..
Question: In ______ Elon Musk joined Tesla.
Answer: 2004

#### 2. MCQ - Multiple Choice Questions
------

**Below is a sample prompt template for MCQs** As said before the examples are based on the some static knowledge. Here we have 2 ``` <Paragraph/Ques-Choice-Ans> ``` pairs. You can have as many as you want to make the generation strong.

* Line 1. Paragraph: Jesus, according to some biblical sources, was born in this town some two millennia ago in Bethlehem. The story begins with wise men who come to the city of Jerusalem after seeing a star that they interpreted as signaling the birth of a new king.
* Line 2. Question: Where was Jesus born ? A) Jerusalem, B) Palestine, C) Bethlehem. D) Tel-Aviv Answer: C
* Line 3. Paragraph: Sachin Tendulkar was born in India 1972. He debuted for Indian cricket in 1988 and retired in 2011.
* Line 4. Question: In what year Sachin started playing cricket? A) 1972, B) 1988, C) 2011, D) 2001. Answer: B_____no_output_____

prompt_template= """
Paragraph: Jesus, according to some biblical sources, was born in this town some two millennia ago in Bethlehem. The story begins with wise men who come to the city of Jerusalem after seeing a star that they interpreted as signaling the birth of a new king.
Question: Where was Jesus born ? A) Jerusalem, B) Palestine, C) Bethlehem. D) Tel-Aviv Answer: C
Paragraph: Sachin Tendulkar was born in India 1972. He debuted for Indian cricket in 1988 and retired in 2011.
Question: In what year Sachin started playing cricket? A) 1972, B) 1988, C) 2011, D) 2001. Answer: B
"""

custom_prompt="""
Paragraph: Elon Musk was a born in South Africa in 1971 and he joined Tesla in 2004.
"""

prompt = prompt_template + custom_prompt
completion = openai.Completion.create(engine="davinci", prompt=prompt, max_tokens=32, temperature=0.7)

print("Generated Question..")
generated = completion.choices[0].text
if "Paragraph" in generated:
   ind = generated.index("Paragraph")
   print(generated[:ind])
Generated Question..
Question: In what year did Elon join Tesla? A) 1971, B) 2004, C) 1992, D) 2012. Answer: B

#### 3. True or False / Boolean Questions
------
Here we have ONLY 1 ``` <Paragraph/Ques-Ans> ``` pair.  But be warned generation might suffer. More the examples in your template Stronger the generations._____no_output_____

prompt_template= """
Paragraph: Jesus, according to some biblical sources, was born in this town some two millennia ago in Bethlehem. The story begins with wise men who come to the city of Jerusalem after seeing a star that they interpreted as signaling the birth of a new king.
Question: Jesus was born in Jerusalem. Answer: False
"""

custom_prompt="""
Paragraph: Elon Musk was a born in South Africa in 1971 and he joined Tesla in 2004.
"""

prompt = prompt_template + custom_prompt
completion = openai.Completion.create(engine="davinci", prompt=prompt, max_tokens=32, temperature=0.7)

print("Generated Question..")
generated = completion.choices[0].text
if "Paragraph" in generated:
   ind = generated.index("Paragraph")
   print(generated[:ind])
Generated Question..
Question: Elon Musk was born in Cape Town.
Answer: True

#### 4. (Just for fun) Lets try generating higher-order probing questions which either needs common sense or deeper inference.
------
Note how we changed the prompt strings to ```<fact/probe-inference>``` pairs_____no_output_____

prompt_template= """
Fact: Jesus, according to some biblical sources, was born in this town some two millennia ago in Bethlehem. The story begins with wise men who come to the city of Jerusalem after seeing a star that they interpreted as signaling the birth of a new king.
Probe: Is Jesus a human being? Inference: No he is a God.
Fact: Sachin Tendulkar was born in India 1972. He debuted for Indian cricket in 1988 and retired in 2011.
Probe: How many years Sachin played cricket? Inference: 23 years.
"""

custom_prompt="""
Fact: Elon Musk started Tesla Motors since 2004. He is the CEO and product architect of Tesla and he is also the Chairman of Musk Foundation, an organization supporting research on renewable energy, human space exploration and pediatrics. At age 12, sold his code for a video game called “Blastar” to a computer magazine for $500.
"""

prompt = prompt_template + custom_prompt
completion = openai.Completion.create(engine="davinci", prompt=prompt, max_tokens=32, temperature=0.7)

print("Generated Probe..")
generated = completion.choices[0].text
if "Fact" in generated:
   ind = generated.index("Fact")
   print(generated[:ind])Generated Probe..
Probe: What is Musk's current age? Inference: 41 years.

### Important Note

- Signup for OpenAI to get your own API key !
- Engine choice: Davinci is not the only engine of choice
```
# list engines
engines = openai.Engine.list()
```
Play with the list using the above snippet and choose the one that best suits your case.
- Number of generations - As you can see, I have limited my generations to only 1. You could iterate over many generations with single prompt
- Play with misc Parameters like ```max_tokens=32, temperature=0.7``` Refer this link  -https://beta.openai.com/docs/api-reference/completions/create
- Feel free to use your creativity and rxpand to other tasks_____no_output_____


# Notebook from tobsecret/Golub_dataset_Class_Prediction_Python
Path: Golub_dataset_Class_Prediction.ipynb

%matplotlib inline
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.neighbors import KNeighborsClassifier
from sklearn.ensemble import BaggingClassifier
from scipy.stats import ttest_ind

_____no_output_____results = pd.DataFrame.from_dict({'Classifier':'1', 'Dataset':'2', 'Training_Error':5, 'Testing_Error':6}, orient='index').T
results_____no_output_____%%cmd
dirMicrosoft Windows [Version 6.1.7601]
Copyright (c) 2009 Microsoft Corporation. Alle Rechte vorbehalten.

(PDSH_mod) C:\Users\Tobsecret\Desktop\Lectures\Topics in Bioinformatics\Module 2 Molecular Classification of Cancer\Golub_dataset_Class_Prediction_Python
> dir
 Datentr„ger in Laufwerk C: ist Acer
 Volumeseriennummer: E6C9-99F6

 Verzeichnis von C:\Users\Tobsecret\Desktop\Lectures\Topics in Bioinformatics\Module 2 Molecular Classification of Cancer\Golub_dataset_Class_Prediction_Python

21.02.2017  16:13    <DIR>          .
21.02.2017  16:13    <DIR>          ..
13.02.2017  21:21    <DIR>          .ipynb_checkpoints
13.02.2017  21:46         1.860.352 data_set_ALL_AML_independent.tsv
14.02.2017  11:02         2.046.810 data_set_ALL_AML_train.tsv
20.02.2017  16:39             6.667 environment.txt
21.02.2017  16:13           231.342 Golub_dataset_Class_Prediction.ipynb
20.02.2017  16:12             1.087 LICENSE
21.02.2017  10:50               986 README.md
13.02.2017  22:18               510 table_ALL_AML_samples.txt
               7 Datei(en),      4.147.754 Bytes
               3 Verzeichnis(se), 341.275.729.920 Bytes frei

(PDSH_mod) C:\Users\Tobsecret\Desktop\Lectures\Topics in Bioinformatics\Module 2 Molecular Classification of Cancer\Golub_dataset_Class_Prediction_Python
>

#### Reading in the datasets
We'll read in the test into the variable **test** and the training dataset into the variable **train**._____no_output_____

test = pd.read_csv('data_set_ALL_AML_independent.tsv', sep='\t', header=0, index_col=0)_____no_output_____train = pd.read_csv('data_set_ALL_AML_train.tsv', sep='\t', header=0, index_col=0)
train.drop(train.columns[len(train.columns)-1], axis=1, inplace=True)_____no_output_____#The sample_table contains the labels AML/ ALL
sample_table = pd.read_csv('table_ALL_AML_samples.txt', sep='\t')
sample_table['Idealized'] = 0
sample_table['Idealized'].loc[sample_table['ALL/AML']=='ALL'] = 1
C:\Users\Tobsecret\Anaconda3\envs\PDSH_mod\lib\site-packages\pandas\core\indexing.py:132: SettingWithCopyWarning:
A value is trying to be set on a copy of a slice from a DataFrame

See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy
  self._setitem_with_indexer(indexer, value)

##### Normalizing the Data
We normalize each element x by subtracting the row minimum and dividing by (row maximum minus row minimum).

x = (x - row_min) / (row_max - row_min)_____no_output_____

norm_train = train.iloc[:,1::2].apply(lambda x: (x - np.min(x))/(np.max(x)-np.min(x)), axis=1)_____no_output_____norm_test = test.iloc[:,1::2].apply(lambda x: (x - np.min(x))/(np.max(x)-np.min(x)), axis=1)
norm_test = norm_test.reindex_axis(sorted(norm_test.columns, key=int), axis=1)_____no_output_____# Sorting the columns numerically
norm_train = norm_train.reindex_axis(sorted(list(norm_train.columns.values), key=int), axis=1)
_____no_output_____

#### K-nearest neighbours
We will try out various different selections of the data and train a separate model for each.
Let's start with the whole data set.
_____no_output_____

norm_train_transp = norm_train.T_____no_output_____neigh = KNeighborsClassifier()
neigh.fit(norm_train_transp, sample_table.iloc[:38,1:2].values.reshape(38))_____no_output_____

Let's score on the training data set to see whether our model is reasonably trained._____no_output_____

neigh.score(norm_train.T, sample_table.iloc[:38,1:2].values.reshape(38))_____no_output_____

Let's score on the test set:_____no_output_____

neigh.score(norm_test.T, sample_table.iloc[38:, 1:2].values.reshape(34))_____no_output_____results = results.append(pd.Series({'Classifier':'K-nearest Neighbours', 'Dataset':'Normalized',
                          'Training_Error':neigh.score(norm_train.T, sample_table.iloc[:38,1:2].values.reshape(38)),
                          'Testing_Error':neigh.score(norm_test.T, sample_table.iloc[38:, 1:2].values.reshape(34))}),
              ignore_index=True)_____no_output_____

#### K-nearest Neighbours Bagging Ensemble
In the following lines we'll try out the BaggingClassifier from scikit-learn and we will populate it with a K-nearest Neighbours Classifier._____no_output_____

bagging_neigh = BaggingClassifier(KNeighborsClassifier(),
                                 max_samples=0.5, max_features=0.5)
bagging_neigh.fit(norm_train.T, sample_table.iloc[:38,1:2].values.reshape(38))
#_____no_output_____results = results.append(pd.Series({'Classifier':'Bagging Ensemble of K-nearest Neighbours', 'Dataset':'Normalized',
                                    'Training_Error':bagging_neigh.score(norm_train.T, sample_table.iloc[:38,1:2].values.reshape(38)),
                                    'Testing_Error':bagging_neigh.score(norm_test.T, sample_table.iloc[38:, 1:2].values.reshape(34))})
                         ,ignore_index=True)_____no_output_____

#### Non-normalized Data
Let's try the same things with only a subset of genes and no normalization besides setting all values below 0 to 1._____no_output_____

non_negative_train = train.iloc[:,1::2].clip_lower(1)
non_negative_train
non_negative_test = test.iloc[:, 1::2].clip_lower(1)_____no_output_____

To separate the rubbish genes from the more predictive genes, we are doing independent sample Welch's t-tests for all genes, comparing the value for each gene in the AML samples to that value for the same gene in the ALL samples._____no_output_____

#The following line computes the independent t-tests between the AML and ALL samples for every gene.
non_negative_train_ttest_pvals = non_negative_train.apply(lambda x: 1-ttest_ind(x.iloc[:26].values.reshape(26,1), x.iloc[26:].values.reshape(12,1), equal_var=False).pvalue , axis=1).iloc[:,:1]_____no_output_____

The violin plot below shows the distribution of the results of the ttests. The values plotted are 1-pvalue for all the genes.
One can see that most genes actually have pretty good p-values. _____no_output_____

sns.violinplot( y="1", data=non_negative_train_ttest_pvals);
_____no_output_____

Now we'll select only the top 3000 genes based on the 1-pvalue scores, the lowest genesin this selection having a value of around 0.8 _____no_output_____

top3000_ttest_mask = non_negative_train_ttest_pvals.sort_values('1', axis=0, ascending=False).head(3000).index_____no_output_____top3000_non_negative_train = non_negative_train.loc[top3000_ttest_mask]
top3000_non_negative_test = non_negative_test.loc[top3000_ttest_mask]_____no_output_____topneigh = KNeighborsClassifier()
topneigh.fit(top3000_non_negative_train.T, sample_table.iloc[:38,1:2].values.reshape(38))
print('Training Prediction Performance: ',
      topneigh.score(top3000_non_negative_train.T, sample_table.iloc[:38,1:2].values.reshape(38)),
      ' Testing Prediction Performance: ',
      topneigh.score(top3000_non_negative_test.T, sample_table.iloc[38:, 1:2].values.reshape(34)))
results = results.append(pd.Series({'Classifier':'K-nearest Neighbours', 'Dataset':'Non-negative',
                          'Training_Error':topneigh.score(top3000_non_negative_train.T, sample_table.iloc[:38, 1:2].values.reshape(38)),
                          'Testing_Error' :topneigh.score(top3000_non_negative_test.T,  sample_table.iloc[38:, 1:2].values.reshape(34))}),
                        ignore_index=True)Training Prediction Performance:  0.973684210526  Testing Prediction Performance:  0.617647058824
top_bagging_neigh = BaggingClassifier(KNeighborsClassifier(),
                                 max_samples=0.5, max_features=0.5)
top_bagging_neigh.fit(top3000_non_negative_train.T, sample_table.iloc[:38,1:2].values.reshape(38))
print('Training Prediction Performance: ',
      top_bagging_neigh.score(top3000_non_negative_train.T, sample_table.iloc[:38,1:2].values.reshape(38)),
      ' Testing Prediction Performance: ',
      top_bagging_neigh.score(top3000_non_negative_test.T, sample_table.iloc[38:, 1:2].values.reshape(34)))

results = results.append(pd.Series({'Classifier':'Bagging Ensemble of K-nearest Neighbours', 'Dataset':'Non-negative',
                          'Training_Error':top_bagging_neigh.score(top3000_non_negative_train.T, sample_table.iloc[:38, 1:2].values.reshape(38)),
                          'Testing_Error' :top_bagging_neigh.score(top3000_non_negative_test.T,  sample_table.iloc[38:, 1:2].values.reshape(34))}),
                        ignore_index=True)Training Prediction Performance:  0.921052631579  Testing Prediction Performance:  0.529411764706
top_bagging_tree = BaggingClassifier(max_samples=0.5, max_features=0.5)

top_bagging_tree.fit(top3000_non_negative_train.T, sample_table.iloc[:38,1:2].values.reshape(38))
print('Training Prediction Performance: ',
      top_bagging_tree.score(top3000_non_negative_train.T, sample_table.iloc[:38,1:2].values.reshape(38)),
      ' Testing Prediction Performance: ',
      top_bagging_tree.score(top3000_non_negative_test.T, sample_table.iloc[38:, 1:2].values.reshape(34)))
results = results.append(pd.Series({'Classifier':'Bagging Ensemble of Decision Trees', 'Dataset':'Non-negative',
                          'Training_Error': top_bagging_tree.score(top3000_non_negative_train.T, sample_table.iloc[:38,1:2].values.reshape(38)),
                          'Testing_Error' :top_bagging_tree.score(top3000_non_negative_test.T, sample_table.iloc[38:, 1:2].values.reshape(34))}),
                        ignore_index=True)Training Prediction Performance:  0.947368421053  Testing Prediction Performance:  0.588235294118
 results = results.iloc[1:,:]_____no_output_____results['Training_Error'] = 1- results['Training_Error']
results['Testing_Error']  = 1-results['Testing_Error']
results_____no_output_____

#### Plotting the Results
Below are plots of the different combinations of data sets and classifiers.
The data sets are *non-negative*, i.e. negative expression values are set to 1 and only the top 3000 genes were considered, and *normalized*, i.e. for each expression value x:

x --> (x - row_min) / (row_max - row_min)_____no_output_____

ax = sns.factorplot(x='Dataset', y='Training_Error', kind='bar', hue='Classifier', data=results)
ax = sns.factorplot(x='Dataset', y='Testing_Error', kind='bar', hue='Classifier', data=results)
_____no_output_____

As we can see, none of the classifiers did as well as the one done by Golub. The classifiers trained on the normalized full training data did better, with the K-nearest neighbours outperforming the bagging classifier with K-nearest neighbours.
If one was to continue the analysis, one would test more different combinations of normalizing/ narrowing down the datasets and test more combinations of classifiers and train them on a subset of the training data and test them on an independent subset of training data to pick the best classifier and use that one to score on the test data._____no_output_____