# Notebook from ankit-kothari/Data-Science-Journey
Path: Machine-Vison/Basics of OpenCV/Identifying_Lableing_Coloring_Contours.ipynb

<code>
import numpy as np
import cv2
import matplotlib.pyplot as plt
# Load and then gray scale image

image = cv2.imread('images/someshapes.jpg')
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)

#cv2.imshow('Identifying Shapes',image)
#cv2.waitKey(0)

ret, thresh = cv2.threshold(gray, 127, 255, 1)
# Extract Contours
contours, hierarchy = cv2.findContours(thresh.copy(), cv2.RETR_LIST, cv2.CHAIN_APPROX_NONE)
for cnt in contours:

    # Get approximate polygons
    approx = cv2.approxPolyDP(cnt, 0.01*cv2.arcLength(cnt,True),True)

    if len(approx) == 3:
        shape_name = "Triangle"
        cv2.drawContours(image,[cnt],0,(0,255,0),-1)

        # Find contour center to place text at the center
        M = cv2.moments(cnt)
        cx = int(M['m10'] / M['m00'])
        cy = int(M['m01'] / M['m00'])
        cv2.putText(image, shape_name, (cx-50, cy), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 0), 2)

    elif len(approx) == 4:
        x,y,w,h = cv2.boundingRect(cnt)
        M = cv2.moments(cnt)
        cx = int(M['m10'] / M['m00'])
        cy = int(M['m01'] / M['m00'])

        # Check to see if 4-side polygon is square or rectangle
        # cv2.boundingRect returns the top left and then width and
        if abs(w-h) <= 3:
            shape_name = "Square"

            # Find contour center to place text at the center
            cv2.drawContours(image, [cnt], 0, (0, 125 ,255), -1)
            cv2.putText(image, shape_name, (cx-50, cy), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 0), 2)
        else:
            shape_name = "Rectangle"

            # Find contour center to place text at the center
            cv2.drawContours(image, [cnt], 0, (0, 0, 255), -1)
            M = cv2.moments(cnt)
            cx = int(M['m10'] / M['m00'])
            cy = int(M['m01'] / M['m00'])
            cv2.putText(image, shape_name, (cx-50, cy), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 0), 2)

    elif len(approx) == 10:
        shape_name = "Star"
        cv2.drawContours(image, [cnt], 0, (255, 255, 0), -1)
        M = cv2.moments(cnt)
        cx = int(M['m10'] / M['m00'])
        cy = int(M['m01'] / M['m00'])
        cv2.putText(image, shape_name, (cx-50, cy), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 0), 2)

    elif len(approx) >= 15:
        shape_name = "Circle"
        cv2.drawContours(image, [cnt], 0, (0, 255, 255), -1)
        M = cv2.moments(cnt)
        cx = int(M['m10'] / M['m00'])
        cy = int(M['m01'] / M['m00'])
        cv2.putText(image, shape_name, (cx-50, cy), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 0), 2)
    #cv2.imshow('Identifying Shapes',image)
    cv2.waitKey(0)

cv2.destroyAllWindows()_____no_output_____plt.imshow(thresh,cmap='gray')_____no_output_____fig = plt.figure(figsize=(16,12))
ax = fig.add_subplot(111)
ax.imshow(image)_____no_output_____
</code>


# Notebook from genepattern/tutorial-materials
Path: 2018-02-05_and_06_12_BroadE_workshop/notebooks/2018-02-05_07 BroadE Classification and Prediction - RNAseq.ipynb

# Classification and Prediction in GenePattern Notebook

This notebook will show you how to use k-Nearest Neighbors (kNN) to build a predictor, use it to classify leukemia subtypes, and assess its accuracy in cross-validation._____no_output_____### K-nearest-neighbors (KNN)
KNN classifies an unknown sample by assigning it the phenotype label most frequently represented among the k nearest known samples.

Additionally, you can select a weighting factor for the 'votes' of the nearest neighbors. For example, one might weight the votes by the reciprocal of the distance between neighbors to give closer neighors a greater vote._____no_output_____
1. Log in to GenePattern

Select GenePattern AWS Beta as the server

Enter your username and password.

Click Login to GenePattern.

_____no_output_____

# Requires GenePattern Notebook: pip install genepattern-notebook
import gp
import genepattern

# Username and password removed for security reasons.
genepattern.GPAuthWidget(genepattern.register_session("https://gp-beta-ami.genepattern.org/gp", "", ""))_____no_output_____

## 2. Run k-Nearest Neighbors Cross Validation_____no_output_____

- Drag [BRCA_HUGO_symbols.preprocessed.gct](https://datasets.genepattern.org/data/TCGA_BRCA/DP_4_BRCA_HUGO_symbols.preprocessed.gct) to the **data filename** field below.
- Drag [BRCA_HUGO_symbols.preprocessed.cls](https://datasets.genepattern.org/data/TCGA_BRCA/Pred_2_BRCA_HUGO_symbols.preprocessed.cls) to the **class filename** field.
- Click **Run**._____no_output_____

knnxvalidation_task = gp.GPTask(genepattern.get_session(0), 'urn:lsid:broad.mit.edu:cancer.software.genepattern.module.analysis:00013')
knnxvalidation_job_spec = knnxvalidation_task.make_job_spec()
knnxvalidation_job_spec.set_parameter("data.filename", "")
knnxvalidation_job_spec.set_parameter("class.filename", "")
knnxvalidation_job_spec.set_parameter("num.features", "10")
knnxvalidation_job_spec.set_parameter("feature.selection.statistic", "0")
knnxvalidation_job_spec.set_parameter("min.std", "")
knnxvalidation_job_spec.set_parameter("num.neighbors", "3")
knnxvalidation_job_spec.set_parameter("weighting.type", "1")
knnxvalidation_job_spec.set_parameter("distance.measure", "1")
knnxvalidation_job_spec.set_parameter("pred.results.file", "<data.filename_basename>.pred.odf")
knnxvalidation_job_spec.set_parameter("feature.summary.file", "<data.filename_basename>.feat.odf")
genepattern.GPTaskWidget(knnxvalidation_task)_____no_output_____

## 4. View a list of features used in the prediction model_____no_output_____

- Select the XXXXXX.KNNXvalidation job result cell by clicking anywhere in it.
- Click on the i icon next to the `<filename>.**feat**.odf` file
- Select "Send to DataFrame"
- You will see a new cell created below the job result cell.
- You will see a table of features, descriptions, and the number of times each feature was included in a model in a cross-validation loop._____no_output_____## 5. View prediction results_____no_output_____

- For the **prediction results file** parameter below, click the down arrow in the file input box.
- Select the `BRCA_HUGO_symbols.preprocessed.pred.odf` file.
- Click **Run**.
- You will see the prediction results in an interactive viewer._____no_output_____

predictionresultsviewer_task = gp.GPTask(genepattern.get_session(0), 'urn:lsid:broad.mit.edu:cancer.software.genepattern.module.visualizer:00019')
predictionresultsviewer_job_spec = predictionresultsviewer_task.make_job_spec()
predictionresultsviewer_job_spec.set_parameter("prediction.results.file", "")
genepattern.GPTaskWidget(predictionresultsviewer_task)_____no_output_____

## References

Breiman, L., Friedman, J. H., Olshen, R. A., & Stone, C. J. 1984. [Classification and regression trees](https://www.amazon.com/Classification-Regression-Wadsworth-Statistics-Probability/dp/0412048418?ie=UTF8&*Version*=1&*entries*=0). Wadsworth & Brooks/Cole Advanced Books & Software, Monterey, CA.

Golub, T.R., Slonim, D.K., Tamayo, P., Huard, C., Gaasenbeek, M., Mesirov, J.P., Coller, H., Loh, M., Downing, J.R., Caligiuri, M.A., Bloomfield, C.D., and Lander, E.S. 1999. Molecular Classification of Cancer: Class Discovery and Class Prediction by Gene Expression. [Science 286:531-537](http://science.sciencemag.org/content/286/5439/531.long).

Lu, J., Getz, G., Miska, E.A., Alvarez-Saavedra, E., Lamb, J., Peck, D., Sweet-Cordero, A., Ebert, B.L., Mak, R.H., Ferrando, A.A, Downing, J.R., Jacks, T., Horvitz, H.R., Golub, T.R. 2005. MicroRNA expression profiles classify human cancers. [Nature 435:834-838](http://www.nature.com/nature/journal/v435/n7043/full/nature03702.html).

Rifkin, R., Mukherjee, S., Tamayo, P., Ramaswamy, S., Yeang, C-H, Angelo, M., Reich, M., Poggio, T., Lander, E.S., Golub, T.R., Mesirov, J.P. 2003. An Analytical Method for Multiclass Molecular Cancer Classification. [SIAM Review 45(4):706-723](http://epubs.siam.org/doi/abs/10.1137/S0036144502411986).

Slonim, D.K., Tamayo, P., Mesirov, J.P., Golub, T.R., Lander, E.S. 2000. Class prediction and discovery using gene expression data. In [Proceedings of the Fourth Annual International Conference on Computational Molecular Biology (RECOMB)](http://dl.acm.org/citation.cfm?id=332564). ACM Press, New York. pp. 263-272._____no_output_____


# Notebook from steeverogers/Deep-Learning
Path: Sequence Models/Dinosaur Island -- Character-level language model/Dinosaurus+Island+--+Character-level+language+model+-+%28final%29+-+learners.ipynb

# Character level language model - Dinosaurus land

Welcome to Dinosaurus Island! 65 million years ago, dinosaurs existed, and in this assignment they are back. You are in charge of a special task. Leading biology researchers are creating new breeds of dinosaurs and bringing them to life on earth, and your job is to give names to these dinosaurs. If a dinosaur does not like its name, it might go beserk, so choose wisely!

Luckily you have learned some deep learning and you will use it to save the day. Your assistant has collected a list of all the dinosaur names they could find, and compiled them into this [dataset](dinos.txt). (Feel free to take a look by clicking the previous link.) To create new dinosaur names, you will build a character level language model to generate new names. Your algorithm will learn the different name patterns, and randomly generate new names. Hopefully this algorithm will keep you and your team safe from the dinosaurs' wrath!

By completing this assignment you will learn:

- How to store text data for processing using an RNN
- How to synthesize data, by sampling predictions at each time step and passing it to the next RNN-cell unit
- How to build a character-level text generation recurrent neural network
- Why clipping the gradients is important

We will begin by loading in some functions that we have provided for you in `rnn_utils`. Specifically, you have access to functions such as `rnn_forward` and `rnn_backward` which are equivalent to those you've implemented in the previous assignment. _____no_output_____

import numpy as np
from utils import *
import random
from random import shuffle_____no_output_____

## 1 - Problem Statement

### 1.1 - Dataset and Preprocessing

Run the following cell to read the dataset of dinosaur names, create a list of unique characters (such as a-z), and compute the dataset and vocabulary size. _____no_output_____

data = open('dinos.txt', 'r').read()
data= data.lower()
chars = list(set(data))
data_size, vocab_size = len(data), len(chars)
print('There are %d total characters and %d unique characters in your data.' % (data_size, vocab_size))_____no_output_____

The characters are a-z (26 characters) plus the "\n" (or newline character), which in this assignment plays a role similar to the `<EOS>` (or "End of sentence") token we had discussed in lecture, only here it indicates the end of the dinosaur name rather than the end of a sentence. In the cell below, we create a python dictionary (i.e., a hash table) to map each character to an index from 0-26. We also create a second python dictionary that maps each index back to the corresponding character character. This will help you figure out what index corresponds to what character in the probability distribution output of the softmax layer. Below, `char_to_ix` and `ix_to_char` are the python dictionaries. _____no_output_____

char_to_ix = { ch:i for i,ch in enumerate(sorted(chars)) }
ix_to_char = { i:ch for i,ch in enumerate(sorted(chars)) }
print(ix_to_char)_____no_output_____

### 1.2 - Overview of the model

Your model will have the following structure:

- Initialize parameters
- Run the optimization loop
    - Forward propagation to compute the loss function
    - Backward propagation to compute the gradients with respect to the loss function
    - Clip the gradients to avoid exploding gradients
    - Using the gradients, update your parameter with the gradient descent update rule.
- Return the learned parameters

 **Figure 1**: Recurrent Neural Network, similar to what you had built in the previous notebook "Building a RNN - Step by Step".

At each time-step, the RNN tries to predict what is the next character given the previous characters. The dataset $X = (x^{\langle 1 \rangle}, x^{\langle 2 \rangle}, ..., x^{\langle T_x \rangle})$ is a list of characters in the training set, while $Y = (y^{\langle 1 \rangle}, y^{\langle 2 \rangle}, ..., y^{\langle T_x \rangle})$ is such that at every time-step $t$, we have $y^{\langle t \rangle} = x^{\langle t+1 \rangle}$. _____no_output_____## 2 - Building blocks of the model

In this part, you will build two important blocks of the overall model:
- Gradient clipping: to avoid exploding gradients
- Sampling: a technique used to generate characters

You will then apply these two functions to build the model._____no_output_____### 2.1 - Clipping the gradients in the optimization loop

In this section you will implement the `clip` function that you will call inside of your optimization loop. Recall that your overall loop structure usually consists of a forward pass, a cost computation, a backward pass, and a parameter update. Before updating the parameters, you will perform gradient clipping when needed to make sure that your gradients are not "exploding," meaning taking on overly large values.

In the exercise below, you will implement a function `clip` that takes in a dictionary of gradients and returns a clipped version of gradients if needed. There are different ways to clip gradients; we will use a simple element-wise clipping procedure, in which every element of the gradient vector is clipped to lie between some range [-N, N]. More generally, you will provide a `maxValue` (say 10). In this example, if any component of the gradient vector is greater than 10, it would be set to 10; and if any component of the gradient vector is less than -10, it would be set to -10. If it is between -10 and 10, it is left alone.

 **Figure 2**: Visualization of gradient descent with and without gradient clipping, in a case where the network is running into slight "exploding gradient" problems.

**Exercise**: Implement the function below to return the clipped gradients of your dictionary `gradients`. Your function takes in a maximum threshold and returns the clipped versions of your gradients. You can check out this [hint](https://docs.scipy.org/doc/numpy-1.13.0/reference/generated/numpy.clip.html)._____no_output_____

### GRADED FUNCTION: clip

def clip(gradients, maxValue):
    '''
    Clips the gradients' values between minimum and maximum.

    Arguments:
    gradients -- a dictionary containing the gradients "dWaa", "dWax", "dWya", "db", "dby"
    maxValue -- everything above this number is set to this number, and everything less than -maxValue is set to -maxValue

    Returns:
    gradients -- a dictionary with the clipped gradients.
    '''

    dWaa, dWax, dWya, db, dby = gradients['dWaa'], gradients['dWax'], gradients['dWya'], gradients['db'], gradients['dby']

    ### START CODE HERE ###
    # clip to mitigate exploding gradients, loop over [dWax, dWaa, dWya, db, dby]. (≈2 lines)
    for gradient in [dWax, dWaa, dWya, db, dby]:
        None
    ### END CODE HERE ###

    gradients = {"dWaa": dWaa, "dWax": dWax, "dWya": dWya, "db": db, "dby": dby}

    return gradients_____no_output_____np.random.seed(3)
dWax = np.random.randn(5,3)*10
dWaa = np.random.randn(5,5)*10
dWya = np.random.randn(2,5)*10
db = np.random.randn(5,1)*10
dby = np.random.randn(2,1)*10
gradients = {"dWax": dWax, "dWaa": dWaa, "dWya": dWya, "db": db, "dby": dby}
gradients = clip(gradients, 10)
print("gradients[\"dWaa\"][1][2] =", gradients["dWaa"][1][2])
print("gradients[\"dWax\"][3][1] =", gradients["dWax"][3][1])
print("gradients[\"dWya\"][1][2] =", gradients["dWya"][1][2])
print("gradients[\"db\"][4] =", gradients["db"][4])
print("gradients[\"dby\"][1] =", gradients["dby"][1])_____no_output_____

** Expected output:**

    **gradients["dWaa"][1][2] **

    10.0

    **gradients["dWax"][3][1]**

    -10.0

    **gradients["dWya"][1][2]**

0.29713815361

    **gradients["db"][4]**

[ 10.]

    **gradients["dby"][1]**

[ 8.45833407]

_____no_output_____### 2.2 - Sampling

Now assume that your model is trained. You would like to generate new text (characters). The process of generation is explained in the picture below:

 **Figure 3**: In this picture, we assume the model is already trained. We pass in $x^{\langle 1\rangle} = \vec{0}$ at the first time step, and have the network then sample one character at a time.

**Exercise**: Implement the `sample` function below to sample characters. You need to carry out 4 steps:

- **Step 1**: Pass the network the first "dummy" input $x^{\langle 1 \rangle} = \vec{0}$ (the vector of zeros). This is the default input before we've generated any characters. We also set $a^{\langle 0 \rangle} = \vec{0}$

- **Step 2**: Run one step of forward propagation to get $a^{\langle 1 \rangle}$ and $\hat{y}^{\langle 1 \rangle}$. Here are the equations:

$$ a^{\langle t+1 \rangle} = \tanh(W_{ax}  x^{\langle t \rangle } + W_{aa} a^{\langle t \rangle } + b)\tag{1}$$

$$ z^{\langle t + 1 \rangle } = W_{ya}  a^{\langle t + 1 \rangle } + b_y \tag{2}$$

$$ \hat{y}^{\langle t+1 \rangle } = softmax(z^{\langle t + 1 \rangle })\tag{3}$$

Note that $\hat{y}^{\langle t+1 \rangle }$ is a (softmax) probability vector (its entries are between 0 and 1 and sum to 1). $\hat{y}^{\langle t+1 \rangle}_i$ represents the probability that the character indexed by "i" is the next character.  We have provided a `softmax()` function that you can use.

- **Step 3**: Carry out sampling: Pick the next character's index according to the probability distribution specified by $\hat{y}^{\langle t+1 \rangle }$. This means that if $\hat{y}^{\langle t+1 \rangle }_i = 0.16$, you will pick the index "i" with 16% probability. To implement it, you can use [`np.random.choice`](https://docs.scipy.org/doc/numpy-1.13.0/reference/generated/numpy.random.choice.html).

Here is an example of how to use `np.random.choice()`:
```python
np.random.seed(0)
p = np.array([0.1, 0.0, 0.7, 0.2])
index = np.random.choice([0, 1, 2, 3], p = p.ravel())
```
This means that you will pick the `index` according to the distribution:
$P(index = 0) = 0.1, P(index = 1) = 0.0, P(index = 2) = 0.7, P(index = 3) = 0.2$.

- **Step 4**: The last step to implement in `sample()` is to overwrite the variable `x`, which currently stores $x^{\langle t \rangle }$, with the value of $x^{\langle t + 1 \rangle }$. You will represent $x^{\langle t + 1 \rangle }$ by creating a one-hot vector corresponding to the character you've chosen as your prediction. You will then forward propagate $x^{\langle t + 1 \rangle }$ in Step 1 and keep repeating the process until you get a "\n" character, indicating you've reached the end of the dinosaur name. _____no_output_____

# GRADED FUNCTION: sample

def sample(parameters, char_to_ix, seed):
    """
    Sample a sequence of characters according to a sequence of probability distributions output of the RNN

    Arguments:
    parameters -- python dictionary containing the parameters Waa, Wax, Wya, by, and b.
    char_to_ix -- python dictionary mapping each character to an index.
    seed -- used for grading purposes. Do not worry about it.

    Returns:
    indices -- a list of length n containing the indexes of the sampled characters.
    """

    # Retrieve parameters and relevant shapes from "parameters" dictionary
    Waa, Wax, Wya, by, b = parameters['Waa'], parameters['Wax'], parameters['Wya'], parameters['by'], parameters['b']
    vocab_size = by.shape[0]
    n_a = Waa.shape[1]

    ### START CODE HERE ###
    # Step 1: Create the one-hot vector x for the first character (initializing the sequence generation). (≈1 line)
    x = None
    # Step 1': Initialize a_prev as zeros (≈1 line)
    a_prev = None

    # Create an empty list of indices, this is the list which will contain the list of indexes of the characters to generate (≈1 line)
    indices = None

    # Idx is a flag to detect a newline character, we initialize it to -1
    idx = -1

    # Loop over time-steps t. At each time-step, sample a character from a probability distribution and append
    # its index to "indexes". We'll stop if we reach 50 characters (which should be very unlikely with a well
    # trained model), which helps debugging and prevents entering an infinite loop.
    counter = 0
    newline_character = char_to_ix['\n']

    while (idx != newline_character and counter != 50):

        # Step 2: Forward propagate x using the equations (1), (2) and (3)
        a = None
        z = None
        y = None

        # for grading purposes
        np.random.seed(counter+seed)

        # Step 3: Sample the index of a character within the vocabulary from the probability distribution y
        idx = None

        # Append the index to "indices"
        None

        # Step 4: Overwrite the input character as the one corresponding to the sampled index.
        x = None
        x[None] = None

        # Update "a_prev" to be "a"
        a_prev = None

        # for grading purposes
        seed += 1
        counter +=1

    ### END CODE HERE ###

    if (counter == 50):
        indices.append(char_to_ix['\n'])

    return indices_____no_output_____np.random.seed(2)
n, n_a = 20, 100
a0 = np.random.randn(n_a, 1)
i0 = 1 # first character is ix_to_char[i0]
Wax, Waa, Wya = np.random.randn(n_a, vocab_size), np.random.randn(n_a, n_a), np.random.randn(vocab_size, n_a)
b, by = np.random.randn(n_a, 1), np.random.randn(vocab_size, 1)
parameters = {"Wax": Wax, "Waa": Waa, "Wya": Wya, "b": b, "by": by}

indexes = sample(parameters, char_to_ix, 0)
print("Sampling:")
print("list of sampled indices:", indexes)
print("list of sampled characters:", [ix_to_char[i] for i in indexes])_____no_output_____

** Expected output:**

    **list of sampled indices:**

    [18, 2, 26, 0]

    **list of sampled characters:**

    ['r', 'b', 'z', '\n']

_____no_output_____## 3 - Building the language model

It is time to build the character-level language model for text generation.

### 3.1 - Gradient descent

In this section you will implement a function performing one step of stochastic gradient descent (with clipped gradients). You will go through the training examples one at a time, so the optimization algorithm will be stochastic gradient descent. As a reminder, here are the steps of a common optimization loop for an RNN:

- Forward propagate through the RNN to compute the loss
- Backward propagate through time to compute the gradients of the loss with respect to the parameters
- Clip the gradients if necessary
- Update your parameters using gradient descent

**Exercise**: Implement this optimization process (one step of stochastic gradient descent).

We provide you with the following functions:

```python
def rnn_forward(X, Y, a_prev, parameters):
    """ Performs the forward propagation through the RNN and computes the cross-entropy loss.
    It returns the loss' value as well as a "cache" storing values to be used in the backpropagation."""
    ....
    return loss, cache

def rnn_backward(X, Y, parameters, cache):
    """ Performs the backward propagation through time to compute the gradients of the loss with respect
    to the parameters. It returns also all the hidden states."""
    ...
    return gradients, a

def update_parameters(parameters, gradients, learning_rate):
    """ Updates parameters using the Gradient Descent Update Rule."""
    ...
    return parameters
```_____no_output_____

# GRADED FUNCTION: optimize

def optimize(X, Y, a_prev, parameters, learning_rate = 0.01):
    """
    Execute one step of the optimization to train the model.

    Arguments:
    X -- list of integers, where each integer is a number that maps to a character in the vocabulary.
    Y -- list of integers, exactly the same as X but shifted one index to the left.
    a_prev -- previous hidden state.
    parameters -- python dictionary containing:
                        Wax -- Weight matrix multiplying the input, numpy array of shape (n_a, n_x)
                        Waa -- Weight matrix multiplying the hidden state, numpy array of shape (n_a, n_a)
                        Wya -- Weight matrix relating the hidden-state to the output, numpy array of shape (n_y, n_a)
                        b --  Bias, numpy array of shape (n_a, 1)
                        by -- Bias relating the hidden-state to the output, numpy array of shape (n_y, 1)
    learning_rate -- learning rate for the model.

    Returns:
    loss -- value of the loss function (cross-entropy)
    gradients -- python dictionary containing:
                        dWax -- Gradients of input-to-hidden weights, of shape (n_a, n_x)
                        dWaa -- Gradients of hidden-to-hidden weights, of shape (n_a, n_a)
                        dWya -- Gradients of hidden-to-output weights, of shape (n_y, n_a)
                        db -- Gradients of bias vector, of shape (n_a, 1)
                        dby -- Gradients of output bias vector, of shape (n_y, 1)
    a[len(X)-1] -- the last hidden state, of shape (n_a, 1)
    """

    ### START CODE HERE ###

    # Forward propagate through time (≈1 line)
    loss, cache = None

    # Backpropagate through time (≈1 line)
    gradients, a = None

    # Clip your gradients between -5 (min) and 5 (max) (≈1 line)
    gradients = None

    # Update parameters (≈1 line)
    parameters = None

    ### END CODE HERE ###

    return loss, gradients, a[len(X)-1]_____no_output_____np.random.seed(1)
vocab_size, n_a = 27, 100
a_prev = np.random.randn(n_a, 1)
Wax, Waa, Wya = np.random.randn(n_a, vocab_size), np.random.randn(n_a, n_a), np.random.randn(vocab_size, n_a)
b, by = np.random.randn(n_a, 1), np.random.randn(vocab_size, 1)
parameters = {"Wax": Wax, "Waa": Waa, "Wya": Wya, "b": b, "by": by}
X = [12,3,5,11,22,3]
Y = [4,14,11,22,25, 26]

loss, gradients, a_last = optimize(X, Y, a_prev, parameters, learning_rate = 0.01)
print("Loss =", loss)
print("gradients[\"dWaa\"][1][2] =", gradients["dWaa"][1][2])
print("np.argmax(gradients[\"dWax\"]) =", np.argmax(gradients["dWax"]))
print("gradients[\"dWya\"][1][2] =", gradients["dWya"][1][2])
print("gradients[\"db\"][4] =", gradients["db"][4])
print("gradients[\"dby\"][1] =", gradients["dby"][1])
print("a_last[4] =", a_last[4])_____no_output_____

** Expected output:**

    **Loss **

    126.503975722

    **gradients["dWaa"][1][2]**

    0.194709315347

    **np.argmax(gradients["dWax"])**

     93

    **gradients["dWya"][1][2]**

     -0.007773876032

    **gradients["db"][4]**

     [-0.06809825]

    **gradients["dby"][1]**

    [ 0.01538192]

    **a_last[4]**

     [-1.]

_____no_output_____### 3.2 - Training the model _____no_output_____Given the dataset of dinosaur names, we use each line of the dataset (one name) as one training example. Every 100 steps of stochastic gradient descent, you will sample 10 randomly chosen names to see how the algorithm is doing. Remember to shuffle the dataset, so that stochastic gradient descent visits the examples in random order.

**Exercise**: Follow the instructions and implement `model()`. When `examples[index]` contains one dinosaur name (string), to create an example (X, Y), you can use this:
```python
        index = j % len(examples)
        X = [None] + [char_to_ix[ch] for ch in examples[index]]
        Y = X[1:] + [char_to_ix["\n"]]
```
Note that we use: `index= j % len(examples)`, where `j = 1....num_iterations`, to make sure that `examples[index]` is always a valid statement (`index` is smaller than `len(examples)`).
The first entry of `X` being `None` will be interpreted by `rnn_forward()` as setting $x^{\langle 0 \rangle} = \vec{0}$. Further, this ensures that `Y` is equal to `X` but shifted one step to the left, and with an additional "\n" appended to signify the end of the dinosaur name. _____no_output_____

# GRADED FUNCTION: model

def model(data, ix_to_char, char_to_ix, num_iterations = 35000, n_a = 50, dino_names = 7, vocab_size = 27):
    """
    Trains the model and generates dinosaur names.

    Arguments:
    data -- text corpus
    ix_to_char -- dictionary that maps the index to a character
    char_to_ix -- dictionary that maps a character to an index
    num_iterations -- number of iterations to train the model for
    n_a -- number of hidden neurons in the softmax layer
    dino_names -- number of dinosaur names you want to sample at each iteration.
    vocab_size -- number of unique characters found in the text, size of the vocabulary

    Returns:
    parameters -- learned parameters
    """

    # Retrieve n_x and n_y from vocab_size
    n_x, n_y = vocab_size, vocab_size

    # Initialize parameters
    parameters = initialize_parameters(n_a, n_x, n_y)

    # Initialize loss (this is required because we want to smooth our loss, don't worry about it)
    loss = get_initial_loss(vocab_size, dino_names)

    # Build list of all dinosaur names (training examples).
    with open("dinos.txt") as f:
        examples = f.readlines()
    examples = [x.lower().strip() for x in examples]

    # Shuffle list of all dinosaur names
    shuffle(examples)

    # Initialize the hidden state of your LSTM
    a_prev = np.zeros((n_a, 1))

    # Optimization loop
    for j in range(num_iterations):

        ### START CODE HERE ###

        # Use the hint above to define one training example (X,Y) (≈ 2 lines)
        index = None
        X = None
        Y = None

        # Perform one optimization step: Forward-prop -> Backward-prop -> Clip -> Update parameters
        # Choose a learning rate of 0.01
        curr_loss, gradients, a_prev = None

        ### END CODE HERE ###

        # Use a latency trick to keep the loss smooth. It happens here to accelerate the training.
        loss = smooth(loss, curr_loss)

        # Every 2000 Iteration, generate "n" characters thanks to sample() to check if the model is learning properly
        if j % 2000 == 0:

            print('Iteration: %d, Loss: %f' % (j, loss) + '\n')

            # The number of dinosaur names to print
            seed = 0
            for name in range(dino_names):

                # Sample indexes and print them
                sampled_indexes = sample(parameters, char_to_ix, seed)
                print_sample(sampled_indexes, ix_to_char)

                seed += 1  # To get the same result for grading purposed, increment the seed by one.

            print('\n')

    return parameters_____no_output_____

Run the following cell, you should observe your model outputting random-looking characters at the first iteration. After a few thousand iterations, your model should learn to generate reasonable-looking names. _____no_output_____

parameters = model(data, ix_to_char, char_to_ix)_____no_output_____

## Conclusion

You can see that your algorithm has started to generate plausible dinosaur names towards the end of the training. At first, it was generating random characters, but towards the end you could see dinosaur names with cool endings. Feel free to run the algorithm even longer and play with hyperparameters to see if you can get even better results. Our implemetation generated some really cool names like `maconucon`, `marloralus` and `macingsersaurus`. Your model hopefully also learned that dinosaur names tend to end in `saurus`, `don`, `aura`, `tor`, etc.

If your model generates some non-cool names, don't blame the model entirely--not all actual dinosaur names sound cool. (For example, `dromaeosauroides` is an actual dinosaur name and is in the training set.) But this model should give you a set of candidates from which you can pick the coolest!

This assignment had used a relatively small dataset, so that you could train an RNN quickly on a CPU. Training a model of the english language requires a much bigger dataset, and usually needs much more computation, and could run for many hours on GPUs. We ran our dinosaur name for quite some time, and so far our favoriate name is the great, undefeatable, and fierce: Mangosaurus!

_____no_output_____## 4 - Writing like Shakespeare

The rest of this notebook is optional and is not graded, but we hope you'll do it anyway since it's quite fun and informative.

A similar (but more complicated) task is to generate Shakespeare poems. Instead of learning from a dataset of Dinosaur names you can use a collection of Shakespearian poems. Using LSTM cells, you can learn longer term dependencies that span many characters in the text--e.g., where a character appearing somewhere a sequence can influence what should be a different character much much later in ths sequence. These long term dependencies were less important with dinosaur names, since the names were quite short.

 Let's become poets!

We have implemented a Shakespeare poem generator with Keras. Run the following cell to load the required packages and models. This may take a few minutes. _____no_output_____

from __future__ import print_function
from keras.callbacks import LambdaCallback
from keras.models import Model, load_model, Sequential
from keras.layers import Dense, Activation, Dropout, Input, Masking
from keras.layers import LSTM
from keras.utils.data_utils import get_file
from keras.preprocessing.sequence import pad_sequences
from shakespeare_utils import *
import sys
import io_____no_output_____

To save you some time, we have already trained a model for ~1000 epochs on a collection of Shakespearian poems called [*"The Sonnets"*](shakespeare.txt). _____no_output_____Let's train the model for one more epoch. When it finishes training for an epoch---this will also take a few minutes---you can run `generate_output`, which will prompt asking you for an input (`<`40 characters). The poem will start with your sentence, and our RNN-Shakespeare will complete the rest of the poem for you! For example, try "Forsooth this maketh no sense " (don't enter the quotation marks). Depending on whether you include the space at the end, your results might also differ--try it both ways, and try other inputs as well.
_____no_output_____

print_callback = LambdaCallback(on_epoch_end=on_epoch_end)

model.fit(x, y, batch_size=128, epochs=1, callbacks=[print_callback])_____no_output_____# Run this cell to try with different inputs without having to re-train the model
generate_output()_____no_output_____

The RNN-Shakespeare model is very similar to the one you have built for dinosaur names. The only major differences are:
- LSTMs instead of the basic RNN to capture longer-range dependencies
- The model is a deeper, stacked LSTM model (2 layer)
- Using Keras instead of python to simplify the code

If you want to learn more, you can also check out the Keras Team's text generation implementation on GitHub: https://github.com/keras-team/keras/blob/master/examples/lstm_text_generation.py.

Congratulations on finishing this notebook! _____no_output_____**References**:
- This exercise took inspiration from Andrej Karpathy's implementation: https://gist.github.com/karpathy/d4dee566867f8291f086. To learn more about text generation, also check out Karpathy's [blog post](http://karpathy.github.io/2015/05/21/rnn-effectiveness/).
- For the Shakespearian poem generator, our implementation was based on the implementation of an LSTM text generator by the Keras team: https://github.com/keras-team/keras/blob/master/examples/lstm_text_generation.py _____no_output_____


# Notebook from dongxulee/lifeCycle
Path: 20200914/solvingHousingProofRead.ipynb

<code>
from scipy.interpolate import interpn
import numpy as np
from multiprocessing import Pool
from functools import partial
import warnings
import math
warnings.filterwarnings("ignore")_____no_output_____# time line
T_min = 0
T_max = 70
T_R = 45
# discounting factor
beta = 1/(1+0.02)
# utility function parameter
gamma = 2
# relative importance of housing consumption and non durable consumption
alpha = 0.8
# parameter used to calculate the housing consumption
kappa = 0.3
# depreciation parameter
delta = 0.025
# housing parameter
chi = 0.3
# uB associated parameter
B = 2
# constant cost
c_h = 0.5
# All the money amount are denoted in thousand dollars
earningShock = [0.8,1.2]
# Define transition matrix of economical states
# GOOD -> GOOD 0.8, BAD -> BAD 0.6
Ps = np.array([[0.6, 0.4],[0.2, 0.8]])
# current risk free interest rate
# r_b = np.array([0.01 ,0.03]) increase the mean by 2%
r_b = np.array([0.03 ,0.05])
# stock return depends on current and future econ states
# r_k = np.array([[-0.2, 0.15],[-0.15, 0.2]])
r_k = np.array([[-0.2, 0.15],[-0.15, 0.20]])
# expected return on stock market, use to calculate 401k dynamics
# r_bar = 0.0667
r_bar = 0.02
# probability of survival
Pa = np.load("prob.npy")
# deterministic income
detEarning = np.load("detEarning.npy")
# probability of employment transition Pe[s, s_next, e, e_next]
Pe = np.array([[[[0.3, 0.7], [0.1, 0.9]], [[0.25, 0.75], [0.05, 0.95]]],
               [[[0.25, 0.75], [0.05, 0.95]], [[0.2, 0.8], [0.01, 0.99]]]])
# tax rate before and after retirement
tau_L = 0.2
tau_R = 0.1
# constant state variables: Purchase value 250k, down payment 50k, mortgage 200k, interest rate 3.6%,
# 55 payment period, 8.4k per period. One housing unit is roughly 1 square feet. Housing price 0.25k/sf
# some variables associate with 401k amount
Nt = [np.sum(Pa[t:]) for t in range(T_max-T_min)]
Dt = [np.ceil(((1+r_bar)**N - 1)/(r_bar*(1+r_bar)**N)) for N in Nt]
# income fraction goes into 401k
yi = 0.005
# mortgate rate
rh = 0.036
D = [((1+rh)**N - 1)/(rh*(1+rh)**N) for N in range(T_max-T_min)]
D[0] = 1
# housing unit
H = 100
# housing price constant
pt = 2*250/1000
# 30k rent 1000 sf
pr = 2*30/1000_____no_output_____# M = 100
# tt = 2
# m = M/D[tt-1]
# for i in range(1,tt):
#     M = M*(1+rh) - m
#     print(M)1.1368683772161603e-13
#Define the utility function
def u(c):
    return (np.float_power(c, 1-gamma) - 1)/(1 - gamma)

#Define the bequeath function, which is a function of wealth
def uB(tb):
    return B*u(tb)

#Calcualte HE
def calHE(x):
    # the input x is a numpy array
    # w, n, M, g_lag, e, s = x
    HE = (H+(1-chi)*(1-delta)*x[:,3])*pt - x[:,2]
    return HE

#Calculate TB
def calTB(x):
    # the input x as a numpy array
    # w, n, M, g_lag, e, s = x
    TB = x[:,0] + x[:,1] + calHE(x)
    return TB_____no_output_____def R(x, a):
    '''
    Input:
        state x: w, n, M, g_lag, e, s
        action a: c, b, k, i, q = a which is a np array
    Output:
        reward value: the length of return should be equal to the length of a
    '''
    w, n, M, g_lag, e, s = x
    reward = np.zeros(a.shape[0])
    # actions with improvement
    i_index = (a[:,4]==1)
    # actions without improvement
    ni_index = (a[:,4]!=1)
    # housing consumption with improvement
    i_h = H + (1-delta)*g_lag + a[i_index][:,3]
    i_Vh = (1+kappa)*i_h
    # housing consumption without improvement
    ni_h = H + (1-delta)*g_lag
    ni_Vh = (1-kappa)*(ni_h-(1-a[ni_index][:,4])*H)
    # combined consumption with and without improvement
    i_C = np.float_power(a[i_index][:,0], alpha) * np.float_power(i_Vh, 1-alpha)
    ni_C = np.float_power(a[ni_index][:,0], alpha) * np.float_power(ni_Vh, 1-alpha)
    reward[i_index] = u(i_C)
    reward[ni_index] = u(ni_C)
    return reward_____no_output_____#Define the earning function, which applies for both employment and unemployment, good econ state and bad econ state
def y(t, x):
    w, n, M, g_lag, e, s = x
    if t <= T_R:
        welfare = 5
        return detEarning[t] * earningShock[int(s)] * e + (1-e) * welfare
    else:
        return detEarning[t]

#Earning after tax and fixed by transaction in and out from 401k account
def yAT(t,x):
    yt = y(t, x)
    w, n, M, g_lag, e, s = x
    if t <= T_R and e == 1:
        # yi portion of the income will be put into the 401k
        return (1-tau_L)*(yt * (1-yi))
    if t <= T_R and e == 0:
        return yt
    else:
        # t > T_R, n/discounting amount will be withdraw from the 401k
        return (1-tau_R)*yt + n/Dt[t]

#Define the evolution of the amount in 401k account
def gn(t, n, x, s_next):
    w, n, M, g_lag, e, s = x
    if t <= T_R and e == 1:
        # if the person is employed, then yi portion of his income goes into 401k
        n_cur = n + y(t, x) * yi
    elif t <= T_R and e == 0:
        # if the perons is unemployed, then n does not change
        n_cur = n
    else:
        # t > T_R, n/discounting amount will be withdraw from the 401k
        n_cur = n - n/Dt[t]
    return (1+r_k[int(s), s_next])*n_cur _____no_output_____def transition(x, a, t):
    '''
    Input: state and action and time
    Output: possible future states and corresponding probability
    '''
    w, n, M, g_lag, e, s = x
    # variables used to collect possible states and probabilities
    x_next = []
    prob_next = []
    # mortgage payment
    m = M/D[T_max-t]
    M_next = M*(1+rh) - m
    for aa in a:
        c,b,k,i,q = aa
        if q == 1:
            g = (1-delta)*g_lag + i
        else:
            g = (1-delta)*g_lag
        for s_next in [0,1]:
            w_next =  b*(1+r_b[int(s)]) + k*(1+r_k[int(s), s_next])
            n_next = gn(t, n, x, s_next)
            if t >= T_R:
                e_next = 0
                x_next.append([w_next, n_next, M_next, g, e_next, s_next])
                prob_next.append(Ps[int(s),s_next])
            else:
                for e_next in [0,1]:
                    x_next.append([w_next, n_next, M_next, g, e_next, s_next])
                    prob_next.append(Ps[int(s),s_next] * Pe[int(s),s_next,int(e),e_next])
    return np.array(x_next), np.array(prob_next)_____no_output_____
</code>


# Notebook from stephtdouglas/colors-and-coding
Path: investigation_path.ipynb

import itertools

import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline
import astropy.io.ascii as at_____no_output_____stars = at.read("sdss_mags.csv")
# Visually check if the possible colors are matplotlib-compatible
print(np.unique(stars["color"]))color
------
  blue
orange
   red
 white
yellow
# Try plotting one magnitude against another
plt.plot(stars['u'],stars['g'],'k.')
plt.xlabel("u")
plt.ylabel("g")_____no_output_____plt.plot(stars['g'],stars['r'],'k.')
plt.xlabel("g")
plt.ylabel("r")_____no_output_____# Plot one magnitude against another, but this time color the stars by their visibly-identified color
for g, r, c in itertools.izip(stars["g"],stars["r"],stars["color"]):
    plt.plot(g,r,'o',mfc=c,ms=15)
plt.xlabel("g",fontsize="x-large")
plt.ylabel("r",fontsize="x-large")_____no_output_____# Define the 5 SDSS bands and their central wavelengths
bands = ["u","g","r","i","z"]
central_wavelengths = [3543,4770,6231,7625,9134]_____no_output_____# Plot the magnitudes vs. the appropriate central wavelengths for each filter
for i,b in enumerate(bands):
    plt.plot(np.ones(len(stars))*central_wavelengths[i],stars[b],'ko',ms=15)_____no_output_____# Also attempted the above with colored points, but it was not any more decipherable_____no_output_____# Plot each star in its own window, and make sure that limits match for ease of comparison
plt.figure(figsize=(10,8))
for i in range(len(stars)):
    plt.subplot(5,4,i+1)
    mags = np.array([stars[i][b] for b in bands])
    plt.plot(central_wavelengths,mags,
             '-o',ms=10,color=stars["color"][i],lw=2)
    plt.ylim(27,12)
#plt.xlabel("g",fontsize="x-large")
#plt.ylabel("r",fontsize="x-large")
_____no_output_____

It's hard to tell the difference between the white and blue stars; they're mostly flat

The red, orange, and yellow distributions all increase towards longer wavelengths.

Can you tell any difference among those?

It looks like the red distributions are steepest/increase the most, while the orange are a little flatter and the yellow are flatter still.

How can we quantify that?_____no_output_____

# Calculate the difference between the magnitudes at the blue and red end
# Subtracting blue from red to get a positive magnitude,
# (but students could very easily see the brighter red wavelengths and try red-blue)
# Students may also try to calculate it as a slope, and do delta-magnitude over delta-wavelength
mag_diff = stars["u"] - stars["z"]

# Sort by this magnitude difference, then print the visible colors in that order
sort_diff = np.argsort(mag_diff)
print(stars["color"][sort_diff])color
------
 white
  blue
 white
 white
  blue
 white
 white
 white
 white
 white
yellow
orange
orange
orange
orange
orange
   red
   red
   red


# Notebook from Kerianne28/ZTF_image_classification
Path: star_gal_MuyGPs_classification.ipynb

# Star Galaxy Classification

### In this notebook we run the un-normalized and normalized datasets through the MuyGPyS classifier (a python classifying function that uses the MuyGPS  Gaussian process hyperparameter estimation method), and compare the resulting accuracies.

**Note:** Must have run `data_normalization.ipynb` to continue._____no_output_____

import numpy as np
from MuyGPyS.examples.classify import do_classify
import pandas as pd
from sklearn.model_selection import train_test_split
import random
from tqdm import tqdm
import matplotlib.pyplot as plt_____no_output_____

### Read in all flattened data (normalized and un-normalized):_____no_output_____

gal_star = pd.read_csv('raw_image_data.csv')
gal_star_norm_1 = pd.read_csv('norm_1_image_data.csv')
gal_star_norm_2 = pd.read_csv('norm_2_image_data.csv')
gal_star_norm_3 = pd.read_csv('norm_3_image_data.csv')
gal_star_norm_4 = pd.read_csv('norm_4_image_data.csv')
gal_star_norm_5 = pd.read_csv('norm_5_image_data.csv')

# Create a list with the name of the variable holding the data,
# and the name you want associated with the data, for each dataset
data_files = [[gal_star, 'Raw data'],
              [gal_star_norm_1, 'Normalized data 1'],
              [gal_star_norm_2, 'Normalized data 2'],
              [gal_star_norm_3, 'Normalized data 3'],
              [gal_star_norm_4, 'Normalized data 4'],
              [gal_star_norm_5, 'Normalized data 5']]_____no_output_____

### Define a function that generates "one-hot" values.

This essentially just takes our truth labels of 0 and 1, and does the following conversions for use in the classifier:
- 0 to [1., -1.]
- 1 to [-1., 1.]_____no_output_____

def generate_onehot_value(values):
    onehot = []
    for val in values:
        if val == 0:
            onehot.append([1., -1.])
        elif val == 1:
            onehot.append([-1., 1.])
    return onehot_____no_output_____

### Run the classifier on each dataset

For each dataset (un-normalized and normalized) in `data_files`, this for loop does the following:
- Separate labels from data
- Split up data between training and testing
    - `test_size` is the fraction of the data you want to use for testing, where 0.5 means half of the data is used for testing and half for training.
    - `random_state` makes each dataset get trained and tested on the same number of stars and galaxies.
- Gets the one-hot values for the testing and training labels
- Gets `train` and `test` into the proper format for the classifier, a dictionary with the keys:
    - 'input':
    - 'output':
    - 'lookup':
- Does the classification (`do_classify`)
- Computes the accuracy of the classifier for the given dataset, by compairing predicted labels to truth labels._____no_output_____

for data, data_label in data_files:
    truth_labels = data.iloc[:, 0].values
    image_data = data.iloc[:, 1:].values

    X_train, X_test, y_train, y_test = train_test_split(image_data, truth_labels, test_size=0.5, random_state=42)

    print("=============== ", data_label, " ===============")
    print('Training data:', len(y_train[y_train==0]), 'stars and', len(y_train[y_train==1]), 'galaxies')
    print('Testing data:', len(y_test[y_test==0]), 'stars and', len(y_test[y_test==1]), 'galaxies')

    onehot_train, onehot_test = generate_onehot_value(y_train), generate_onehot_value(y_test)

    train = {'input': X_train, 'output': onehot_train, 'lookup': y_train}
    test = {'input': X_test, 'output': onehot_test, 'lookup': y_test}

    print("Running Classifier on", data_label)
    #Switch verbose to True for more output
    muygps, nbrs_lookup, surrogate_predictions = do_classify(test_features=np.array(test['input']),
                                                             train_features=np.array(train['input']),
                                                             train_labels=np.array(train['output']),
                                                             nn_count=30, verbose=False)
    predicted_labels = np.argmax(surrogate_predictions, axis=1)
    print("Total accuracy for", data_label, ":", np.around((np.sum(predicted_labels == np.argmax(test["output"], axis=1))/len(predicted_labels))*100, 3), '%')_____no_output_____

***Note:*** Each time you run the classifier will result in different accuracies._____no_output_____### As you can see, all 5 normalization techniques do much better than the un-normalized data, with some performing better than others.

### Things you can try, to see how they affect the classifier accuracy:
- Play around with different values of `test_size`. What does testing on more or less data do?
- Play around with different parameters that are passed to `do_classify`. Start with `nn_count` and `embed_dim`(For what those arguments are, and a full list of all of the arguments you can pass to do_classify, look at the function `do_classify` in `/MuyGPyS/examples/classify.py`).
- Try generating more cutouts using `generating_ZTF_cutouts_from_ra_dec.ipynb`. How does having more testing and training data affects the classifier?
- Play around with the parameters used to make the cutouts. What happens if you remove blend cuts? Can the classifier classify blends? What is you increase the seeing limit? Can the classifier classify images with bad atmoshperic quality?_____no_output_____

_____no_output_____## **Optional Step:**
### Running each dataset through the classifier multiple times, testing and training on varying amounts of data, different random states, and plotting the accuracy outcomes

- Each time you run the following steps, you change:
    - `test_size`: This is used in `train_test_split`, and changes the size of the testing and training datasets, which effects the accuracy of the classifier.
    - `random_state`: This is used in `train_test_split`, and changes the ratio of how many stars-to-galaxies get tested on.
- You can set how many times to run the classifier with varying test sizes and random states by setting `num_runs`, and you can manually change the test_size values by editing `test_size_values`._____no_output_____

test_size_values = [.2, .25, .33, .4, .5, .75]
num_runs = 3_____no_output_____def run_classifier(image_data, truth_labels, test_size, state):
    X_train, X_test, y_train, y_test = train_test_split(image_data, truth_labels, test_size=test_size, random_state=state)
    onehot_train, onehot_test = generate_onehot_value(y_train), generate_onehot_value(y_test)
    train = {'input': X_train, 'output': onehot_train, 'lookup': y_train}
    test = {'input': X_test, 'output': onehot_test, 'lookup': y_test}
    #Switch verbose to True for more output
    muygps, nbrs_lookup, surrogate_predictions= do_classify(test_features=np.array(test['input']),
                                                            train_features=np.array(train['input']),
                                                            train_labels=np.array(train['output']),
                                                            nn_count=30, verbose=False)
    predicted_labels = np.argmax(surrogate_predictions, axis=1)
    accuracy = (np.sum(predicted_labels == np.argmax(test["output"], axis=1))/len(predicted_labels))*100
    return accuracy_____no_output_____accuracies = pd.DataFrame({'test_size': test_size_values})

# Setting progress bar for each time the classifier will be run during this step
pbar = tqdm(total=len(data_files)*num_runs*len(test_size_values), desc='Running classifier', leave=True)

for data, data_label in data_files:
    truth_labels = data.iloc[:, 0].values
    image_data = data.iloc[:, 1:].values
    all_acc_dataset = []
    for test_size in test_size_values:
        acc = []
        idx = 1
        while idx <= num_runs:
            accuracy = run_classifier(image_data, truth_labels, test_size, state=random.randint(0, 10000))
            acc.append(accuracy)
            pbar.update(1)
            idx += 1
        avg_acc = np.average(acc)
        all_acc_dataset.append(avg_acc)
    temp_df = pd.DataFrame({str(data_label): all_acc_dataset})
    accuracies = pd.concat([accuracies, temp_df], axis=1)
display(accuracies)_____no_output_____plt.figure(figsize=(12,8))

for data, data_labels in data_files:
    plt.plot(accuracies['test_size'].values, accuracies[data_labels].values, label=data_labels)

plt.legend(fontsize=12)
plt.tick_params(labelsize=14)
plt.xlabel("Test size (as a ratio to full data size)", fontsize=18)
plt.ylabel("Accuracy [%]", fontsize=18)
plt.show()_____no_output_____


# Notebook from DaltonGOO/Python-Code-Challenges
Path: Python Code Challenges/Get_the_middle_character.ipynb

# Resources

### Python

[Python 3 Documentation](https://docs.python.org/3/library/)

### General

[Stackoverflow](https://stackoverflow.com/)

### YouTube vids

[Kalle Hallden](https://www.youtube.com/channel/UCWr0mx597DnSGLFk1WfvSkQ)

[PyCon 2019](https://www.youtube.com/channel/UCxs2IIVXaEHHA4BtTiWZ2mQ)

[Tech With Tim](https://www.youtube.com/channel/UC4JX40jDee_tINbkjycV4Sg)

[Python Programmer](https://www.youtube.com/user/consumerchampion)

[sentdex](https://www.youtube.com/user/sentdex)

### Markdown links
[Markdown Cheatsheet](https://github.com/adam-p/markdown-here/wiki/Markdown-Cheatsheet)

[Markdown Guide](https://www.markdownguide.org/)

[Markdown Table Generator](https://www.tablesgenerator.com/markdown_tables)

### Code

```df.reset_index(self, level=None, drop=False, inplace=False, col_level=0, col_fill='')```

```df = pd.DataFrame(np.random.randint(500,4000,size=(200, 1)), columns=list('A'))```

```df['randNumCol'] = np.random.randint(1, 6, df.shape[0])```

```
# Declare a list that is to be converted into a column
tradcounthvac = xyzhvac.x.count()
tradehvac = tradcounthvac * ['hvac']

tradcountelec = xyzelec.x.count()
tradeelec = tradcountelec * ['elec']

# Using 'Trade' as the column name
# and equating it to the list
xyzhvac['Trade'] = tradehvac
xyzelec['Trade'] = tradeelec
```

### Packages

```! pip install pandas-profiling```

```! pip install plotly```

```! pip install cufflinks```

```! pip install plotly==4.2.1```

```!pip install dovpanda```

```import numpy as np```

```import pandas as pd```

```import pandas_profiling```

```import plotly.graph_objects as go```

```import dovpanda```

### pandas
[What can you do with the new ‘Pandas’?](https://towardsdatascience.com/what-can-you-do-with-the-new-pandas-2d24cf8d8b4b)

[Reordering Pandas DataFrame Columns: Thumbs Down On Standard Solutions](https://towardsdatascience.com/reordering-pandas-dataframe-columns-thumbs-down-on-standard-solutions-1ff0bc2941d5)

[pandas Documentation](https://pandas.pydata.org/pandas-docs/stable/)

[7 practical pandas tips when you start working with the library](https://towardsdatascience.com/7-practical-pandas-tips-when-you-start-working-with-the-library-e4a9205eb443)

[dataframe transpose](https://www.geeksforgeeks.org/python-pandas-dataframe-transpose/)

[Combining DataFrames with Pandas](https://datacarpentry.org/python-ecology-lesson/05-merging-data/)

[dovpanda](https://github.com/dovpanda-dev/dovpanda)

[Selecting Subsets of Data in Pandas: Part 1](https://medium.com/dunder-data/selecting-subsets-of-data-in-pandas-6fcd0170be9c)

[10 simple Python tips to speed up your data analysis](https://thenextweb.com/syndication/2020/10/12/10-simple-python-tips-to-speed-up-your-data-analysis/)

[15 Tips and Tricks to use in Jupyter Notebooks](https://towardsdatascience.com/15-tips-and-tricks-to-use-jupyter-notebook-more-efficiently-ef05ede4e4b9)

```result = df.transpose() ```

5 functions to examine your data: ```df.head()', df.describe(), df.info(), df.shape, df.sum(), df['Trade'].value_counts() ```

Reports: ```pandas_profiling.ProfileReport(df)```

Import: ```import pandas_profiling```

Save a dataframe to a csv```df.to_csv```

Create a Pandas Dataframe```df = pd.DataFrame(data) ```

Read a csv file```pd.read_csv('')```

Read a excel file```pd.read_excel('')```

All rows that have a sepal length greater than 6 are dangerous ```df['is_dangerous'] = np.where(df['sepal length (cm)']>6, 'yes', 'no')```

Max columns option ```pd.set_option('display.max_columns', 500)```

Max row option ```pd.set_option('display.max_rows', 500)```

to see columns ```df.columns```

replace strings ```df.columns = df.columns.str.replace(' \(cm\)', '').str.replace(' ', '_')```

### plotly

[plotly Graphing Libraries](https://plot.ly/python/)

[Different Colors for Bars in Barchart by their Value](https://community.plot.ly/t/different-colors-for-bars-in-barchart-by-their-value/6527)

### Scikit-Learn

[A beginner’s guide to Linear Regression in Python with Scikit-Learn](https://towardsdatascience.com/a-beginners-guide-to-linear-regression-in-python-with-scikit-learn-83a8f7ae2b4f)

### Notes_____no_output_____

<code>
# Codewars
# https://www.codewars.com/kata/56747fd5cb988479af000028/train/python
# Get the Middle Character_____no_output_____def get_middle(s):
    if len(s) % 2 == 0:
        return s[int(len(s) / 2)-1] + s[int(len(s) / 2)]
    else:
        return s[int(len(s) / 2)]
    _____no_output_____get_middle("test")_____no_output_____
</code>


# Notebook from alexlyttle/asteroloc8
Path: notebooks/nuprior.ipynb

<code>
import numpy as np

from matplotlib import pyplot as plt
from asteroloc8.prior import NuPrior_____no_output_____tic261154649 = NuPrior(
    teff_spec=4947., teff_spec_err=92.,
    logg_spec=2.4565, logg_spec_err=0.05,
    plx=1.44, plx_err=0.01,
    jmag=8.293, jmag_err=0.03,
    hmag=7.694, hmag_err=0.023,
    kmag=7.559, kmag_err=0.027,
)_____no_output_____a = tic261154649.gaiascalnmx(mass='giants', N_samples=10000)
print('numax_50th = ', a[0][0])
print('numax_std = ', a[0][1])numax_50th =  30.28591078595059
numax_std =  2.1089309617327188
plt.hist(a[1]);_____no_output_____b = tic261154649.specnmx()
print('numax_50th = ', b[0][0])
print('numax_std = ', b[0][1])numax_50th =  34.50953960565552
numax_std =  4.1676418452268145
plt.hist(b[1]);_____no_output_____infile='DR16_APOTIC_NS_TMshort2.txt'_____no_output_____outfile = infile.replace('.txt','_out.txt')_____no_output_____stars=np.loadtxt(infile, usecols=(0,3,4,5,6,7,8,9,10,11,12,13,14), skiprows=1)_____no_output_____ids,ateff,atefferr,alogg,aloggerr, gparallax, gparallaxerr, tj, tjerr,th, therr,tk, tkerr=zip(*stars) _____no_output_____open(outfile,'w').write('## TICID  specnumax  specnumaxerr gaianumax  gaianumaxerr \n')_____no_output_____for i in range(len(ids)):
    star=NuPrior(teff_spec=ateff[i], teff_spec_err=atefferr[i], logg_spec=alogg[i], logg_spec_err=aloggerr[i], plx=gparallax[i], plx_err=gparallaxerr[i],
        jmag=tj[i], jmag_err=tjerr[i], hmag=th[i], hmag_err=therr[i], kmag=tk[i], kmag_err=tkerr[i])

    if gparallax[i] > 0:
        gaiastar=star.gaiascalnmx(mass='giants')
        gaianumaxstar=gaiastar[0][0]
        gaianumaxrangestar=gaiastar[0][1]
    else:
        gaianumaxstar=-9999
        gaianumaxrangestar=-9999

    if (ateff[i] > 0 and alogg[i] > -10):
        specstar=star.specnmx()
        specnumaxstar=specstar[0][0]
        specnumaxrangestar=specstar[0][1]
    else:
        specnumaxstar=-9999
        specnumaxrangestar=-9999

    lin='   %.0f'%ids[i]+'\t'+'   %.6f'%specnumaxstar+'\t'+'   %.6f'%specnumaxrangestar+'\t'+'   %.6f'%gaianumaxstar+'\t'+'   %.6f'%gaianumaxrangestar+'\n'
    open(outfile,'a').write(lin)     /Users/ajl573/Repos/alexlyttle/asteroloc8/asteroloc8/prior.py:217: RuntimeWarning: invalid value encountered in log10
  MK = Kmag-(5*np.log10(1000/parallax)-5)

</code>


# Notebook from maxime1310/scVI
Path: docs/examples/scVI-dev.ipynb

# Importing required packages and functions_____no_output_____

cd ../../home/jeff/git/scVI
import os

import numpy as np
import seaborn as sns
from sklearn.manifold import TSNE
import matplotlib.pyplot as plt
%matplotlib inline

from scvi.dataset import CortexDataset, RetinaDataset

from scvi.metrics.clustering import entropy_batch_mixing, get_latent
from scvi.metrics.differential_expression import de_stats
from scvi.metrics.imputation import imputation

from scvi.models import VAE, SVAEC

from scvi.inference import VariationalInference_____no_output_____

# Loading data

Here we load the CORTEX dataset described in
* Zeisel, Amit, et al. "Cell types in the mouse cortex and hippocampus revealed by single-cell RNA-seq." Science 347.6226 (2015): 1138-1142.

Please see our data loading Jupyter notebook for more examples of data loading -- scVI has many "built-in" datasets, as well as support for loading arbitrary .csv, .loom, and .h5ad (AnnData) files._____no_output_____

gene_dataset = CortexDataset()File data/expression.bin already downloaded
Preprocessing Cortex data
Finished preprocessing Cortex data

# Training_____no_output_____* __n_epochs__: Maximum number of epochs to train the model. If the likelihood change is small than a set threshold training will stop automatically.
* __lr__: learning rate. Set to 0.001 here.
* __use_batches__: If the value of true than batch information is used in the training. Here it is set to false because the cortex data only contains one batch.
* __use_cuda__: Set to true to use CUDA.
_____no_output_____

n_epochs=500
lr=1e-3
use_batches=False
use_cuda=True_____no_output_____

**Train the model and output model likelihood every 5 epochs **_____no_output_____

vae = VAE(gene_dataset.nb_genes, n_batch=gene_dataset.n_batches * use_batches)
infer = VariationalInference(vae,
                             gene_dataset,
                             train_size=0.9,
                             use_cuda=use_cuda,
                             frequency=5)
infer.train(n_epochs=n_epochs, lr=lr)training: 100%|██████████| 500/500 [01:15<00:00,  6.62it/s]

** Plotting the likelihood change across the 500 epochs of training: blue for training error and orange for testing error.**_____no_output_____

ll_train = infer.history["ll_train"]
ll_test = infer.history["ll_test"]
x = np.linspace(0,500,(len(ll_train)))
plt.plot(x, ll_train)
plt.plot(x, ll_test)
plt.ylim(1150,1600)
plt.show()_____no_output_____

# Visualizing the latent space_____no_output_____

infer.show_t_sne('sequential', n_samples=False, color_by='labels')_____no_output_____

# Imputation
Parameters:
* The rate of simulated dropout is defined by __rate__, here set ot 0.3

To check the imputation accuracy of the model, we iterate over each batch of input count matrix (n_sample\*n_genes), randomly induce dropouts by changing non-zero values in the matrix (__sample_batch__) to zero, and obtain the imputed value (__px_rate__) from the trained VAE model at those positions. We then plot the distribution of absolute errors between the imputed value and the true value at the dropout positions.

Note: The imputed value __px_rate__ is the rate parameter (expected value) of the Zero-Inflated Negative Binomial (ZINB) distribution._____no_output_____

imputation_errors = infer.imputation_errors('test', rate=0.3)_____no_output_____plt.hist(np.log10(imputation_errors))_____no_output_____

# Differential Expression
From the trained VAE model we can sample the gene expression rate for each gene in each cell. For the two populations of interest, we can then randomly sample pairs of cells, one from each population to compare their expression rate for a gene. The degree of differential expression is measured by __logit(p/(1-p))__ where __p__ is the probability of a cell from population A having a higher expression than a cell from population B. We can form the null distribution of the DE values by sampling pairs randomly from the combined population.

The following example is implemented for the cortext dataset, vary __cell_types__ and __genes_of_interest__ for other datasets. _____no_output_____**1. Set population A and population B for comparison**_____no_output_____

cell_types = np.array(['astrocytes_ependymal', 'endothelial-mural', 'interneurons', 'microglia',
                       'oligodendrocytes', 'pyramidal CA1', 'pyramidal SS'], dtype=np.str)
# oligodendrocytes (#4) VS pyramidal CA1 (#5)
couple_celltypes = (4, 5)  # the couple types on which to study DE

print("\nDifferential Expression A/B for cell types\nA: %s\nB: %s\n" %
      tuple((cell_types[couple_celltypes[i]] for i in [0, 1])))
Differential Expression A/B for cell types
A: oligodendrocytes
B: pyramidal CA1

**2. Define parameters**
* __M_sampling__: the number of times to sample __px_scales__ from the vae model for each gene in each cell.
* __M_permutation__: Number of pairs sampled from the px_scales values for comparison._____no_output_____

M_sampling = 100
M_permutation = 100000
permutation = False_____no_output_____

**3. Sample from the gene expression level from all cells**
Note: The expectation of the ZINB distribution __px_rate ~ library_size * px_scale__, so __px_scale__ could be understood as the mean gene expression level of each cell after adjusting for the library size factor. _____no_output_____

px_scale, all_labels = infer.differential_expression_stats('train')_____no_output_____

**4. Extract the sampled gene expression level for the two populations of interest, and create indexes for the samples**_____no_output_____

sample_rate_a = px_scale[all_labels.view(-1) == couple_celltypes[0]].view(-1, px_scale.size(1)).cpu().detach().numpy()
sample_rate_b = px_scale[all_labels.view(-1) == couple_celltypes[1]].view(-1, px_scale.size(1)).cpu().detach().numpy()

list_1 = list(np.arange(sample_rate_a.shape[0]))
list_2 = list(sample_rate_a.shape[0] + np.arange(sample_rate_b.shape[0]))

samples = np.vstack((sample_rate_a, sample_rate_b))_____no_output_____

**5. Compute whether a gene is differentially expressed by computing pairs of cells from population A and population B**_____no_output_____

u, v = np.random.choice(list_1, size=M_permutation), np.random.choice(list_2, size=M_permutation)
first_set = samples[u]
second_set = samples[v]
res1 = np.mean(first_set >= second_set, 0)
res1 = np.log(res1 + 1e-8) - np.log(1 - res1 + 1e-8)_____no_output_____

**6. Obtaining the null value by comparing pairs sampled from the combined population**_____no_output_____

u, v = (np.random.choice(list_1 + list_2, size=M_permutation),
    np.random.choice(list_1 + list_2, size=M_permutation))
first_set = samples[u]
second_set = samples[v]
res2 = np.mean(first_set >= second_set, 0)
res2 = np.log(res2 + 1e-8) - np.log(1 - res2 + 1e-8)_____no_output_____

**7. Print out the differential expression value from both the true comparison and the permuted comparison**_____no_output_____

genes_of_interest = ["Thy1", "Mbp"]
gene_names = infer.gene_dataset.gene_names
result = [(gene_name, res1[np.where(gene_names == gene_name.upper())[0]][0],res2[np.where(gene_names == gene_name.upper())[0]][0]) for gene_name in genes_of_interest]
print('\n'.join([gene_name + " : " + str(r1) + " , "+ str(r2) for (gene_name, r1,r2) in result]))Thy1 : -5.057694442863545 , 0.004000005253345873
Mbp : 5.133726741796103 , -0.0020000006266669423

**8. Plot the null distribution of the DE values**_____no_output_____

plt.hist(res2)_____no_output_____

# Correction for batch effects

First we load the RETINA dataset that is described in
* Shekhar, Karthik, et al. "Comprehensive classification of retinal bipolar neurons by single-cell transcriptomics." Cell 166.5 (2016): 1308-1323._____no_output_____

gene_dataset = RetinaDataset()File data/retina.loom already downloaded
Preprocessing dataset
Finished preprocessing dataset
n_epochs=100
lr=1e-3
use_batches=True
use_cuda=True

### Train the model and output model likelihood every 5 epochs
vae = VAE(gene_dataset.nb_genes, n_batch=gene_dataset.n_batches * use_batches)
infer = VariationalInference(vae,
                             gene_dataset,
                             train_size=0.9,
                             use_cuda=use_cuda,
                             frequency=5)
infer.train(n_epochs=n_epochs, lr=lr)training: 100%|██████████| 100/100 [13:27<00:00,  8.07s/it]
# Plotting the likelihood change across the 500 epochs of training: blue for training error and orange for testing error.

ll_train = infer.history["ll_train"]
ll_test = infer.history["ll_test"]
x = np.linspace(0,500,(len(ll_train)))
plt.plot(x, ll_train)
plt.plot(x, ll_test)
plt.ylim(min(ll_train)-50, 3500)
plt.show()_____no_output_____

** Computing batch mixing **_____no_output_____

print("Entropy batch mixing :", infer.entropy_batch_mixing('sequential'))Entropy batch mixing : 0.6587664259373629

** Coloring by batch and cell type **_____no_output_____

# obtaining latent space in the same order as the input data

infer.show_t_sne('sequential',n_samples=1000, color_by='batches and labels')_____no_output_____


# Notebook from sul-dlss-labs/ai-etd
Path: 07_bio_prep.ipynb

<code>
import datetime
import pickle
import re

import lxml.etree as etree
import requests

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from nltk.corpus import stopwords
from sentence_transformers import SentenceTransformer
from sklearn.cluster import KMeans
sbert_model = SentenceTransformer('bert-base-nli-mean-tokens')
stop_words_list = stopwords.words('english')
ns = {"mods": "http://www.loc.gov/mods/v3"}_____no_output_____def get_mods(druid: str):
    purl_url = f"https://purl.stanford.edu/{druid}.mods"
    try:
        purl_result = requests.get(purl_url)
    except:
        print(f"Request error for {druid} with {purl_url}")
        return ''
    try:
        purl_xml = etree.XML(purl_result.text.encode())
    except etree.XMLSyntaxError:
        print(f"\nXML Syntax Error for {druid}")
        return ''
    return purl_xml

def get_abstract(purl_xml):
    abstract = purl_xml.find("mods:abstract", namespaces=ns)
    if abstract is not None:
        return abstract.text

def get_department(purl_xml):
    corporate_names = purl_xml.findall("mods:name[@type='corporate']/mods:namePart", namespaces=ns)
    department = []
    for name in corporate_names:
        if name.text.startswith("Stanford University"):
            continue
        department.append(name.text)
    return ','.join(department)

special_char_re = re.compile(r'[^a-zA-Z]')

def cleanup(term: str) -> str:
    cleaned = []
    for char in term.split():
        cleaned_char = special_char_re.sub(' ', char).lower()
        if cleaned_char in stop_words_list:
            continue
        cleaned.append(cleaned_char)
    return ' '.join(cleaned)_____no_output_____biology_df = pd.read_pickle("data/biology.pkl")_____no_output_____biology_df.head()_____no_output_____biology_df['abstracts'].isnull().sum()_____no_output_____missing_info = []
for i,row in enumerate(biology_df['abstracts'].isnull()):
    if row:
        missing_info.append(biology_df.iloc[i])_____no_output_____len(missing_info)_____no_output_____missing_info[0]_____no_output_____xml = get_mods('bz168ds1940')_____no_output_____get_department(xml)_____no_output_____get_abstract(xml)_____no_output_____missing_info[0]['abstracts'] = get_abstract(xml)_____no_output_____missing_info[0]['druids']_____no_output_____def populate_missing(missing_info):
    start = datetime.datetime.utcnow()
    print(f"Starting at {start} for {len(missing_info)}")
    for i,row in enumerate(missing_info):
        druid = row['druids']
        xml = get_mods(druid)
        if len(xml) < 1:
            print(f"XML is empty for {druid}, index {i}")
            continue
        row['departments'] = get_department(xml)
        abstract = get_abstract(xml)
        if abstract is None:
            print(f"No abstract for {druid}, index {i}")
            abstract = ''
        row['abstracts'] = abstract
        row['abstracts_cleaned'] = cleanup(abstract)

    end = datetime.datetime.utcnow()
    print(f"Finished at {end} total time {(end-start).seconds / 60.} mins")_____no_output_____populate_missing()Starting at 2020-10-21 20:42:47.660303 for 247
No abstract for dg421jb1685, index 15
Finished at 2020-10-21 20:44:04.694893 total time 1.2833333333333334 mins
missing_info[156]['druids'] = 'yq505mb7923'_____no_output_____print(missing_info[15])druids                         dg421jb1685
abstracts
abstracts_cleaned
departments          Department of Biology
Name: 216, dtype: object
missing_info[15]['departments'] = 'Department of Biology'_____no_output_____biology_df.tail()_____no_output_____biology_df.to_pickle("./data/biology.pkl")_____no_output_____biology_embeddings = sbert_model.encode(biology_df['abstracts_cleaned'])_____no_output_____wcss = []
for i in range(1,50):
    kmeans = KMeans(n_clusters=i, init = 'k-means++', random_state=42)
    kmeans.fit(biology_embeddings)
    wcss.append(kmeans.inertia_)_____no_output_____plt.figure(figsize=(10,8))
plt.plot(range(1,50), wcss, marker = 'o', linestyle='--')
plt.xlabel('Number of clusters')
plt.ylabel('WCSS')
plt.title('K-means Clustering')
plt.show()_____no_output_____clustering_model = KMeans(n_clusters=18)
clustering_model.fit(biology_embeddings)
cluster_assignment = clustering_model.labels______no_output_____clustered_sentences = [[] for i in range(18)]
for abstract_id, cluster_id in enumerate(cluster_assignment):
    clustered_sentences[cluster_id].append({ 'abstract': biology_df.iloc[abstract_id]['abstracts'],
                                             'druid': biology_df.iloc[abstract_id]['druids'] })_____no_output_____for i,row in enumerate(clustered_sentences):
    print(f"Cluster {i} size {len(row)}")Cluster 0 size 48
Cluster 1 size 83
Cluster 2 size 94
Cluster 3 size 84
Cluster 4 size 63
Cluster 5 size 80
Cluster 6 size 41
Cluster 7 size 43
Cluster 8 size 84
Cluster 9 size 3
Cluster 10 size 39
Cluster 11 size 55
Cluster 12 size 55
Cluster 13 size 91
Cluster 14 size 84
Cluster 15 size 81
Cluster 16 size 52
Cluster 17 size 63
print(clustered_sentences[9][0])
print(f"Last abstract")
print(clustered_sentences[9][-1]){'abstract': '', 'druid': 'dg421jb1685'}
Last abstract
{'abstract': 'Abstract to be provided.', 'druid': 'bz491tn4091'}
with open('./data/biology-bert-embedding.pkl', 'wb+') as fo:
    pickle.dump(biology_embeddings, fo)_____no_output_____import pandas as pd
bio_df = pd.read_pickle("../biology-etds/data/biology.pkl")_____no_output_____len(bio_df)_____no_output_____
</code>


# Notebook from katarinagresova/genomic_ML_playground
Path: experiments/subwords_tokenization/cDNA_subwords_vocab2048.ipynb

If you would like more explanation to Data preparation part, please go to cDNA_subwords_vocab64 notebook._____no_output_____# Setup_____no_output_____

!pip show fastaiName: fastai
Version: 2.2.7
Summary: fastai simplifies training fast and accurate neural nets using modern best practices
Home-page: https://github.com/fastai/fastai/tree/master/
Author: Jeremy Howard, Sylvain Gugger, and contributors
Author-email: info@fast.ai
License: Apache Software License 2.0
Location: /opt/conda/lib/python3.7/site-packages
Requires: pandas, pillow, scipy, fastcore, matplotlib, torchvision, spacy, pip, requests, scikit-learn, fastprogress, torch, pyyaml, packaging
Required-by:
!pip show biopythonName: biopython
Version: 1.78
Summary: Freely available tools for computational molecular biology.
Home-page: https://biopython.org/
Author: The Biopython Contributors
Author-email: biopython@biopython.org
License: UNKNOWN
Location: /opt/conda/lib/python3.7/site-packages
Requires: numpy
Required-by: bio
from Bio import SeqIO
from fastai.text.all import *_____no_output_____

# Data preparation_____no_output_____

!wget -nc http://ftp.ensembl.org/pub/release-103/fasta/homo_sapiens/cdna/Homo_sapiens.GRCh38.cdna.abinitio.fa.gz
!yes n | gunzip Homo_sapiens.GRCh38.cdna.abinitio.fa.gzFile ‘Homo_sapiens.GRCh38.cdna.abinitio.fa.gz’ already there; not retrieving.

gzip: Homo_sapiens.GRCh38.cdna.abinitio.fa already exists;	not overwritten
yes: standard output: Broken pipe

# Token preparation_____no_output_____

with open("Homo_sapiens.GRCh38.cdna.abinitio.fa", "rt") as handle:
  txts = L(str(record.seq).lower() for record in SeqIO.parse(handle, "fasta"))_____no_output_____txt = txts[0]
txts = txts[1:10001]_____no_output_____SPECIAL_TOKENS = 2
ALPHABET = 4
VOCAB_SIZE = 2048 + SPECIAL_TOKENS + ALPHABET
tokenizer = SubwordTokenizer(vocab_sz=VOCAB_SIZE, special_toks=[], cache_dir='tmp/vocab2048', lang='dna')
tokenizer.setup(txts)_____no_output_____toks = first(tokenizer([txt]))
print(coll_repr(toks, 30))(#210) ['▁atgga','aagaggaa','agaagaa','aagaa','tttcc','aataa','gttac','aacaa','acttt','tcacc','attct','aaagaa','ccca','cttt','cctt','atcaac','caagctg','ggcttc','tctct','agtga','ctcct','attct','agcct','ttccc','cagaaa','cagaga','gtgt','taa','tcctgg','tga'...]
txt[:100]_____no_output_____tkn = Tokenizer(tokenizer, rules=[], sep='')
print(coll_repr(tkn(txt), 30))(#210) ['▁atgga','aagaggaa','agaagaa','aagaa','tttcc','aataa','gttac','aacaa','acttt','tcacc','attct','aaagaa','ccca','cttt','cctt','atcaac','caagctg','ggcttc','tctct','agtga','ctcct','attct','agcct','ttccc','cagaaa','cagaga','gtgt','taa','tcctgg','tga'...]
toks_all = txts.map(tkn)_____no_output_____

# Tokens analysis_____no_output_____

from operator import add

tokens = reduce(add, toks_all)_____no_output_____

## Top 10 most common tokens_____no_output_____

import collections

elements_count = collections.Counter(tokens)
print(elements_count.most_common(10))[('tga', 16335), ('tag', 9722), ('taa', 9384), ('ctga', 8428), ('atga', 7842), ('ag', 7732), ('ttga', 6836), ('tgtga', 6109), ('cgg', 5413), ('cccag', 5359)]

Eight most common tokens correnspond to stop codons._____no_output_____## Distribution of occurences_____no_output_____

counts = []
for count in elements_count:
    counts.append(elements_count[count])_____no_output_____from matplotlib import pyplot

pyplot.figure(figsize=(10,4))
pyplot.hist(counts, bins=range(min(counts), max(counts) + 500, 500))
pyplot.xlabel("occurence count")
pyplot.ylabel("distinct subword count")
pyplot.show()_____no_output_____

## Distribution of lengths of subwords_____no_output_____

lengths = []
vocab = set(tokens)
for token in vocab:
    lengths.append(len(token))_____no_output_____from matplotlib import pyplot

pyplot.figure(figsize=(4,6))
pyplot.hist(lengths, bins=range(min(lengths), max(lengths) + 1, 1))
pyplot.xlabel("length")
pyplot.ylabel("count")
pyplot.show()_____no_output_____

## Longest words in vocabulary_____no_output_____

vocab = list(vocab)
vocab.sort(key=len)_____no_output_____print(vocab[-1])
print(vocab[-2])
print(vocab[-3])gaagaagaagaagaag
aatggaatcgaatgga
aatggaatggaatgga

Longest subwords are repeats. And two of them differ only in one letter._____no_output_____## Least freaquent subwords in training data_____no_output_____

print(elements_count.most_common()[-1])
print(elements_count.most_common()[-2])
print(elements_count.most_common()[-3])
print(elements_count.most_common()[-4])('t', 86)
('c', 116)
('a', 142)
('g', 148)

All four least frequent subwords are all four bases. I think it could mean, that vocabulary has the right size and subwords are really representing all input sequences. And there is no need to tokenize them by individual bases._____no_output_____