# Notebook from utshabkg/Machine-Learning-Projects
Path: End to End ML Project/.ipynb_checkpoints/A Full ML Project with useful features-checkpoint.ipynb

# Author: Kazi Amit Hasan

Department of Computer Science & Engineering,

Rajshahi University of Engineering & Technology (RUET)

Website: https://amithasanshuvo.github.io/

Linkedin: https://www.linkedin.com/in/kazi-amit-hasan-514443140/

Email: kaziamithasan89@gmail.com

### Comment:
This notebook represents the chapter two(2) of Hands-On Machine Learning with Scikit-Learn, Keras, and TensorFlow: Concepts, Tools, and Techniques to Build Intelligent Systems book. I tried to implemented each examples while I was reading and practing this book.

Please give your feedback how I can improve the notebook and plase put a star on github to support. Happy Learning!

##### Reference
Hands-On Machine Learning with Scikit-Learn, Keras, and TensorFlow: Concepts, Tools, and Techniques to Build Intelligent Systems book (The best book I ever got on Data Sciene <3)

Language: Python

##### Topics Covered:
1. Downloading and playing with the dataset
2. Data Visualization
3. Correaltion Analysis
4. Data Cleaning and Handling Missing Values
5. Handling text and Categorical attributes

##### Todo list:
1. Feature Scaling
2. Pipelines
3. Train and Evaluating
4. Cross Validation
5. Tuning
6. Grid Search
7. Randomized Search
8. Ensemble method
9. Do the exercises_____no_output_____## Downloading and playing with the dataset_____no_output_____

#Getting the dataset
import os
import tarfile
import urllib

DOWNLOAD_ROOT = "https://raw.githubusercontent.com/ageron/handson-ml2/master/"
HOUSING_PATH = os.path.join("datasets", "housing")
HOUSING_URL = DOWNLOAD_ROOT + "datasets/housing/housing.tgz"

def fetch_housing_data(housing_url=HOUSING_URL, housing_path=HOUSING_PATH):
    if not os.path.isdir(housing_path):
        os.makedirs(housing_path)
    tgz_path = os.path.join(housing_path, "housing.tgz")
    urllib.request.urlretrieve(housing_url, tgz_path)
    housing_tgz = tarfile.open(tgz_path)
    housing_tgz.extractall(path=housing_path)
    housing_tgz.close()_____no_output_____fetch_housing_data()_____no_output_____
import pandas as pd

def load_housing_data(housing_path=HOUSING_PATH):
    csv_path = os.path.join(housing_path, "housing.csv")
    return pd.read_csv(csv_path)_____no_output_____#Showing the first 5 rows
housing = load_housing_data()
housing.head()_____no_output_____#info of dataset
housing.info_____no_output_____#Value counts of ocean_proximity column
housing['ocean_proximity'].value_counts()_____no_output_____#shape of our data
housing.shape_____no_output_____#Summary of each numerical attributes
housing.describe()_____no_output_____#Showwing the correlations
housing.corr()
_____no_output_____#Showing the columns
housing.columns_____no_output_____#Plotting histograms for each numerical attributes
%matplotlib inline

import matplotlib.pyplot as plt
housing.hist(bins =50, figsize=(20,15))

plt.show()

#Slighly over 1000 distrcts have a median_house_value about 500000 usd_____no_output_____housing.hist(column='population')
_____no_output_____# random_state parameter always generate the same shuffle indices. If the dataset
# is not big enough then there's a chance of sampling bias
from sklearn.model_selection import train_test_split
train_set, test_set = train_test_split(housing, test_size = 0.2, random_state=42)

_____no_output_____

## Data Visualization_____no_output_____

# Geographical data of all districts
housing.plot (kind ='scatter', x='longitude', y= 'latitude')_____no_output_____housing.plot (kind ='scatter', x='longitude', y= 'latitude', alpha = 0.1)
# adding alpha for better visualization. This helps to visualize the high density data points_____no_output_____housing.plot(kind="scatter", x="longitude", y="latitude", alpha=0.4,
    s=housing["population"]/100, label="population", figsize=(10,10),
    c="median_house_value", cmap=plt.get_cmap("jet"), colorbar=True,
    sharex=False)
plt.legend()

# This represents that housing prices are varies with locations and population density
_____no_output_____

## Correaltion Analysis_____no_output_____

corr_matrix = housing.corr()_____no_output_____print (corr_matrix)                    longitude  latitude  housing_median_age  total_rooms  \
longitude            1.000000 -0.924664           -0.108197     0.044568
latitude            -0.924664  1.000000            0.011173    -0.036100
housing_median_age  -0.108197  0.011173            1.000000    -0.361262
total_rooms          0.044568 -0.036100           -0.361262     1.000000
total_bedrooms       0.069608 -0.066983           -0.320451     0.930380
population           0.099773 -0.108785           -0.296244     0.857126
households           0.055310 -0.071035           -0.302916     0.918484
median_income       -0.015176 -0.079809           -0.119034     0.198050
median_house_value  -0.045967 -0.144160            0.105623     0.134153

                    total_bedrooms  population  households  median_income  \
longitude                 0.069608    0.099773    0.055310      -0.015176
latitude                 -0.066983   -0.108785   -0.071035      -0.079809
housing_median_age       -0.320451   -0.296244   -0.302916      -0.119034
total_rooms               0.930380    0.857126    0.918484       0.198050
total_bedrooms            1.000000    0.877747    0.979728      -0.007723
population                0.877747    1.000000    0.907222       0.004834
households                0.979728    0.907222    1.000000       0.013033
median_income            -0.007723    0.004834    0.013033       1.000000
median_house_value        0.049686   -0.024650    0.065843       0.688075

                    median_house_value
longitude                    -0.045967
latitude                     -0.144160
housing_median_age            0.105623
total_rooms                   0.134153
total_bedrooms                0.049686
population                   -0.024650
households                    0.065843
median_income                 0.688075
median_house_value            1.000000
# Correlation of median_house_value with other attributes
corr_matrix['median_house_value'].sort_values(ascending = False)_____no_output_____# another way to check correlation using pandas

from pandas.plotting import scatter_matrix
attributes = ['median_house_value', 'median_income','total_rooms','housing_median_age']
scatter_matrix(housing[attributes], figsize = (12,8))_____no_output_____housing.plot (kind='scatter', x="median_income",y='median_house_value',alpha = 0.1)

# It means corr is very strong. _____no_output_____# Creating new attributes

housing['room_per-_household'] = housing['total_rooms']/housing['households']
housing['bedrooms_per_rooms'] = housing['total_bedrooms']/housing['total_rooms']
housing['population_per_household'] = housing['population']/housing['households']_____no_output_____# Now the corr matrix will look something like this
corr_matrix = housing.corr()
corr_matrix['median_house_value'].sort_values(ascending = False)

# This means bedrooms_per_room attributee is much more correlated with mediamn house value than total num of rooms
# Lower bedrooms has high price_____no_output_____housing.plot(kind="scatter",
             x="room_per-_household",
             y="median_house_value",
             alpha=0.2)
plt.axis([0, 5, 0, 520000])

_____no_output_____

## Data Cleaning and Handling Missing Values_____no_output_____

# Data Cleaning and handling missing values

#housing.dropna(subset=['total_bedrooms'])

# we can use this. but sklearn also provide e good function_____no_output_____from sklearn.impute import SimpleImputer
imputer = SimpleImputer(strategy ='median')

# this help to take care of missing values.
# medians only be computed on numerical values ,so we need a copy of data without text attributes i.e ocean_proximity_____no_output_____housing_num = housing.drop('ocean_proximity', axis=1)_____no_output_____imputer.fit(housing_num)
# fitting the imputer instances to training data. It only computed the median of each attributes._____no_output_____housing_num.median().values
_____no_output_____X = imputer.transform(housing_num)

# Transforming the train set by replacing the missing values with new medians_____no_output_____imputer.strategy
_____no_output_____housing_tr = pd.DataFrame(X, columns=housing_num.columns, index=housing.index)_____no_output_____housing_tr.loc[housing_num.index.values]
# Putting back into Pandas df_____no_output_____housing_tr = pd.DataFrame(X, columns=housing_num.columns,
                          index=housing_num.index)_____no_output_____housing_tr.head()
_____no_output_____

## Handling text and Categorical attributes_____no_output_____

housing_cat = housing[["ocean_proximity"]]
housing_cat.head(10)_____no_output_____# Earlier we left the text attribute (ocean_proximity), now we have to work on that

# Converting them into labels

from sklearn.preprocessing import OrdinalEncoder
ordinal_encoder = OrdinalEncoder()
housing_cat_encoded = ordinal_encoder.fit_transform(housing_cat)
housing_cat_encoded[:10]_____no_output_____ordinal_encoder.categories_
_____no_output_____# Convert int to categorical values into onehot vectors
from sklearn.preprocessing import OneHotEncoder

cat_encoder = OneHotEncoder()
housing_cat_1hot = cat_encoder.fit_transform(housing_cat)
housing_cat_1hot_____no_output_____housing_cat_1hot.toarray()
_____no_output_____


# Notebook from sigvehaug/MLwPython
Path: Course_4.ipynb

# Data Science Fundamentals 5

Basic introduction on how to perform typical machine learning tasks with Python.

Prepared by Mykhailo Vladymyrov & Aris Marcolongo,
Science IT Support, University Of Bern, 2020

This work is licensed under CC0.

# Part 4._____no_output_____

from matplotlib import  pyplot as plt
import numpy as np
from imageio import imread
import pandas as pd
from time import time as timer

import tensorflow as tf

%matplotlib inline
from matplotlib import animation
from IPython.display import HTML_____no_output_____

# 1. Classification of the F-MNIST dataset with neural network_____no_output_____## 0. Load data_____no_output_____We will create a model for classification of the F-MNIST dataset that we go acquainted with in previous sessions. We will normalize the inputs to have values $\in[0,1]$:_____no_output_____

fashion_mnist = tf.keras.datasets.fashion_mnist
(x_train, y_train), (x_test, y_test) = fashion_mnist.load_data()
x_train = x_train/255
x_test = x_test/255

class_names = ['T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat',
               'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle boot']Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/train-labels-idx1-ubyte.gz
32768/29515 [=================================] - 0s 0us/step
Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/train-images-idx3-ubyte.gz
26427392/26421880 [==============================] - 0s 0us/step
Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/t10k-labels-idx1-ubyte.gz
8192/5148 [===============================================] - 0s 0us/step
Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/t10k-images-idx3-ubyte.gz
4423680/4422102 [==============================] - 0s 0us/step

## 1. Building a neural network_____no_output_____

print(x_train[0].shape)(28, 28)

The size of each image sample $-\; 28\times28\text{ pixels}\;-\;$ defines the input size for our neural network. Network's output - probabilities of belonging to each of the 10 classes._____no_output_____The following creates a 'model'. It is an object containing the ML model itself - a simple 3-layer fully connected neural network, optimization parameters, as well as the interface for model training._____no_output_____

model = tf.keras.models.Sequential([
  tf.keras.layers.Flatten(input_shape=(28, 28)),
  tf.keras.layers.Dense(10, activation='softmax')
])

model.compile(optimizer='adam',
              loss='sparse_categorical_crossentropy',
              metrics=['accuracy'])_____no_output_____

Model summary provides information about the model's layers and trainable parameters_____no_output_____

model.summary()Model: "sequential"
_________________________________________________________________
Layer (type)                 Output Shape              Param #
=================================================================
flatten (Flatten)            (None, 784)               0
_________________________________________________________________
dense (Dense)                (None, 10)                7850
=================================================================
Total params: 7,850
Trainable params: 7,850
Non-trainable params: 0
_________________________________________________________________

## 2. Model training_____no_output_____The `fit` function is the interface for model training.
Here one can specify training and validation datasets, minibatch size, and the number of training epochs.

Here during training we also save the trained models checkpoints after each epoch of training._____no_output_____

save_path = 'save/mnist_{epoch}.ckpt'
save_callback = tf.keras.callbacks.ModelCheckpoint(filepath=save_path, save_weights_only=True)

hist = model.fit(x=x_train, y=y_train,
                 epochs=50, batch_size=128,
                 validation_data=(x_test, y_test),
                 callbacks=[save_callback])Epoch 1/50
469/469 [==============================] - 2s 4ms/step - loss: 0.7509 - accuracy: 0.7534 - val_loss: 0.5770 - val_accuracy: 0.8064
Epoch 2/50
469/469 [==============================] - 2s 4ms/step - loss: 0.5161 - accuracy: 0.8283 - val_loss: 0.5155 - val_accuracy: 0.8245
Epoch 3/50
469/469 [==============================] - 2s 4ms/step - loss: 0.4758 - accuracy: 0.8393 - val_loss: 0.4922 - val_accuracy: 0.8305
Epoch 4/50
469/469 [==============================] - 2s 4ms/step - loss: 0.4515 - accuracy: 0.8484 - val_loss: 0.4780 - val_accuracy: 0.8364
Epoch 5/50
469/469 [==============================] - 2s 4ms/step - loss: 0.4378 - accuracy: 0.8512 - val_loss: 0.4704 - val_accuracy: 0.8375
Epoch 6/50
469/469 [==============================] - 2s 4ms/step - loss: 0.4279 - accuracy: 0.8539 - val_loss: 0.4603 - val_accuracy: 0.8400
Epoch 7/50
469/469 [==============================] - 2s 4ms/step - loss: 0.4206 - accuracy: 0.8559 - val_loss: 0.4580 - val_accuracy: 0.8394
Epoch 8/50
469/469 [==============================] - 2s 4ms/step - loss: 0.4138 - accuracy: 0.8584 - val_loss: 0.4591 - val_accuracy: 0.8375
Epoch 9/50
469/469 [==============================] - 2s 4ms/step - loss: 0.4102 - accuracy: 0.8589 - val_loss: 0.4512 - val_accuracy: 0.8416
Epoch 10/50
469/469 [==============================] - 2s 4ms/step - loss: 0.4056 - accuracy: 0.8616 - val_loss: 0.4472 - val_accuracy: 0.8416
Epoch 11/50
469/469 [==============================] - 2s 4ms/step - loss: 0.4022 - accuracy: 0.8620 - val_loss: 0.4486 - val_accuracy: 0.8429
Epoch 12/50
469/469 [==============================] - 2s 4ms/step - loss: 0.3990 - accuracy: 0.8625 - val_loss: 0.4475 - val_accuracy: 0.8432
Epoch 13/50
469/469 [==============================] - 2s 4ms/step - loss: 0.3965 - accuracy: 0.8634 - val_loss: 0.4612 - val_accuracy: 0.8398
Epoch 14/50
469/469 [==============================] - 2s 4ms/step - loss: 0.3937 - accuracy: 0.8632 - val_loss: 0.4401 - val_accuracy: 0.8464
Epoch 15/50
469/469 [==============================] - 2s 4ms/step - loss: 0.3906 - accuracy: 0.8657 - val_loss: 0.4436 - val_accuracy: 0.8450
Epoch 16/50
469/469 [==============================] - 2s 4ms/step - loss: 0.3899 - accuracy: 0.8652 - val_loss: 0.4538 - val_accuracy: 0.8412
Epoch 17/50
469/469 [==============================] - 2s 4ms/step - loss: 0.3886 - accuracy: 0.8656 - val_loss: 0.4463 - val_accuracy: 0.8437
Epoch 18/50
469/469 [==============================] - 2s 4ms/step - loss: 0.3867 - accuracy: 0.8662 - val_loss: 0.4440 - val_accuracy: 0.8450
Epoch 19/50
469/469 [==============================] - 2s 4ms/step - loss: 0.3843 - accuracy: 0.8664 - val_loss: 0.4431 - val_accuracy: 0.8435
Epoch 20/50
469/469 [==============================] - 2s 4ms/step - loss: 0.3831 - accuracy: 0.8670 - val_loss: 0.4398 - val_accuracy: 0.8466
Epoch 21/50
469/469 [==============================] - 2s 4ms/step - loss: 0.3825 - accuracy: 0.8680 - val_loss: 0.4391 - val_accuracy: 0.8465
Epoch 22/50
469/469 [==============================] - 2s 4ms/step - loss: 0.3803 - accuracy: 0.8676 - val_loss: 0.4423 - val_accuracy: 0.8451
Epoch 23/50
469/469 [==============================] - 2s 4ms/step - loss: 0.3793 - accuracy: 0.8677 - val_loss: 0.4460 - val_accuracy: 0.8457
Epoch 24/50
469/469 [==============================] - 2s 4ms/step - loss: 0.3794 - accuracy: 0.8685 - val_loss: 0.4394 - val_accuracy: 0.8474
Epoch 25/50
469/469 [==============================] - 2s 4ms/step - loss: 0.3788 - accuracy: 0.8682 - val_loss: 0.4373 - val_accuracy: 0.8478
Epoch 26/50
469/469 [==============================] - 2s 4ms/step - loss: 0.3755 - accuracy: 0.8687 - val_loss: 0.4408 - val_accuracy: 0.8452
Epoch 27/50
469/469 [==============================] - 2s 4ms/step - loss: 0.3762 - accuracy: 0.8694 - val_loss: 0.4381 - val_accuracy: 0.8474
Epoch 28/50
469/469 [==============================] - 2s 4ms/step - loss: 0.3751 - accuracy: 0.8693 - val_loss: 0.4374 - val_accuracy: 0.8465
Epoch 29/50
469/469 [==============================] - 2s 4ms/step - loss: 0.3741 - accuracy: 0.8705 - val_loss: 0.4449 - val_accuracy: 0.8429
Epoch 30/50
469/469 [==============================] - 2s 4ms/step - loss: 0.3727 - accuracy: 0.8699 - val_loss: 0.4453 - val_accuracy: 0.8440
Epoch 31/50
469/469 [==============================] - 2s 5ms/step - loss: 0.3718 - accuracy: 0.8702 - val_loss: 0.4396 - val_accuracy: 0.8451
Epoch 32/50
469/469 [==============================] - 2s 4ms/step - loss: 0.3728 - accuracy: 0.8699 - val_loss: 0.4425 - val_accuracy: 0.8451
Epoch 33/50
469/469 [==============================] - 2s 4ms/step - loss: 0.3715 - accuracy: 0.8706 - val_loss: 0.4363 - val_accuracy: 0.8475
Epoch 34/50
469/469 [==============================] - 2s 4ms/step - loss: 0.3707 - accuracy: 0.8711 - val_loss: 0.4382 - val_accuracy: 0.8471
Epoch 35/50
469/469 [==============================] - 2s 4ms/step - loss: 0.3692 - accuracy: 0.8707 - val_loss: 0.4433 - val_accuracy: 0.8430
Epoch 36/50
469/469 [==============================] - 2s 4ms/step - loss: 0.3693 - accuracy: 0.8699 - val_loss: 0.4410 - val_accuracy: 0.8445
Epoch 37/50
469/469 [==============================] - 2s 4ms/step - loss: 0.3679 - accuracy: 0.8710 - val_loss: 0.4446 - val_accuracy: 0.8426
Epoch 38/50
469/469 [==============================] - 2s 4ms/step - loss: 0.3691 - accuracy: 0.8716 - val_loss: 0.4379 - val_accuracy: 0.8471
Epoch 39/50
469/469 [==============================] - 2s 4ms/step - loss: 0.3669 - accuracy: 0.8711 - val_loss: 0.4419 - val_accuracy: 0.8467
Epoch 40/50
469/469 [==============================] - 2s 4ms/step - loss: 0.3669 - accuracy: 0.8719 - val_loss: 0.4388 - val_accuracy: 0.8454
Epoch 41/50
469/469 [==============================] - 2s 4ms/step - loss: 0.3669 - accuracy: 0.8711 - val_loss: 0.4432 - val_accuracy: 0.8460
Epoch 42/50
469/469 [==============================] - 2s 4ms/step - loss: 0.3661 - accuracy: 0.8713 - val_loss: 0.4393 - val_accuracy: 0.8456
Epoch 43/50
469/469 [==============================] - 2s 4ms/step - loss: 0.3665 - accuracy: 0.8715 - val_loss: 0.4388 - val_accuracy: 0.8458
Epoch 44/50
469/469 [==============================] - 2s 4ms/step - loss: 0.3649 - accuracy: 0.8723 - val_loss: 0.4404 - val_accuracy: 0.8453
Epoch 45/50
469/469 [==============================] - 2s 4ms/step - loss: 0.3652 - accuracy: 0.8724 - val_loss: 0.4406 - val_accuracy: 0.8467
Epoch 46/50
469/469 [==============================] - 2s 4ms/step - loss: 0.3638 - accuracy: 0.8716 - val_loss: 0.4376 - val_accuracy: 0.8471
Epoch 47/50
469/469 [==============================] - 2s 4ms/step - loss: 0.3638 - accuracy: 0.8722 - val_loss: 0.4389 - val_accuracy: 0.8460
Epoch 48/50
469/469 [==============================] - 2s 4ms/step - loss: 0.3643 - accuracy: 0.8726 - val_loss: 0.4380 - val_accuracy: 0.8467
Epoch 49/50
469/469 [==============================] - 2s 4ms/step - loss: 0.3622 - accuracy: 0.8727 - val_loss: 0.4375 - val_accuracy: 0.8475
Epoch 50/50
469/469 [==============================] - 2s 4ms/step - loss: 0.3626 - accuracy: 0.8722 - val_loss: 0.4433 - val_accuracy: 0.8444
fig, axs = plt.subplots(1, 2, figsize=(10,5))
axs[0].plot(hist.epoch, hist.history['loss'])
axs[0].plot(hist.epoch, hist.history['val_loss'])
axs[0].legend(('training loss', 'validation loss'), loc='lower right')
axs[1].plot(hist.epoch, hist.history['accuracy'])
axs[1].plot(hist.epoch, hist.history['val_accuracy'])

axs[1].legend(('training accuracy', 'validation accuracy'), loc='lower right')
plt.show()_____no_output_____

Current model performance can be evaluated on a dataset:_____no_output_____

model.evaluate(x_test,  y_test, verbose=2)313/313 - 1s - loss: 0.4433 - accuracy: 0.8444

We can test trained model on an image:_____no_output_____

im_id = 0
y_pred = model(x_test)

y_pred_most_probable = np.argmax(y_pred[im_id])
print('true lablel: ', y_test[im_id],
      '; predicted: ',  y_pred_most_probable,
      f'({class_names[y_pred_most_probable]})')
plt.imshow(x_test[im_id], cmap='gray');true lablel:  9 ; predicted:  9 (Ankle boot)

As well as inspect on which samples does the model fail:_____no_output_____

y_pred_most_probable_all = np.argmax(y_pred, axis=1)
wrong_pred_map = y_pred_most_probable_all!=y_test
wrong_pred_idx = np.arange(len(wrong_pred_map))[wrong_pred_map]

im_id = wrong_pred_idx[0]

y_pred_most_probable = y_pred_most_probable_all[im_id]
print('true lablel: ', y_test[im_id],
      f'({class_names[y_test[im_id]]})',
      '; predicted: ',  y_pred_most_probable,
      f'({class_names[y_pred_most_probable]})')
plt.imshow(x_test[im_id], cmap='gray');true lablel:  7 (Sneaker) ; predicted:  5 (Sandal)

## 3. Loading trained model_____no_output_____

model.load_weights('save/mnist_1.ckpt')
model.evaluate(x_test,  y_test, verbose=2)

model.load_weights('save/mnist_12.ckpt')
model.evaluate(x_test,  y_test, verbose=2)

model.load_weights('save/mnist_18.ckpt')
model.evaluate(x_test,  y_test, verbose=2)313/313 - 1s - loss: 0.5770 - accuracy: 0.8064
313/313 - 1s - loss: 0.4475 - accuracy: 0.8432
313/313 - 1s - loss: 0.4440 - accuracy: 0.8450

## 4. Inspecting trained variables_____no_output_____We can obtain the trained variables from model layers:_____no_output_____

l = model.get_layer(index=1)
w, b = l.weights

w = w.numpy()
b = b.numpy()
print(w.shape, b.shape)
w = w.reshape((28,28,-1)).transpose((2, 0, 1))(784, 10) (10,)

Let's visualize first 5:_____no_output_____

n = 10
fig, axs = plt.subplots(1, n, figsize=(4.1*n,4))
for i, wi in enumerate(w[:n]):
  axs[i].imshow(wi, cmap='gray')
  axs[i].set_title(class_names[i])_____no_output_____

## 6. Inspecting gradients_____no_output_____We can also evaluate the gradients of each output with respect to an input:_____no_output_____

idx = 112
inp_v = x_train[idx:idx+1]  # use some image to compute gradients with respect to

inp = tf.constant(inp_v)  # create tf constant tensor
with tf.GradientTape() as tape:  # gradient tape for gradint evaluation
  tape.watch(inp)  # take inp as variable
  preds = model(inp) # evaluate model output

grads = tape.jacobian(preds, inp)  # evaluate d preds[i] / d inp[j]
print(grads.shape, '<- (Batch_preds, preds[i], Batch_inp, inp[y], inp[x])')
grads = grads.numpy()[0,:,0](1, 10, 1, 28, 28) <- (Batch_preds, preds[i], Batch_inp, inp[y], inp[x])
print('prediction:', np.argmax(preds[0]))
fig, axs = plt.subplots(1, 11, figsize=(4.1*11,4))
axs[0].imshow(inp_v[0])
axs[0].set_title('raw')
vmin,vmax = grads.min(), grads.max()
for i, g in enumerate(grads):
  axs[i+1].imshow(g, cmap='gray', vmin=vmin, vmax=vmax)
  axs[i+1].set_title(r'$\frac{\partial\;P(digit\,%d)}{\partial\;input}$' % i, fontdict={'size':16})prediction: 6

## EXERCISE 1: Train deeper network for F-MNIST classification_____no_output_____Make a deeper model, with wider layers. Remember to `'softmax'` activation in the last layer, as required for the classification task to encode pseudoprobabilities. In the other layers you could use `'relu'`.

Try to achieve 90% accuracy.
Does your model overfit?_____no_output_____

# 1. create model
# 2. train the model
# 3. plot the loss and accuracy evolution during training
# 4. evaluate model in best point (before overfitting)_____no_output_____

# 2. Extras and Q&A_____no_output__________no_output__________no_output__________no_output_____


# Notebook from edopelawi/monte-carlo-covid19-socialdistancing
Path: monte-carlo-sim.ipynb

# Monte Carlo Simulation for Social Distancing to Prevent COVID-19 Infection

This notebook holds the modelling and analysis for the Monte Carlo Simulation._____no_output_____## Model simulation_____no_output_____

# Dependencies

import pandas as pd
import numpy as np
import seaborn as sns

from grid import Grid

sns.set_style('whitegrid')
_____no_output_____# Simulation setup

# Grid size
grid_height = 100
grid_width = 100

# Simulation parameters
population_size = 1000
infection_rate = 0.009
mobility = 0.75
p_death = 0.06
infection_duration = 4
max_evolutions = 1000

# Percentage of stationary population (per simulation)
stationary_percentages = np.arange(0, 1.25, 0.25).round(2).tolist()

# Simulation per stationary population
# TODO: Update this to 10000 for proper run
run_per_stat_percentage = 10
[0.0, 0.25, 0.5, 0.75, 1.0]
# Running simulation

# DataFrame creation

df = pd.DataFrame({
    "Stationary Percentage": [],
    "Death Rate": [],
    "Max Infection Rate": [],
    "Max Infection Rate Period": [],
    "Max Infected Count": [],
    "Max Infected Count Period": [],
    "Total Infection Rate": [],
    "Stop Period": []
})

for stat_percentage in stationary_percentages:

    death_rates = []

    max_infection_rates = []
    max_infection_rate_periods = []

    max_infected_counts = []
    max_infected_count_periods = []

    total_infection_rates = []

    stop_periods = []

    for i in range(run_per_stat_percentage):

        if (i+1)%10 == 0:
            print(f"runs performed: {i+1} for stat_percentage: {stat_percentage}")

        last_evolution = 0

        max_infection_rate = 0
        max_infection_rate_period = 0

        max_infected_count = 0
        max_infected_count_period = 0

        last_dead_person_count = 0
        last_immune_person_count = 0
        last_infected_person_count = 0

        g = Grid(grid_height, grid_width, population_size, infection_rate,
                 stat_percentage, mobility, p_death, infection_duration)

        for evolution in range(max_evolutions):

            if g.is_outbreak_stable():
                last_evolution = evolution
                break

            g.perform_evolution()

            current_snapshot = g.get_current_snapshot()

            current_dead_person_count = current_snapshot["dead_person_count"]
            current_immune_person_count = current_snapshot["immune_person_count"]
            current_infected_person_count = current_snapshot["infected_person_count"]

            current_infection_rate = (current_infected_person_count - last_infected_person_count)
            current_infection_rate += (current_immune_person_count - last_immune_person_count)
            current_infection_rate += (current_dead_person_count - last_dead_person_count)

            if (current_infection_rate > max_infection_rate):
                max_infection_rate = current_infection_rate
                max_infection_rate_period = evolution

            if (current_infected_person_count > max_infected_count):
                max_infected_count = current_infected_person_count
                max_infected_count_period = evolution

            last_dead_person_count = current_dead_person_count
            last_immune_person_count = current_immune_person_count
            last_infected_person_count = current_infected_person_count

        final_snapshot = g.get_current_snapshot()

        final_dead_person_count = final_snapshot["dead_person_count"]
        final_immune_person_count = final_snapshot["immune_person_count"]
        final_infected_person_count = final_snapshot["infected_person_count"]

        total_infection_rate = final_dead_person_count + final_immune_person_count + final_infected_person_count

        death_rates.append(final_dead_person_count)

        max_infection_rates.append(max_infection_rate)
        max_infection_rate_periods.append(max_infection_rate_period + 1)

        max_infected_counts.append(max_infected_count)
        max_infected_count_periods.append(max_infected_count_period + 1)

        total_infection_rates.append(total_infection_rate)

        stop_periods.append(last_evolution + 1)

    new_df = pd.DataFrame({
        "Stationary Percentage": np.repeat(stat_percentage, run_per_stat_percentage),
        "Death Rate": death_rates,
        "Max Infection Rate": max_infection_rates,
        "Max Infection Rate Period": max_infection_rate_periods,
        "Max Infected Count": max_infected_counts,
        "Max Infected Count Period": max_infected_count_periods,
        "Total Infection Rate": total_infection_rates,
        "Stop Period": stop_periods
    })

    df = df.append(new_df, ignore_index=True)_____no_output_____

## Analysis_____no_output_____

# Please run this before the next cell if you want to replace the df with data from pickles.

pickle_files = [
    "picklefiles/simulation_df_i0.25_s0_part1.pkl",
    "picklefiles/simulation_df_i0.25_s0_part2.pkl",
    "picklefiles/simulation_df-i0.25-s0.25.pkl",
    "picklefiles/simulation_df-i0.25-s0.5-1.pkl"
]

df = pd.DataFrame({
    "Stationary Percentage": [],
    "Death Rate": [],
    "Max Infection Rate": [],
    "Max Infection Rate Period": [],
    "Max Infected Count": [],
    "Max Infected Count Period": [],
    "Total Infection Rate": [],
    "Stop Period": []
})

for pickle in pickle_files:
    new_df = pd.read_pickle(pickle)
    df = df.append(new_df, ignore_index=True)_____no_output_____# Store the Dataframe

# Uncomment the command below to store the current Dataframe to Pickle.
# To read the result, run df = pd.read_pickle(file_name)

# df.to_pickle("simulation_df.pkl")_____no_output_____# Display the dataframe

display(df)_____no_output_____# Show histogram for the death rate, max infection rate, and total infection rate for each stationary percentage

import matplotlib.pyplot as plt

def show_histogram_plot(stat_percentage):

    stat_df = df.loc[df["Stationary Percentage"] == stat_percentage]

    show_category_histogram_plot(stat_percentage, stat_df, "Death Rate")
    show_category_histogram_plot(stat_percentage, stat_df, "Total Infection Rate")
    show_category_histogram_plot(stat_percentage, stat_df, "Max Infection Rate")
    show_category_histogram_plot(stat_percentage, stat_df, "Max Infection Rate Period")
    show_category_histogram_plot(stat_percentage, stat_df, "Max Infected Count")
    show_category_histogram_plot(stat_percentage, stat_df, "Max Infected Count Period")
    show_category_histogram_plot(stat_percentage, stat_df, "Stop Period")

def show_category_histogram_plot(stat_percentage, dataframe, category):

    # To make the distribution shows the kernel distribution, change the kde value to True.
    sns.distplot(dataframe[category], kde=False)
    plt.suptitle(f"Stationery Percentage: {stat_percentage}")
    plt.savefig(f"images/histogram_{category}_s{stat_percentage}.png")
    plt.show()

for stat_percentage in stationary_percentages:

    show_histogram_plot(stat_percentage)
    _____no_output_____# Show graphs for each category against each stationary percentage

mean_df = df.groupby("Stationary Percentage").mean()

display(mean_df)_____no_output_____# Plot for Categories for each mean

def show_line_plot(dataframe, category):
    sns.lineplot(data=dataframe[category])
    plt.suptitle(f"{category}")
    plt.savefig(f"images/lineplot_{category}_s{stat_percentage}.png")
    plt.show()

categories = [
    "Death Rate", "Max Infection Rate", "Max Infection Rate Period",
    "Max Infected Count","Max Infected Count Period",
    "Total Infection Rate","Stop Period"
]

for category in categories:
    show_line_plot(mean_df, category)_____no_output_____


# Notebook from GTPB/3DAROC21
Path: assets/material/Notebooks/Day3/11-3D_Models_production_and_analysis.ipynb

## Modeling a region of the chromatin_____no_output_______running time__: < 60 min

After the optimization step we have to create an ensemble of models using the optimal set of parameters that maximize our correlation with the input data.

### PSC cell_____no_output_____

%%bash

tadbit describe -w ../results/PSC_rep1/ -t 13,-----------------.
| MODELED_REGIONs |
,----.-------.-------.--------.------------.--------.-------.-------.
| Id | JOBid |  Type | PATHid |  PARAM_md5 |   RESO |   BEG |   END |
|----+-------+-------+--------+------------+--------+-------+-------|
|  1 |    26 | OPTIM |     62 | 49e20a90c8 | 10,000 | 3,395 | 3,545 |
'----^-------^-------^--------^------------^--------^-------^-------'

From the models produced (n_models) we will tell TADbit to conserve a number of them (n_keep) that best satisfy the imposed restraints._____no_output_____The final ensemble is built with the `tadbit model` and the `--model` flag. To pass to tadbit which matrix we want to use we pass the `jobid` of the previously generated text matrix. The following command takes around 20 minutes with 8 cpus:_____no_output_____

%%bash

tadbit model -w ../results/PSC_rep1/ \
    --reso 10000 \
    --crm chr3 \
    --beg 33950000 --end 35450000 \
    --nmodels 2000 --nkeep 1000 \
    --cpu 8 \
    --jobid 25 \
    --model     o Loading Hi-C matrix

        Modeling
********

          - Region: Chromosome chr3 from 33950000 to 35450000 at resolution 10 kb (150 particles)

     o Loading optimized parameters
Loaded UpFreq:   0.6 LowFreq:   -0.8 MaxDist:    250 scale:  0.01 cutoff:      2 Correlation:0.72
       1/1         0.6    -0.8     250   0.01     2.0 | 0.7301

The result is the generated ensemble of models in a format that can be loaded in the python TADbit library stored in the `07_model/<id>` subfolder._____no_output_____

%%bash

tadbit describe -w ../results/PSC_rep1/ -t 13,-----------------.
| MODELED_REGIONs |
,----.-------.-------.--------.------------.--------.-------.-------.
| Id | JOBid |  Type | PATHid |  PARAM_md5 |   RESO |   BEG |   END |
|----+-------+-------+--------+------------+--------+-------+-------|
|  1 |    26 | OPTIM |     62 | 49e20a90c8 | 10,000 | 3,395 | 3,545 |
|  2 |    27 | MODEL |     72 | a66efe4b3f | 10,000 | 3,395 | 3,545 |
'----^-------^-------^--------^------------^--------^-------^-------'

`tadbit model` provides some out-of-the-box analysis of the generated models. To produce them we use the tool again with the `--analyze` flag:_____no_output_____

%%bash

tadbit model -w ../results/PSC_rep1 \
    --analyzeNumber of singletons excluded from clustering: 40 (total singletons: 42)
Total number of clusters: 77
   Cluster #1 has 249 models [top model: 1831]
   Cluster #2 has 221 models [top model: 590]
   Cluster #3 has 36 models [top model: 902]
   Cluster #4 has 33 models [top model: 23]
   Cluster #5 has 28 models [top model: 1478]
   Cluster #6 has 27 models [top model: 199]
   Cluster #7 has 26 models [top model: 950]
   Cluster #8 has 23 models [top model: 1412]
   Cluster #9 has 15 models [top model: 286]
   Cluster #10 has 15 models [top model: 1151]
   Cluster #11 has 14 models [top model: 634]
   Cluster #12 has 11 models [top model: 739]
   Cluster #13 has 11 models [top model: 1598]
   Cluster #14 has 11 models [top model: 552]
   Cluster #15 has 11 models [top model: 399]
   Cluster #16 has 11 models [top model: 620]
   Cluster #17 has 11 models [top model: 1720]
   Cluster #18 has 9 models [top model: 312]
   Cluster #19 has 9 models [top model: 1203]
   Cluster #20 has 8 models [top model: 1477]
   Cluster #21 has 7 models [top model: 1729]
   Cluster #22 has 7 models [top model: 1480]
   Cluster #23 has 7 models [top model: 1304]
   Cluster #24 has 6 models [top model: 1135]
   Cluster #25 has 6 models [top model: 248]
   Cluster #26 has 6 models [top model: 234]
   Cluster #27 has 5 models [top model: 1325]
   Cluster #28 has 5 models [top model: 489]
   Cluster #29 has 5 models [top model: 827]
   Cluster #30 has 4 models [top model: 568]
   Cluster #31 has 4 models [top model: 1844]
   Cluster #32 has 4 models [top model: 1747]
   Cluster #33 has 4 models [top model: 224]
   Cluster #34 has 4 models [top model: 887]
   Cluster #35 has 4 models [top model: 400]
   Cluster #36 has 4 models [top model: 1622]
   Cluster #37 has 3 models [top model: 359]
   Cluster #38 has 3 models [top model: 1146]
   Cluster #39 has 3 models [top model: 29]
   Cluster #40 has 3 models [top model: 1157]
   Cluster #41 has 3 models [top model: 616]
   Cluster #42 has 3 models [top model: 1793]
   Cluster #43 has 3 models [top model: 1544]
   Cluster #44 has 3 models [top model: 512]
   Cluster #45 has 3 models [top model: 708]
   Cluster #46 has 3 models [top model: 1893]
   Cluster #47 has 3 models [top model: 702]
   Cluster #48 has 3 models [top model: 833]
   Cluster #49 has 3 models [top model: 1353]
   Cluster #50 has 3 models [top model: 1438]
   Cluster #51 has 3 models [top model: 203]
   Cluster #52 has 2 models [top model: 1489]
   Cluster #53 has 2 models [top model: 478]
   Cluster #54 has 2 models [top model: 147]
   Cluster #55 has 2 models [top model: 935]
   Cluster #56 has 2 models [top model: 138]
   Cluster #57 has 2 models [top model: 1751]
   Cluster #58 has 2 models [top model: 1995]
   Cluster #59 has 2 models [top model: 424]
   Cluster #60 has 2 models [top model: 862]
   Cluster #61 has 2 models [top model: 1458]
   Cluster #62 has 2 models [top model: 1910]
   Cluster #63 has 2 models [top model: 223]
   Cluster #64 has 2 models [top model: 265]
   Cluster #65 has 2 models [top model: 494]
   Cluster #66 has 2 models [top model: 920]
   Cluster #67 has 2 models [top model: 915]
   Cluster #68 has 2 models [top model: 1062]
   Cluster #69 has 2 models [top model: 1308]
   Cluster #70 has 2 models [top model: 971]
   Cluster #71 has 2 models [top model: 1999]
   Cluster #72 has 2 models [top model: 369]
   Cluster #73 has 2 models [top model: 1824]
   Cluster #74 has 2 models [top model: 502]
   Cluster #75 has 2 models [top model: 153]
   Cluster #76 has 2 models [top model: 1142]
   Cluster #77 has 2 models [top model: 1111]

%%bash

ls ../results/PSC_rep1/07_model/a66efe4b3f_chr3_3396-3545/a66efe4b3f.json
a66efe4b3f.models
a66efe4b3f_accessibility.dat
a66efe4b3f_accessibility.pdf
a66efe4b3f_clusters.pdf
a66efe4b3f_consistency.dat
a66efe4b3f_consistency.pdf
a66efe4b3f_contact.dat
a66efe4b3f_corre_real.pdf
a66efe4b3f_density.dat
a66efe4b3f_density.pdf
a66efe4b3f_interactions.dat
a66efe4b3f_interactions.pdf
a66efe4b3f_obj-func.pdf
a66efe4b3f_pL.dat
a66efe4b3f_wang.dat
a66efe4b3f_wang.pdf
a66efe4b3f_zscores.pdf
cfg_250_0.6_-0.8_0.01
models

### B cell_____no_output_____

%%bash

tadbit describe -w ../results/B_rep1/ -t 13,-----------------.
| MODELED_REGIONs |
,----.-------.-------.--------.------------.--------.-------.-------.
| Id | JOBid |  Type | PATHid |  PARAM_md5 |   RESO |   BEG |   END |
|----+-------+-------+--------+------------+--------+-------+-------|
|  1 |    17 | OPTIM |     49 | adf8bdcee7 | 10,000 | 3,395 | 3,545 |
'----^-------^-------^--------^------------^--------^-------^-------'
%%bash

tadbit model -w ../results/B_rep1/ \
    --reso 10000 \
    --crm chr3 \
    --beg 33950000 --end 35450000 \
    --nmodels 2000 --nkeep 1000 \
    --cpu 16 \
    --jobid 16 \
    --model     o Loading Hi-C matrix

        Modeling
********

          - Region: Chromosome chr3 from 33950000 to 35450000 at resolution 10 kb (150 particles)

     o Loading optimized parameters
Loaded UpFreq:  -0.8 LowFreq:   -1.4 MaxDist:    175 scale:  0.01 cutoff:      4 Correlation:nan
       1/1        -0.8    -1.4     175   0.01     2.0 | 0.6373
%%bash

tadbit describe -w ../results/B_rep1/ -t 13,-----------------.
| MODELED_REGIONs |
,----.-------.-------.--------.------------.--------.-------.-------.
| Id | JOBid |  Type | PATHid |  PARAM_md5 |   RESO |   BEG |   END |
|----+-------+-------+--------+------------+--------+-------+-------|
|  1 |    17 | OPTIM |     49 | adf8bdcee7 | 10,000 | 3,395 | 3,545 |
|  2 |    18 | MODEL |     62 | 42d6ff17ca | 10,000 | 3,395 | 3,545 |
'----^-------^-------^--------^------------^--------^-------^-------'
%%bash

tadbit model -w ../results/B_rep1 \
    --analyzeNumber of singletons excluded from clustering: 281 (total singletons: 306)
Total number of clusters: 187
   Cluster #1 has 27 models [top model: 777]
   Cluster #2 has 19 models [top model: 1495]
   Cluster #3 has 15 models [top model: 1641]
   Cluster #4 has 12 models [top model: 1372]
   Cluster #5 has 11 models [top model: 689]
   Cluster #6 has 10 models [top model: 1367]
   Cluster #7 has 10 models [top model: 1532]
   Cluster #8 has 9 models [top model: 235]
   Cluster #9 has 9 models [top model: 856]
   Cluster #10 has 9 models [top model: 807]
   Cluster #11 has 8 models [top model: 1315]
   Cluster #12 has 8 models [top model: 668]
   Cluster #13 has 8 models [top model: 470]
   Cluster #14 has 8 models [top model: 716]
   Cluster #15 has 8 models [top model: 1327]
   Cluster #16 has 8 models [top model: 147]
   Cluster #17 has 8 models [top model: 1542]
   Cluster #18 has 7 models [top model: 1165]
   Cluster #19 has 7 models [top model: 1266]
   Cluster #20 has 7 models [top model: 1752]
   Cluster #21 has 6 models [top model: 1648]
   Cluster #22 has 6 models [top model: 1739]
   Cluster #23 has 6 models [top model: 748]
   Cluster #24 has 6 models [top model: 1927]
   Cluster #25 has 5 models [top model: 1571]
   Cluster #26 has 5 models [top model: 1437]
   Cluster #27 has 5 models [top model: 197]
   Cluster #28 has 5 models [top model: 557]
   Cluster #29 has 5 models [top model: 899]
   Cluster #30 has 5 models [top model: 569]
   Cluster #31 has 5 models [top model: 115]
   Cluster #32 has 5 models [top model: 1902]
   Cluster #33 has 5 models [top model: 927]
   Cluster #34 has 5 models [top model: 838]
   Cluster #35 has 5 models [top model: 1714]
   Cluster #36 has 4 models [top model: 1909]
   Cluster #37 has 4 models [top model: 616]
   Cluster #38 has 4 models [top model: 1182]
   Cluster #39 has 4 models [top model: 559]
   Cluster #40 has 4 models [top model: 1653]
   Cluster #41 has 4 models [top model: 720]
   Cluster #42 has 4 models [top model: 174]
   Cluster #43 has 4 models [top model: 189]
   Cluster #44 has 4 models [top model: 1750]
   Cluster #45 has 4 models [top model: 172]
   Cluster #46 has 4 models [top model: 593]
   Cluster #47 has 4 models [top model: 20]
   Cluster #48 has 4 models [top model: 127]
   Cluster #49 has 4 models [top model: 1525]
   Cluster #50 has 4 models [top model: 384]
   Cluster #51 has 4 models [top model: 622]
   Cluster #52 has 4 models [top model: 1506]
   Cluster #53 has 4 models [top model: 1142]
   Cluster #54 has 4 models [top model: 1712]
   Cluster #55 has 4 models [top model: 1946]
   Cluster #56 has 4 models [top model: 1942]
   Cluster #57 has 4 models [top model: 1841]
   Cluster #58 has 4 models [top model: 919]
   Cluster #59 has 4 models [top model: 1505]
   Cluster #60 has 4 models [top model: 1387]
   Cluster #61 has 4 models [top model: 1273]
   Cluster #62 has 4 models [top model: 1138]
   Cluster #63 has 4 models [top model: 756]
   Cluster #64 has 4 models [top model: 1857]
   Cluster #65 has 4 models [top model: 1487]
   Cluster #66 has 3 models [top model: 298]
   Cluster #67 has 3 models [top model: 979]
   Cluster #68 has 3 models [top model: 1001]
   Cluster #69 has 3 models [top model: 1126]
   Cluster #70 has 3 models [top model: 369]
   Cluster #71 has 3 models [top model: 317]
   Cluster #72 has 3 models [top model: 1892]
   Cluster #73 has 3 models [top model: 635]
   Cluster #74 has 3 models [top model: 1335]
   Cluster #75 has 3 models [top model: 711]
   Cluster #76 has 3 models [top model: 278]
   Cluster #77 has 3 models [top model: 1867]
   Cluster #78 has 3 models [top model: 1422]
   Cluster #79 has 3 models [top model: 969]
   Cluster #80 has 3 models [top model: 659]
   Cluster #81 has 3 models [top model: 138]
   Cluster #82 has 3 models [top model: 1042]
   Cluster #83 has 3 models [top model: 1621]
   Cluster #84 has 3 models [top model: 1617]
   Cluster #85 has 3 models [top model: 1298]
   Cluster #86 has 3 models [top model: 754]
   Cluster #87 has 3 models [top model: 444]
   Cluster #88 has 3 models [top model: 978]
   Cluster #89 has 3 models [top model: 1481]
   Cluster #90 has 3 models [top model: 954]
   Cluster #91 has 3 models [top model: 1829]
   Cluster #92 has 3 models [top model: 1738]
   Cluster #93 has 3 models [top model: 1139]
   Cluster #94 has 3 models [top model: 91]
   Cluster #95 has 3 models [top model: 211]
   Cluster #96 has 3 models [top model: 101]
   Cluster #97 has 3 models [top model: 1480]
   Cluster #98 has 3 models [top model: 1155]
   Cluster #99 has 3 models [top model: 324]
   Cluster #100 has 3 models [top model: 1978]
   Cluster #101 has 3 models [top model: 859]
   Cluster #102 has 3 models [top model: 958]
   Cluster #103 has 3 models [top model: 1265]
   Cluster #104 has 3 models [top model: 274]
   Cluster #105 has 3 models [top model: 912]
   Cluster #106 has 3 models [top model: 1868]
   Cluster #107 has 3 models [top model: 1464]
   Cluster #108 has 3 models [top model: 776]
   Cluster #109 has 2 models [top model: 1751]
   Cluster #110 has 2 models [top model: 1350]
   Cluster #111 has 2 models [top model: 220]
   Cluster #112 has 2 models [top model: 767]
   Cluster #113 has 2 models [top model: 1465]
   Cluster #114 has 2 models [top model: 1074]
   Cluster #115 has 2 models [top model: 1974]
   Cluster #116 has 2 models [top model: 924]
   Cluster #117 has 2 models [top model: 985]
   Cluster #118 has 2 models [top model: 1280]
   Cluster #119 has 2 models [top model: 73]
   Cluster #120 has 2 models [top model: 1570]
   Cluster #121 has 2 models [top model: 598]
   Cluster #122 has 2 models [top model: 1433]
   Cluster #123 has 2 models [top model: 672]
   Cluster #124 has 2 models [top model: 17]
   Cluster #125 has 2 models [top model: 1549]
   Cluster #126 has 2 models [top model: 1596]
   Cluster #127 has 2 models [top model: 1225]
   Cluster #128 has 2 models [top model: 304]
   Cluster #129 has 2 models [top model: 264]
   Cluster #130 has 2 models [top model: 892]
   Cluster #131 has 2 models [top model: 1645]
   Cluster #132 has 2 models [top model: 1536]
   Cluster #133 has 2 models [top model: 802]
   Cluster #134 has 2 models [top model: 964]
   Cluster #135 has 2 models [top model: 1449]
   Cluster #136 has 2 models [top model: 1658]
   Cluster #137 has 2 models [top model: 242]
   Cluster #138 has 2 models [top model: 939]
   Cluster #139 has 2 models [top model: 532]
   Cluster #140 has 2 models [top model: 1717]
   Cluster #141 has 2 models [top model: 608]
   Cluster #142 has 2 models [top model: 1818]
   Cluster #143 has 2 models [top model: 1371]
   Cluster #144 has 2 models [top model: 790]
   Cluster #145 has 2 models [top model: 788]
   Cluster #146 has 2 models [top model: 1]
   Cluster #147 has 2 models [top model: 1684]
   Cluster #148 has 2 models [top model: 1860]
   Cluster #149 has 2 models [top model: 1960]
   Cluster #150 has 2 models [top model: 1833]
   Cluster #151 has 2 models [top model: 1649]
   Cluster #152 has 2 models [top model: 1100]
   Cluster #153 has 2 models [top model: 1566]
   Cluster #154 has 2 models [top model: 1467]
   Cluster #155 has 2 models [top model: 1324]
   Cluster #156 has 2 models [top model: 501]
   Cluster #157 has 2 models [top model: 1014]
   Cluster #158 has 2 models [top model: 1708]
   Cluster #159 has 2 models [top model: 116]
   Cluster #160 has 2 models [top model: 722]
   Cluster #161 has 2 models [top model: 1743]
   Cluster #162 has 2 models [top model: 1528]
   Cluster #163 has 2 models [top model: 58]
   Cluster #164 has 2 models [top model: 445]
   Cluster #165 has 2 models [top model: 1713]
   Cluster #166 has 2 models [top model: 5]
   Cluster #167 has 2 models [top model: 1272]
   Cluster #168 has 2 models [top model: 823]
   Cluster #169 has 2 models [top model: 1580]
   Cluster #170 has 2 models [top model: 469]
   Cluster #171 has 2 models [top model: 761]
   Cluster #172 has 2 models [top model: 1746]
   Cluster #173 has 2 models [top model: 141]
   Cluster #174 has 2 models [top model: 1361]
   Cluster #175 has 2 models [top model: 1504]
   Cluster #176 has 2 models [top model: 1707]
   Cluster #177 has 2 models [top model: 760]
   Cluster #178 has 2 models [top model: 950]
   Cluster #179 has 2 models [top model: 514]
   Cluster #180 has 2 models [top model: 1827]
   Cluster #181 has 2 models [top model: 1409]
   Cluster #182 has 2 models [top model: 599]
   Cluster #183 has 2 models [top model: 1151]
   Cluster #184 has 2 models [top model: 294]
   Cluster #185 has 2 models [top model: 1311]
   Cluster #186 has 2 models [top model: 1768]
   Cluster #187 has 2 models [top model: 874]

%%bash

ls ../results/B_rep1/07_model/42d6ff17ca_chr3_3396-3545/42d6ff17ca.json
42d6ff17ca.models
42d6ff17ca_accessibility.dat
42d6ff17ca_accessibility.pdf
42d6ff17ca_clusters.pdf
42d6ff17ca_consistency.dat
42d6ff17ca_consistency.pdf
42d6ff17ca_contact.dat
42d6ff17ca_corre_real.pdf
42d6ff17ca_density.dat
42d6ff17ca_density.pdf
42d6ff17ca_interactions.dat
42d6ff17ca_interactions.pdf
42d6ff17ca_obj-func.pdf
42d6ff17ca_pL.dat
42d6ff17ca_wang.dat
42d6ff17ca_wang.pdf
42d6ff17ca_zscores.pdf
cfg_175_-0.8_-1.4_0.01
models

### Structural models analysis

The first task of the `--analysis` flag is to compute subpopulations of structures by clustering the ensemble by similarity using the Markov Cluster Algorithm (MCL). The analysis dendogram allows us to view the different clusters population and their OF values._____no_output_____

from IPython.display import IFrame_____no_output_____IFrame("../results/PSC_rep1/07_model/a66efe4b3f_chr3_3396-3545/a66efe4b3f_clusters.pdf", width=900, height=800)_____no_output_____IFrame("../results/B_rep1/07_model/42d6ff17ca_chr3_3396-3545/42d6ff17ca_clusters.pdf", width=900, height=800)_____no_output_____

Inside the `models` subfolder we find the coordinates of all the models in xyz and Chimera (https://www.cgl.ucsf.edu/chimera/) format organized by clusters._____no_output_____

%%bash

ls ../results/PSC_rep1/07_model/a66efe4b3f_chr3_3396-3545/models/Singletons
Singletons.lst
Singletons_superimpose.cmd
best.cmm
best.xyz
centroid.cmm
centroid.xyz
cl_1
cl_1.lst
cl_10
cl_10.lst
cl_10_superimpose.cmd
cl_11
cl_11.lst
cl_11_superimpose.cmd
cl_12
cl_12.lst
cl_12_superimpose.cmd
cl_13
cl_13.lst
cl_13_superimpose.cmd
cl_14
cl_14.lst
cl_14_superimpose.cmd
cl_15
cl_15.lst
cl_15_superimpose.cmd
cl_16
cl_16.lst
cl_16_superimpose.cmd
cl_17
cl_17.lst
cl_17_superimpose.cmd
cl_18
cl_18.lst
cl_18_superimpose.cmd
cl_19
cl_19.lst
cl_19_superimpose.cmd
cl_1_superimpose.cmd
cl_2
cl_2.lst
cl_20
cl_20.lst
cl_20_superimpose.cmd
cl_21
cl_21.lst
cl_21_superimpose.cmd
cl_22
cl_22.lst
cl_22_superimpose.cmd
cl_23
cl_23.lst
cl_23_superimpose.cmd
cl_24
cl_24.lst
cl_24_superimpose.cmd
cl_25
cl_25.lst
cl_25_superimpose.cmd
cl_26
cl_26.lst
cl_26_superimpose.cmd
cl_27
cl_27.lst
cl_27_superimpose.cmd
cl_28
cl_28.lst
cl_28_superimpose.cmd
cl_29
cl_29.lst
cl_29_superimpose.cmd
cl_2_superimpose.cmd
cl_3
cl_3.lst
cl_30
cl_30.lst
cl_30_superimpose.cmd
cl_31
cl_31.lst
cl_31_superimpose.cmd
cl_32
cl_32.lst
cl_32_superimpose.cmd
cl_33
cl_33.lst
cl_33_superimpose.cmd
cl_34
cl_34.lst
cl_34_superimpose.cmd
cl_35
cl_35.lst
cl_35_superimpose.cmd
cl_36
cl_36.lst
cl_36_superimpose.cmd
cl_37
cl_37.lst
cl_37_superimpose.cmd
cl_38
cl_38.lst
cl_38_superimpose.cmd
cl_39
cl_39.lst
cl_39_superimpose.cmd
cl_3_superimpose.cmd
cl_4
cl_4.lst
cl_40
cl_40.lst
cl_40_superimpose.cmd
cl_41
cl_41.lst
cl_41_superimpose.cmd
cl_42
cl_42.lst
cl_42_superimpose.cmd
cl_43
cl_43.lst
cl_43_superimpose.cmd
cl_44
cl_44.lst
cl_44_superimpose.cmd
cl_45
cl_45.lst
cl_45_superimpose.cmd
cl_46
cl_46.lst
cl_46_superimpose.cmd
cl_47
cl_47.lst
cl_47_superimpose.cmd
cl_48
cl_48.lst
cl_48_superimpose.cmd
cl_49
cl_49.lst
cl_49_superimpose.cmd
cl_4_superimpose.cmd
cl_5
cl_5.lst
cl_50
cl_50.lst
cl_50_superimpose.cmd
cl_51
cl_51.lst
cl_51_superimpose.cmd
cl_52
cl_52.lst
cl_52_superimpose.cmd
cl_53
cl_53.lst
cl_53_superimpose.cmd
cl_54
cl_54.lst
cl_54_superimpose.cmd
cl_55
cl_55.lst
cl_55_superimpose.cmd
cl_56
cl_56.lst
cl_56_superimpose.cmd
cl_57
cl_57.lst
cl_57_superimpose.cmd
cl_58
cl_58.lst
cl_58_superimpose.cmd
cl_59
cl_59.lst
cl_59_superimpose.cmd
cl_5_superimpose.cmd
cl_6
cl_6.lst
cl_60
cl_60.lst
cl_60_superimpose.cmd
cl_61
cl_61.lst
cl_61_superimpose.cmd
cl_62
cl_62.lst
cl_62_superimpose.cmd
cl_63
cl_63.lst
cl_63_superimpose.cmd
cl_64
cl_64.lst
cl_64_superimpose.cmd
cl_65
cl_65.lst
cl_65_superimpose.cmd
cl_66
cl_66.lst
cl_66_superimpose.cmd
cl_67
cl_67.lst
cl_67_superimpose.cmd
cl_68
cl_68.lst
cl_68_superimpose.cmd
cl_69
cl_69.lst
cl_69_superimpose.cmd
cl_6_superimpose.cmd
cl_7
cl_7.lst
cl_70
cl_70.lst
cl_70_superimpose.cmd
cl_71
cl_71.lst
cl_71_superimpose.cmd
cl_72
cl_72.lst
cl_72_superimpose.cmd
cl_73
cl_73.lst
cl_73_superimpose.cmd
cl_74
cl_74.lst
cl_74_superimpose.cmd
cl_75
cl_75.lst
cl_75_superimpose.cmd
cl_76
cl_76.lst
cl_76_superimpose.cmd
cl_77
cl_77.lst
cl_77_superimpose.cmd
cl_7_superimpose.cmd
cl_8
cl_8.lst
cl_8_superimpose.cmd
cl_9
cl_9.lst
cl_9_superimpose.cmd
%%bash

head ../results/PSC_rep1/07_model/a66efe4b3f_chr3_3396-3545/models/centroid.xyz -n 14# ASSEMBLY        :  None
# CELL TYPE       :  None
# CHROMOSOME      :  chr3
# END             :  35450000
# EXPERIMENT TYPE :  Hi-C
# IDENTIFIER      :  None
# PROJECT         :  None
# RESOLUTION      :  10000
# RESTRICTION ENZYME :  None
# SPECIES         :  UNKNOWN
# START           :  33950000
1	chr3:33953396-33963395	-497.769	-9996.501	481.934
2	chr3:33963396-33973395	-520.384	-9977.995	569.415
3	chr3:33973396-33983395	-415.309	-10021.954	509.441
from pytadbit.utils.extraviews      import plot_3d_model
import numpy as np

centroid = np.loadtxt("../results/PSC_rep1/07_model/a66efe4b3f_chr3_3396-3545/models/centroid.xyz",
                     usecols=[2,3,4])

x, y, z = centroid[:,0], centroid[:,1], centroid[:,2]
plot_3d_model(x, y, z, particle_size=50, alpha_part=0.7, azimuth=20, elevation=100.)

centroid = np.loadtxt("../results/B_rep1/07_model/42d6ff17ca_chr3_3396-3545/models/centroid.xyz",
                     usecols=[2,3,4])

x, y, z = centroid[:,0], centroid[:,1], centroid[:,2]
plot_3d_model(x, y, z, particle_size=50, alpha_part=0.7, azimuth=75, elevation=100.)_____no_output_____

The z-score plot shows how the Hi-C values converted to z-scores are applied as restraints:_____no_output_____

IFrame("../results/PSC_rep1/07_model/a66efe4b3f_chr3_3396-3545/a66efe4b3f_zscores.pdf", width=900, height=600)_____no_output_____IFrame("../results/B_rep1/07_model/42d6ff17ca_chr3_3396-3545/42d6ff17ca_zscores.pdf", width=900, height=300)_____no_output_____

The IMP objective function (OF) of the model that best satisfies the imposed restraints during the Monte Carlo optimization:_____no_output_____

IFrame("../results/PSC_rep1/07_model/a66efe4b3f_chr3_3396-3545/a66efe4b3f_obj-func.pdf", width=800, height=800)_____no_output_____

We can check the correlation of models with the original HiC matrix in the correlation plot.

In the plot "Real vs modelled data" we should see a positive correlation of the contacts in the models with the frequency of interaction of the pairs of beads in the HiC matrix. High interaction frequency between two loci in the matrix is reflected by the fact of having a high proportion of models where the beads representing those two loci are "in contact" (distance lower than the cutoff). _____no_output_____

IFrame("../results/PSC_rep1/07_model/a66efe4b3f_chr3_3396-3545/a66efe4b3f_corre_real.pdf", width=900, height=350)_____no_output_____IFrame("../results/B_rep1/07_model/42d6ff17ca_chr3_3396-3545/42d6ff17ca_corre_real.pdf", width=900, height=350)_____no_output_____

Model consistency gives a measure of the variability of the particles accross a set of models. Particles in the same position accross different models are considered consistent if their distance is less than the given cutoff._____no_output_____

IFrame("../results/PSC_rep1/07_model/a66efe4b3f_chr3_3396-3545/a66efe4b3f_consistency.pdf", width=900, height=400)_____no_output_____IFrame("../results/B_rep1/07_model/42d6ff17ca_chr3_3396-3545/42d6ff17ca_consistency.pdf", width=900, height=400)_____no_output_____

From the 3D models, the DNA density (or local compactness) can be calculated as the ratio of the bin size (in base pairs) and the distances between consequtive particles in the models. The higher the density the more compact DNA for the region. As this measure varies dramatically from particle to particle, one can calculate it using running averages.

In the modelling we have used a scale of 0.01 nm/bp; that means that if we expect 100 bp/nm of chromatin in each bead and between two consecutives beads.

_____no_output_____

IFrame("../results/PSC_rep1/07_model/a66efe4b3f_chr3_3396-3545/a66efe4b3f_density.pdf", width=900, height=400)_____no_output_____IFrame("../results/B_rep1/07_model/42d6ff17ca_chr3_3396-3545/42d6ff17ca_density.pdf", width=900, height=400)_____no_output_____

Walking_angle plots the angle between triplets of contiguous particles. The higher are these values the straighter are the models.

_____no_output_____

IFrame("../results/PSC_rep1/07_model/a66efe4b3f_chr3_3396-3545/a66efe4b3f_wang.pdf", width=900, height=400)_____no_output_____IFrame("../results/B_rep1/07_model/42d6ff17ca_chr3_3396-3545/42d6ff17ca_wang.pdf", width=900, height=400)_____no_output_____

Interactions plot (particles closer than the given cutoff)

_____no_output_____

IFrame("../results/PSC_rep1/07_model/a66efe4b3f_chr3_3396-3545/a66efe4b3f_interactions.pdf", width=900, height=400)_____no_output_____IFrame("../results/B_rep1/07_model/42d6ff17ca_chr3_3396-3545/42d6ff17ca_interactions.pdf", width=900, height=400)_____no_output_____

The accessibility is calculated by considering a mesh surface around the model and checking if each point of this mesh could be replaced by an object (i.e. a protein) represented as a sphere of a given radius.

Outer part of the model can be excluded from the estimation of accessible surface because contacts from this outer part to particles outside the model are unknown. To exclude the outer contour a sphere with a higher radius (superradius) is first tested in the mesh before proceding to the accessibility calculation.

_____no_output_____

IFrame("../results/PSC_rep1/07_model/a66efe4b3f_chr3_3396-3545/a66efe4b3f_accessibility.pdf", width=900, height=400)_____no_output_____IFrame("../results/B_rep1/07_model/42d6ff17ca_chr3_3396-3545/42d6ff17ca_accessibility.pdf", width=900, height=400)_____no_output_____

### Questions

- Would you expect that running TADbit twice with the same parameters leads to the same set of models? Why yes? Why no?
- Can you comment on the conditions in which the TADbit models may result in highly variable or highly consistent structures?
- Can you comment on the main structural differences between the models obtained in B-cells and PSC?_____no_output_____


# Notebook from vutuanhai237/QuantumTomography
Path: codes/graph_ansatz/plot_compare_ansatz.ipynb

import pandas as pd
import numpy as np
from matplotlib import pyplot as plt_____no_output_____

### Survay losss_____no_output_____

state = 'w'
ansatz = 'linear_ansatz_'
path = "../../experiments/" + ansatz + state + "/" + str(3) + '/fidelities_sgd.csv'
fidelities1 = pd.read_csv(path, sep=",", header=None)
fidelities1 = fidelities1.applymap(lambda s: complex(s.replace('i', 'j'))).values
path = "../../experiments/" + ansatz + state + "/" + str(3) + '/fidelities_adam.csv'
fidelities2 = pd.read_csv(path, sep=",", header=None)
fidelities2 = fidelities2.applymap(lambda s: complex(s.replace('i', 'j'))).values
path = "../../experiments/" + ansatz + state + "/" + str(3) + '/fidelities_qng.csv'
fidelities3 = pd.read_csv(path, sep=",", header=None)
fidelities3 = fidelities3.applymap(lambda s: complex(s.replace('i', 'j'))).values
plt.ylim(0, 1.1)
plt.plot(fidelities1, label = 'SGD')
plt.plot(fidelities2, label = 'Adam')
plt.plot(fidelities3, label = 'QNG')
# plt.savefig(state.upper() + ' state - ' + ansatz + '.eps', dpi = 1000)
plt.legend()

C:\Users\haime\AppData\Local\Programs\Python\Python38\lib\site-packages\matplotlib\cbook\__init__.py:1298: ComplexWarning: Casting complex values to real discards the imaginary part
  return np.asarray(x, float)
C:\Users\haime\AppData\Local\Programs\Python\Python38\lib\site-packages\matplotlib\cbook\__init__.py:1298: ComplexWarning: Casting complex values to real discards the imaginary part
  return np.asarray(x, float)
C:\Users\haime\AppData\Local\Programs\Python\Python38\lib\site-packages\matplotlib\cbook\__init__.py:1298: ComplexWarning: Casting complex values to real discards the imaginary part
  return np.asarray(x, float)
state = 'ghz'
ansatz = 'linear_ansatz_'
path = "../../experiments/" + ansatz + state + "/" + str(3) + '/loss_values_sgd.csv'
loss1 = pd.read_csv(path, sep=",", header=None).to_numpy()
path = "../../experiments/" + ansatz + state + "/" + str(3) + '/loss_values_adam.csv'
loss2 = pd.read_csv(path, sep=",", header=None).to_numpy()
path = "../../experiments/" + ansatz + state + "/" + str(3) + '/loss_values_qng.csv'
loss3 = pd.read_csv(path, sep=",", header=None).to_numpy()
plt.plot(loss1, label = 'SGD')
plt.plot(loss2, label = 'Adam')
plt.plot(loss3, label = 'QNG')
plt.savefig(state.upper() + ' state - ' + ansatz + '.eps', dpi = 1000)
plt.legend()_____no_output_____losss = []

for i in range(2, 11):
    path = "../experiments/linear_ansatz_w/" + str(i) + '/loss_values_qng.csv'
    loss = pd.read_csv(path, sep=",", header=None).to_numpy()
    plt.plot(loss, label = str(i) + ' qubits')
plt.legend()
plt.show()_____no_output_____losss = []

for i in range(2, 7):
    path = "../experiments/star_ansatz_ghz/" + str(i) + '/loss_values_sgd.csv'
    loss = pd.read_csv(path, sep=",", header=None).to_numpy()
    plt.plot(loss, label = str(i) + ' qubits')

plt.xlabel('Iteration')
plt.ylim((0,0.2))
plt.yticks(np.arange(0, 0.25, 0.05))
plt.ylabel('Value')
plt.title('Loss')
plt.legend()
plt.show()_____no_output_____qubits = range(2, 7)
star_ansatz = []
for i in qubits:
    path = "../experiments/star_ansatz_" + 'ghz' + "/" + str(i) + '/loss_values_' + 'sgd' + '.csv'
    loss = pd.read_csv(path, sep=",", header=None).to_numpy()
    star_ansatz.append((loss[-1]))

plt.ylim((0,0.2))
plt.xticks(np.arange(0, 7, 1))
plt.yticks(np.arange(0, 0.2, 0.05))
plt.plot(qubits, star_ansatz, marker='s', label = 'Star ansatz')
_____no_output_____def get_loss(reconstruted_state, optimizer):
    losss_linear, losss_star, losss_polygon = [], [], []
    qubits = range(2, 11)
    for i in qubits:
        path = "../experiments/linear_ansatz_" + reconstruted_state + "/" + str(i) + '/loss_values_' + optimizer + '.csv'
        loss = pd.read_csv(path, sep=",", header=None).to_numpy()
        losss_linear.append(np.min(loss))
    for i in qubits:
        path = "../experiments/star_ansatz_" + reconstruted_state + "/" + str(i) + '/loss_values_' + optimizer + '.csv'
        loss = pd.read_csv(path, sep=",", header=None).to_numpy()
        losss_star.append(np.min(loss))
    for i in qubits:
        path = "../experiments/polygon_ansatz_" + reconstruted_state + "/" + str(i) + '/loss_values_' + optimizer + '.csv'
        loss = pd.read_csv(path, sep=",", header=None).to_numpy()
        losss_polygon.append(np.min(loss))
    plt.plot(qubits, losss_linear, marker='o', color = 'blue', label = 'Linear ansatz')
    plt.plot(qubits, losss_star, marker='s', color = 'orange', label = 'Star ansatz')
    plt.plot(qubits, losss_polygon, marker='^', color = 'green', label = 'Polygon ansatz')
    plt.xlabel('Number of qubits')
    plt.yticks(np.arange(0, 1.2, 0.2))
    plt.ylabel('Fubini-study loss')
    # plt.text(3.5, 0.5, optimizer.upper())
    plt.title(reconstruted_state.upper() + ' state preparation')
    plt.legend()
    plt.savefig(reconstruted_state.upper() + ' state - ' + optimizer.upper() + ' optimizer.eps', dpi = 1000)
    plt.show()

# get_loss('ghz', 'qng')_____no_output_____get_loss('ghz', 'adam')The PostScript backend does not support transparency; partially transparent artists will be rendered opaque.
get_loss('ghz', 'sgd')The PostScript backend does not support transparency; partially transparent artists will be rendered opaque.
get_loss('w', 'adam')The PostScript backend does not support transparency; partially transparent artists will be rendered opaque.
def get_loss(reconstruted_state, optimizer):
    losss_linear, losss_star, losss_polygon = [], [], []
    qubits = range(2, 11)
    for i in qubits:
        path = "../experiments/star_ansatz_" + reconstruted_state + "/" + str(i) + '/loss_values_' + optimizer + '.csv'
        loss = pd.read_csv(path, sep=",", header=None).to_numpy()
        losss_star.append(np.min(loss))

    plt.plot(qubits, losss_star, label = reconstruted_state)
    plt.xlabel('Number of qubits')
    plt.yticks(np.arange(0, 1.4, 0.2))
    plt.ylabel('Fubini-study loss')
    # plt.title(reconstruted_state.upper() + ' state - ' + optimizer.upper() + ' optimizer')
    plt.legend()

get_loss('ghz', 'sgd')
get_loss('w', 'sgd')
plt.show()_____no_output_____num_layers = range(2, 6)
for i in num_layers:
    if i == 2:
        path = "../experiments/star_ansatz_ghz/7/loss_values_qng.csv"
    else:
        path = "../experiments/star_ansatz_ghz/7/loss_values_qng" + str(i) + ".csv"
    loss = pd.read_csv(path, sep=",", header=None).to_numpy()
    plt.plot(loss, label = i)
plt.legend()_____no_output_____num_layers = range(2, 6)
for i in num_layers:
    if i == 2:
        path = "../experiments/polygon_ansatz_w/7/loss_values_qng.csv"
    else:
        path = "../experiments/polygon_ansatz_w/7/loss_values_qng" + str(i) + ".csv"
    loss = pd.read_csv(path, sep=",", header=None).to_numpy()[:100]
    plt.plot(loss, label = i)
plt.legend()_____no_output_____

### Barren plataeus_____no_output_____

losss_polygon_ghz, losss_polygon_w, losss_star_ghz, losss_star_w, losss_linear_ghz, losss_linear_w = [], [], [], [], [], []
num_layers = range(2, 6)
for i in num_layers:
    if i == 2:
        path = "../experiments/polygon_ansatz_w/7/loss_values_qng.csv"
    else:
        path = "../experiments/polygon_ansatz_w/7/loss_values_qng" + str(i) + ".csv"
    loss = pd.read_csv(path, sep=",", header=None).to_numpy()
    losss_polygon_w.append(np.min(loss))
for i in num_layers:
    if i == 2:
        path = "../experiments/polygon_ansatz_ghz/7/loss_values_qng.csv"
    else:
        path = "../experiments/polygon_ansatz_ghz/7/loss_values_qng" + str(i) + ".csv"
    loss = pd.read_csv(path, sep=",", header=None).to_numpy()
    losss_polygon_ghz.append(np.min(loss))
for i in num_layers:
    if i == 2:
        path = "../experiments/star_ansatz_ghz/7/loss_values_qng.csv"
    else:
        path = "../experiments/star_ansatz_ghz/7/loss_values_qng" + str(i) + ".csv"
    loss = pd.read_csv(path, sep=",", header=None).to_numpy()
    losss_star_ghz.append(np.min(loss))
for i in num_layers:
    if i == 2:
        path = "../experiments/star_ansatz_w/7/loss_values_qng.csv"
    else:
        path = "../experiments/star_ansatz_w/7/loss_values_qng" + str(i) + ".csv"
    loss = pd.read_csv(path, sep=",", header=None).to_numpy()
    losss_star_w.append(np.min(loss))
for i in num_layers:
    if i == 2:
        path = "../experiments/linear_ansatz_ghz/7/loss_values_qng.csv"
    else:
        path = "../experiments/linear_ansatz_ghz/7/loss_values_qng" + str(i) + ".csv"
    loss = pd.read_csv(path, sep=",", header=None).to_numpy()
    losss_linear_ghz.append(np.min(loss))
for i in num_layers:
    if i == 2:
        path = "../experiments/linear_ansatz_w/7/loss_values_qng.csv"
    else:
        path = "../experiments/linear_ansatz_w/7/loss_values_qng" + str(i) + ".csv"
    loss = pd.read_csv(path, sep=",", header=None).to_numpy()
    losss_linear_w.append(np.min(loss))

plt.plot(num_layers, losss_linear_ghz, marker='o', color='blue', label = 'Linear ansatz - GHZ state')
plt.plot(num_layers, losss_linear_w, marker='o', color='orange', label = 'Linear ansatz - W state')
plt.plot(num_layers, losss_star_ghz, marker='^', color='blue', label = 'Star ansatz - GHZ state')
plt.plot(num_layers, losss_star_w, marker='^', color='orange',label = 'Star ansatz - W state')
plt.plot(num_layers, losss_polygon_ghz, marker='v', color='blue', label = 'Polygon ansatz - GHZ state')
plt.plot(num_layers, losss_polygon_w, marker='v', color='orange', label = 'Polygon ansatz - W state')
plt.xlabel('Number of layers')
plt.xticks(np.arange(2, 6, 1))
plt.yticks(np.arange(0, 1.2, 0.2))
plt.ylabel('Fubini-study loss')
# plt.title(reconstruted_state.upper() + ' state - ' + optimizer.upper() + ' optimizer')
plt.legend()
plt.savefig('compare_layer.eps', format = 'eps', dpi = 1000)The PostScript backend does not support transparency; partially transparent artists will be rendered opaque.

### Adding noise_____no_output_____

losss_star_ghz_1, losss_star_ghz_2 = [], []
losss_star_ghz_3, losss_star_ghz_4, losss_star_ghz_5 = [], [], []
num_noise = [0.0, 0.001, 0.005, 0.01, 0.015, 0.02, 0.025, 0.03, 0.035]
for i in num_noise:
    path = "../experiments/star_ansatz_ghz/7/loss_values_qng1_" + str(i) + ".csv"
    loss = pd.read_csv(path, sep=",", header=None).to_numpy()
    losss_star_ghz_1.append(np.min(loss))
for i in num_noise:
    path = "../experiments/star_ansatz_ghz/7/loss_values_qng2_" + str(i) + ".csv"
    loss = pd.read_csv(path, sep=",", header=None).to_numpy()
    losss_star_ghz_2.append(np.min(loss))
for i in num_noise:
    path = "../experiments/star_ansatz_ghz/7/loss_values_qng3_" + str(i) + ".csv"
    loss = pd.read_csv(path, sep=",", header=None).to_numpy()
    losss_star_ghz_3.append(np.min(loss))
for i in num_noise:
    path = "../experiments/star_ansatz_ghz/7/loss_values_qng4_" + str(i) + ".csv"
    loss = pd.read_csv(path, sep=",", header=None).to_numpy()
    losss_star_ghz_4.append(np.min(loss))
for i in num_noise:
    path = "../experiments/star_ansatz_ghz/7/loss_values_qng5_" + str(i) + ".csv"
    loss = pd.read_csv(path, sep=",", header=None).to_numpy()
    losss_star_ghz_5.append(np.min(loss))
plt.plot(num_noise, losss_star_ghz_1, marker='o', color='blue', label = '1 layer')
plt.plot(num_noise, losss_star_ghz_2, marker='o', color='orange', label = '2 layers')
plt.plot(num_noise, losss_star_ghz_3, marker='o', color='green', label = '3 layers')
plt.plot(num_noise, losss_star_ghz_4, marker='o', color='red', label = '4 layers')
plt.plot(num_noise, losss_star_ghz_5, marker='o', color='brown', label = '5 layers')
plt.xlabel('Prob noise')
plt.xticks(np.arange(0, 0.1, 0.01))
plt.yticks(np.arange(0, 1.2, 0.2))
plt.ylabel('Fubini-study loss')
plt.title('Star ansatz - GHZ state')
# plt.title(reconstruted_state.upper() + ' state - ' + optimizer.upper() + ' optimizer')
plt.legend()_____no_output_____losss_star_ghz_mitigating, losss_star_ghz_nomitigating = [], []
num_noise = [0.00, 0.01, 0.02, 0.03, 0.04]
for i in num_noise:
    path = "./loss_values_qngm2_" + str(i) + ".csv"
    loss = pd.read_csv(path, sep=",", header=None).to_numpy()
    losss_star_ghz_mitigating.append(np.min(loss))
    print(np.min(loss))
for i in num_noise:
    path = "./loss_values_qngnm2_" + str(i) + ".csv"
    loss = pd.read_csv(path, sep=",", header=None).to_numpy()
    losss_star_ghz_nomitigating.append(np.min(loss))

plt.plot(num_noise, losss_star_ghz_mitigating, marker='o', color='red', label = 'Mitigated')
plt.plot(num_noise, losss_star_ghz_nomitigating, marker='o', color='brown', label = 'Not mitigated')
plt.xlabel('Prob noise')
plt.xticks(np.arange(0, 0.06, 0.01))
plt.yticks(np.arange(0, 0.7, 0.2))
plt.ylabel('Fubini-study distance')
plt.title('Star ansatz - GHZ state')
# plt.title(reconstruted_state.upper() + ' state - ' + optimizer.upper() + ' optimizer')
plt.legend()
plt.savefig('compare_mitigating2.eps', format = 'eps', dpi = 1000)0.0
0.0
0.0
0.0
0.0
path = "./loss_values_qngm2_0.04.csv"
loss = pd.read_csv(path, sep=",", header=None).to_numpy()
thetass = pd.read_csv("./thetass_qngm2_0.04.csv", sep=",", header=None).to_numpy()

imin = np.argmin(loss)
thetas = thetass[imin]

import qiskit, qtm.nqubit
num_qubits = 5
num_layers = 2
qc = qiskit.QuantumCircuit(num_qubits, num_qubits)
qc = qtm.nqubit.create_star2graph_state(qc, thetas, num_layers)
psi = qiskit.quantum_info.Statevector.from_instruction(qc)
rho_psi = qiskit.quantum_info.DensityMatrix(psi)
print(psi)Statevector([-7.08711347e-01+0.j,  1.00203401e-03+0.j,  1.37367906e-03+0.j,
              1.29969380e-05+0.j,  1.66033081e-06+0.j, -5.37137698e-05+0.j,
             -4.25119626e-05+0.j,  1.43529491e-03+0.j,  2.03290397e-05+0.j,
             -1.61529267e-03+0.j, -3.65592592e-06+0.j,  1.78332961e-06+0.j,
              3.86176693e-06+0.j, -1.14245343e-05+0.j,  1.19991057e-03+0.j,
              3.23874763e-03+0.j,  2.01923239e-03+0.j,  1.18598776e-03+0.j,
             -3.93034292e-06+0.j, -1.36414524e-06+0.j,  1.14608143e-05+0.j,
             -4.96313578e-07+0.j, -3.30031207e-03+0.j,  6.69459235e-05+0.j,
             -1.44767715e-03+0.j,  1.95786285e-05+0.j, -7.92948071e-04+0.j,
              4.62569238e-06+0.j, -3.61981277e-05+0.j,  2.47223440e-03+0.j,
              2.64788061e-03+0.j, -7.05461907e-01+0.j],
            dims=(2, 2, 2, 2, 2))

path = "./loss_values_qngnm2_0.01.csv"
loss1 = pd.read_csv(path, sep=",", header=None).to_numpy()
path = "./loss_values_qngnm2_0.02.csv"
loss2 = pd.read_csv(path, sep=",", header=None).to_numpy()
path = "./loss_values_qngnm2_0.03.csv"
loss3 = pd.read_csv(path, sep=",", header=None).to_numpy()
path = "./loss_values_qngnm2_0.04.csv"
loss4 = pd.read_csv(path, sep=",", header=None).to_numpy()
plt.plot(loss1, ':', color = 'orange', label = 'no mitigating ' + str(i))
plt.plot(loss2, ':', color = 'blue', label = 'no mitigating ' + str(i))
plt.plot(loss3, ':', color = 'green', label = 'no mitigating ' + str(i))
plt.plot(loss4, ':', color = 'red', label = 'no mitigating ' + str(i))
path = "./loss_values_qngm2_0.01.csv"
loss1 = pd.read_csv(path, sep=",", header=None).to_numpy()
path = "./loss_values_qngm2_0.02.csv"
loss2 = pd.read_csv(path, sep=",", header=None).to_numpy()
path = "./loss_values_qngm2_0.03.csv"
loss3 = pd.read_csv(path, sep=",", header=None).to_numpy()
path = "./loss_values_qngm2_0.04.csv"
loss4 = pd.read_csv(path, sep=",", header=None).to_numpy()
plt.plot(loss1, color = 'orange', label = 'mitigating ' + str(i))
plt.plot(loss2, color = 'blue', label = 'mitigating ' + str(i))
plt.plot(loss3, color = 'green', label = 'mitigating ' + str(i))
plt.plot(loss4, color = 'red', label = 'mitigating ' + str(i))

plt.legend()
plt.savefig('compare_mitigating.eps', format = 'eps', dpi = 1000)The PostScript backend does not support transparency; partially transparent artists will be rendered opaque.
path = "./loss_values_qngm2_0.01.csv"
loss1 = pd.read_csv(path, sep=",", header=None).to_numpy()
path = "./loss_values_qngm2_0.02.csv"
loss2 = pd.read_csv(path, sep=",", header=None).to_numpy()
path = "./loss_values_qngm2_0.03.csv"
loss3 = pd.read_csv(path, sep=",", header=None).to_numpy()
path = "./loss_values_qngm2_0.04.csv"
loss4 = pd.read_csv(path, sep=",", header=None).to_numpy()
plt.plot(loss1, color = 'orange', label = 'mitigating ' + str(i))
plt.plot(loss2, color = 'blue', label = 'mitigating ' + str(i))
plt.plot(loss3, color = 'green', label = 'mitigating ' + str(i))
plt.plot(loss4, color = 'red', label = 'mitigating ' + str(i))
plt.ylim(0, 0.2)
plt.legend()_____no_output_____


# Notebook from yuqian-deng/pydata-book
Path: ch07_notes.ipynb

Database-style DataFrame Merges_____no_output_____

from pandas import DataFrame, Series
import pandas as pd
import numpy as np
df1 = DataFrame({'key': ['b', 'b', 'a', 'c', 'a', 'a', 'b'],
                 'data1':range(7)})_____no_output_____df2 = DataFrame({'key': ['a', 'b', 'd'],'data2': range(3)})_____no_output_____pd.merge(df1,df2)_____no_output_____pd.merge(df1, df2, on='key')_____no_output_____df3 = DataFrame({'lkey': ['b', 'b', 'a', 'c', 'a', 'a', 'b'], 'data1':range(7)})_____no_output_____df4 = DataFrame({'rkey':['a','b','d'],
                'data2':range(3)})_____no_output_____#By default merge does an 'inner' join
#Other possible options are 'left','right','outer'
pd.merge(df3,df4, left_on='lkey', right_on='rkey')_____no_output_____df1 = DataFrame({'key': ['b', 'b', 'a', 'c', 'a', 'b'], 'data1': range(6)})_____no_output_____df2 = DataFrame({'key': ['a', 'b', 'a', 'b', 'd'], 'data2':range(5)})_____no_output_____pd.merge(df1, df2, on='key', how='left')_____no_output_____pd.merge(df1, df2, how='inner')_____no_output_____left = DataFrame({'key1': ['foo', 'foo', 'bar'],'key2': ['one', 'two', 'one'],
                  'lval': [1, 2, 3]})
right = DataFrame({'key1': ['foo', 'foo', 'bar', 'bar'],'key2': ['one', 'one', 'one', 'two'],
                   'rval': [4, 5, 6, 7]})_____no_output_____pd.merge(left,right,on=['key1','key2'],how='outer')_____no_output_____pd.merge(left, right, on='key1')_____no_output_____pd.merge(left, right, on='key1', suffixes=('_left','_right'))_____no_output_____

Merging on Index_____no_output_____

left1 = DataFrame({'key': ['a', 'b', 'a', 'a', 'b', 'c'], 'value':range(6)})
right1 = DataFrame({'group_val': [3.5, 7]}, index=['a', 'b'])_____no_output_____pd.merge(left1, right1, left_on='key',right_index=True)_____no_output_____pd.merge(left1, right1, left_on='key',right_index=True, how='outer')_____no_output_____lefth = DataFrame({'key1': ['Ohio', 'Ohio', 'Ohio', 'Nevada', 'Nevada'],
                   'key2': [2000, 2001, 2002, 2001, 2002],
                   'data': np.arange(5.)})_____no_output_____righth = DataFrame(np.arange(12).reshape((6, 2)),
                   index=[['Nevada', 'Nevada', 'Ohio', 'Ohio', 'Ohio', 'Ohio'],
                          [2001, 2000, 2000, 2000, 2001, 2002]],
                   columns=['event1', 'event2'])
righth_____no_output_____pd.merge(lefth, righth, left_on=['key1','key2'], right_index=True, how='outer')_____no_output_____left2 = DataFrame([[1., 2.], [3., 4.], [5., 6.]], index=['a', 'c', 'e'],
                  columns=['Ohio', 'Nevada'])
right2 = DataFrame([[7., 8.], [9., 10.], [11., 12.], [13, 14]],
                   index=['b', 'c', 'd', 'e'], columns=['Missouri', 'Alabama'])_____no_output_____pd.merge(left2, right2, how='outer', left_index=True, right_index=True)_____no_output_____left2.join(right2, how='outer')_____no_output_____left1.join(right1, on='key')_____no_output_____another = DataFrame([[7., 8.], [9., 10.], [11., 12.], [16., 17.]],
                    index=['a', 'c', 'e', 'f'], columns=['New York', 'Oregon'])_____no_output_____left2.join([right2,another])_____no_output_____left2.join([right2, another], how='outer')_____no_output_____

Concatenating along an axis_____no_output_____

arr = np.arange(12).reshape((3,4))
arr_____no_output_____np.concatenate([arr,arr],axis=1)_____no_output_____s1 = Series([0,1], index=['a','b'])
s2 = Series([2,3,4], index=['c','d','e'])
s3 = Series([5,6], index=['f','g'])_____no_output_____pd.concat([s1,s2,s3])_____no_output_____pd.concat([s1,s2,s3], axis=1)_____no_output_____s4 = pd.concat([s1*5,s3])
print(s4)
s1a    0
b    5
f    5
g    6
dtype: int64
pd.concat([s1,s4],axis=1)_____no_output_____pd.concat([s1,s4], axis=1, join='inner')_____no_output_____pd.concat([s1,s4], axis=1).reindex(['a','c','b','e'])_____no_output_____#To make the concatenated pieces identifiable
result = pd.concat([s1,s2,s3], keys=['one','two','three'])
result_____no_output_____result.unstack()_____no_output_____pd.concat([s1,s2,s3], axis=1, keys=['one','two','three'])_____no_output_____df1 = DataFrame(np.arange(6).reshape(3, 2), index=['a', 'b', 'c'],
                columns=['one', 'two'])
df2 = DataFrame(5 + np.arange(4).reshape(2, 2), index=['a', 'c'],
                columns=['three', 'four'])_____no_output_____pd.concat([df1,df2], axis=1, keys=['level1','level2'])_____no_output_____pd.concat({'level1':df1,'level2':df2}, axis=1)_____no_output_____pd.concat([df1, df2], axis=1, keys=['level1', 'level2'],
          names=['upper','lower'])_____no_output_____df1 = DataFrame(np.random.randn(3, 4), columns=['a', 'b', 'c', 'd'])
df2 = DataFrame(np.random.randn(2, 3), columns=['b', 'd', 'a'])_____no_output_____pd.concat([df1,df2], ignore_index=True)_____no_output_____

Combining Data with Overlap_____no_output_____

a = Series([np.nan, 2.5, np.nan, 3.5, 4.5, np.nan],
           index=['f', 'e', 'd', 'c', 'b', 'a'])
b = Series(np.arange(len(a), dtype=np.float64),
           index=['f', 'e', 'd', 'c', 'b', 'a'])_____no_output_____b[-1] = np.nan_____no_output_____type(np.where(pd.isnull(a), b, a))_____no_output_____#"patching" missing data in the calling object with data from object passed
b[:-2].combine_first(a[2:])_____no_output_____df1 = DataFrame({'a': [1., np.nan, 5., np.nan],
                 'b': [np.nan, 2., np.nan, 6.],
                 'c': range(2, 18, 4)})
df2 = DataFrame({'a': [5., 4., np.nan, 3., 7.],
                 'b': [np.nan, 3., 4., 6., 8.]})_____no_output_____df1.combine_first(df2)_____no_output_____

Reshaping and Pivoting_____no_output_____Reshaping with Hierarchical Indexing_____no_output_____

data = DataFrame(np.arange(6).reshape((2, 3)),
                 index=pd.Index(['Ohio', 'Colorado'], name='state'),
                 columns=pd.Index(['one', 'two', 'three'], name='number'))_____no_output_____result = data.stack()_____no_output_____#by default the innermost level is unstacked
result.unstack()_____no_output_____result.unstack(0)_____no_output_____result.unstack('state')_____no_output_____s1 = Series([0, 1, 2, 3], index=['a', 'b', 'c', 'd'])
s2 = Series([4, 5, 6], index=['c', 'd', 'e'])
data2 = pd.concat([s1, s2], keys=['one', 'two'])
data2_____no_output_____data2.unstack()_____no_output_____data2.unstack().stack()_____no_output_____data2.unstack().stack(dropna=False)_____no_output_____df = DataFrame({'left': result, 'right': result + 5},
               columns=pd.Index(['left', 'right'], name='side'))
result_____no_output_____df_____no_output_____df.unstack('state')_____no_output_____df.unstack('state').stack('side')_____no_output_____

Pivoting "long" to "wide" format_____no_output_____

#'pivot' is just a shortcut for creating a hierarchical index using set_index
#and reshaping with unstack_____no_output_____

Data Transformation

Filtering, cleaning, and other transformations_____no_output_____Removing Duplicates_____no_output_____

data = DataFrame({'k1': ['one'] * 3 + ['two'] * 4,
                  'k2': [1, 1, 2, 3, 3, 4, 4]})
data.duplicated()_____no_output_____data.drop_duplicates()_____no_output_____data['v1'] = range(7)
data.drop_duplicates(['k1'])_____no_output_____data.drop_duplicates(['k1','k2'], keep='last')_____no_output_____

Transforming Data using a function or mapping_____no_output_____

data = DataFrame({'food': ['bacon', 'pulled pork', 'bacon', 'Pastrami',
                           'corned beef', 'Bacon', 'pastrami', 'honey ham',
                           'nova lox'],
                  'ounces': [4, 3, 12, 6, 7.5, 8, 3, 5, 6]})_____no_output_____meat_to_animal = {
'bacon': 'pig',
'pulled pork': 'pig',
'pastrami': 'cow',
'corned beef': 'cow',
'honey ham': 'pig',
'nova lox': 'salmon'
}_____no_output_____data['animal'] = data['food'].map(str.lower).map(meat_to_animal)_____no_output_____data['food'].map(lambda x: meat_to_animal[x.lower()])_____no_output_____

Replacing values_____no_output_____

data = Series([1., -999., 2., -999., -1000., 3.])_____no_output_____data.replace(-999, np.nan)_____no_output_____data.replace([-999,1000],np.nan)_____no_output_____data.replace([-999,1000],[np.nan,0])_____no_output_____data.replace({-999:np.nan,1000:0})_____no_output_____

Renaming axis indexes_____no_output_____

data = DataFrame(np.arange(12).reshape((3, 4)),
                 index=['Ohio', 'Colorado', 'New York'],
                 columns=['one', 'two', 'three', 'four'])_____no_output_____data.index.map(str.upper)_____no_output_____data.index = data.index.map(str.upper)_____no_output_____data_____no_output_____data.rename(index=str.title, columns=str.upper)_____no_output_____data_____no_output_____data.rename(index={'OHIO': 'INDIANA'},columns={'three': 'peekaboo'})_____no_output______ = data.rename(index={'OHIO': 'INDIANA'}, inplace=True)_____no_output_____data_____no_output_____

Discretization and binning

Continuous data is often discretized or otherwised separated into “bins” for analysis._____no_output_____

ages = [20, 22, 25, 27, 21, 23, 37, 31, 61, 45, 41, 32]_____no_output_____bins = [18, 25, 35, 60, 100]
cats = pd.cut(ages,bins)_____no_output_____cats_____no_output_____pd.value_counts(cats)_____no_output_____pd.cut(ages, [18,26,36,61,100], right=False)_____no_output_____group_names = ['Youth', 'YoungAdult', 'MiddleAged', 'Senior']
pd.cut(ages, bins, labels=group_names)_____no_output_____data = np.random.rand(20)_____no_output_____pd.cut(data,4,precision=2)_____no_output_____data = np.random.randn(1000)_____no_output_____cats = pd.qcut(data,4) # Cut into quantiles_____no_output_____cats_____no_output_____pd.value_counts(cats)_____no_output_____pd.qcut(data,[0, 0.1, 0.5, 0.9, 1.])_____no_output_____pd.value_counts(pd.qcut(data,[0, 0.1, 0.5, 0.9, 1.]))_____no_output_____

Detecting and filtering outliers_____no_output_____

np.random.seed(12345)
data = DataFrame(np.random.randn(1000,4))
data.describe()_____no_output_____col = data[3]
col[np.abs(col)>3]_____no_output_____data[(np.abs(data)>3).any(1)]_____no_output_____data[(np.abs(data)>3)][:6]_____no_output_____# to cap values outside the interval -3 to 3
data[np.abs(data)>3]=np.sign(data)*3_____no_output_____data.describe()_____no_output_____

Permutation and random sampling_____no_output_____

df = DataFrame(np.arange(5 * 4).reshape(5, 4))_____no_output_____sampler = np.random.permutation(5)_____no_output_____sampler_____no_output_____df.take(sampler)_____no_output_____# To select a random subset without replacement, one way is to slice off the
# first k elements of the array returned by permutation, where k is the desired sample size
df.take(np.random.permutation(len(df))[:3])_____no_output_____# To generate a sample with replacement
# the fastest way is to use np.random.randint to draw random integers
bag = np.array([5, 7, -1, 6, 4])
sampler = np.random.randint(0, len(bag), size=10)
sampler_____no_output_____draws = bag.take(sampler)
draws_____no_output_____

Computing indicator/Dummy variables_____no_output_____

df = DataFrame({'key': ['b', 'b', 'a', 'c', 'a', 'b'],
                'data1': range(6)})
pd.get_dummies(df['key'])_____no_output_____dummies = pd.get_dummies(df['key'], prefix='key')
df_with_dummy = df[['data1']].join(dummies)
df_with_dummy_____no_output_____mnames = ['movie_id', 'title', 'genres']_____no_output_____pwd_____no_output_____cd datasetsC:\Users\yuqia\pydata-book\datasets
cd movielensC:\Users\yuqia\pydata-book\datasets\movielens
movies = pd.read_table('movies.dat', sep='::', header=None, names=mnames, engine='python')
movies[:10]_____no_output_____genre_iter = (set(x.split('|')) for x in movies['genres'])_____no_output_____type(genre_iter)_____no_output_____genres=sorted(set.union(*genre_iter))_____no_output_____dummies = DataFrame(np.zeros((len(movies), len(genres))),columns=genres)_____no_output_____for i, gen in enumerate(movies.genres):
    dummies.loc[i, gen.split('|')]=1_____no_output_____movies_windic = movies.join(dummies.add_prefix('Genre_'))_____no_output_____movies_windic.iloc[0]_____no_output_____values = np.random.rand(10)_____no_output_____values_____no_output_____bins = [0, 0.2, 0.4, 0.6, 0.8, 1]_____no_output_____pd.get_dummies(pd.cut(values, bins))_____no_output_____

String Manipulation_____no_output_____String object methods_____no_output_____

val = 'a,b, guido'_____no_output_____val.split(',')_____no_output_____pieces = [x.strip() for x in val.split(',')]
pieces_____no_output_____pieces = [x.strip() for x in val.split(',')]
pieces_____no_output_____first, second, third = pieces_____no_output_____first + '::' + second + '::' + third_____no_output_____'::'.join(pieces)_____no_output_____'guido' in val_____no_output_____val.index(',')_____no_output_____val.find(':')_____no_output_____val.index(':')_____no_output_____val.count(',')_____no_output_____val.replace(',','::')_____no_output_____val.replace(',','')_____no_output_____

Regular Expressions_____no_output_____

import re_____no_output_____text = "foo bar\t baz \tqux"_____no_output_____re.split('\s+', text)_____no_output_____regex = re.compile('\s+')_____no_output_____regex.split(text)_____no_output_____regex.findall(text)_____no_output_____text = """Dave dave@google.com
Steve steve@gmail.com
Rob rob@gmail.com
Ryan ryan@yahoo.com
"""
pattern = r'[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,4}'
# re.IGNORECASE makes the regex case-insensitive
regex = re.compile(pattern, flags=re.IGNORECASE)_____no_output_____regex.findall(text)_____no_output_____type(regex)_____no_output_____m = regex.search(text)
m_____no_output_____text[m.start():m.end()]_____no_output_____print(regex.match(text))None
print(regex.sub('REDACTED',text))Dave REDACTED
Steve REDACTED
Rob REDACTED
Ryan REDACTED

pattern = r'([A-Z0-9._%+-]+)@([A-Z0-9.-]+)\.([A-Z]{2,4})'
regex = re.compile(pattern, flags=re.IGNORECASE)_____no_output_____m = regex.match('wesm@bright.net')
m.groups()_____no_output_____regex.findall(text)_____no_output_____print(regex.sub(r'Username: \1, Domain: \2, Suffix: \3', text))Dave Username: dave, Domain: google, Suffix: com
Steve Username: steve, Domain: gmail, Suffix: com
Rob Username: rob, Domain: gmail, Suffix: com
Ryan Username: ryan, Domain: yahoo, Suffix: com

type(regex.sub(r'Username: \1, Domain: \2, Suffix: \3', text))_____no_output_____regex = re.compile(r"""
(?P<username>[A-Z0-9._%+-]+)
@
(?P<domain>[A-Z0-9.-]+)
\.
(?P<suffix>[A-Z]{2,4})""", flags=re.IGNORECASE|re.VERBOSE)_____no_output_____m = regex.match('wesm@bright.net')_____no_output_____m.groupdict()_____no_output_____

Vectorized string functions in pandas_____no_output_____

data = {'Dave': 'dave@google.com', 'Steve': 'steve@gmail.com',
        'Rob': 'rob@gmail.com', 'Wes': np.nan}
data = Series(data)_____no_output_____data.isnull()_____no_output_____data.str.contains('gmail')_____no_output_____pattern_____no_output_____data.str.findall(pattern, flags=re.IGNORECASE)_____no_output_____matches = data.str.findall(pattern, flags=re.IGNORECASE)
matches_____no_output_____type(matches)_____no_output_____type(matches.str[0])_____no_output_____matches = matches.str[0]_____no_output_____matches.str.get(1)_____no_output_____data.str[:5]_____no_output_____

Example: USDA food database_____no_output_____

pwd_____no_output_____cd -C:\Users\yuqia\pydata-book\datasets
cd datasets[WinError 2] The system cannot find the file specified: 'datasets'
C:\Users\yuqia\pydata-book\datasets
cd usda_foodC:\Users\yuqia\pydata-book\datasets\usda_food
import json
db = json.load(open('database.json'))_____no_output_____len(db)_____no_output_____db[0].keys()_____no_output_____db[0]['nutrients'][0]_____no_output_____nutrients = DataFrame(db[0]['nutrients'])_____no_output_____nutrients[:7]_____no_output_____info_keys = ['description','group','id','manufacturer']
info = DataFrame(db, columns=info_keys)_____no_output_____info[:5]_____no_output_____info_____no_output_____pd.value_counts(info.group)[:10]_____no_output_____nutrients = []
for rec in db:
    fnuts = DataFrame(rec['nutrients'])
    fnuts['id'] = rec['id']
    nutrients.append(fnuts)

nutrients = pd.concat(nutrients, ignore_index=True)_____no_output_____nutrients_____no_output_____nutrients.duplicated().sum()_____no_output_____nutrients = nutrients.drop_duplicates()_____no_output_____col_mapping1 = {'description' : 'food',
              'group' : 'fgroup'}
col_mapping2 = {'description' : 'nutrient',
               'group' : 'nutgroup'}
info = info.rename(columns=col_mapping1, copy=False)
nutrients = nutrients.rename(columns=col_mapping2, copy=False)_____no_output_____nutrients_____no_output_____info_____no_output_____ndata = pd.merge(nutrients, info, on='id', how='outer')_____no_output_____ndata_____no_output_____result = ndata.groupby(['nutrient','fgroup'])['value'].quantile(0.5)_____no_output_____result_____no_output_____result['Zinc, Zn'].sort_values().plot(kind='barh')_____no_output_____#find which food is most dense in each nutrient
by_nutrient = ndata.groupby(['nutgroup','nutrient'])

get_maximum = lambda x: x.xs(x.value.idxmax())
get_minimum = lambda x: x.xs(x.value.idxmin())

max_foods = by_nutrient.apply(get_maximum)

max_foods.food = max_foods.food.str[:50]_____no_output_____max_foods.loc['Amino Acids']['food']_____no_output_____max_foods_____no_output_____