# Notebook from Mohitsharma44/ucsl17
Path: notebooks/05-00 Statistics/UCSL2016 continuous distributions demo.ipynb

author__ = "__fbbianco__"
''' CUSP 2016 UCSL
    Continuous probability distributions demos
'''
import numpy as np
import matplotlib.pyplot as plt
import pylab as pl
import pylab as pl
import time
import json
import os

from IPython import display

# this imports some of the pylab settings I like for plotting
#if you want to run the notebook you can comment it out, or download the files
#here and set up the environmentail variable PUI2015
s = json.load( open(os.getenv('PUI2015')+"/fbb_matplotlibrc.json") )

pl.rcParams.update(s)
pl.rcParams['font.size'] = 20

%pylab inlinePopulating the interactive namespace from numpy and matplotlib

# Evolution of samples drawn under different distributions as the distribution parameters and sample size change_____no_output_____### Binomial B(100,p), change p_____no_output_____

from IPython import display
np.random.seed(111)
n = 100
N = 1000
pl.figure(figsize=(10, 6))

#for loop over the parameter p
for p in [0.1, 0.25, 0.5, 0.75, 0.9]:
    pl.clf()

    pl.hist(np.random.binomial(n, p, N), alpha = 0.8,
            label = "Binomial n=%d, p=%.1f, N=%d"%(n, p, N),
            color = 'SteelBlue', normed = True)
    pl.ylabel("PMF", fontsize = 20)
    pl.xlabel("x", fontsize = 20)
    pl.xlim(-1,100)
    pl.legend(fontsize = 20)

    display.display(pl.gcf())
    display.clear_output(wait = True)
    time.sleep(1)_____no_output_____

### Poisson distribution P(lambda), changing  lambda_____no_output_____

from scipy.stats import poisson

np.random.seed(111)
pl.figure(figsize=(10, 6))

# loop pver the parameter lambda
for lmbd in [1, 5, 10, 25, 50]:
    pl.clf()
    pl.hist(np.random.poisson(lmbd, size=1000), alpha = 0.8,
            label = r"Poisson $\lambda=%.1f$"%(lmbd),
            color = 'SteelBlue', normed = True)
    pl.xlim(-1, 100)
    pl.ylabel("PMF", fontsize = 20)
    pl.xlabel("x", fontsize = 20)
    pl.legend(fontsize = 20)
    display.display(pl.gcf())
    display.clear_output(wait = True)
    time.sleep(1)

# overplot the theoretical distribution: the pmf
x = range(0,100)

# the next line of code is a bit complicated: I want the distribution to be plotted with steps
# this indicates that the distribution is NOT DEFINED for non integer values of x
pl.plot(np.array([[xx,xx] for xx in x]).flatten()[1:],
        np.array([[poisson.pmf(xx, lmbd),poisson.pmf(xx, lmbd)] for xx in x]).flatten()[:-1],
        '-', color = 'IndianRed')
display.display(pl.gcf())
display.clear_output(wait = True)_____no_output_____

### Gaussian distribution N(0,1), changing sample size_____no_output_____

#it is very easy to write a gaussian distribution from scratch
def gaussian(x, mu, sig):
    norm = 1.0 / np.sqrt(2.0 * np.pi) / sig
    return norm * np.exp(-np.power(x - mu, 2.) / (2 * np.power(sig, 2.)))

np.random.seed(111)
pl.figure(figsize=(10, 6))
for n in [10, 50, 100, 500, 1000]:

    pl.clf()
    pl.ylabel("PDF", fontsize = 20)
    pl.xlabel("x", fontsize = 20)
    pl.hist(np.random.randn(n), color = 'IndianRed',
            normed = True, alpha = 0.8,
            label = r"Gaussian $N$(0,1) N=%d"%(N))
    pl.xlim(-3.5, 3.5)
    pl.ylim(0, 0.8)
    pl.legend(fontsize = 20)
    display.display(pl.gcf())
    display.clear_output(wait = True)

#overplot the theoretical distribution: the pmf
x = np.arange(-3.5, 3.5, 0.1)
pl.plot(x, gaussian(x, 0, 1), color = 'SteelBlue')
display.display(pl.gcf())
display.clear_output(wait = True)_____no_output_____

## Gaussian distribution N(mu, sigma), changing mu and sigma (sample size = 1000)
_____no_output_____

#it is very easy to write a gaussian distribution from scratch
def gaussian(x, mu, sig):
    norm = 1.0 / np.sqrt(2.0 * np.pi) / sig
    return norm * np.exp(-np.power(x - mu, 2.) / (2 * np.power(sig, 2.)))

muold, sigmaold = np.nan, np.nan
np.random.seed(111)
pl.figure(figsize=(10,6))
n = 1000

# loop over mu-sigma pairs with the zip function:
# the n-th loop iteration will take the n-th entry in the mu list,
# and the n-th entry in the sigmas list
for mu,sigma in zip([0,1,5,5,5,10,100], [0.5,0.5,0.5,1,5,5,5]):
    pl.clf()

    # add an if statemeth that prevents us from trying to plot
    # the previous PDF if we are at the first loop step
    # i could have used a try/ecvept too to make it more pythonic
    if not np.isnan(muold):
        pl.plot(x, gaussian(x, muold, sigmaold), color='SteelBlue',
            alpha = 0.5,
            label = r"$N$(%d, %.1f) N=%d"%(muold, sigmaold, N))

    pl.hist(np.random.normal(mu, sigma,n), color = 'IndianRed',
            normed = True, alpha=0.8,
            label = r"$N$(%d, %.1f) N=%d"%(mu, sigma, N))

    pl.xlim(mu - 8 * sigma, mu + 8 * sigma)
    pl.ylabel("PDF", fontsize = 20)
    pl.xlabel("x", fontsize = 20)
    pl.xticks(fontsize = 20)
    pl.yticks(fontsize = 20)
    pl.legend(fontsize = 20)
    time.sleep(1)

    #overplot the theoretical distribution: the pdf
    x = np.linspace(mu-4*sigma, mu+4*sigma, 100)
    pl.plot(x, gaussian(x, mu, sigma), color='SteelBlue',
           label=r"$N$(%d, %.1f) N=%d"%(mu, sigma, N))
    display.display(pl.gcf())
    display.clear_output(wait=True)
    time.sleep(1)
    muold, sigmaold  = mu, sigma

pl.ylabel("probability")_____no_output_____

# Central Limit Theorem demonstration with binomial distribution

for ANY distribution, as the sample size
increases,

the distribution of sample means
converges to the population mean

and the distribution of the means is itself Gaussian _____no_output_____

np.random.seed(111)
n = 100
p = 0.5
fig = pl.figure(figsize=(15,6))

sampSizes = np.array([3,10,20,50,100,200,500,1000])
means = np.zeros(sampSizes.shape) * np.nan

# loop over sample size
for i, N in enumerate(sampSizes):
    fig.clf()
    ax1 = fig.add_subplot(121)
    ax2 = fig.add_subplot(122)
    sample = np.random.binomial(n, p, N)

    heights = ax1.hist(sample, alpha = 0.8,
            label = "Binomial n=%d, p=%.1f, N=%d"%(n, p, N),
            color = 'SteelBlue', normed = True)
    ax1.set_xlim(25, 75)

    ax1.set_ylabel("PMF", fontsize = 15)
    ax1.set_xlabel("x", fontsize = 15)
    ax1.plot([100*0.5, 100*0.5], [0, pl.ylim()[1]], "IndianRed")
    means[i] = sample.mean()
    ax1.plot([means[-1], means[-1]], [0, pl.ylim()[1]], "k",
           label = "Sample mean")

    ax1.legend(fontsize = 18)
    ax1.set_ylim(0, heights[0].max()*1.3)
    ax1.text(100*0.47, heights[0].max()*0.5, "Pop mean", rotation=90,
           color="IndianRed", fontsize=18)
    ax2.plot(sampSizes, means, '-o', label = "sample mean")
    ax2.set_xlabel("sample size N")
    ax2.set_ylabel("sample mean", fontsize=18)
    ax2.plot(sampSizes, np.ones(sampSizes.shape) * n * p, label="Pop mean")
    ax2.set_ylim(47, 52)
    ax2.text(400, 50.3, "Pop mean",
           color="IndianRed", fontsize=18)
    display.display(pl.gcf())
    display.clear_output(wait=True)
    time.sleep(1)

binmeans = [np.random.binomial(n, p, 1000).mean() for N in np.arange(1,1000)]
pl.xlabel("X")
    pl.ylabel("PDF")
pl.figure(figsize = (10,6))
patch = pl.hist(binmeans, bins=30, label = "Distribution of means")
pl.plot(np.arange(48, 52, 0.1),
        gaussian(np.arange(48, 52, 0.1), n*p, 0.15)*35, 'k', label = "Gaussian")
pl.legend(fontsize = 18)_____no_output_____

# the Poisson Distribution approaches Gaussian for large $lambda$, with mean lamdba & stdev lambda (and so does the binomial!) _____no_output_____

from scipy.stats import poisson

np.random.seed(111)
pl.figure(figsize = (10,6))
for lmbd in [1, 2, 3, 5, 10, 25]:

    pl.clf()
    pl.hist(np.random.poisson(lmbd, size=1000), alpha = 0.8,
            label=r"Poisson $\lambda=%.1f$"%(lmbd),
            color = 'SteelBlue', normed = True)
    pl.xlim(-40,40)
    pl.ylim(0, pl.ylim()[1]*1.2)

    time.sleep(1)

    #overplot the theoretical distribution: the pmf
    x = np.arange(-40,40,1)
    pl.plot(x, gaussian(x, lmbd, np.sqrt(lmbd)), color='IndianRed',
            label = r"$N$(%d,%.2f)"%(lmbd, np.sqrt(lmbd)))
    pl.legend(fontsize = 18, ncol=2)
    pl.xlabel("X")
    pl.ylabel("PDF")
    display.display(pl.gcf())
    display.clear_output(wait=True)
    time.sleep(1)_____no_output__________no_output_____


# Notebook from ANILPETER/DataCamp
Path: 6. Importing Data in Python (Part 2)/Ch_1_Importing_flat_files_from_the_web/Ch1_Importing_flat_files_from_the_web.ipynb

# 1. Importing flat files from the web _____no_output_____### 1.1 Importing flat files from the web: your turn!

You are about to import your first file from the web! The flat file you will import will be `'winequality-red.csv'` from the University of California, Irvine's [Machine Learning repository](http://archive.ics.uci.edu/ml/index.php). The flat file contains tabular data of physiochemical properties of red wine, such as pH, alcohol content and citric acid content, along with wine quality rating.

The URL of the file is

>'https://s3.amazonaws.com/assets.datacamp.com/production/course_1606/datasets/winequality-red.csv'

After you import it, you'll check your working directory to confirm that it is there and then you'll load it into a `pandas` DataFrame.

**Instructions**
- Import the function `urlretrieve` from the subpackage urllib.request.
- Assign the URL of the file to the variable `url`.
- Use the function `urlretrieve()` to save the file locally as `'winequality-red.csv'`.
- Execute the remaining code to load `'winequality-red.csv'` in a pandas DataFrame and to print its head to the shell._____no_output_____

# Import package
from urllib.request import urlretrieve

# Import pandas
import pandas as pd

# Assign url of file: url

url = 'https://s3.amazonaws.com/assets.datacamp.com/production/course_1606/datasets/winequality-red.csv'

# Save file locally
urlretrieve(url,'winequality-red.csv')

# Read file into a DataFrame and print its head
df = pd.read_csv('winequality-red.csv', sep=';')
print(df.head())   fixed acidity  volatile acidity  citric acid  residual sugar  chlorides  \
0            7.4              0.70         0.00             1.9      0.076
1            7.8              0.88         0.00             2.6      0.098
2            7.8              0.76         0.04             2.3      0.092
3           11.2              0.28         0.56             1.9      0.075
4            7.4              0.70         0.00             1.9      0.076

   free sulfur dioxide  total sulfur dioxide  density    pH  sulphates  \
0                 11.0                  34.0   0.9978  3.51       0.56
1                 25.0                  67.0   0.9968  3.20       0.68
2                 15.0                  54.0   0.9970  3.26       0.65
3                 17.0                  60.0   0.9980  3.16       0.58
4                 11.0                  34.0   0.9978  3.51       0.56

   alcohol  quality
0      9.4        5
1      9.8        5
2      9.8        5
3      9.8        6
4      9.4        5

### 1.2 Opening and reading flat files from the web

You have just imported a file from the web, saved it locally and loaded it into a DataFrame. If you just wanted to load a file from the web into a DataFrame without first saving it locally, you can do that easily using `pandas`. In particular, you can use the function `pd.read_csv()` with the URL as the first argument and the separator `sep` as the second argument.

The URL of the file, once again, is

>'https://s3.amazonaws.com/assets.datacamp.com/production/course_1606/datasets/winequality-red.csv'

**Instructions**
- Assign the URL of the file to the variable `url`.
- Read file into a DataFrame df using `pd.read_csv()`, recalling that the separator in the file is `';'`.
- Print the head of the DataFrame `df`.
- Execute the rest of the code to plot histogram of the first feature in the DataFrame `df`._____no_output_____

# Import packages
import matplotlib.pyplot as plt
import pandas as pd

# Assign url of file: url
url = 'https://s3.amazonaws.com/assets.datacamp.com/production/course_1606/datasets/winequality-red.csv'

# Read file into a DataFrame: df

df = pd.read_csv(url,sep=';')
# Print the head of the DataFrame
print(df.head())

# Plot first column of df
pd.DataFrame.hist(df.iloc[:, 0:1])
plt.xlabel('fixed acidity (g(tartaric acid)/dm$^3$)')
plt.ylabel('count')
plt.show()   fixed acidity  volatile acidity  citric acid  residual sugar  chlorides  \
0            7.4              0.70         0.00             1.9      0.076
1            7.8              0.88         0.00             2.6      0.098
2            7.8              0.76         0.04             2.3      0.092
3           11.2              0.28         0.56             1.9      0.075
4            7.4              0.70         0.00             1.9      0.076

   free sulfur dioxide  total sulfur dioxide  density    pH  sulphates  \
0                 11.0                  34.0   0.9978  3.51       0.56
1                 25.0                  67.0   0.9968  3.20       0.68
2                 15.0                  54.0   0.9970  3.26       0.65
3                 17.0                  60.0   0.9980  3.16       0.58
4                 11.0                  34.0   0.9978  3.51       0.56

   alcohol  quality
0      9.4        5
1      9.8        5
2      9.8        5
3      9.8        6
4      9.4        5

### 1.3 Importing non-flat files from the web
Congrats! You've just loaded a flat file from the web into a DataFrame without first saving it locally using the pandas function `pd.read_csv()`. This function is super cool because it has close relatives that allow you to load all types of files, not only flat ones. In this interactive exercise, you'll use `pd.read_excel()` to import an Excel spreadsheet.

The URL of the spreadsheet is

>'http://s3.amazonaws.com/assets.datacamp.com/course/importing_data_into_r/latitude.xls'

Your job is to use `pd.read_excel()` to read in all of its sheets, print the sheet names and then print the head of the first sheet using its name, not its index.

Note that the output of `pd.read_excel()` is a Python dictionary with sheet names as keys and corresponding DataFrames as corresponding values.

**Instructions**

- Assign the URL of the file to the variable `url`.
- Read the file in url into a dictionary `xl` using `pd.read_excel()` recalling that, in order to import all sheets you need to pass None to the argument `sheetname`.
- Print the names of the sheets in the Excel spreadsheet; these will be the keys of the dictionary `xl`.
- Print the head of the first sheet using the sheet name, not the index of the sheet! The sheet name is `'1700'`_____no_output_____

# Import package
import pandas as pd

# Assign url of file: url

url = 'http://s3.amazonaws.com/assets.datacamp.com/course/importing_data_into_r/latitude.xls'

# Read in all sheets of Excel file: xl
xl = pd.read_excel(url,sheet_name=None)

# Print the sheetnames to the shell
print(xl.keys())

# Print the head of the first sheet (using its name, NOT its index)
print(xl['1700'].head())
odict_keys(['1700', '1900'])
                 country       1700
0            Afghanistan  34.565000
1  Akrotiri and Dhekelia  34.616667
2                Albania  41.312000
3                Algeria  36.720000
4         American Samoa -14.307000

# 2. HTTP requests to import files from the web_____no_output_____### 2.1 Performing HTTP requests in Python using urllib

Now that you know the basics behind HTTP GET requests, it's time to perform some of your own. In this interactive exercise, you will ping our very own DataCamp servers to perform a GET request to extract information from our teach page, `"http://www.datacamp.com/teach/documentation"`.

In the next exercise, you'll extract the HTML itself. Right now, however, you are going to package and send the request and then catch the response.

**Instructions**
- Import the functions urlopen and Request from the subpackage `urllib.request`.
- Package the request to the url `"http://www.datacamp.com/teach/documentation"` using the function `Request()` and assign it to `request`.
- Send the request and catch the response in the variable response with the function `urlopen()`.
- Run the rest of the code to see the datatype of response and to `close` the connection!_____no_output_____

# Import packages
from urllib.request import urlopen , Request

# Specify the url
url = "http://www.datacamp.com/teach/documentation"

# This packages the request:
request = Request(url)

# Sends the request and catches the response:
response = urlopen(request)

# Print the datatype of response
print(type(response))

# Be polite and close the response!
response.close()<class 'http.client.HTTPResponse'>

### Printing HTTP request results in Python using urllib

You have just packaged and sent a GET request to `"http://www.datacamp.com/teach/documentation"` and then caught the response. You saw that such a response is a `http.client.HTTPResponse object`. The question remains: what can you do with this response?

Well, as it came from an HTML page, you could read it to extract the HTML and, in fact, such a `http.client.HTTPResponse` object has an associated `read()` method. In this exercise, you'll build on your previous great work to extract the response and print the HTML.

**Instructions**
- Send the request and catch the response in the variable response with the function `urlopen()`, as in the previous exercise.
- Extract the response using the `read()` method and store the result in the variable `html`.
- Print the string `html`.
- Hit submit to perform all of the above and to close the response: be tidy!_____no_output_____

# Import packages
from urllib.request import urlopen, Request

# Specify the url
url = "http://www.datacamp.com/teach/documentation"

# This packages the request
request = Request(url)

# Sends the request and catches the response:

response = urlopen(request)
# Extract the response: html

html = response.read()
# Print the html
print(html)

# Be polite and close the response!
response.close()b'<!doctype html>\n\n  \n    <link href="https://fonts.intercomcdn.com" rel="preconnect" crossorigin>\n      \n      \n    <meta charset="utf-8">\n    <meta http-equiv="X-UA-Compatible" content="IE=edge">\n    <title>DataCamp Help Center</title>\n    <meta name="description" content="">\n    <meta name="viewport" content="width=device-width, initial-scale=1">\n\n      <link rel="alternate" href="http://instructor-support.datacamp.com/en/" hreflang="en">\n\n      <meta name="intercom:trackingEvent" content="{"name":"Viewed Help Center","metadata":{"action":"viewed","object":"educate_home","place":"help_center","owner":"educate","default_locale":"en","current_locale":"en","is_default_locale":true}}" />\n\n    <link rel="stylesheet" media="all" href="https://static.intercomassets.com/alexandria/assets/application-00b67c7c87b72734807679ffe67cf68adf711566c397121c5d3432b6d9ff1657.css" />\n    <link rel="canonical" href="http://instructor-support.datacamp.com/en/"/>\n\n        <link href="https://static.intercomassets.com/assets/educate/educate-favicon-64x64-at-2x-52016a3500a250d0b118c0a04ddd13b1a7364a27759483536dd1940bccdefc20.png" rel="shortcut icon" type="image/png" />\n      \n\n      <meta property="og:title" content="DataCamp Help Center" />\n  <meta name="twitter:title" content="DataCamp Help Center" />\n\n\n<meta property="og:type" content="website" />\n<meta property="og:image" content="" />\n\n<meta name="twitter:image" content="" />\n\n  \n  \n    \n
\n
\n
\n
\n
\n            \n                \n            \n
\n
\n
\n
\n              <svg width="14" height="14" viewBox="0 0 14 14" xmlns="http://www.w3.org/2000/svg"><title>Group 65</title><g stroke="#FFF" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"><path d="M11.5 6.73v6.77H.5v-11h7.615M4.5 9.5l7-7M13.5 5.5v-5h-5"/></g></svg>Go to DataCamp\n            \n
\n
\n
\n
Advice and answers from the DataCamp Team
\n      <form action="/en/" autocomplete="off" class="header__form search">\n        <input type="text" autocomplete="off" class="search__input js__search-input o__ltr" placeholder="Search for articles..." tabindex="1" name="q" value="">\n
\n          <button type="submit" class="search__submit o__ltr"></button>\n          \n            <svg class="interface-icon" xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 16 16">\n              <path d="M8.018 6.643L5.375 4 4 5.375l2.643 2.643L4 10.643 5.375 12l2.643-2.625L10.625 12 12 10.643 9.357 8.018 12 5.375 10.643 4z" />\n            </svg>\n          \n      </form>\n
\n
\n
\n\n\n
\n

\n
\n      \n
\n
\n            <svg role=\'img\' viewBox=\'0 0 48 48\'><g id="chat-star" stroke-width="2" fill="none" fill-rule="evenodd" stroke-linejoin="round"><path d="M20 34.942c-2.083-.12-4.292-.42-6-.942L3 39l4-9c-3.858-3.086-6-7.246-6-12C1 8.61 10.328 1 21.835 1 33.343 1 43 8.61 43 18c0 1.044-.117 2.065-.342 3.057"></path><path d="M36.016 25L40 33h7l-6 5 3 9-8-5.494L28 47l3-9-6-5h7l4.016-8z"></path></g></svg>\n
\n
\n
Getting Started
\n
Everything you need to know to begin your DataCamp journey!
\n
\n
\n        \n\n        \n\n        \n\n      +2\n
\n
\n
\n      \n        11 articles in this collection\n      \n
\n      Written by  Becca Robins,  David Venturi,  Yashas Roy and 2 others\n
\n
\n
\n\n
\n
\n      \n
\n
\n      \n
\n
\n            <svg role=\'img\' viewBox=\'0 0 48 48\'><g id="devices-laptop" stroke-width="2" fill="none" fill-rule="evenodd" stroke-linecap="round"><path d="M41 31H7V11h34v20z"></path><path d="M3 35V10a3 3 0 0 1 3-3h36a3 3 0 0 1 3 3v25m-16 0v2H19v-2H1v4a2 2 0 0 0 2 2h42a2 2 0 0 0 2-2v-4H29z" stroke-linejoin="round"></path></g></svg>\n
\n
\n
Courses
\n
Everything you need to know about creating DataCamp courses.
\n
\n
\n        \n\n        \n\n        \n\n      +12\n
\n
\n
\n      \n        89 articles in this collection\n      \n
\n      Written by  Nick Carchedi,  Vincent Vankrunkelsven,  Becca Robins and 12 others\n
\n
\n
\n\n
\n
\n      \n
\n
\n      \n
\n
\n            <svg role=\'img\' viewBox=\'0 0 48 48\'><g id="tools-dashboard" stroke-width="2" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"><path d="M27 31a3 3 0 0 1-6 0 3 3 0 0 1 6 0zm-.88-2.12l9.9-9.9M5 32h4m34 .002L39 32m2.553-8.27l-3.696 1.53M31.27 13.447l-1.53 3.695M24 12v4m-7.27-2.553l1.53 3.695m-7.694.422l2.826 2.83M6.447 23.73l3.695 1.53"></path><path d="M24 8C11.297 8 1 18.3 1 31v9h46v-9C47 18.3 36.703 8 24 8z"></path></g></svg>\n
\n
\n
Daily Practice
\n
Everything you need to know about creating DataCamp Daily Practice.
\n
\n
\n        \n\n
\n
\n
\n      \n        15 articles in this collection\n      \n
\n      Written by  Anneleen Beckers\n
\n
\n
\n\n
\n
\n      \n
\n
\n      \n
\n
\n            <svg role=\'img\' viewBox=\'0 0 48 48\'><g id="book-opened2"><path d="M24 11c0-3.866 10.297-7 23-7v33c-12.703 0-23 3.134-23 7 0-3.866-10.3-7-23-7V4c12.7 0 23 3.134 23 7zm0 0v32m-5-27.52c-3.22-1.232-7.773-2.128-13-2.48m13 8.48c-3.22-1.232-7.773-2.128-13-2.48m13 8.48c-3.22-1.232-7.773-2.128-13-2.48m13 8.48c-3.22-1.23-7.773-2.127-13-2.48m23-15.52c3.223-1.232 7.773-2.128 13-2.48m-13 8.48c3.223-1.232 7.773-2.128 13-2.48m-13 8.48c3.223-1.232 7.773-2.128 13-2.48m-13 8.48c3.223-1.23 7.773-2.127 13-2.48" stroke-width="2" fill="none" stroke-linecap="round" stroke-linejoin="round"></path></g></svg>\n
\n
\n
Projects
\n
Everything you need to know about creating DataCamp projects.
\n
\n
\n        \n\n        \n\n
\n
\n
\n      \n        20 articles in this collection\n      \n
\n      Written by  David Venturi and  Richie Cotton\n
\n
\n
\n\n
\n
\n      \n
\n
\n      \n
\n
\n            <svg role=\'img\' viewBox=\'0 0 48 48\'><g id="book-bookmark" stroke-width="2" fill="none" fill-rule="evenodd" stroke-linecap="round"><path d="M35 31l-6-6-6 6V7h12v24z"></path><path d="M35 9h6v38H11a4 4 0 0 1-4-4V5" stroke-linejoin="round"></path><path d="M39 9V1H11a4 4 0 0 0 0 8h12" stroke-linejoin="round"></path></g></svg>\n
\n
\n
Course Editor Basics
\n
Everything you need to know to get going with our online course editor.
\n
\n
\n        \n\n        \n\n        \n\n
\n
\n
\n      \n        5 articles in this collection\n      \n
\n      Written by  Nick Carchedi,  Amy Peterson, and  Sara Billen\n
\n
\n
\n\n
\n
\n      \n
\n
\n      \n
\n
\n            <svg role=\'img\' viewBox=\'0 0 48 48\'><g id="comms-mail" stroke-width="2" fill="none" fill-rule="evenodd" stroke-linejoin="round"><path d="M47 3L1 22l18 7L47 3z"></path><path d="M47 3l-8 37-20-11L47 3zM19 29v16l7-12"></path></g></svg>\n
\n
\n
Tips & Tricks
\n
Become a DataCamp wizard!
\n
\n
\n        \n\n
\n
\n
\n      \n        6 articles in this collection\n      \n
\n      Written by  Becca Robins\n
\n
\n
\n\n
\n
\n      \n
\n
\n      \n
\n
\n            <svg role=\'img\' viewBox=\'0 0 48 48\'><g id="chat-question" fill="none" fill-rule="evenodd"><path d="M47 21.268c0 10.363-10.297 18.765-23 18.765-2.835 0-5.55-.418-8.058-1.184L2.725 45 7.9 34.668c-4.258-3.406-6.9-8.15-6.9-13.4C1 10.904 11.297 2.502 24 2.502s23 8.402 23 18.766z" stroke-width="2" stroke-linejoin="round"></path><path d="M25 28.502a2 2 0 1 0 0 4 2 2 0 0 0 0-4" fill="#231F1F"></path><path d="M19 17.75c0-3.312 2.686-6.124 6-6.124 3.313 0 6 2.626 6 5.938 0 3.315-2.687 5.938-6 5.938V26" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"></path></g></svg>\n
\n
\n
Frequently Asked Questions (FAQ)
\n
Common questions that arise during content creation.
\n
\n
\n        \n\n        \n\n        \n\n      +3\n
\n
\n
\n      \n        52 articles in this collection\n      \n
\n      Written by  Nick Carchedi,  Becca Robins,  Richie Cotton and 3 others\n
\n
\n
\n\n
\n
\n      \n
\n
\n      \n
\n
\n            <svg role=\'img\' viewBox=\'0 0 48 48\'><g id="tools-edit"><path d="M14.932 43.968L2 47l3.033-12.93 31.2-31.203a4 4 0 0 1 5.658 0l4.247 4.243a4 4 0 0 1 0 5.656L14.932 43.968zm29.84-29.735L34.82 4.28m7.125 12.782L31.992 7.11M15.436 43.465l-9.9-9.9" stroke-width="2" fill="none" stroke-linecap="round" stroke-linejoin="round"></path></g></svg>\n
\n
\n
Miscellaneous
\n
Have a question for DataCamp, but not about creating content? You'll probably find the answer here.
\n
\n
\n        \n\n        \n\n        \n\n      +1\n
\n
\n
\n      \n        10 articles in this collection\n      \n
\n      Written by  Becca Robins,  Jen Bricker,  Lisa Monteleone and 1 other\n
\n
\n
\n\n
\n
\n      \n
\n
\n
\n
\n    \n
\n
\n
\n
\n          \n              \n          \n
\n
\n            \n            We run on Intercom\n
\n
\n
\n
\n\n\n    \n  \n  \n\n    \n\n    \n  \n\n'

Performing HTTP requests in Python using requests
Now that you've got your head and hands around making `HTTP` requests using the `urllib` package, you're going to figure out how to do the same using the higher-level requests library. You'll once again be pinging DataCamp servers for their `"http://www.datacamp.com/teach/documentation"` page.

Note that unlike in the previous exercises using urllib, you don't have to close the connection when using requests!

**Instructions**
- Import the package `requests`.
- Assign the URL of interest to the variable `url`.
- Package the request to the URL, send the request and catch the response with a single function `requests.get()`, assigning the response to the variable `r`.
- Use the `text` attribute of the object r to return the HTML of the webpage as a string; store the result in a variable `text`.
- Hit submit to print the HTML of the webpage._____no_output_____

# Import package
import requests

# Specify the url: url
url = 'http://www.datacamp.com/teach/documentation'

# Packages the request, send the request and catch the response: r
r = requests.get(url)

# Extract the response: text
text = r.text

# Print the html
print(text)<!doctype html>

    <link href="https://fonts.intercomcdn.com" rel="preconnect" crossorigin>

    <meta charset="utf-8">
    <meta http-equiv="X-UA-Compatible" content="IE=edge">
    <title>DataCamp Help Center</title>
    <meta name="description" content="">
    <meta name="viewport" content="width=device-width, initial-scale=1">

      <link rel="alternate" href="http://instructor-support.datacamp.com/en/" hreflang="en">

      <meta name="intercom:trackingEvent" content="{"name":"Viewed Help Center","metadata":{"action":"viewed","object":"educate_home","place":"help_center","owner":"educate","default_locale":"en","current_locale":"en","is_default_locale":true}}" />

    <link rel="stylesheet" media="all" href="https://static.intercomassets.com/alexandria/assets/application-00b67c7c87b72734807679ffe67cf68adf711566c397121c5d3432b6d9ff1657.css" />
    <link rel="canonical" href="http://instructor-support.datacamp.com/en/"/>

        <link href="https://static.intercomassets.com/assets/educate/educate-favicon-64x64-at-2x-52016a3500a250d0b118c0a04ddd13b1a7364a27759483536dd1940bccdefc20.png" rel="shortcut icon" type="image/png" />

      <meta property="og:title" content="DataCamp Help Center" />
  <meta name="twitter:title" content="DataCamp Help Center" />

<meta property="og:type" content="website" />
<meta property="og:image" content="" />

<meta name="twitter:image" content="" />

              <svg width="14" height="14" viewBox="0 0 14 14" xmlns="http://www.w3.org/2000/svg"><title>Group 65</title><g stroke="#FFF" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"><path d="M11.5 6.73v6.77H.5v-11h7.615M4.5 9.5l7-7M13.5 5.5v-5h-5"/></g></svg>Go to DataCamp

Advice and answers from the DataCamp Team

      <form action="/en/" autocomplete="off" class="header__form search">
        <input type="text" autocomplete="off" class="search__input js__search-input o__ltr" placeholder="Search for articles..." tabindex="1" name="q" value="">

          <button type="submit" class="search__submit o__ltr"></button>

            <svg class="interface-icon" xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 16 16">
              <path d="M8.018 6.643L5.375 4 4 5.375l2.643 2.643L4 10.643 5.375 12l2.643-2.625L10.625 12 12 10.643 9.357 8.018 12 5.375 10.643 4z" />
            </svg>

      </form>

            <svg role='img' viewBox='0 0 48 48'><g id="chat-star" stroke-width="2" fill="none" fill-rule="evenodd" stroke-linejoin="round"><path d="M20 34.942c-2.083-.12-4.292-.42-6-.942L3 39l4-9c-3.858-3.086-6-7.246-6-12C1 8.61 10.328 1 21.835 1 33.343 1 43 8.61 43 18c0 1.044-.117 2.065-.342 3.057"></path><path d="M36.016 25L40 33h7l-6 5 3 9-8-5.494L28 47l3-9-6-5h7l4.016-8z"></path></g></svg>

Getting Started

Everything you need to know to begin your DataCamp journey!

      +2

        11 articles in this collection

      Written by  Becca Robins,  David Venturi,  Yashas Roy and 2 others

            <svg role='img' viewBox='0 0 48 48'><g id="devices-laptop" stroke-width="2" fill="none" fill-rule="evenodd" stroke-linecap="round"><path d="M41 31H7V11h34v20z"></path><path d="M3 35V10a3 3 0 0 1 3-3h36a3 3 0 0 1 3 3v25m-16 0v2H19v-2H1v4a2 2 0 0 0 2 2h42a2 2 0 0 0 2-2v-4H29z" stroke-linejoin="round"></path></g></svg>

Courses

Everything you need to know about creating DataCamp courses.

      +12

        89 articles in this collection

      Written by  Nick Carchedi,  Vincent Vankrunkelsven,  Becca Robins and 12 others

            <svg role='img' viewBox='0 0 48 48'><g id="tools-dashboard" stroke-width="2" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"><path d="M27 31a3 3 0 0 1-6 0 3 3 0 0 1 6 0zm-.88-2.12l9.9-9.9M5 32h4m34 .002L39 32m2.553-8.27l-3.696 1.53M31.27 13.447l-1.53 3.695M24 12v4m-7.27-2.553l1.53 3.695m-7.694.422l2.826 2.83M6.447 23.73l3.695 1.53"></path><path d="M24 8C11.297 8 1 18.3 1 31v9h46v-9C47 18.3 36.703 8 24 8z"></path></g></svg>

Daily Practice

Everything you need to know about creating DataCamp Daily Practice.

        15 articles in this collection

      Written by  Anneleen Beckers

            <svg role='img' viewBox='0 0 48 48'><g id="book-opened2"><path d="M24 11c0-3.866 10.297-7 23-7v33c-12.703 0-23 3.134-23 7 0-3.866-10.3-7-23-7V4c12.7 0 23 3.134 23 7zm0 0v32m-5-27.52c-3.22-1.232-7.773-2.128-13-2.48m13 8.48c-3.22-1.232-7.773-2.128-13-2.48m13 8.48c-3.22-1.232-7.773-2.128-13-2.48m13 8.48c-3.22-1.23-7.773-2.127-13-2.48m23-15.52c3.223-1.232 7.773-2.128 13-2.48m-13 8.48c3.223-1.232 7.773-2.128 13-2.48m-13 8.48c3.223-1.232 7.773-2.128 13-2.48m-13 8.48c3.223-1.23 7.773-2.127 13-2.48" stroke-width="2" fill="none" stroke-linecap="round" stroke-linejoin="round"></path></g></svg>

Projects

Everything you need to know about creating DataCamp projects.

        20 articles in this collection

      Written by  David Venturi and  Richie Cotton

            <svg role='img' viewBox='0 0 48 48'><g id="book-bookmark" stroke-width="2" fill="none" fill-rule="evenodd" stroke-linecap="round"><path d="M35 31l-6-6-6 6V7h12v24z"></path><path d="M35 9h6v38H11a4 4 0 0 1-4-4V5" stroke-linejoin="round"></path><path d="M39 9V1H11a4 4 0 0 0 0 8h12" stroke-linejoin="round"></path></g></svg>

Course Editor Basics

Everything you need to know to get going with our online course editor.

        5 articles in this collection

      Written by  Nick Carchedi,  Amy Peterson, and  Sara Billen

            <svg role='img' viewBox='0 0 48 48'><g id="comms-mail" stroke-width="2" fill="none" fill-rule="evenodd" stroke-linejoin="round"><path d="M47 3L1 22l18 7L47 3z"></path><path d="M47 3l-8 37-20-11L47 3zM19 29v16l7-12"></path></g></svg>

Tips & Tricks

Become a DataCamp wizard!

        6 articles in this collection

      Written by  Becca Robins

            <svg role='img' viewBox='0 0 48 48'><g id="chat-question" fill="none" fill-rule="evenodd"><path d="M47 21.268c0 10.363-10.297 18.765-23 18.765-2.835 0-5.55-.418-8.058-1.184L2.725 45 7.9 34.668c-4.258-3.406-6.9-8.15-6.9-13.4C1 10.904 11.297 2.502 24 2.502s23 8.402 23 18.766z" stroke-width="2" stroke-linejoin="round"></path><path d="M25 28.502a2 2 0 1 0 0 4 2 2 0 0 0 0-4" fill="#231F1F"></path><path d="M19 17.75c0-3.312 2.686-6.124 6-6.124 3.313 0 6 2.626 6 5.938 0 3.315-2.687 5.938-6 5.938V26" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"></path></g></svg>

Frequently Asked Questions (FAQ)

Common questions that arise during content creation.

      +3

        52 articles in this collection

      Written by  Nick Carchedi,  Becca Robins,  Richie Cotton and 3 others

            <svg role='img' viewBox='0 0 48 48'><g id="tools-edit"><path d="M14.932 43.968L2 47l3.033-12.93 31.2-31.203a4 4 0 0 1 5.658 0l4.247 4.243a4 4 0 0 1 0 5.656L14.932 43.968zm29.84-29.735L34.82 4.28m7.125 12.782L31.992 7.11M15.436 43.465l-9.9-9.9" stroke-width="2" fill="none" stroke-linecap="round" stroke-linejoin="round"></path></g></svg>

Miscellaneous

Have a question for DataCamp, but not about creating content? You'll probably find the answer here.

      +1

        10 articles in this collection

      Written by  Becca Robins,  Jen Bricker,  Lisa Monteleone and 1 other

            We run on Intercom

# 3. Scraping the web in Python _____no_output_____### 3.1 Parsing HTML with BeautifulSoup

In this interactive exercise, you'll learn how to use the BeautifulSoup package to parse, prettify and extract information from HTML. You'll scrape the data from the webpage of Guido van Rossum, Python's very own [Benevolent Dictator for Life](https://en.wikipedia.org/wiki/Benevolent_dictator_for_life). In the following exercises, you'll prettify the HTML and then extract the text and the hyperlinks.

The URL of interest is `url = 'https://www.python.org/~guido/'`.

**Instructions**
- Import the function `BeautifulSoup` from the package `bs4`.
- Assign the URL of interest to the variable `url`.
- Package the request to the URL, send the request and catch the response with a single function `requests.get()`, assigning the response to the variable `r`.
- Use the text attribute of the object `r` to return the HTML of the webpage as a string; store the result in a variable `html_doc`.
- Create a BeautifulSoup object `soup` from the resulting HTML using the function `BeautifulSoup()`.
- Use the method `prettify()` on `soup` and assign the result to `pretty_soup`.
- Hit submit to print to prettified HTML to your shell!
_____no_output_____

# Import packages
import requests
from bs4 import BeautifulSoup

# Specify url: url
url = 'https://www.python.org/~guido/'

# Package the request, send the request and catch the response: r
r = requests.get(url)

# Extracts the response as html: html_doc
html_doc = r.text

# Create a BeautifulSoup object from the HTML: soup
soup = BeautifulSoup(html_doc)

# Prettify the BeautifulSoup object: pretty_soup
pretty_soup = soup.prettify()

# Print the response
print(pretty_soup)

  <title>
   Guido's Personal Home Page
  </title>

   Guido van Rossum - Personal Home Page

     "Gawky and proud of it."

    Who
I Am

   Read
my

    "King's
Day Speech"

   for some inspiration.

   I am the author of the

    Python

   programming language.  See also my

    resume

   and my

    publications list

   , a

    brief bio

   , assorted

    writings

   ,

    presentations

   and

    interviews

   (all about Python), some

    pictures of me

   ,

    my new blog

   , and
my

    old
blog

   on Artima.com.  I am

    @gvanrossum

   on Twitter.

   In January 2013 I joined

    Dropbox

   .  I work on various Dropbox
products and have 50% for my Python work, no strings attached.
Previously, I have worked for Google, Elemental Security, Zope
Corporation, BeOpen.com, CNRI, CWI, and SARA.  (See
my

    resume

   .)  I created Python while at CWI.

   How to Reach Me

   You can send email for me to guido (at) python.org.
I read everything sent there, but if you ask
me a question about using Python, it's likely that I won't have time
to answer it, and will instead refer you to
help (at) python.org,

    comp.lang.python

   or

    StackOverflow

   .  If you need to
talk to me on the phone or send me something by snail mail, send me an
email and I'll gladly email you instructions on how to reach me.

   My Name

   My name often poses difficulties for Americans.

    Pronunciation:

   in Dutch, the "G" in Guido is a hard G,
pronounced roughly like the "ch" in Scottish "loch".  (Listen to the

    sound clip

   .)  However, if you're
American, you may also pronounce it as the Italian "Guido".  I'm not
too worried about the associations with mob assassins that some people
have. :-)

    Spelling:

   my last name is two words, and I'd like to keep it
that way, the spelling on some of my credit cards notwithstanding.
Dutch spelling rules dictate that when used in combination with my
first name, "van" is not capitalized: "Guido van Rossum".  But when my
last name is used alone to refer to me, it is capitalized, for
example: "As usual, Van Rossum was right."

    Alphabetization:

   in America, I show up in the alphabet under
"V".  But in Europe, I show up under "R".  And some of my friends put
me under "G" in their address book...

   More Hyperlinks

    Here's a collection of

     essays

    relating to Python
that I've written, including the foreword I wrote for Mark Lutz' book
"Programming Python".

    I own the official

     Python license.

   The Audio File Formats FAQ

   I was the original creator and maintainer of the Audio File Formats
FAQ.  It is now maintained by Chris Bagwell
at

    http://www.cnpbagwell.com/audio-faq

   .  And here is a link to

    SOX

   , to which I contributed
some early code.

   "On the Internet, nobody knows you're
a dog."

### 3.2Turning a webpage into data using BeautifulSoup: getting the text

As promised, in the following exercises, you'll learn the basics of extracting information from HTML soup. In this exercise, you'll figure out how to extract the text from the BDFL's webpage, along with printing the webpage's title.

**Instructions**

- In the sample code, the HTML response object html_doc has already been created: your first task is to Soupify it using the function `BeautifulSoup()` and to assign the resulting soup to the variable `soup`.
- Extract the title from the HTML `soup` using the attribute title and assign the result to `guido_title`.
- Print the title of Guido's webpage to the shell using the `print()` function.
- Extract the text from the HTML soup soup using the method `get_text()` and assign to `guido_text`.
- Hit submit to print the text from Guido's webpage to the shell._____no_output_____

# Import packages
import requests
from bs4 import BeautifulSoup

# Specify url: url
url = 'https://www.python.org/~guido/'

# Package the request, send the request and catch the response: r
r = requests.get(url)

# Extract the response as html: html_doc
html_doc = r.text

# Create a BeautifulSoup object from the HTML: soup
soup = BeautifulSoup(html_doc)

# Get the title of Guido's webpage: guido_title
guido_title = soup.title

# Print the title of Guido's webpage to the shell
print(guido_title)

# Get Guido's text: guido_text
guido_text = soup.text

# Print Guido's text to the shell
print(guido_text)<title>Guido's Personal Home Page</title>

Guido's Personal Home Page

Guido van Rossum - Personal Home Page

"Gawky and proud of it."
Who
I Am
Read
my "King's
Day Speech" for some inspiration.

I am the author of the Python
programming language.  See also my resume
and my publications list, a brief bio, assorted writings, presentations and interviews (all about Python), some
pictures of me,
my new blog, and
my old
blog on Artima.com.  I am
@gvanrossum on Twitter.

In January 2013 I joined
Dropbox.  I work on various Dropbox
products and have 50% for my Python work, no strings attached.
Previously, I have worked for Google, Elemental Security, Zope
Corporation, BeOpen.com, CNRI, CWI, and SARA.  (See
my resume.)  I created Python while at CWI.

How to Reach Me
You can send email for me to guido (at) python.org.
I read everything sent there, but if you ask
me a question about using Python, it's likely that I won't have time
to answer it, and will instead refer you to
help (at) python.org,
comp.lang.python or
StackOverflow.  If you need to
talk to me on the phone or send me something by snail mail, send me an
email and I'll gladly email you instructions on how to reach me.

My Name
My name often poses difficulties for Americans.

Pronunciation: in Dutch, the "G" in Guido is a hard G,
pronounced roughly like the "ch" in Scottish "loch".  (Listen to the
sound clip.)  However, if you're
American, you may also pronounce it as the Italian "Guido".  I'm not
too worried about the associations with mob assassins that some people
have. :-)

Spelling: my last name is two words, and I'd like to keep it
that way, the spelling on some of my credit cards notwithstanding.
Dutch spelling rules dictate that when used in combination with my
first name, "van" is not capitalized: "Guido van Rossum".  But when my
last name is used alone to refer to me, it is capitalized, for
example: "As usual, Van Rossum was right."

Alphabetization: in America, I show up in the alphabet under
"V".  But in Europe, I show up under "R".  And some of my friends put
me under "G" in their address book...

More Hyperlinks

Here's a collection of essays relating to Python
that I've written, including the foreword I wrote for Mark Lutz' book
"Programming Python".
I own the official
Python license.

The Audio File Formats FAQ
I was the original creator and maintainer of the Audio File Formats
FAQ.  It is now maintained by Chris Bagwell
at http://www.cnpbagwell.com/audio-faq.  And here is a link to
SOX, to which I contributed
some early code.

"On the Internet, nobody knows you're
a dog."

### 3.3Turning a webpage into data using BeautifulSoup: getting the hyperlinks
In this exercise, you'll figure out how to extract the URLs of the hyperlinks from the BDFL's webpage. In the process, you'll become close friends with the soup method `find_all()`.

**Instructions**
- Use the method `find_all()` to find all hyperlinks in soup, remembering that hyperlinks are defined by the HTML tag `<a\>` but passed to `find_all()` without angle brackets; store the result in the variable `a_tags`.
- The variable a_tags is a results set: your job now is to enumerate over it, using a for loop and to print the actual URLs of the hyperlinks; to do this, for every element link in `a_tags`, you want to print() `link.get('href')`._____no_output_____

# Import packages
import requests
from bs4 import BeautifulSoup

# Specify url
url = 'https://www.python.org/~guido/'

# Package the request, send the request and catch the response: r
r = requests.get(url)

# Extracts the response as html: html_doc
html_doc = r.text

# create a BeautifulSoup object from the HTML: soup
soup = BeautifulSoup(html_doc)

# Print the title of Guido's webpage
print(soup.title)

# Find all 'a' tags (which define hyperlinks): a_tags
a_tags = soup.find_all('a')

# Print the URLs to the shell
for link in a_tags:
    print(link.get('href'))<title>Guido's Personal Home Page</title>
pics.html
pics.html
http://www.washingtonpost.com/wp-srv/business/longterm/microsoft/stories/1998/raymond120398.htm
http://metalab.unc.edu/Dave/Dr-Fun/df200004/df20000406.jpg
http://neopythonic.blogspot.com/2016/04/kings-day-speech.html
http://www.python.org
Resume.html
Publications.html
bio.html
http://legacy.python.org/doc/essays/
http://legacy.python.org/doc/essays/ppt/
interviews.html
pics.html
http://neopythonic.blogspot.com
http://www.artima.com/weblogs/index.jsp?blogger=12088
https://twitter.com/gvanrossum
http://www.dropbox.com
Resume.html
http://groups.google.com/groups?q=comp.lang.python
http://stackoverflow.com
guido.au
http://legacy.python.org/doc/essays/
images/license.jpg
http://www.cnpbagwell.com/audio-faq
http://sox.sourceforge.net/
images/internetdog.gif


# Notebook from lrsoenksen/CL_RNA_SynthBio
Path: src/pre-processing/1_GPML_Graphic_toehold_V1.ipynb

<code>
## Import Libraries
# General system libraries
import os
import numpy as np
import pandas as pd
from time import time
from IPython.display import Image

# Multiprocessing
import multiprocessing

# DNA/RNA Analysis Libraries (Biopython, ViennaRNA, pysster)
# Biopython Lib
import Bio
from Bio import SeqIO
from Bio.Seq import Seq
from Bio.SeqRecord import SeqRecord
from Bio.Alphabet import generic_rna, generic_dna, generic_protein, IUPAC
# ViennaRNA Lib
import RNA
# pysster Lib
from pysster import utils
from pysster.Data import Data
from pysster.Grid_Search import Grid_Search
from pysster.One_Hot_Encoder import One_Hot_Encoder
from pysster.Alphabet_Encoder import Alphabet_Encoder
#RNAssp
import src.RNAssp.rna as rnassp

# Import TPOT libs
from tpot import TPOTRegressor

# Import Tensorflow
import tensorflow as tf

# Import Keras
from keras.models import Sequential, load_model
from keras.layers import Activation, Dropout, Flatten, Dense, Input
from keras.preprocessing import image
from keras.applications.imagenet_utils import decode_predictions, preprocess_input
from keras.preprocessing.image import ImageDataGenerator, img_to_array, load_img
from keras.layers import Convolution2D, MaxPooling2D, ZeroPadding2D
from keras import optimizers
from keras import applications
from keras.models import Model
#from keras.utils import multi_gpu_model
from keras.callbacks import EarlyStopping
from keras_tqdm import TQDMNotebookCallback
from keras.utils.np_utils import to_categorical

# Import sklearn libs
from sklearn import preprocessing
from sklearn.preprocessing import OneHotEncoder
from sklearn.model_selection import train_test_split
from sklearn.metrics import explained_variance_score, mean_absolute_error
from sklearn.metrics import mean_squared_error, mean_squared_log_error
from sklearn.metrics import median_absolute_error, r2_score

# Math & Visualization Libs
import seaborn as sns
import matplotlib.pyplot as plt
from scipy import stats

# Warnings
import warningsUsing TensorFlow backend.
# Function definition to create complementary matrix of RNA molecule
def one_hot_special_complementarity_directional_matrix(m, show=False):
    """Produce complementarity matrix for the given RNA molecule.
    by Luis Soenksen 2019-02-01
    Complementary bases (according to Watson-Crick) are assigned the following values:
    G-C or C-G are assigned 3 = [0 0 0 1],
    A-U or U-A are assigned 2 = [0 0 1 0],
    G-U or U-G are assigned 1 = [0 1 0 0],
    NonWCpairs are assigned 0 = [1 0 0 0],
    G-C or C-G are assigned 3 = [0 0 0 1],
    A-U or U-A are assigned 2 = [0 0 1 0],
    G-U or U-G are assigned 1 = [0 1 0 0],
    NonWCpairs are assigned 0 = [1 0 0 0],

    Args:
        m: Molecule object.
        show (bool): Make a matrix plot of the result.

    Returns:
        p_oh: One-Hot Encoded Categorical Complementarity-directional Matrix
        p: Categorical(integer) Complementarity-directional Matrix
    """
    from keras.utils import to_categorical

    l = len(m.seq)
    p = np.zeros((l, l), dtype='float64')
    for i in range(l):
        for j in range(l):
            if m.seq[i] == 'G' and m.seq[j] == 'C' :
                p[i, j] = 6
            if m.seq[i] == 'A' and m.seq[j] == 'U' :
                p[i, j] = 4
            if m.seq[i] == 'G' and m.seq[j] == 'U' :
                p[i, j] = 2

            # By default... if m.seq[i] == m.seq[j] ; p[i, j] = 0

            if m.seq[i] == 'C' and m.seq[j] == 'G':
                p[i, j] = 5
            if m.seq[i] == 'U' and m.seq[j] == 'A':
                p[i, j] = 3
            if m.seq[i] == 'U' and m.seq[j] == 'G':
                p[i, j] = 1

    if show:
        %matplotlib ipympl
        fig = plt.figure()
        ax = fig.add_subplot(111)
        cmap = plt.get_cmap('jet', np.max(p)-np.min(p)+1)
        pos = ax.matshow(p, interpolation='nearest', cmap=cmap)
        ax.set_xticks(np.arange(l))
        ax.set_yticks(np.arange(l))
        ax.set_xticklabels([i for i in m.seq])
        ax.set_yticklabels([i for i in m.seq])

        # Add colorbar to make it easy to read the energy levels
        cbar = plt.colorbar(pos, ticks=np.arange(np.min(p),np.max(p)+1))
        plt.show()

    p_oh = to_categorical(p[:,:])

    return p_oh, p_____no_output_____# The RNA sequence
seq = "GAGUAGUGGAACCAGGCUAUGUUUGUGACUCGCAGACUAACA"
# compute minimum free energy (MFE) and corresponding structure
(ss, mfe) = RNA.fold(seq)
# print output
print ("%s\n%s [ %6.2f ]" % (seq, ss, mfe))

#Show
m = rnassp.Molecule(seq, ss)
seq_pair = rnassp.pair_matrix(m, show=True)
seq_p_oh, seq_p = one_hot_special_complementarity_directional_matrix(m, show=True)GAGUAGUGGAACCAGGCUAUGUUUGUGACUCGCAGACUAACA
..(((((........)))))(((((((...)))))))..... [  -8.80 ]
#Check one-hot
seq_p_oh_____no_output_____
</code>