# Notebook from ebruayyurek/derin_ogrenme
Path: _notebooks/2020-02-01-Python.ipynb

# Python
> Makine Öğrenmesi ve Derin Öğrenme için gerekli olan Python konuları.

- toc: true
- badges: true
- comments: true
- categories: [jupyter]
- image: images/chart-preview.png_____no_output_____#  Değişkenler_____no_output_____Variables are containers for holding data and they're defined by a name and value.

_____no_output_____

# Integer variable
x = 5
print (x)
print (type(x))
5
<class 'int'>

We can change the value of a variable by simply assigning a new value to it._____no_output_____

# String variable
x = "hello"
print (x)
print (type(x))hello
<class 'str'>

There are many different types of variables: integers, floats, strings, boolean etc._____no_output_____

# int variable
x = 5
print (x, type(x))5 <class 'int'>
# float variable
x = 5.0
print (x, type(x))5.0 <class 'float'>
# text variable
x = "5"
print (x, type(x))5 <class 'str'>
# boolean variable
x = True
print (x, type(x))True <class 'bool'>

We can also do operations with variables._____no_output_____

# Variables can be used with each other
a = 1
b = 2
c = a + b
print (c)3

We should always know what types of variables we're dealing with so we can do the right operations with them. Here's a common mistake that can happen if we're using the wrong variable type._____no_output_____

# int variables
a = 5
b = 3
print (a + b)8
# string variables
a = "5"
b = "3"
print (a + b)53

#  Lists_____no_output_____Lists are an ordered, mutable (changeable) collection of values that are *comma separated* and enclosed by *square brackets*. A list can be comprised of many different types of variables (below is a list with an integer, string and a float).

_____no_output_____

# Creating a list
x = [3, "hello", 1.2]
print (x)[3, 'hello', 1.2]
# Length of a list
len(x)_____no_output_____

You can add to a list by using the **append** function._____no_output_____

# Adding to a list
x.append(7)
print (x)
print (len(x))[3, 'hello', 1.2, 7]
4
# Replacing items in a list
x[1] = "bye"
print (x)[3, 'bye', 1.2, 7]
# Operations
y = [2.4, "world"]
z = x + y
print (z)[3, 'bye', 1.2, 7, 2.4, 'world']

### Indexing and Slicing_____no_output_____Indexing and slicing from lists allow us to retrieve specific values within lists. Note that indices can be positive (starting from 0) or negative (-1 and lower, where -1 is the last item in the list).

_____no_output_____

# Indexing
x = [3, "hello", 1.2]
print ("x[0]: ", x[0])
print ("x[1]: ", x[1])
print ("x[-1]: ", x[-1]) # the last item
print ("x[-2]: ", x[-2]) # the second to last itemx[0]:  3
x[1]:  hello
x[-1]:  1.2
x[-2]:  hello
# Slicing
print ("x[:]: ", x[:]) # all indices
print ("x[1:]: ", x[1:]) # index 1 to the end of the list
print ("x[1:2]: ", x[1:2]) # index 1 to index 2 (not including index 2)
print ("x[:-1]: ", x[:-1]) # index 0 to last index (not including last index)x[:]:  [3, 'hello', 1.2]
x[1:]:  ['hello', 1.2]
x[1:2]:  ['hello']
x[:-1]:  [3, 'hello']

# Tuples_____no_output_____Tuples are collections that are ordered and immutable (unchangeable). You will use these to store values that will never be changed._____no_output_____

# Creating a tuple
x = (3.0, "hello") # tuples start and end with ()
print (x)(3.0, 'hello')
# Adding values to a tuple
x = x + (5.6, 4)
print (x)(3.0, 'hello', 5.6, 4)
# Try to change (it won't work and you'll get an error)
x[0] = 1.2_____no_output_____

# Dictionaries_____no_output_____Dictionaries are an unordered, mutable and indexed collection of key-value pairs. You can retrieve values based on the key and a dictionary cannot have two of the same keys.

_____no_output_____

# Creating a dictionary
person = {'name': 'Goku',
          'eye_color': 'brown'}
print (person)
print (person['name'])
print (person['eye_color']){'name': 'Goku', 'eye_color': 'brown'}
Goku
brown
# Changing the value for a key
person['eye_color'] = 'green'
print (person){'name': 'Goku', 'eye_color': 'green'}
# Adding new key-value pairs
person['age'] = 24
print (person){'name': 'Goku', 'eye_color': 'green', 'age': 24}
# Length of a dictionary
print (len(person))3

# If statements_____no_output_____We can use `if` statements to conditionally do something. The conditions are defined by the words `if`, `elif` (which stands for else if) and `else`. We can have as many `elif` statements as we want. The indented code below each condition is the code that will execute if the condition is `True`.

_____no_output_____

# If statement
x = 4
if x < 1:
    score = 'low'
elif x <= 4: # elif = else if
    score = 'medium'
else:
    score = 'high'
print (score)medium
# If statement with a boolean
x = True
if x:
    print ("it worked")it worked

# Loops_____no_output_____### For Loops_____no_output_____A `for` loop can iterate over a collection of values (lists, tuples, dictionaries, etc.) The indented code is executed for each item in the collection of values._____no_output_____

# For loop
veggies = ["carrots", "broccoli", "beans"]
for veggie in veggies:
    print (veggie)carrots
broccoli
beans

When the loop encounters the `break` command, the loop will terminate immediately. If there were more items in the list, they will not be processed._____no_output_____

# `break` from a for loop
veggies = ["carrots", "broccoli", "beans"]
for veggie in veggies:
    if veggie == "broccoli":
        break
    print (veggie)carrots

When the loop encounters the `continue` command, the loop will skip all other operations for that item in the list only. If there were more items in the list, the loop will continue normally._____no_output_____

# `continue` to the next iteration
veggies = ["carrots", "broccoli", "beans"]
for veggie in veggies:
    if veggie == "broccoli":
        continue
    print (veggie)carrots
beans

### While Loops_____no_output_____A `while` loop can perform repeatedly as long as a condition is `True`. We can use `continue` and `break` commands in `while` loops as well._____no_output_____

# While loop
x = 3
while x > 0:
    x -= 1 # same as x = x - 1
    print (x)2
1
0

# Functions_____no_output_____Functions are a way to modularize reusable pieces of code. They're defined by the keyword `def` which stands for definition and they can have the following components.

_____no_output_____

# Define the function
def add_two(x):
    """Increase x by 2.""" # explains what this function will do
    x += 2
    return x_____no_output_____

Here are the components that may be required when we want to use the function. we need to ensure that the function name and the input parameters match with how we defined the function above.

_____no_output_____

# Use the function
score = 0
new_score = add_two(x=score)
print (new_score)2

A function can have as many input parameters and outputs as we want._____no_output_____

# Function with multiple inputs
def join_name(first_name, last_name):
    """Combine first name and last name."""
    joined_name = first_name + " " + last_name
    return joined_name_____no_output_____# Use the function
first_name = "Goku"
last_name = "Mohandas"
joined_name = join_name(first_name=first_name,
                        last_name=last_name)
print (joined_name)Goku Mohandas

**NOTE**: It's good practice to always use keyword argument when using a function so that it's very clear what input variable belongs to what function input parameter. On a related note, you will often see the terms ***args** and ****kwargs** which stand for arguments and keyword arguments. You can extract them when they are passed into a function. The significance of the * is that any number of arguments and keyword arguments can be passed into the function._____no_output_____

def f(*args, **kwargs):
    x = args[0]
    y = kwargs.get('y')
    print (f"x: {x}, y: {y}")_____no_output_____f(5, y=2)x: 5, y: 2

# Classes_____no_output_____Classes are object constructors and are a fundamental component of object oriented programming in Python. They are composed of a set of functions that define the class and it's operations.

_____no_output_____### `__init__()` function_____no_output_____The `init` function is used when an instance of the class is initialized. _____no_output_____

# Creating the class
class Pet(object):
    """Class object for a pet."""

    def __init__(self, species, name):
        """Initialize a Pet."""
        self.species = species
        self.name = name_____no_output_____# Creating an instance of a class
my_dog = Pet(species="dog",
             name="Scooby")
print (my_dog)
print (my_dog.name)<__main__.Pet object at 0x7fe487e9c358>
Scooby

### `__str()__` function_____no_output_____The `print (my_dog)` command printed something not so relevant to us. Let's fix that with the `__str()__` function._____no_output_____

# Creating the class
class Pet(object):
    """Class object for a pet."""

    def __init__(self, species, name):
        """Initialize a Pet."""
        self.species = species
        self.name = name

    def __str__(self):
        """Output when printing an instance of a Pet."""
        return f"{self.species} named {self.name}"_____no_output_____# Creating an instance of a class
my_dog = Pet(species="dog",
             name="Scooby")
print (my_dog)
print (my_dog.name)dog named Scooby
Scooby

**NOTE**: Classes can be customized with `magic` functions like, `__str__`, to enable powerful operations. We'll be exploring additional built-in functions in subsequent notebooks (like `__iter__` and `__getitem__`) but if you're curious, here is a [tutorial](https://rszalski.github.io/magicmethods/) on more magic methods._____no_output_____### Object methods_____no_output_____

# Creating the class
class Pet(object):
    """Class object for a pet."""

    def __init__(self, species, name):
        """Initialize a Pet."""
        self.species = species
        self.name = name

    def __str__(self):
        """Output when printing an instance of a Pet."""
        return f"{self.species} named {self.name}"

    def change_name(self, new_name):
        """Change the name of your Pet."""
        self.name = new_name_____no_output_____# Creating an instance of a class
my_dog = Pet(species="dog",
             name="Scooby")
print (my_dog)
print (my_dog.name)dog named Scooby
Scooby
# Using a class's function
my_dog.change_name(new_name="Scrappy")
print (my_dog)
print (my_dog.name)dog named Scrappy
Scrappy

### Inheritance_____no_output_____Inheritance allows you to inherit all the properties and methods from another class (the parent). Notice how we inherited the initialized variables from the parent `Pet` class like species and name. We also inherited the `change_name` function. But for the `__str__` function, we define our own version to overwrite the `Pet` class' `__str__` function._____no_output_____

class Dog(Pet):
    def __init__(self, species, name, breed):
        super().__init__("dog", name)
        self.breed = breed

    def __str__(self):
        return f"{self.breed} named {self.name}"_____no_output_____scooby = Dog(species="dog", breed="Great Dane", name="Scooby")
print (scooby)Great Dane named Scooby
scooby.change_name('Scooby Doo')
print (scooby)Great Dane named Scooby Doo

# Decorators_____no_output_____Recall that functions allow us to modularize code and reuse them. However, we'll often want to add some functionality before or after the main function executes and we may want to do this for many different functions. Instead of adding more code to the original function, we can use decorators!

*    **decorators**: augment a function with pre/post-processing. Decorators wrap around the main function and allow us to operate on the inputs and or outputs._____no_output_____Suppose we have a function called `operations` which increments the input value x by 1._____no_output_____

def operations(x):
    """Basic operations."""
    x += 1
    return x_____no_output_____operations(x=1)_____no_output_____

Now let's say we want to increment our input x by 1 before and after the `operations` function executes and, to illustrate this example, let's say the increments have to be separate steps. Here's how we would do it by changing the original code:_____no_output_____

def operations(x):
    """Basic operations."""
    x += 1
    x += 1
    x += 1
    return x_____no_output_____operations(x=1)_____no_output_____

We were able to achieve what we want but we now increased the size of our `operations` function and if we want to do the same incrementation for any other function, we have to add the same code to all of those as well ... not very efficient. To solve this, let's create a decorator called `add` which increments `x` by 1 before and after the main function `f` executes._____no_output_____### Creating a decorator function_____no_output_____The decorator function accepts a function `f` which is the function we wish to wrap around (in our case, it's `operations`). The output of the decorator is its `wrapper` function which receives the arguments and keyword arguments passed to function `f`.

Inside the `wrapper` function, we can extract the input parameters `[line 5]` passed to function `f` and make any changes we want `[line 6]`. Then the function `f` is executed `[line 7]` and then we can make changes to the outputs as well `[line 8]`. Finally, the `wrapper` function will return some value(s) `[line 9]` which is what the decorator returns as well since it returns `wrapper`. _____no_output_____

# Decorator
def add(f):
    def wrapper(*args, **kwargs):
        """Wrapper function for @add."""
        x = kwargs.pop('x') # .get() if not altering x
        x += 1 # executes before function f
        x = f(*args, **kwargs, x=x)
        x += 1 # executes after function f
        return x
    return wrapper_____no_output_____

We can use this decorator by simply adding it to the top of our main function preceded by the `@` symbol. _____no_output_____

@add
def operations(x):
    """Basic operations."""
    x += 1
    return x_____no_output_____operations(x=1)_____no_output_____

Suppose we wanted to debug and see what function actually executed with `operations`. _____no_output_____

operations.__name__, operations.__doc_______no_output_____

The function name and docstring are not what we're looking for but it appears this way because the `wrapper` function is what was executed. In order to fix this, Python offers `functools.wraps` which carries the main function's metadata. _____no_output_____

from functools import wraps_____no_output_____# Decorator
def add(f):
    @wraps(f)
    def wrap(*args, **kwargs):
        """Wrapper function for @add."""
        x = kwargs.pop('x')
        x += 1
        x = f(*args, **kwargs, x=x)
        x += 1
        return x
    return wrap_____no_output_____@add
def operations(x):
    """Basic operations."""
    x += 1
    return x_____no_output_____operations.__name__, operations.__doc_______no_output_____

Awesome! We were able to decorate our main function `operation` to achieve the customization we wanted without actually altering the function. We can reuse our decorator for other functions that may need the same customization!

This was a dummy example to show how decorators work but we'll be using them heavily during our production ML lessons. A simple scenario would be using decorators to create uniform JSON responses from each API endpoint without including the bulky code in each endpoint._____no_output_____# Callbacks_____no_output_____Decorators allow for customized operations before and after the main function's execution but what about in between? Suppose we want to conditionally/situationally do some operations. Instead of writing a whole bunch of if-statements and make our functions bulky, we can use callbacks!

*   **callbacks**: conditional/situational processing within the function._____no_output_____Our callbacks will be classes that have functions with key names that will execute at various periods during the main function's execution. The function names are up to us but we need to invoke the same callback functions within our main function._____no_output_____

# Callback
class x_tracker(object):
    def __init__(self, x):
        self.history = []
    def at_start(self, x):
        self.history.append(x)
    def at_end(self, x):
        self.history.append(x)_____no_output_____

We can pass in as many callbacks as we want and because they have appropriately named functions, they will be invoked at the appropriate times. _____no_output_____

def operations(x, callbacks=[]):
    """Basic operations."""
    for callback in callbacks:
        callback.at_start(x)
    x += 1
    for callback in callbacks:
        callback.at_end(x)
    return x_____no_output_____x = 1
tracker = x_tracker(x=x)
operations(x=x, callbacks=[tracker])_____no_output_____tracker.history_____no_output_____

# Putting it all together_____no_output_____decorators + callbacks = powerful customization *before*, *during* and *after* the main function’s execution without increasing its complexity. We will be using this duo to create powerful ML training scripts that are highly customizable in future lessons.

_____no_output_____

from functools import wraps_____no_output_____# Decorator
def add(f):
    @wraps(f)
    def wrap(*args, **kwargs):
        """Wrapper function for @add."""
        x = kwargs.pop('x') # .get() if not altering x
        x += 1 # executes before function f
        x = f(*args, **kwargs, x=x)
        # can do things post function f as well
        return x
    return wrap_____no_output_____# Callback
class x_tracker(object):
    def __init__(self, x):
        self.history = [x]
    def at_start(self, x):
        self.history.append(x)
    def at_end(self, x):
        self.history.append(x)_____no_output_____# Main function
@add
def operations(x, callbacks=[]):
    """Basic operations."""
    for callback in callbacks:
        callback.at_start(x)
    x += 1
    for callback in callbacks:
        callback.at_end(x)
    return x_____no_output_____x = 1
tracker = x_tracker(x=x)
operations(x=x, callbacks=[tracker])_____no_output_____tracker.history_____no_output_____

# Additional resources_____no_output_____* **Python 3**: This was a very quick look at Python but it's good enough for practical machine learning and we'll be learning more in future lessons. If you want to learn more, check out this [free Python3 course](https://www.w3schools.com/python/default.asp)._____no_output_____---
Share and discover ML projects at Made With ML.

             _____no_output_____


# Notebook from Dell-AI/Recommendation-System-in-Spark
Path: Recommendation_Comparison/Mini_Movie_Recommender_ALS_plus_Trees.ipynb

# Some functions are from or based on work done by Kevin Liao in the below notebook
# https://github.com/KevinLiao159/MyDataSciencePortfolio/blob/master/movie_recommender/movie_recommendation_using_ALS.ipynb

# Intialization
import os
import time

import warnings
warnings.filterwarnings("ignore", message="numpy.dtype size changed")

# spark imports
from pyspark.sql import SparkSession
from pyspark.sql.functions import UserDefinedFunction, explode, desc
from pyspark.sql.types import StringType, ArrayType
from pyspark.ml.recommendation import ALS
from pyspark.ml.linalg import Vectors
from pyspark.ml.feature import VectorAssembler
from pyspark.ml.regression import GBTRegressor, GBTRegressionModel

# data science imports
import numpy as np
import pandas as pd
from sklearn.preprocessing import MultiLabelBinarizer_____no_output_____data_path = 'hdfs:///user/andrew/'_____no_output_____%%time
# Read in data through spark since the data is sored in hadoop and format the columns
# Convert to pandas dataframes for easier and faster manipulation
from pyspark.sql.types import *
from pyspark.sql import SQLContext, Row
from pyspark.sql.functions import *
sqlContext = SQLContext(sc)

Rating = Row("user_id", "item_id", "label") # Ignore timestamp

movies = sqlContext.read.parquet(data_path + 'movie_metadata_OHE')
movies_df = movies.toPandas()
movies_df = movies_df.set_index(movies_df.item_id) # set index so no sorting errors occur

# movies_gp = sqlContext.read.parquet('hdfs:///user/andrew/movie_genre_and_people_metadata_ohe_subset')
movies_gp = movies.drop('title', 'imdb_id', 'imdb_rating', 'imdb_votes', 'metascore', 'runtime', 'year')

ratings = sc.textFile(data_path + 'ratings.dat')\
    .map(lambda line: line.split("::")[0:3])\
    .map(lambda line: map(int, line))\
    .map(lambda r: Rating(*r))
ratings = sqlContext.createDataFrame(ratings)CPU times: user 59.6 s, sys: 2.17 s, total: 1min 1s
Wall time: 2min 2s
# User input function - takes user input data, strpis it down, and calls other functions on that data
# Takes in user age, gender, list of favorite movies
# All movies in the list of favorite movies will be rated 5 stars
def new_user_input(fav_movies, all_ratings, movies, spark_context,
                   sqlContext = None, num_recs = 10, age = None,
                   gender = None, movies_gp = None, movies_df = None):
    # age should be an integer in 1 - 100
    # gender should be M or F
    # fav_movies should be in the form of ["Iron Man", "The Shawshank Redemption", "Robin Hood"]
    #    If there are multiple versions of the movie and the user wishes for one other than the most recent one, they
    #    should specify with a year in parenthesis, like "Robin Hood (1993)"

    # collect favorite movie ids
    print 'Collecting favorite movie IDs'
    movieIds = get_movieId(movies_df, fav_movies)
    print 'Favorite movies in the available set'
    print movies_df[['item_id', 'title', 'year']].loc[movieIds]

    print 'Adding ratings to full set'
    # add new user movie ratings to all ratings dataframe
    all_ratings_updated, new_user_ratings = add_new_user_to_data(all_ratings, movieIds, spark_context)
    del all_ratings

    print 'Creating prediction set'
    # get all unrated movies for user (unnecessary in Spark 2.2+, instead use the recommendForAllUsers(num_to_rec) method)
    all_user_unrated = get_inference_data(all_ratings_updated, movieIds)

    print 'Training ALS model'
    # train ALS model, then predict movie ratings
    als = ALS(seed = 42, regParam = 0.1, maxIter = 15, rank = 12,
          userCol = "user_id", itemCol = "item_id", ratingCol = "label")
    als_model = als.fit(all_ratings_updated)
    del all_ratings_updated

    print 'Making Predictions'
    # keep top 30 predictions
    full_predictions_sorted = als_model.transform(all_user_unrated).sort(desc('prediction'))
    als_top_n_predictions = full_predictions_sorted.take(num_recs)
    # extract movie ids
    als_top_n_ids = [r[1] for r in als_top_n_predictions]

    als_movie_recs = movies.filter(movies.item_id.isin(als_top_n_ids)).select('title', 'year')
    print ''
    print 'ALS Recommendations'
    print als_movie_recs.toPandas()

    # format data for prediction using GBTs
    # create user_id x item_id matrix need to get data in the form of user_id, item_id, label, then pivot
    # filter movies_gp dataframe by the movieIds. pivot new_user_ratings into a vector,
    # then multiply by the filtered movies_gp dataframe; divide by binarized user ratings;
    # this should now be a vector of user preferences.
    # join a OHE age, gender, and possibly occupation, to the user preferences
    user_summary = get_user_preferences(user_ratings = new_user_ratings, movieIds = movieIds,
                                        movies_gp = movies_gp, age = age, gender = gender, sqlContext = sqlContext)

    # Extract movie ids from the top 5*num_recs for Gradient Boosted Trees prediction
    als_top_3xn_predictions = full_predictions_sorted.take(3*num_recs)
    als_top_3xn_ids = [r[1] for r in als_top_3xn_predictions]
    all_user_unrated_top_3xn = all_user_unrated.filter(all_user_unrated.item_id.isin(als_top_3xn_ids))
    top_3xn_movies_metadata = movies.filter(movies.item_id.isin(als_top_3xn_ids))

    # lastly, replicate the user pref rows for each rated movieId, then join with the filtered movies dataframe
    # (MAKE SURE ALL COLUMNS ARE ORDERED AND NAMED CORRECTLY)
    unrated_with_movie_metadata = all_user_unrated_top_3xn \
                                    .join(top_3xn_movies_metadata, on = 'item_id', how = 'left')
    unrated_with_full_metadata = unrated_with_movie_metadata \
                                    .join(user_summary, on = 'user_id', how = 'left') \
                                    .drop('user_id', 'title', 'imdb_id')

    # the GBT model takes in the rows as vectors, so the columns must be converted to the feature space
    unrated_with_full_metadata_rdd = unrated_with_full_metadata.rdd.map(lambda x: (x[0], Vectors.dense(x[1:])))
    unrated_metadata_features = sqlContext.createDataFrame(unrated_with_full_metadata_rdd, schema = ['item_id', 'features'])

    # import the GBT model, in this case a GBTRegressionModel with tree depth of 10
    GBTRegD10Model = GBTRegressionModel.load(data_path + 'GBTRegD10Model')

    # use pre-trained GBT model to predict movie ratings
    gbtr_preds = GBTRegD10Model.transform(unrated_metadata_features)

    # sort by predicted rating, and keep top recommend top n
    gbtr_top_n_predictions = gbtr_preds.sort(desc('prediction')).take(num_recs)
    # extract movie ids
    gbtr_top_n_ids = [r[0] for r in gbtr_top_n_predictions]

    gbtr_movie_recs = movies.filter(movies.item_id.isin(gbtr_top_n_ids)).select('title', 'year')
    print ''
    print 'GBTR Recommendations'
    print gbtr_movie_recs.toPandas()_____no_output_____def get_movieId(movies_df, fav_movie_list):
    """
    return all movieId(s) of user's favorite movies

    Parameters
    ----------
    df_movies: spark Dataframe, movies data

    fav_movie_list: list, user's list of favorite movies

    Return
    ------
    movieId_list: list of movieId(s)
    """
    movieId_list = []
    for movie in fav_movie_list:
        # Remove first word in specific cases. This is because sometimes the first word is listed
        # at the end of the movie title and would not be found by the search if it is included.
        if movie[0:4] == 'The ':
            movie = movie[4:]
        elif movie[0:3] == 'An ':
            movie = movie[3:]
        elif movie[0:3] == 'La ':
            movie = movie[3:]
        elif movie[0:2] == 'A ':
            movie = movie[3:]

        # If a year is provided by the user, remove it from the title and use it in the filtering.
        if movie[-6:-5] == '(':
            year = int(movie[-5:-1])
            movie = movie[0:-7]
            movieIds = movies_df.item_id[(movies_df.title.str.contains(movie)) & (movies_df.year == year)]
            movieId_list.extend(movieIds)
        # If no year is provided, determine if the title is a single word.
        # If it is a single work, find an exact match
        elif len(movie.split(' ')) == 1:
            movieIds = movies_df.item_id[movies_df.title == movie]
            movieId_list.extend(movieIds)
        # Otherwise search for the movie title
        else:
            movieIds = movies_df.item_id[movies_df.title.str.contains(movie)]
            movieId_list.extend(movieIds)
    return movieId_list_____no_output_____def add_new_user_to_data(train_data, movieIds, spark_context):
    """
    add new rows with new user, user's movie and ratings to
    existing train data

    Parameters
    ----------
    train_data: Spark DataFrame, ratings data

    movieIds: spark DataFrame, single column of movieId(s)

    spark_context: Spark Context object

    Return
    ------
    new train data with the new user's rows
    """
    # Get new user id
    new_id = train_data.agg({"user_id": "max"}).collect()[0][0] + 1
    # Get max rating
    max_rating = train_data.agg({"label": "max"}).collect()[0][0]
    # Create new user sdf for max rating
    user_rows_max = [(new_id, movieId, max_rating) for movieId in movieIds]
    new_sdf_max = spark_context.parallelize(user_rows_max).toDF(['user_id', 'item_id', 'label'])
    # Return new train data
    return train_data.union(new_sdf_max), new_sdf_max_____no_output_____def get_inference_data(train_data, movieIds):
    """
    return a rdd with the userid and all movies (except ones in movieId_list)

    Parameters
    ----------
    train_data: spark RDD, ratings data

    df_movies: spark Dataframe, movies data

    movieId_list: list, list of movieId(s)

    Return
    ------
    inference data: Spark RDD
    """
    # Get new user id
    new_id = train_data.agg({"user_id": "max"}).collect()[0][0]
    # Get the set of movies that have not been rated by the user
    distinct_unrated_items = ratings.select('item_id').distinct().filter(~col('item_id').isin(movieIds))
    # Create the dataset for the new user with their unrated movies
    user_unrated = distinct_unrated_items.withColumn('user_id', lit(new_id)).select('user_id', 'item_id')
    return user_unrated_____no_output_____def get_user_preferences(user_ratings, movieIds, movies_gp, age, gender, sqlContext):
    user_demog = pd.DataFrame({'gender_M': 0, 'gender_F': 0, 'age_group_1': 0,
                               'age_group_18': 0, 'age_group_25': 0, 'age_group_35': 0,
                               'age_group_45': 0, 'age_group_50': 0, 'age_group_56': 0}, index = [0])
    # Bin user by age
    if age < 18:
        user_demog.age_group_1[0] = 1
    elif age < 25:
        user_demog.age_group_18[0] = 1
    elif age < 35:
        user_demog.age_group_25[0] = 1
    elif age < 45:
        user_demog.age_group_35[0] = 1
    elif age < 50:
        user_demog.age_group_45[0] = 1
    elif age < 56:
        user_demog.age_group_50[0] = 1
    else:
        user_demog.age_group_56[0] = 1
    # Binarize gender
    if gender == 'M':
        user_demog.gender_M[0] = 1
    else:
        user_demog.gender_F[0] = 1

    # new_user_ratings
    # Create the user-item matrix for the new user
    pivoted_user_ratings_df = user_ratings.toPandas() \
                                            .pivot(index='user_id',
                                                   columns='item_id',
                                                   values='label') \
                                            .fillna(0)
    pivoted_user_ratings_df_binary = pivoted_user_ratings_df / pivoted_user_ratings_df

    # Reduce the movie set to only the user's rate movies
    movies_gp_filtered = movies_gp.filter(col('item_id').isin(movieIds))
    movies_gp_filtered_df = movies_gp_filtered.toPandas()
    movies_gp_filtered_df = movies_gp_filtered_df.set_index('item_id')

    # Create user profile
    user_summary_total = pivoted_user_ratings_df.dot(movies_gp_filtered_df)
    user_summary_count = pivoted_user_ratings_df_binary.dot(movies_gp_filtered_df)
    user_summary_avg = (user_summary_total / user_summary_count).fillna(0)
    user_summary_avg = user_summary_avg.add_suffix('_avg_rating').reset_index()

    # Create Spark dataframe output for full user profile
    user_summary = pd.concat([user_summary_avg, user_demog], axis = 1)
    sorted_columns = list(user_summary.columns.sort_values())
    user_summary_sdf = sqlContext.createDataFrame(user_summary[sorted_columns])
    return user_summary_sdf_____no_output_____

### Step by Step Walkthrough of Main Function (to show runtime)_____no_output_____

%%time
fav_movies = ['Tinker Tailor Soldier Spy', 'Shawshank Redemption', 'Lord of the Rings']
# collect favorite movie ids
print 'Collecting favorite movie IDs'
movieIds = get_movieId(movies_df, fav_movies)
if movies_df is not None:
    print 'Favorite movies in the available set'
    print movies_df[['item_id', 'title', 'year']].loc[movieIds]Collecting favorite movie IDs
Favorite movies in the available set
         item_id                      title  year
item_id
318          318  Shawshank Redemption, The  1994
2116        2116     Lord of the Rings, The  1978
CPU times: user 13.6 ms, sys: 2.06 ms, total: 15.7 ms
Wall time: 14.9 ms
%%time
print 'Adding ratings to full set'
# add new user movie ratings to all ratings dataframe
all_ratings_updated, user_ratings = add_new_user_to_data(ratings, movieIds, sc)Adding ratings to full set
CPU times: user 43.4 ms, sys: 4.89 ms, total: 48.3 ms
Wall time: 12 s
%%time
print 'Creating prediction set'
# get all unrated movies for user (unnecessary in Spark 2.2+, instead use the recommendForAllUsers(num_to_rec) method)
all_user_unrated = get_inference_data(all_ratings_updated, movieIds)Creating prediction set
CPU times: user 14.8 ms, sys: 4.14 ms, total: 18.9 ms
Wall time: 4.19 s
%%time
print 'Training ALS model'
# train ALS model, then predict movie ratings
als = ALS(seed = 42, regParam = 0.1, maxIter = 15, rank = 12,
      userCol = "user_id", itemCol = "item_id", ratingCol = "label")
als_model = als.fit(all_ratings_updated)
del all_ratings_updatedTraining ALS model
CPU times: user 41 ms, sys: 11.5 ms, total: 52.5 ms
Wall time: 8.09 s
%%time
print 'Making Predictions'
# keep top 15 predictions
num_recs = 15
full_predictions = als_model.transform(all_user_unrated)
als_top_n_predictions = full_predictions.sort(desc('prediction')).take(num_recs)
# extract movie ids
als_top_n_ids = [r[1] for r in als_top_n_predictions]

als_movie_recs = movies.filter(movies.item_id.isin(als_top_n_ids)).select('title', 'year')
print 'ALS Recommendations'
print als_movie_recs.toPandas()Making Predictions
ALS Recommendations
                                        title  year
0                            Ulysses (Ulisse)  1954
1          Schlafes Bruder (Brother of Sleep)  1995
2                             Foreign Student  1994
3                             Window to Paris  1994
4                      Across the Sea of Time  1995
5                         Usual Suspects, The  1995
6                                  Braveheart  1995
7                               Smashing Time  1967
8                    Very Thought of You, The  1998
9                       Saltmen of Tibet, The  1997
10                          Bewegte Mann, Der  1994
11  Midaq Alley (Callejn de los milagros, El)  1995
12         Star Wars: Episode IV - A New Hope  1977
13         Life Is Beautiful (La Vita  bella)  1997
14                         American History X  1998
CPU times: user 39.7 ms, sys: 4.22 ms, total: 43.9 ms
Wall time: 44 s
%%time
# import GBT model input data format
# Create user_id x item_id matrix need to get data in the form of user_id, item_id, label, then pivot
# filter movies_gp dataframe by the movieIds. pivot new_user_ratings into a vector,
# then multiply by the filtered movies_gp dataframe; divide by binarized user ratings;
# this should now be a vector of user preferences.
# join a OHE age, gender, and possibly occupation, to the user preferences
user_summary_sdf = get_user_preferences(user_ratings = user_ratings, movieIds = movieIds,
                                        movies_gp = movies_gp, age = 26, gender = 'M',
                                        sqlContext = sqlContext)CPU times: user 2.2 s, sys: 1.08 s, total: 3.28 s
Wall time: 1min 6s
%%time
als_top_3xn_predictions = full_predictions.sort(desc('prediction')).take(3*num_recs)
als_top_3xn_ids = [r[1] for r in als_top_3xn_predictions]
all_user_unrated_top_3xn = all_user_unrated.filter(all_user_unrated.item_id.isin(als_top_3xn_ids))
top_3xn_movies_metadata = movies.filter(movies.item_id.isin(als_top_3xn_ids))CPU times: user 40.2 ms, sys: 9.43 ms, total: 49.6 ms
Wall time: 5.55 s
%%time
# lastly, replicate the user pref rows for each rated movieId, then join with the filtered movies dataframe
# (MAKE SURE ALL COLUMNS ARE ORDERED AND NAMED CORRECTLY)
unrated_with_movie_metadata = all_user_unrated_top_3xn \
                                .join(top_3xn_movies_metadata, on = 'item_id', how = 'left')
unrated_with_full_metadata = unrated_with_movie_metadata \
                                .join(user_summary_sdf, on = 'user_id', how = 'left') \
                                .drop('user_id', 'title', 'imdb_id')CPU times: user 3.25 ms, sys: 937 µs, total: 4.19 ms
Wall time: 8.22 s
%%time
# convert predictors to "features" and it is ready for prediction.
unrated_with_full_metadata_rdd = unrated_with_full_metadata.rdd.map(lambda x: (x[0], Vectors.dense(x[1:])))
unrated_metadata_features = sqlContext.createDataFrame(unrated_with_full_metadata_rdd, schema = ['item_id', 'features'])CPU times: user 59.1 ms, sys: 14.1 ms, total: 73.3 ms
Wall time: 5min 16s
%%time
GBTRegD10Model = GBTRegressionModel.load(data_path + 'GBTRegD10Model')
gbtr_preds = GBTRegD10Model.transform(unrated_metadata_features)CPU times: user 2.83 ms, sys: 1.25 ms, total: 4.08 ms
Wall time: 13.3 s
%%time
gbtr_top_n_predictions = gbtr_preds.sort(desc('prediction')).take(num_recs)
# extract movie ids
gbtr_top_n_ids = [r[0] for r in gbtr_top_n_predictions]

gbtr_movie_recs = movies.filter(movies.item_id.isin(gbtr_top_n_ids)).select('title', 'year')
print 'GBTR Recommendations'
print gbtr_movie_recs.toPandas()GBTR Recommendations
                                 title  year
0                     Ulysses (Ulisse)  1954
1   Schlafes Bruder (Brother of Sleep)  1995
2                      Foreign Student  1994
3                      Window to Paris  1994
4               Across the Sea of Time  1995
5                             Lamerica  1994
6                        Smashing Time  1967
7                       Big Trees, The  1952
8                      Acid House, The  1998
9             Very Thought of You, The  1998
10                   Julien Donkey-Boy  1999
11               Saltmen of Tibet, The  1997
12                   Bewegte Mann, Der  1994
13                        Mother Night  1996
14               Dream With the Fishes  1997
CPU times: user 56.6 ms, sys: 9.68 ms, total: 66.3 ms
Wall time: 2min 33s

### Full Function Recommendation Examples_____no_output_____

%%time
fav_movies = ['Iron Man', 'Tinker Tailor Soldier Spy', 'Shawshank Redemption', 'Lord of the Rings', 'Harry Potter',
             'The Family Stone', 'Shaun of the Dead', 'Up', 'A View to a Kill']
new_user_input(fav_movies = fav_movies, all_ratings = ratings,
               movies = movies, spark_context = sc, sqlContext = sqlContext,
               num_recs = 10, age = 26, gender = 'M', movies_gp = movies_gp, movies_df = movies_df)Collecting favorite movie IDs
Favorite movies in the available set
         item_id                      title  year
item_id
318          318  Shawshank Redemption, The  1994
2116        2116     Lord of the Rings, The  1978
2376        2376          View to a Kill, A  1985
Adding ratings to full set
Creating prediction set
Training ALS model
Making Predictions

ALS Recommendations
                                       title  year
0                           Ulysses (Ulisse)  1954
1         Schlafes Bruder (Brother of Sleep)  1995
2                            Foreign Student  1994
3                            Window to Paris  1994
4                     Across the Sea of Time  1995
5                   Very Thought of You, The  1998
6                      Saltmen of Tibet, The  1997
7                          Bewegte Mann, Der  1994
8  Midaq Alley (Callejn de los milagros, El)  1995
9         Star Wars: Episode IV - A New Hope  1977

GBTR Recommendations
                                        title  year
0                            Ulysses (Ulisse)  1954
1                             Foreign Student  1994
2                      Across the Sea of Time  1995
3                               Smashing Time  1967
4                              Big Trees, The  1952
5                    Very Thought of You, The  1998
6  Mating Habits of the Earthbound Human, The  1998
7                                 Boys Life 2  1997
8                         One Magic Christmas  1985
9                           Bewegte Mann, Der  1994
CPU times: user 2.34 s, sys: 1.2 s, total: 3.54 s
Wall time: 10min
%%time
fav_movies = ['Tinker Tailor Soldier Spy', 'Shawshank Redemption', 'Lord of the Rings']
new_user_input(fav_movies = fav_movies, all_ratings = ratings,
               movies = movies, spark_context = sc, sqlContext = sqlContext,
               num_recs = 10, age = 26, gender = 'M', movies_gp = movies_gp, movies_df = movies_df)Collecting favorite movie IDs
Favorite movies in the available set
         item_id                      title  year
item_id
318          318  Shawshank Redemption, The  1994
2116        2116     Lord of the Rings, The  1978
Adding ratings to full set
Creating prediction set
Training ALS model
Making Predictions

ALS Recommendations
                                title  year
0                    Ulysses (Ulisse)  1954
1                     Foreign Student  1994
2                     Window to Paris  1994
3              Across the Sea of Time  1995
4                          Braveheart  1995
5                       Smashing Time  1967
6               Saltmen of Tibet, The  1997
7                   Bewegte Mann, Der  1994
8  Life Is Beautiful (La Vita  bella)  1997
9                  American History X  1998

GBTR Recommendations
                                title  year
0                    Ulysses (Ulisse)  1954
1  Schlafes Bruder (Brother of Sleep)  1995
2                     Foreign Student  1994
3              Across the Sea of Time  1995
4                       Smashing Time  1967
5                     Acid House, The  1998
6            Very Thought of You, The  1998
7                   Julien Donkey-Boy  1999
8               Saltmen of Tibet, The  1997
9                   Bewegte Mann, Der  1994
CPU times: user 2.44 s, sys: 1.2 s, total: 3.64 s
Wall time: 10min 14s
%%time
fav_movies = ['Frozen', 'Tangled', 'Oceans Eleven', 'Toy Story', 'The Princess Bride',
              'The Incredibles', 'Castle in the Sky', 'Monsters, Inc']
new_user_input(fav_movies = fav_movies, all_ratings = ratings,
               movies = movies, spark_context = sc, sqlContext = sqlContext,
               num_recs = 10, age = 26, gender = 'M', movies_gp = movies_gp, movies_df = movies_df)Collecting favorite movie IDs
Favorite movies in the available set
         item_id                title  year
item_id
3114        3114          Toy Story 2  1999
1              1            Toy Story  1995
1197        1197  Princess Bride, The  1987
Adding ratings to full set
Creating prediction set
Training ALS model
Making Predictions

ALS Recommendations
                                             title  year
0                              Wrong Trousers, The  1993
1                          Raiders of the Lost Ark  1981
2                                 Schindler's List  1993
3                                  Foreign Student  1994
4  Wallace & Gromit: The Best of Aardman Animation  1996
5                                          Sanjuro  1962
6                        Leather Jacket Love Story  1997
7                                   Close Shave, A  1995
8               Star Wars: Episode IV - A New Hope  1977
9                        Shawshank Redemption, The  1994

GBTR Recommendations
                                 title  year
0   Schlafes Bruder (Brother of Sleep)  1995
1              Raiders of the Lost Ark  1981
2                      Foreign Student  1994
3                  Usual Suspects, The  1995
4                     Sixth Sense, The  1999
5            Leather Jacket Love Story  1997
6                           Sting, The  1973
7          Gate of Heavenly Peace, The  1995
8                          Rear Window  1954
9  Inheritors, The (Die Siebtelbauern)  1998
CPU times: user 2.41 s, sys: 1.16 s, total: 3.57 s
Wall time: 10min 10s
%%time
fav_movies = ['Frozen', 'Tangled', 'Oceans Eleven', 'Toy Story', 'The Princess Bride',
              'The Incredibles', 'Castle in the Sky', 'Monsters, Inc']
new_user_input(fav_movies = fav_movies, all_ratings = ratings,
               movies = movies, spark_context = sc, sqlContext = sqlContext,
               num_recs = 10, age = 8, gender = 'F', movies_gp = movies_gp, movies_df = movies_df)Collecting favorite movie IDs
Favorite movies in the available set
         item_id                title  year
item_id
3114        3114          Toy Story 2  1999
1              1            Toy Story  1995
1197        1197  Princess Bride, The  1987
Adding ratings to full set
Creating prediction set
Training ALS model
Making Predictions

ALS Recommendations
                                             title  year
0                              Wrong Trousers, The  1993
1                          Raiders of the Lost Ark  1981
2                                 Schindler's List  1993
3                                  Foreign Student  1994
4  Wallace & Gromit: The Best of Aardman Animation  1996
5                                          Sanjuro  1962
6                        Leather Jacket Love Story  1997
7                                   Close Shave, A  1995
8               Star Wars: Episode IV - A New Hope  1977
9                        Shawshank Redemption, The  1994

GBTR Recommendations
                                 title  year
0   Schlafes Bruder (Brother of Sleep)  1995
1              Raiders of the Lost Ark  1981
2                      Foreign Student  1994
3                  Usual Suspects, The  1995
4                     Sixth Sense, The  1999
5            Leather Jacket Love Story  1997
6                           Sting, The  1973
7          Gate of Heavenly Peace, The  1995
8                          Rear Window  1954
9  Inheritors, The (Die Siebtelbauern)  1998
CPU times: user 2.36 s, sys: 1.22 s, total: 3.58 s
Wall time: 10min 17s
%%time
fav_movies = ['The Sound of Music', 'Blackhawk Down', 'Pearl Harbor', 'Toy Story', 'The Princess Bride',
              'Foreign Student', 'Star Wars', 'The Shining', 'Rear Window', 'Groundhog Day', 'Ghostbusters',
              'Robin Hood (1993)', 'Die Hard']
new_user_input(fav_movies = fav_movies, all_ratings = ratings,
               movies = movies, spark_context = sc, sqlContext = sqlContext,
               num_recs = 10, age = 40, gender = 'M', movies_gp = movies_gp, movies_df = movies_df)Collecting favorite movie IDs
Favorite movies in the available set
         item_id                                           title  year
item_id
1035        1035                             Sound of Music, The  1965
3114        3114                                     Toy Story 2  1999
1              1                                       Toy Story  1995
1197        1197                             Princess Bride, The  1987
572          572                                 Foreign Student  1994
1196        1196  Star Wars: Episode V - The Empire Strikes Back  1980
1210        1210      Star Wars: Episode VI - Return of the Jedi  1983
2628        2628       Star Wars: Episode I - The Phantom Menace  1999
260          260              Star Wars: Episode IV - A New Hope  1977
904          904                                     Rear Window  1954
1265        1265                                   Groundhog Day  1993
2716        2716                                    Ghostbusters  1984
520          520                       Robin Hood: Men in Tights  1993
1036        1036                                        Die Hard  1988
165          165                      Die Hard: With a Vengeance  1995
1370        1370                                      Die Hard 2  1990
Adding ratings to full set
Creating prediction set
Training ALS model
Making Predictions

ALS Recommendations
                                title  year
0                         Chushingura  1962
1                    Ulysses (Ulisse)  1954
2  Schlafes Bruder (Brother of Sleep)  1995
3             Raiders of the Lost Ark  1981
4                    Sixth Sense, The  1999
5                             Bandits  1997
6                         Matrix, The  1999
7  Indiana Jones and the Last Crusade  1989
8                           Firelight  1997
9           Shawshank Redemption, The  1994

GBTR Recommendations
                                       title  year
0                           Ulysses (Ulisse)  1954
1         Schlafes Bruder (Brother of Sleep)  1995
2                            Window to Paris  1994
3                     Across the Sea of Time  1995
4                              Smashing Time  1967
5                                    Bandits  1997
6                  Ayn Rand: A Sense of Life  1997
7                  Leather Jacket Love Story  1997
8                                  Firelight  1997
9  Midaq Alley (Callejn de los milagros, El)  1995
CPU times: user 2.8 s, sys: 1.29 s, total: 4.09 s
Wall time: 10min 8s
sc.stop()_____no_output_____


# Notebook from Novartus/ScrapBook
Path: Parse/BeautifulSoup 4/BeautifulSoup_Filters.ipynb

import re, requests,webbrowser
from bs4 import BeautifulSoup_____no_output_____resp = requests.get("http://books.toscrape.com/") #For Scrapping Practice we will use http://toscrape.com/
resp_____no_output_____soup = BeautifulSoup(resp.text, "lxml")
print(soup.prettify())<!DOCTYPE html>

  <title>
   All products | Books to Scrape - Sandbox
  </title>
  <meta content="text/html; charset=utf-8" http-equiv="content-type"/>
  <meta content="24th Jun 2016 09:29" name="created"/>
  <meta content="" name="description"/>
  <meta content="width=device-width" name="viewport"/>
  <meta content="NOARCHIVE,NOCACHE" name="robots"/>

  <link href="static/oscar/favicon.ico" rel="shortcut icon"/>
  <link href="static/oscar/css/styles.css" rel="stylesheet" type="text/css"/>
  <link href="static/oscar/js/bootstrap-datetimepicker/bootstrap-datetimepicker.css" rel="stylesheet"/>
  <link href="static/oscar/css/datetimepicker.css" rel="stylesheet" type="text/css"/>

       Books to Scrape

       We love being scraped!

       Home

      All products

          Books

            Travel

            Mystery

            Historical Fiction

            Sequential Art

            Classics

            Philosophy

            Romance

            Womens Fiction

            Fiction

            Childrens

            Religion

            Nonfiction

            Music

            Default

            Science Fiction

            Sports and Games

            Add a comment

            Fantasy

            New Adult

            Young Adult

            Science

            Poetry

            Paranormal

            Art

            Psychology

            Autobiography

            Parenting

            Adult Fiction

            Humor

            Horror

            History

            Food and Drink

            Christian Fiction

            Business

            Biography

            Thriller

            Contemporary

            Spirituality

            Academic

            Self Help

            Historical

            Christian

            Suspense

            Short Stories

            Novels

            Health

            Politics

            Cultural

            Erotica

            Crime

        All products

      <form class="form-horizontal" method="get">

        1000

       results - showing

        1

       to

        20

       .
      </form>

         Warning!

        This is a demo website for web scraping purposes. Prices and ratings here were randomly assigned and have no real meaning.

             A Light in the ...

             Â£51.77

             In stock

            <form>
             <button class="btn btn-primary btn-block" data-loading-text="Adding..." type="submit">
              Add to basket
             </button>
            </form>

             Tipping the Velvet

             Â£53.74

             In stock

            <form>
             <button class="btn btn-primary btn-block" data-loading-text="Adding..." type="submit">
              Add to basket
             </button>
            </form>

             Soumission

             Â£50.10

             In stock

            <form>
             <button class="btn btn-primary btn-block" data-loading-text="Adding..." type="submit">
              Add to basket
             </button>
            </form>

             Sharp Objects

             Â£47.82

             In stock

            <form>
             <button class="btn btn-primary btn-block" data-loading-text="Adding..." type="submit">
              Add to basket
             </button>
            </form>

             Sapiens: A Brief History ...

             Â£54.23

             In stock

            <form>
             <button class="btn btn-primary btn-block" data-loading-text="Adding..." type="submit">
              Add to basket
             </button>
            </form>

             The Requiem Red

             Â£22.65

             In stock

            <form>
             <button class="btn btn-primary btn-block" data-loading-text="Adding..." type="submit">
              Add to basket
             </button>
            </form>

             The Dirty Little Secrets ...

             Â£33.34

             In stock

            <form>
             <button class="btn btn-primary btn-block" data-loading-text="Adding..." type="submit">
              Add to basket
             </button>
            </form>

             The Coming Woman: A ...

             Â£17.93

             In stock

            <form>
             <button class="btn btn-primary btn-block" data-loading-text="Adding..." type="submit">
              Add to basket
             </button>
            </form>

             The Boys in the ...

             Â£22.60

             In stock

            <form>
             <button class="btn btn-primary btn-block" data-loading-text="Adding..." type="submit">
              Add to basket
             </button>
            </form>

             The Black Maria

             Â£52.15

             In stock

            <form>
             <button class="btn btn-primary btn-block" data-loading-text="Adding..." type="submit">
              Add to basket
             </button>
            </form>

             Starving Hearts (Triangular Trade ...

             Â£13.99

             In stock

            <form>
             <button class="btn btn-primary btn-block" data-loading-text="Adding..." type="submit">
              Add to basket
             </button>
            </form>

             Shakespeare's Sonnets

             Â£20.66

             In stock

            <form>
             <button class="btn btn-primary btn-block" data-loading-text="Adding..." type="submit">
              Add to basket
             </button>
            </form>

             Set Me Free

             Â£17.46

             In stock

            <form>
             <button class="btn btn-primary btn-block" data-loading-text="Adding..." type="submit">
              Add to basket
             </button>
            </form>

             Scott Pilgrim's Precious Little ...

             Â£52.29

             In stock

            <form>
             <button class="btn btn-primary btn-block" data-loading-text="Adding..." type="submit">
              Add to basket
             </button>
            </form>

             Rip it Up and ...

             Â£35.02

             In stock

            <form>
             <button class="btn btn-primary btn-block" data-loading-text="Adding..." type="submit">
              Add to basket
             </button>
            </form>

             Our Band Could Be ...

             Â£57.25

             In stock

            <form>
             <button class="btn btn-primary btn-block" data-loading-text="Adding..." type="submit">
              Add to basket
             </button>
            </form>

             Olio

             Â£23.88

             In stock

            <form>
             <button class="btn btn-primary btn-block" data-loading-text="Adding..." type="submit">
              Add to basket
             </button>
            </form>

             Mesaerion: The Best Science ...

             Â£37.59

             In stock

            <form>
             <button class="btn btn-primary btn-block" data-loading-text="Adding..." type="submit">
              Add to basket
             </button>
            </form>

             Libertarianism for Beginners

             Â£51.33

             In stock

            <form>
             <button class="btn btn-primary btn-block" data-loading-text="Adding..." type="submit">
              Add to basket
             </button>
            </form>

             It's Only the Himalayas

             Â£45.17

             In stock

            <form>
             <button class="btn btn-primary btn-block" data-loading-text="Adding..." type="submit">
              Add to basket
             </button>
            </form>

           Page 1 of 50

            next

soup.find("p")
#OR soup.p
#First P only _____no_output_____soup.findAll("p")
#All p in page_____no_output_____soup.findAll(href = "index.html")_____no_output_____soup.findAll(href=re.compile('[^"]+\.html'))  #Finding all 'index.html' files from page_____no_output_____for tag in soup.findAll(href=re.compile('[^"]+\.html')):
    print(tag)Books to Scrape
Home

                                Books

                                Travel

                                Mystery

                                Historical Fiction

                                Sequential Art

                                Classics

                                Philosophy

                                Romance

                                Womens Fiction

                                Fiction

                                Childrens

                                Religion

                                Nonfiction

                                Music

                                Default

                                Science Fiction

                                Sports and Games

                                Add a comment

                                Fantasy

                                New Adult

                                Young Adult

                                Science

                                Poetry

                                Paranormal

                                Art

                                Psychology

                                Autobiography

                                Parenting

                                Adult Fiction

                                Humor

                                Horror

                                History

                                Food and Drink

                                Christian Fiction

                                Business

                                Biography

                                Thriller

                                Contemporary

                                Spirituality

                                Academic

                                Self Help

                                Historical

                                Christian

                                Suspense

                                Short Stories

                                Novels

                                Health

                                Politics

                                Cultural

                                Erotica

                                Crime

A Light in the ...

Tipping the Velvet

Soumission

Sharp Objects

Sapiens: A Brief History ...

The Requiem Red

The Dirty Little Secrets ...

The Coming Woman: A ...

The Boys in the ...

The Black Maria

Starving Hearts (Triangular Trade ...

Shakespeare's Sonnets

Set Me Free

Scott Pilgrim's Precious Little ...

Rip it Up and ...

Our Band Could Be ...

Olio

Mesaerion: The Best Science ...

Libertarianism for Beginners

It's Only the Himalayas
next
soup.find('img', attrs= {'class':re.compile('^thum')}) #Finding a Class which has thumbnail property_____no_output_____soup.findAll('img', attrs= {'class':re.compile('^thum')}) #Finding All Class which has thumbnail property_____no_output_____soup.findAll(['a'])_____no_output_____for tag in soup.findAll(True):
    print(tag.name)html
head
title
meta
meta
meta
meta
meta
link
link
link
link
body
header
div
div
div
a
small
div
div
ul
li
a
li
div
aside
div
div
ul
li
a
ul
li
a
li
a
li
a
li
a
li
a
li
a
li
a
li
a
li
a
li
a
li
a
li
a
li
a
li
a
li
a
li
a
li
a
li
a
li
a
li
a
li
a
li
a
li
a
li
a
li
a
li
a
li
a
li
a
li
a
li
a
li
a
li
a
li
a
li
a
li
a
li
a
li
a
li
a
li
a
li
a
li
a
li
a
li
a
li
a
li
a
li
a
li
a
li
a
li
a
li
a
div
div
h1
div
div
form
div
strong
strong
strong
section
div
strong
div
ol
li
article
div
a
img
p
i
i
i
i
i
h3
a
div
p
p
i
form
button
li
article
div
a
img
p
i
i
i
i
i
h3
a
div
p
p
i
form
button
li
article
div
a
img
p
i
i
i
i
i
h3
a
div
p
p
i
form
button
li
article
div
a
img
p
i
i
i
i
i
h3
a
div
p
p
i
form
button
li
article
div
a
img
p
i
i
i
i
i
h3
a
div
p
p
i
form
button
li
article
div
a
img
p
i
i
i
i
i
h3
a
div
p
p
i
form
button
li
article
div
a
img
p
i
i
i
i
i
h3
a
div
p
p
i
form
button
li
article
div
a
img
p
i
i
i
i
i
h3
a
div
p
p
i
form
button
li
article
div
a
img
p
i
i
i
i
i
h3
a
div
p
p
i
form
button
li
article
div
a
img
p
i
i
i
i
i
h3
a
div
p
p
i
form
button
li
article
div
a
img
p
i
i
i
i
i
h3
a
div
p
p
i
form
button
li
article
div
a
img
p
i
i
i
i
i
h3
a
div
p
p
i
form
button
li
article
div
a
img
p
i
i
i
i
i
h3
a
div
p
p
i
form
button
li
article
div
a
img
p
i
i
i
i
i
h3
a
div
p
p
i
form
button
li
article
div
a
img
p
i
i
i
i
i
h3
a
div
p
p
i
form
button
li
article
div
a
img
p
i
i
i
i
i
h3
a
div
p
p
i
form
button
li
article
div
a
img
p
i
i
i
i
i
h3
a
div
p
p
i
form
button
li
article
div
a
img
p
i
i
i
i
i
h3
a
div
p
p
i
form
button
li
article
div
a
img
p
i
i
i
i
i
h3
a
div
p
p
i
form
button
li
article
div
a
img
p
i
i
i
i
i
h3
a
div
p
p
i
form
button
div
ul
li
li
a
footer
script
script
script
script
script
script
script
def has_src_and_no_href(tag):
    return tag.has_attr("src") and not tag.has_attr("href")_____no_output_____soup.findAll(has_src_and_no_href)_____no_output_____def only_index(href):
    return href and re.compile("books").search(href)_____no_output_____soup.findAll(href =only_index)_____no_output_____for link in soup.findAll(href=only_index):
    print(link["href"])catalogue/category/books_1/index.html
catalogue/category/books/travel_2/index.html
catalogue/category/books/mystery_3/index.html
catalogue/category/books/historical-fiction_4/index.html
catalogue/category/books/sequential-art_5/index.html
catalogue/category/books/classics_6/index.html
catalogue/category/books/philosophy_7/index.html
catalogue/category/books/romance_8/index.html
catalogue/category/books/womens-fiction_9/index.html
catalogue/category/books/fiction_10/index.html
catalogue/category/books/childrens_11/index.html
catalogue/category/books/religion_12/index.html
catalogue/category/books/nonfiction_13/index.html
catalogue/category/books/music_14/index.html
catalogue/category/books/default_15/index.html
catalogue/category/books/science-fiction_16/index.html
catalogue/category/books/sports-and-games_17/index.html
catalogue/category/books/add-a-comment_18/index.html
catalogue/category/books/fantasy_19/index.html
catalogue/category/books/new-adult_20/index.html
catalogue/category/books/young-adult_21/index.html
catalogue/category/books/science_22/index.html
catalogue/category/books/poetry_23/index.html
catalogue/category/books/paranormal_24/index.html
catalogue/category/books/art_25/index.html
catalogue/category/books/psychology_26/index.html
catalogue/category/books/autobiography_27/index.html
catalogue/category/books/parenting_28/index.html
catalogue/category/books/adult-fiction_29/index.html
catalogue/category/books/humor_30/index.html
catalogue/category/books/horror_31/index.html
catalogue/category/books/history_32/index.html
catalogue/category/books/food-and-drink_33/index.html
catalogue/category/books/christian-fiction_34/index.html
catalogue/category/books/business_35/index.html
catalogue/category/books/biography_36/index.html
catalogue/category/books/thriller_37/index.html
catalogue/category/books/contemporary_38/index.html
catalogue/category/books/spirituality_39/index.html
catalogue/category/books/academic_40/index.html
catalogue/category/books/self-help_41/index.html
catalogue/category/books/historical_42/index.html
catalogue/category/books/christian_43/index.html
catalogue/category/books/suspense_44/index.html
catalogue/category/books/short-stories_45/index.html
catalogue/category/books/novels_46/index.html
catalogue/category/books/health_47/index.html
catalogue/category/books/politics_48/index.html
catalogue/category/books/cultural_49/index.html
catalogue/category/books/erotica_50/index.html
catalogue/category/books/crime_51/index.html


# Notebook from dibay/Mask-wearing-policy-COVID-19
Path: maskwearing-covid.ipynb

# Bibliography
- Introduction
- Background or Related Work
- Research questions or hypotheses
- Methodology
- Findings
- Discussion (including Limitations and Implications)
- Conclusion
- References

# Introduction:

We are living in a critical era as the numbers of the new COVID_19 cases and death due to COVID-19 are increasing worldwide and in the US. Mask wearing as part of the social distancing measures is recommended as to subside the spread of this virus. However, twelve states in the US have very limited mask wearing requirements and South Dakota has absolutely no requirement. In this project I am trying to evaluate the effect of mask wearing policy on COVID-19 burden. To do so I will focus on the data from two neighbor states (Georgia and Alabama) with different mask wearing policies to minimize the effect of confounding factors (e.g. weather, demographics).

# Background and Related Work:

Coronavirus disease 2019 (COVID-19) is caused by a novel coronavirus. It results in severe [acute respiratory syndrome coronavirus 2](https://www.cdc.gov/coronavirus/2019-ncov/cases-updates/about-epidemiology/identifying-source-outbreak.html) (1). This was first identified in an outbreak of respiratory illness cases in Wuhan City, Hubei Province, China (1). So far COVID-19 cases are approximately [73 million worldwide including 16 million cases in the United States](https://coronavirus.jhu.edu/map.html) (2). Per a WHO report on November 8th, [new cases have increased by 8%, compared to the previous week while new deaths have increased by 21%](https://www.who.int/publications/m/item/weekly-epidemiological-update---10-november-2020) (3). The global and the US death toll due to COVID-19 by December 15th have been [1,624,819 and 300,723](https://coronavirus.jhu.edu/map.html) respectively (2).

Mask wearing as part of the social distancing policies have been suggested as [one of the most effective prevention measures](https://www.cdc.gov/coronavirus/2019-ncov/prevent-getting-sick/cloth-face-cover-guidance.html) (4). However, [12 states in the US have very limited mask wearing policies and one states (South Dakota) has no policy around wearing a mask](https://masks4all.co/what-states-require-masks/) (5). In this project I am trying to understand the impact of the mask wearing policy on the burden (i.e. new daily cases and death) of COVID-19.
I chose Georgia and Alabama to investigate the mask wearing policies. Georgia only passed mask wearing policy for certain employees while Alabama mandated mask wearing from July 17th (5). These two states are neighbors which helps controlling for the differences in weather in addition to other factors such as ethnicity, education whcih are very similar for [Georgia](https://www.census.gov/quickfacts/GA) and [Alabama](https://www.census.gov/quickfacts/AL) (6,7). Per CDC [minorities are at higher risk of COVID-19](https://www.cdc.gov/coronavirus/2019-ncov/community/health-equity/race-ethnicity.html) and thus it is important to make the comparison of mask mandating policies between states that are similar/close in ethnicity distribution (8).

# Research Question and Hypothesis:

1- What is the trend of new cases from July 17th to November 11th for Georgia and for Alabama
It is expected to observe a steeper line for Georgia compared to Alabama since Georgia mask policy is very limited, but Alabama had the policy in place since July 17th.

2- Is the number of the new cases more in Georgia compared to Alabama from July 17th to November 11th
Hypothesis: The number of cases is significantly higher in Georgia compared to Alabama

3- Is the accumulative number of the death more in Georgia compared to Alabama from July 17th to November 11th
Hypothesis: The number of deaths is significantly higher in Georgia compared to Alabama

# Methodology:

### Dataset:

The ["United States COVID-19 Cases and Deaths by State over Time"](https://healthdata.gov/dataset/united-states-covid-19-cases-and-deaths-state-over-time) dataset provides detail daily data on the cases and death of COVID-19 for all the states from January to November 11th 2020. All the data in this dataset is unidentified and there is no way to track back this information. This data is available through a government website. The source should be acknowledged when this data is used for analysis.
Some of the variables in this dataset that will be used in this analysis are as follow:

|Variables   | Description|
|------------|------------|
| state |Jurisdiction|
| submission date |Date of counts|
| new_casw |Number of new_case_trend.pngnew cases|
| new_death| Number of new deaths|

### Source:

https://healthdata.gov/harvest_source/datacdcgov

### License:

http://opendefinition.org/licenses/odc-odbl/

### Statistical Analysis:

All the analysis will be done on a subset of data from July 17th (the day that a comprehensive mask wearing policy got effective in Alabama) to November 11th. The number of new cases and new deaths will be normalized to the population in each state and all the statistics and models will run on the normalized data. [The Georgia’s population estimates by July 1, 2019 is 10,617,423](https://www.census.gov/quickfacts/GA) (6) and [the Alabama’s population estimates by July 1, 2019 is 4,903,185](https://www.census.gov/quickfacts/AL) (7).

Descriptive statistics for the two states for new cases and deaths will be provided from July 17th to November 11th. Time series graph will be provided from July 17th to November 11th for Georgia and Alabama to compare the trend between the two states. Repeated measures ANOVA test will be used to compare the numbers of daily new cases and new deaths between the two states.

It is important to note that per CDC ["because these provisional counts are subject to change, including updates to data reported previously, adjustments can occur. These adjustments can result in fewer total numbers of cases and deaths compared with the previous data, which means that new numbers of cases or deaths can include negative values that reflect such adjustments."](https://data.cdc.gov/Case-Surveillance/United-States-COVID-19-Cases-and-Deaths-by-State-o/9mfq-cb36) (9). Thus, for the purpose of timeseries graph, the rows with negative number of cases and deaths will be dropped but for ANOVA test where the mean number of the cases and deaths will be testes, no rows will be dropped to account for the adjustment in the number of cases and deaths over time.

# Findings:

The mean of new daily new cases is higher in Alabama (253.6 daily new cases per 1,000,000) compared to Georgia (195.2 daily new cases per 1,000,000) contrary to what was expected. However, the mean of new daily deaths is less in Alabama compared to Georgia (3.4 vs. 4.6 cases per 1,000,000 respectively). The mean and standard deviations of daily new cases and new deaths are provided in Table 1.

#### Table 1. Descriptive statistics of daily new cases and new deaths per 1,000,000 in Alabama and Georgia

|     |  New Cases  |New Death |
|------|------------|----------|
|      | Mean(SD) |Mean(SD)|
| Alabama| 253.6 (108.3)|3.4 (3.0)|
|Georgia|195.2 (92.3)| 4.6 (5.0)|

Time series graphs for daily new cases did not show a decline of cases in Alabama compared to Georgia, contrary to what was expected. However, daily new death graph shows that overall Alabama had lower number of deaths and the trend was more stable compared to Georgia.

Results from repeated measured ANOVA shows a statistically difference between daily number of cases (F statistics: 19.684, P- value <0.001) and daily number of death (F statistics: 19.684, P-value:0.029) comparing Alabama to Georgia. The number of daily new cases is higher in Alabama compared to Georgia contrary to our hypothesis, but number of new deaths is lower in Alabama compared to Georgia aligned with our hypothesis.

# Discussion:

Contrary to the expectations, the number of daily new cases were higher in Alabama compared to Georgia. However, the daily number of new deaths were lower in Alabama compared to Georgia, as expected. It is important to note that with choosing Alabama and Georgia as neighbor states that have different mask wearing policies, I was able to minimize the differences in some of the confounding factors such as temperature and demographic configurations which is a strength of this project. Also, with normalizing the data to the population of the states, the population confounding effect of population density was minimized to some level. However, there are confounding factors that could not be accounted for in this analysis which is a limitation. For example, [social distancing policies include mask wearing but its not limited to that. Social distancing measures include stay at home order, mandatory quarantine, non-essential business closure, large gathering ban, restaurant limits, bar closue](https://www.kff.org/coronavirus-covid-19/issue-brief/state-data-and-policy-actions-to-address-coronavirus/) (10). Even though the main difference in social distancing policies comparing these two states is mask wearing, however, from March to November other social distancing policies got effected and some were lifted in these two states which can directly affect the results. The fact that we are seeing more cases in Alabama compared to Georgia could be the result of other social distancing policies that got effected in Geprgia and were lifted in Alabama. Also, it is important to note that Georgia has a mask wearing policy for the employees which can affect the rest of the population positively (e.g. more people wear masks). Finally, while number of new cases are higher in Alabama, the death is significantly lower in this state.

# Conclusions:

In conclusion the findings from this research support the hypothesis that mask wearing policy could be associated with lower number of death and thus burden of COVID-19. Eventhough no causality can be inferred, but results support advocating for maskwearing policies to decrease the burden of COVID-19.

# References:

1- Center for Disease Control Prevention: https://www.cdc.gov/coronavirus/2019-ncov/cases-updates/about-epidemiology/identifying-source-outbreak.html

2- John Hopkins University. Corona Virus Resource Center: https://coronavirus.jhu.edu/map.html

3- WHO: https://www.who.int/publications/m/item/weekly-epidemiological-update---10-november-2020

4- Center for Disease Control Prevention: https://www.cdc.gov/coronavirus/2019-ncov/prevent-getting-sick/cloth-face-cover-guidance.html

5- https://masks4all.co/what-states-require-masks/

6- https://www.census.gov/quickfacts/GA

7- https://www.census.gov/quickfacts/AL

8- https://www.cdc.gov/coronavirus/2019-ncov/community/health-equity/race-ethnicity.html

9- https://data.cdc.gov/Case-Surveillance/United-States-COVID-19-Cases-and-Deaths-by-State-o/9mfq-cb36

10- https://www.kff.org/coronavirus-covid-19/issue-brief/state-data-and-policy-actions-to-address-coronavirus/_____no_output_____

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import statsmodels.api as sm
import statsmodels.formula.api as smf
from statsmodels.stats.anova import anova_lm
import pingouin as pg_____no_output_____

$\large\text{Step 1: Select the dataset}$
_____no_output_____The ["United States COVID-19 Cases and Deaths by State over Time"](https://healthdata.gov/dataset/united-states-covid-19-cases-and-deaths-state-over-time) dataset provides detail daily data on the cases and death of COVID-19 for all the states from January to November 11th 2020. All the data in this dataset is unidentified and there is no way to track back this information. This data is available through a government website.

1- Loading the dataset after dowloading it._____no_output_____

covid_data=  pd.read_csv('/Users/sdibay/Downloads/United_States_COVID-19_Cases_and_Deaths_by_State_over_Time.csv')_____no_output_____

$\large\text{Step 2: Processing the data}$
_____no_output_____2- Dropping all states except Georgia and Alabama_____no_output_____

covid_data_subset = covid_data[(covid_data.state == 'AL')|(covid_data.state == 'GA')]_____no_output_____

3- Making sure that the date has data type since I will subset the data to cew cases and death between July 17th to November 11th. _____no_output_____

#changing the submission date to date type
covid_data_subset['submission_date']= pd.to_datetime(covid_data_subset['submission_date'])<ipython-input-4-c167b098e8ab>:2: SettingWithCopyWarning:
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  covid_data_subset['submission_date']= pd.to_datetime(covid_data_subset['submission_date'])

4- Checking dataset and specifically quick descriptive analysis on the new_case and new_death columns_____no_output_____

covid_data_subset.head(3)_____no_output_____covid_data_subset.groupby('state')['new_case'].describe()_____no_output_____covid_data_subset.groupby('state')['new_death'].describe()_____no_output_____

5- Dropping dates before July 17th since this is the date that mask wearing law got effective in Alabama so we compare cases/deaths in two sates after this date.

6- We see that we have negative numbers in the dataset for new cases and new death. Per CDC ["Because these provisional counts are subject to change, including updates to data reported previously, adjustments can occur. These adjustments can result in fewer total numbers of cases and deaths compared with the previous data, which means that new numbers of cases or deaths can include negative values that reflect such adjustments."](https://data.cdc.gov/Case-Surveillance/United-States-COVID-19-Cases-and-Deaths-by-State-o/9mfq-cb36). For the purpose of the timeseries graph,  rows with negative new cases and new deaths are dropped. However, for the purpose of hypothesis testing, the negative cases and deaths will not be dropped to adjust for the counts in number of new cases and new deaths as discussed above._____no_output_____

#drop rows with date before July 17th
res= covid_data_subset[covid_data_subset["submission_date"] > '2020-07-17'] _____no_output_____

7- Reseting indexes to be able to properly access the data and create new columns_____no_output_____

# reseting indexes
res= res.reset_index(drop=True)_____no_output_____

8- Creating new cases per 1,000,000 and new death per 1,000,000 for two states normalized for the population in these states. This step is done so the numbers in two states become comparable.

[Georgia population estimates by July 1, 2019 is 10,617,423](https://www.census.gov/quickfacts/GA)

[Alabama population estimates by July 1, 2019 is	4,903,185](https://www.census.gov/quickfacts/AL)_____no_output_____

# creating two columns to calculate new cases and new deaths per 1,000,000

# create emply 'new_case_per1000000' column
res['new_case_per1000000'] = ""

# create emply 'new_death_per1000000' column
res['new_death_per1000000'] = ""

# for loop that iterates through new_case and new_death columns and create the normalized new_case_per1000000
#and new_death per 1000000 columns
for i in range(len(res)):
    if (res.at[i,'state']=='AL'):
        res.at[i,'new_case_per1000000'] =  res.at[i,'new_case']/4903185.0*1000000.0
        res.at[i,'new_death_per1000000'] =  res.at[i,'new_death']/4903185.0*1000000.0
    if (res.at[i,'state']=='GA'):
        res.at[i,'new_case_per1000000'] =  res.at[i,'new_case']/10617423.0*1000000.0
        res.at[i,'new_death_per1000000'] =  res.at[i,'new_death']/10617423.0*1000000.0_____no_output_____

9- Saving two columns as numerical to make sure the ANOVA models work_____no_output_____

# changing 'new_case_per1000000' from previous step to numeric
res['new_case_per1000000']=pd.to_numeric(res['new_case_per1000000'])

# changing 'new_death_per1000000' from previous step to numeric
res['new_death_per1000000']=pd.to_numeric(res['new_death_per1000000'])_____no_output_____

10- Making sure rows that have negative new_case and new_death are ommited for the purpose of timeseries graph._____no_output_____

res_one= res[res["new_case_per1000000"] > 0]
res_one= res[res["new_death_per1000000"] > 0] _____no_output_____

$\large\text{Step 3: Descriptive Statistical Analysis}$
_____no_output_____Describing the normalized new cases and new death for two states. The mean of new daily new cases is higher in Alabama compared to Georgia contrary to what was expected. The minimum of daily new cases however is lower in Alabama compared to Georgia (0.40 vs. 1.32 cases per 1,000,000).  _____no_output_____

# mean, and range
res.groupby('state')['new_case_per1000000'].describe()_____no_output_____# standard deviation for new cases
res.groupby('state')['new_case_per1000000'].std()_____no_output_____res.groupby('state')['new_death_per1000000'].describe()_____no_output_____# standard deviation for new deaths
res.groupby('state')['new_death_per1000000'].std()_____no_output_____

Creating distributions of daily new_cases per 1,000,0000 and new_death per 1,000,000 from July 17th to November 11th.
We can observe that the distributions of the days with certain number of new cases is differnt between the states. _____no_output_____

#plotting the distribution of new cases for Georgia and Alabama
sns.displot(res, x="new_case_per1000000", hue="state")
plt.legend(title='States', loc='upper right', labels=['Georgia', 'Alabama'])
plt.title('Distribution of Covid-19 daily new cases per 1,000,000 from July 17th 2020 to November 11th 2020 in Georgia and Alabama ')_____no_output_____

Creating distributions of daily new_death per 1,000,0000 and new_death per 1,000,000 from July 17th to November 11th.
We can observe that the distributions of the days with certain number of deaths is differnt between the states. Georgia shows more number._____no_output_____

#plotting the distribution of new death for Georgia and Alabama
sns.displot(res, x="new_death_per1000000", hue="state")
plt.legend(title='States', loc='upper right', labels=['Georgia', 'Alabama'])
plt.title('Distribution of Covid-19 daily new deaths per 1,000,000 from July 17th 2020 to November 11th 2020 in Georgia and Alabama')_____no_output_____

Creating the new cases per 1,000,000 time series graph for two states from July 17th 2020 to November 11th 2020_____no_output_____

sns.set_theme(style="whitegrid")
sns.set_context("notebook", font_scale=0.25, rc={"lines.linewidth": 2.5})
sns.set(rc={'figure.figsize':(11.7,8.27)})
# Load an example dataset with long-form data
# Plot the responses for different events and regions
sns.lineplot(x="submission_date", y="new_case_per1000000",
             hue="state",
             data=res_one).set_title('COVID-19 daily new cases per 1,000,000 trend from July 17th 2020 to November 11th 2020 in Georgia and Alabama', fontsize=14)
plt.legend(title='States', loc='upper right', labels=['Alabama', 'Georgia'])_____no_output_____

Creating the new deaths per 1,000,000 time series graph for two states from July 17th 2020 to November 11th 2020_____no_output_____

sns.set_theme(style="whitegrid")
sns.set_context("notebook", font_scale=0.25, rc={"lines.linewidth": 2.5})
sns.set(rc={'figure.figsize':(11.7,8.27)})
# Load an example dataset with long-form data
# Plot the responses for different events and regions
sns.lineplot(x="submission_date", y="new_death_per1000000",
             hue="state",
             data=res_one).set_title('COVID-19 daily new deaths per 1,000,000 trend from July 17th to November 11th 2020 in Georgia and Alabama', fontsize=14)
plt.legend(title='States', loc='upper right', labels=['Alabama', 'Georgia'])_____no_output_____

$\large\text{Step 4: Hypothesis testing}$
_____no_output_____Running repeated measures ANOVA test to compare the mean number of new cases between two states over time_____no_output_____

# repeated measured ANOVA testing to compare the mean of new cases per 1,000,000 comparing two states
res.anova(dv="new_case_per1000000", between=['state']).round(3)
_____no_output_____

Running repeated measures ANOVA test to compare the mean number of new deaths between two states over time_____no_output_____

## repeated measured ANOVA testing to compare the mean of new deaths per 1,000,000 comparing two states
res.anova(dv="new_death_per1000000", between=['state']).round(3)_____no_output_____