# Notebook from UIUC-iSchool-DataViz/spring2019online
Path: _site/week06/prep_notebook_week06.ipynb

# Activity #1 - intro to traitlets and object oriented programming
* So first, we'll do the relatively dry stuff - playing with some concepts of object oriented programming_____no_output_____

# it is possible you will have to install this
import traitlets_____no_output_____# ok, lets make a simple object class
class MyObject(traitlets.HasTraits):
    # lets use the unicode trait to let our object have a name
    name = traitlets.Unicode("unnamed")
    # lets also give our object an age
    age = traitlets.Int(0)_____no_output_____# now, lets create an object we can manipulate out of this class
my_obj = MyObject()
my_obj_____no_output_____# so far, nothing is defined yet and the object has the
#  class's defaults
my_obj.name, my_obj.age_____no_output_____# lets create another object
# and here we can show how it can
#  be created and initialized
obj2 = MyObject(name = "Bob", age = 47)
obj2.name, obj2.age_____no_output_____# now lets create a function that can act on
#  one of the variables of our object
#  in particluar, its going to change the name in
#  my object
def name_changed(change):
    print(change['new']) # this is a format that is required of a traitlets class

# lets tell traitlets that we are going to change something
my_obj.observe(name_changed, ['name'])_____no_output_____my_obj.name = "Linda"Linda
# note: right now this seems overly complicated
#  why would we do this?  can't I just simply
# assign the name to the object's name??
#  whats with this whole function name_changed
#  and .observe?

# so, we can actually associate actions with changes
# Behold:
def name_changed2(change):
    print("I just changed '%s' to '%s'" % (change['old'], change['new']))

# lets apply this to our other object we created
obj2.name_____no_output_____obj2.observe(name_changed2, ['name'])_____no_output_____obj2.name = "Gene"I just changed 'Bob' to 'Gene'
# we can even take a look at the nitty gritty
# details of how this change is taking place
# "under the hood" with traitlets

# lets create a new object
obj3 = MyObject()_____no_output_____# and now, instead of specifying an action
#  with our change, lets look at what
# change really is:
def trait_change(change):
    print("Change dictionary:")
    print(change)

obj3.observe(trait_change)_____no_output_____obj3.name = "Something Else"
# so we can see that there are these weird "new" and "old" names
#  this is tracking how things change
#  the "owner" is just pointint to obj3 (of class MyObject)Change dictionary:
{'owner': <__main__.MyObject object at 0x109b473d0>, 'new': u'Something Else', 'old': u'unnamed', 'name': 'name', 'type': 'change'}
# lets try another one:
obj3.age = 10Change dictionary:
{'owner': <__main__.MyObject object at 0x109b473d0>, 'new': 10, 'old': 0, 'name': 'age', 'type': 'change'}

### An aside: string printing_____no_output_____

# just as an aside, there are fancy ways to print out strings
#  and numbers in one line

# here is one example
"String formatting is sticking stuff %s the middle: %0.5f" % ("in", 10.135)
# so "in" is placed at %s and 10.135 is placed at %0.5f_____no_output_____# we can also insert strings as place holder numbers:
"{2} nicer way {0} to do this, {1}".format("is", "see", "The")_____no_output_____

### Take aways
* this was just a quick overview of traitlets
* traitlets are another way to use interactivity in python - we can create classes and then track changes to those classes with our "change" functions_____no_output_____# Activity #2 - More details of ipywidgets
* We've already done a LOT of things with ipywidgets
* we'll get into some more of the nitty gritty details of different things we can use with widgets now_____no_output_____

# import, per usual
import ipywidgets

# so, this is how we've used widgets before for the most part
@ipywidgets.interact(name = ['Linda', 'Tina', 'Louise'])
def print_name(name):
    print(name) # just a simple print out_____no_output_____# but now, lets go into ipywidgets in a bit more detail
# and look at specific functions

# for example, we can create a little display
# that increments integer numbers
itext = ipywidgets.IntText()_____no_output_____# to be able to display our widget
from IPython.display import display
# note: depending on the version of jupyter
#  you are using, you may or maynot have
#  to use "display" to display things
#  we'll do it both ways, starting
#  with display

# lets show the thing!
display(itext)
# you can see there are little numbers at the
#  end that we can toggle up and down_____no_output_____# note that if I make another display of itext,
# this one and the previous one have values that are
# tied together
display(itext)_____no_output_____# also, the value of itext is then stored - so
#  we could in theory generate a toggle and then
# do stuff with its value
itext.value
# note if I go up and change the toggle value
#  I have to re-run this cell to print out the
# newly stored value_____no_output_____# I can also set the value "by hand"
itext.value = 10
# once I run this cell, now the toggle values are
# updated above_____no_output_____# we can also do fun things like create progress bars
ip = ipywidgets.IntProgress()_____no_output_____display(ip)_____no_output_____# so, its probably hard to see, but there is indeed a
# bar there
# I can show this better by setting the progress
# to 90% by hand:
ip.value = 90_____no_output_____# we can also check out more of the interer counters,
#  like this nifty slider
irange = ipywidgets.IntRangeSlider(min = -10, max = 10, step = 1)_____no_output_____display(irange)_____no_output_____# again, the value of the slider is stored
#  if we want to use it
irange.value_____no_output_____

### BY THEIR POWERS COMBINED!!_____no_output_____

# we can also mix traitlets & ipython widgets
#  in neat ways
#  here, we'll use the ipywidgets function
#  link to associate 2 different things

# a new instance of my class as m
m = MyObject(name = "Bob", age = 47)
# lets link an widget label with a traitlet class value
l = ipywidgets.Label()

# lets link the traitelet name with the
# widget label, l
ipywidgets.link( (m, "name"), (l, "value") )_____no_output_____# now if we display this widget label
#  we see the traitlet object's name
display(l)_____no_output_____# we can even update the traitlet name & the
# widget label updates as well
m.name = "Linda"_____no_output_____

### More ipywidgets fun_____no_output_____

# we can use ipywidgets to make clickers
button1 = ipywidgets.Button(description = "I am a Clicker")_____no_output_____display(button1)
# we note that nothing happens when we press this button_____no_output_____# now, lets make a function that can act when we press
# this button
def say_click(event):
    print("I have clicked. Click.")_____no_output_____# now we can tell our button that we've created to
# say click when we click
button1.on_click(say_click)
# we'll note that this is "back reactive" - so now
# if we go up to our displayed button and press click
# it says it has clicked were before it did nothing_____no_output_____# we can use ipywidgets to create text boxes
ta1 = ipywidgets.Textarea("Hi, this is a box of text. (1)")

ta2 = ipywidgets.Textarea("Hi, this is a box of text. (2)")

ta3 = ipywidgets.Textarea("Hi, this is a box of text. (3)")

ta4 = ipywidgets.Textarea("Hi, this is a box of text. (4)")_____no_output_____# first we'll just display the first one
display(ta1)_____no_output_____# we can organize these text boxes in a few ways
#  one way is to use ipywidgets to display things
#  in tabular form:
tabs = ipywidgets.Tab([ta1, ta2, ta3, ta4])_____no_output_____display(tabs)_____no_output_____# or we can "accordian" them:
acc = ipywidgets.Accordion([ta1, ta2, ta3, ta4])_____no_output_____display(acc)_____no_output_____# or we can organize them into horizontal boxes
ipywidgets.HBox([ta1, ta2, ta3, ta4])
# note this means a few of them are off screen so
# this might not be the best option_____no_output_____# or we can organize them in a vertical manner
ipywidgets.VBox([ta1, ta2, ta3, ta4])_____no_output_____# or, if we want to be super fancy we
# can organize them in a verticle box of
# sets of horizontal boxes
ipywidgets.VBox( [ipywidgets.HBox([ta1, ta2]),
                  ipywidgets.HBox([ta3, ta4])] )_____no_output_____# we can even insert a label in our array of
# horizontal boxes and see how ipywidgets would
#  place this text
ipywidgets.VBox( [ipywidgets.HBox([ta1, ta2]),
                  ipywidgets.Label("Hello there!"),
                  ipywidgets.HBox([ta3, ta4])] )_____no_output_____

* So, those were just some toy examples
* Lets see how we can create something interactive with some of these tools that will update a variable based on the interactivity that we have built_____no_output_____

# lets start by making a progress bar again:
ip = ipywidgets.IntProgress()
# now, lets add in a button that will add 10
button_plus = ipywidgets.Button(description = "+10")
# and one that will subtract 10
button_minus = ipywidgets.Button(description = "-10")

# lets see how this looks in a horizontal box row:
ipywidgets.HBox([button_minus, ip, button_plus])
# we note if we click these, nothing happens
#  this is because we haven't associated actions
#  to our clicks_____no_output_____# so, no matter what we press, we don't see anything happening
ip.value_____no_output_____# so, lets now associate a change in the value of our
# progress bar when we click the down button:
def click_down(event):
    ip.value -= 10

# and lets tie this change in value to
# the click with the "on_click" function of our down button
button_minus.on_click(click_down)_____no_output_____# same type of thing, but for our up button
def click_up(event):
    ip.value += 10
button_plus.on_click(click_up)_____no_output_____# lets try this again:
ipywidgets.HBox([button_minus, ip, button_plus])
# note also again that these associations with
# these up and down click functions are in a sense
# "back reactive" - so now our previous instance of this
# clicker and progress bar updates as well!_____no_output_____

* ok, lets make some more fun linking examples_____no_output_____

# lets first make a slider
islider = ipywidgets.IntSlider(min = 0, max = 10, step = 1, orientation = 'vertical')_____no_output_____# lets give this slider a base color that
# is sort of purple-y, using a hex code
islider.style.handle_color = "#750075"_____no_output_____islider # note here (and above with our boxes) I'm not using "display"
# note this slider slides up and down,
# nothing too exciting_____no_output_____# lets create a new widget object called a color picker:
cp = ipywidgets.ColorPicker()
cp
# when we show this we can click on the little
# box and it popus up a color picker we can mess around with_____no_output_____# now, lets link the slider and the color picker
ipywidgets.link( (cp, 'value'), (islider.style, 'handle_color'))_____no_output_____ipywidgets.VBox([cp, islider])
# note how previous instances of these
# objects are tied together
#  also note how the .link function
# sort of intuitievly knows how to link these
#  two interactive widgets together_____no_output_____# just a few more fun examples and then we'll move
# on to bqplot

# we can create a date picker:
ipywidgets.DatePicker()_____no_output_____# we can also make a widget that does allows
#  us to "press play" or interact with
# playable things like music or animation
ipywidgets.Play()
# of course, we haven't linked any functions to
# it now, but lets try that_____no_output_____# as a quick example, lets link an integer slider to
# our play button

# first, lets define the range of our play button
play = ipywidgets.Play(interval = 50, value = 50, min = 1, max = 100, step = 1, description = "Press Play")
# so, we start at 50, and create min and max value around that

# now, lets create our slider
slider = ipywidgets.IntSlider()

# now, lets associate the min and max of the play interval
#  with the min and max of the integer slider
ipywidgets.link((play, 'min'), (slider, 'min'))
ipywidgets.link((play, 'max'), (slider, 'max'))
ipywidgets.link((play, 'value'), (slider, 'value'))
# note that we don't have to tell the .link function
# how to link these objects - it knows intuatively
# how to link the play widget min & max to the
# slider min and max

# lets display them side by side
ipywidgets.HBox([play, slider])_____no_output_____# note if we display the slider on its own,
# its still linked to the play button above
slider_____no_output_____

### Take aways
* so, we went through some toy examples for how we can link widgets together
* these may have seemed like some silly examples, but these basic ideas are used to build up higher-level interactivity packages like bqplot that we are going to use now_____no_output_____# Activity #3: Bqplot!_____no_output_____

# note: it is possible you will have to pip-install this or anaconda install this
#  also: you might have to close and reopen this notebook if when we get to
# "display" below, no figure pops up

# hopefully, you won't have to restart jupyter but maybe

import bqplot
import numpy as np_____no_output_____

Now we are going to mess around with some of the declaritive programming type options that bqplot can use.  This will rely heavily on the "Grammar of Graphics" constructs._____no_output_____

# lets first start by creating data elements for our graphic
# just some random numbers:
x = np.arange(100)
y = np.random.random(100) + 5_____no_output_____# now we'll define some scale objects which will
# determine how lines will be drawn on our canvas
x_sc = bqplot.LinearScale()
y_sc = bqplot.LinearScale()_____no_output_____# now we are going to use GoG type calls to
# define what lines to actually draw combining
# information about our data and our scales
lines = bqplot.Lines(x = x, y = y, scales = {'x': x_sc, 'y': y_sc})_____no_output_____# now, we are going to define what axis we want placed around
# the lines that we draw
# we'll draw both x & y axis
ax_x = bqplot.Axis(scale = x_sc, label = 'X Value')
ax_y = bqplot.Axis(scale = y_sc, label = 'Y Value', orientation = 'vertical')_____no_output_____# finally, we combine all these things together into a
# bonified figure:
fig = bqplot.Figure(marks = [lines], axes = [ax_x, ax_y])
#display(fig)
# if you don't see the following fig, here is where
#  you might have to close and reopen your notebook
fig # note: just "fig" instead of "display(fig)" may also be an option for you_____no_output_____# ok, but this isn't interactive in anyway
# lets make it!!
pz = bqplot.interacts.PanZoom( scales = {'x': [x_sc], 'y': [y_sc]})_____no_output_____fig = bqplot.Figure(marks = [lines], axes = [ax_x, ax_y], interaction = pz)
display(fig)
# note that if I pan and zoom, the figure updates. Ooooo. fancy
#  note also, that the above figure also reacts as well
# this is because we are using the same lines & ax's objects (I think)_____no_output_____# lets see an example of where this can fail

# first lets make an x from 0-10 in 100 steps
x = np.mgrid[0.0:10.0:100j]
# and 2 y variables
y1 = x * 2
y2 = x**2_____no_output_____x_sc = bqplot.LinearScale(min = 1, max = 10)
# lets do one y-scale over linear and 1 over log
y_sc1 = bqplot.LinearScale(min = 1, max = 20)
y_sc2 = bqplot.LogScale(min = 1, max = 100)_____no_output_____# lets genrate lines for each y value
lines1 = bqplot.Lines(x = x, y = y1, scales = {'x': x_sc, 'y': y_sc1})
lines2 = bqplot.Lines(x = x, y = y2, scales = {'x': x_sc, 'y': y_sc2})_____no_output_____# and lets plot an x axis like before
ax_x = bqplot.Axis(scale = x_sc, label = 'X Value')
# and one y axis on the left
ax_y1 = bqplot.Axis(scale = y_sc1, label = 'Y1 Value',
                    orientation = 'vertical')
# and one y-axis on the right
ax_y2 = bqplot.Axis(scale = y_sc2, label = 'Y2 Value',
                    orientation = 'vertical', side = 'right')_____no_output_____# lets allow pan and zoom
pz = bqplot.interacts.PanZoom(scales = {'x': [x_sc],
                                        'y': [y_sc1, y_sc2]})
#bqplot.interacts.PanZoom?
fig = bqplot.Figure(marks = [lines1, lines2],
                    axes = [ax_x, ax_y1, ax_y2], interaction=pz)
#display(fig)
fig
# now we note if we zoom out too far, or pan to too negative of the x-axis
# we lose a line

# why? because the line is log-scaled, and log(numbers < 0) is undefined

# this is a way in which declaritive programming can fail because there
#  aren't obvious options to inhibit pan&zoom to a positive range_____no_output_____

FYI lots of more fun notebooks here: https://github.com/dmadeka/PyGotham-2017

From video tutorial here: https://www.youtube.com/watch?v=rraXF0EjRC8_____no_output_____

# ok, lets do another quick interactive example using a scatter plot
x = np.random.random(100)
y = np.random.random(100)_____no_output_____x_sc = bqplot.LinearScale()
y_sc = bqplot.LinearScale()

x_ax = bqplot.Axis(scale = x_sc, label = 'X')
y_ax = bqplot.Axis(scale = y_sc, label = 'Y', orientation = 'vertical')_____no_output_____scatters = bqplot.Scatter(x = x,
                          y = y,
                          scales = {'x': x_sc, 'y': y_sc})_____no_output_____# now, lets create a selector to select points along the x-axis
selector = bqplot.interacts.FastIntervalSelector(
            scale = x_sc, marks = [scatters])
# we can also highlight what points we are selecting
scatters.unselected_style={'opacity': 0.8}
scatters.selected_style={'fill': 'red', 'stroke': 'yellow'}
# note we are selecing along the x-scale
fig = bqplot.Figure(marks = [scatters], axes = [x_ax, y_ax], interaction = selector)
display(fig)
# this might depend on what computer you are in, but on my mac, I click
#  to start selecting and then double click to "lock in" my selected
# region_____no_output_____# we can then print out what region is selected
selector.selected_____no_output_____# now, lets try some interactive histogramming of our buildings data
import pandas as pd
buildings = pd.read_csv("/Users/jillnaiman1/Downloads/building_inventory.csv",
                        na_values = {'Year Acquired': 0,
                                     'Year Constructed': 0,
                                     'Square Footage': 0})_____no_output_____# since buildings is our data, we don't have to do anything
#  with that, but we do need to create our scales and
# axes like we've been doing before:
# (1)
x_sc = bqplot.LinearScale()
y_sc = bqplot.LinearScale()
x_ax = bqplot.Axis(scale = x_sc)
y_ax = bqplot.Axis(scale = y_sc, orientation = 'vertical')

# (2) now, lets do an interactive rebinning, but lets
# use bqplot and a slider widget to do it
hist = bqplot.Hist(sample = buildings["Year Acquired"],
            scales = {'sample': x_sc, 'count': y_sc},
                   bins = 128, normalized = True,
                   colors = ["#FFFFFF"])

# lets also create a slider like we've done before
islider = ipywidgets.IntSlider(min = 8, max = 128, step = 1)
# and lets link our sider and our bins of our histogram
ipywidgets.link((islider, 'value'), (hist, 'bins'))
# construct a fig
#fig = bqplot.Figure(marks = [hist], axes = [x_ax, y_ax])
# ***RUN NEXT CELL BEFORE ADDING 2ND HIST

# (3) ok, but maybe we want to see our original histogram
#  underneath, lets add this to our figure
hist2 = bqplot.Hist(sample = buildings["Year Acquired"],
                   opacity = 0.1, normalized = True,
            scales = {'sample': x_sc, 'count': y_sc},
                  bins = 128)
fig = bqplot.Figure(marks = [hist, hist2], axes = [x_ax, y_ax])

# for 2 & 3
#display(ipywidgets.VBox([fig, islider]))
ipywidgets.VBox([fig, islider])_____no_output_____

# Activity #4: Wealth of Nations plot
* originially from the TedTalk: https://www.ted.com/talks/hans_rosling_shows_the_best_stats_you_ve_ever_seen
* found on Rosling's website: https://www.ted.com/talks/hans_rosling_shows_the_best_stats_you_ve_ever_seen
* We're going to make a tool similar to GapMinders:https://www.gapminder.org/world/
* Much of this is, in more detail, in the PyGothum-2017 github: https://github.com/dmadeka/PyGotham-2017
* This will talk to javascript on the backend to mimic the output of another plotting package d3.js, but we don't have to learn about d3.js (just now) and can instead rely on our current Python knowledge_____no_output_____

# import pandas if we have not
import pandas as pd

# lets start off our plot at the initial year of 1800
initial_year = 1800_____no_output_____# we'll read in our datafile and apply
# some pre-written cleaning routines
# get out the data we want for our plotting
from wealth_of_nations import process_data, get_min_max, get_data

# grab data
data = process_data('/Users/jillnaiman1/Downloads/nations.json')

data_____no_output_____# grab min & max values of our variables of interest
income_min, income_max, life_exp_min, life_exp_max, pop_min, pop_max = get_min_max(data)_____no_output_____# lets allow for a mouse-over interaction
# for silly:
import bqplot
tt = bqplot.Tooltip(fields=['name', 'x', 'y'],
                    labels=['Country Name',
                            'Income per Capita', 'Life Expectancy'])
#bqplot.Tooltip?_____no_output_____# we will label what year is being plotted, just like in the Gabminder plot
year_label = bqplot.Label(x=[0.75], y=[0.10],
                   font_size=52, font_weight='bolder',
                   colors=['orange'],
                   text=[str(initial_year)], enable_move=True)_____no_output_____# we'll define our scales like before
# here we scale our x & y axis to the scales of the min and max of our data
x_sc = bqplot.LogScale(min=income_min, max=income_max)
y_sc = bqplot.LinearScale(min=life_exp_min, max=life_exp_max)

# this is just something to color-code each circle by the region it corresponds to
#  (for example, asia, south america, africa, etc)
# the colors call is just mapping each catagorical variable to a color
c_sc = bqplot.OrdinalColorScale(domain=data['region'].unique().tolist(),
                                colors=bqplot.CATEGORY10[:6])

# finally, we want the size of each of our dots to correspond to the population of
# each country
#size_sc = bqplot.LinearScale(min=pop_min, max=pop_max)#, mid_range=0.1)
size_sc = bqplot.LinearScale(max=1326856173.0, min=2128.0)
#bqplot.LinearScale?_____no_output_____# create and label our x & y axis
ax_y = bqplot.Axis(label='Life Expectancy', scale=y_sc,
                   orientation='vertical', side='left',
                   grid_lines='solid')
ax_x = bqplot.Axis(label='Income per Capita', scale=x_sc,
                   grid_lines='solid')_____no_output_____# now we'll use another little function from our library above to grab
# data for our initial setup (year = 1800)
# Start with the first year's data
cap_income, life_exp, pop = get_data(data,initial_year,initial_year)_____no_output_____# now lets make our scatter plot!
wealth_scat = bqplot.Scatter(x=cap_income, y=life_exp,
                             color=data['region'], size=pop,
                      names=data['name'], display_names=False,
                      scales={'x': x_sc, 'y': y_sc, 'color': c_sc,
                              'size': size_sc},
                      default_size=4112, tooltip=tt,
                             animate=True, stroke='Black',
                      unhovered_style={'opacity': 0.5})
# much of these calls are things we've seen before, others will allow fun things
#  like animation and also the ability to click on our plot and interact with it_____no_output_____# for our initial, 1800 view, we'll just allow the first "line" of the evolution of the
# each nation's track to be displayed... this is essentially a place holder (visible = false)
nation_line = bqplot.Lines(x=data['income'][0],
                           y=data['lifeExpectancy'][0],
                           colors=['Gray'],
                       scales={'x': x_sc, 'y': y_sc}, visible=False)_____no_output_____# milliseconds of time between changes we make
time_interval = 10_____no_output_____# create the figure &
fig = bqplot.Figure(marks=[wealth_scat, year_label, nation_line],
                    axes=[ax_x, ax_y],
             title='Health and Wealth of Nations',
                    animation_duration=time_interval)

# lets control the size in pixels too
fig.layout.min_width = '960px'
fig.layout.min_height = '640px'_____no_output_____# we'll use our friend the int slider to slide through years
# for silly:
import ipywidgets
year_slider = ipywidgets.IntSlider(min=1800, max=2008, step=1, description='Year', value=initial_year)_____no_output_____# make sure we define what happens when we change the year on our slider
def year_changed(change):
    wealth_scat.x, wealth_scat.y, wealth_scat.size = get_data(data,year_slider.value,initial_year)
    #wealth_scat.size+=1000
    year_label.text = [str(year_slider.value)]

year_slider.observe(year_changed, 'value')_____no_output_____# now we'll say what happens when we hover over an object
# we'll use "change" again to make it such that if a
# user hovers over a country, the countries "life line"
#  is visible
def hover_changed(change):
    if change.new is not None:
        nation_line.x = data['income'][change.new + 1]
        nation_line.y = data['lifeExpectancy'][change.new + 1]
        nation_line.visible = True
    else:
        nation_line.visible = False

wealth_scat.observe(hover_changed, 'hovered_point')_____no_output_____# finally, lets add a little play button so we can animate
#  what happens in time, just like on the d3.js plot
play_button = ipywidgets.Play(min=1800, max=2008, interval=time_interval)
# note, we use "jslink" because the "backend" here is javascript
#  bqplot is just interacting with javascript
ipywidgets.jslink((play_button, 'value'), (year_slider, 'value'))_____no_output_____# finally, lets put it all together!!

ipywidgets.VBox([ipywidgets.HBox([play_button, year_slider]), fig])_____no_output_____pop_min, pop_max_____no_output_____import bqplot
bqplot.Tooltip?_____no_output_____


# Notebook from Madhu612/Practical-Data-Science-Specialization
Path: 1_ Analyze Datasets and Train ML Models using AutoML/Week1/project/C1_W1_Assignment.ipynb

# Register and visualize dataset

### Introduction

In this lab you will ingest and transform the customer product reviews dataset. Then you will use AWS data stack services such as AWS Glue and Amazon Athena for ingesting and querying the dataset. Finally you will use AWS Data Wrangler to analyze the dataset and plot some visuals extracting insights._____no_output_____### Table of Contents

- [1. Ingest and transform the public dataset](#c1w1-1.)
  - [1.1. List the dataset files in the public S3 bucket](#c1w1-1.1.)
    - [Exercise 1](#c1w1-ex-1)
  - [1.2. Copy the data locally to the notebook](#c1w1-1.2.)
  - [1.3. Transform the data](#c1w1-1.3.)
  - [1.4 Write the data to a CSV file](#c1w1-1.4.)
- [2. Register the public dataset for querying and visualizing](#c1w1-2.)
  - [2.1. Register S3 dataset files as a table for querying](#c1w1-2.1.)
    - [Exercise 2](#c1w1-ex-2)
  - [2.2. Create default S3 bucket for Amazon Athena](#c1w1-2.2.)
- [3. Visualize data](#c1w1-3.)
  - [3.1. Preparation for data visualization](#c1w1-3.1.)
  - [3.2. How many reviews per sentiment?](#c1w1-3.2.)
    - [Exercise 3](#c1w1-ex-3)
  - [3.3. Which product categories are highest rated by average sentiment?](#c1w1-3.3.)
  - [3.4. Which product categories have the most reviews?](#c1w1-3.4.)
    - [Exercise 4](#c1w1-ex-4)
  - [3.5. What is the breakdown of sentiments per product category?](#c1w1-3.5.)
  - [3.6. Analyze the distribution of review word counts](#c1w1-3.6.)_____no_output_____Let's install the required modules first._____no_output_____

# please ignore warning messages during the installation
!pip install --disable-pip-version-check -q sagemaker==2.35.0
!pip install --disable-pip-version-check -q pandas==1.1.4
!pip install --disable-pip-version-check -q awswrangler==2.7.0
!pip install --disable-pip-version-check -q numpy==1.18.5
!pip install --disable-pip-version-check -q seaborn==0.11.0
!pip install --disable-pip-version-check -q matplotlib===3.3.3/opt/conda/lib/python3.7/site-packages/secretstorage/dhcrypto.py:16: CryptographyDeprecationWarning: int_from_bytes is deprecated, use int.from_bytes instead
  from cryptography.utils import int_from_bytes
/opt/conda/lib/python3.7/site-packages/secretstorage/util.py:25: CryptographyDeprecationWarning: int_from_bytes is deprecated, use int.from_bytes instead
  from cryptography.utils import int_from_bytes
WARNING: Running pip as root will break packages and permissions. You should install packages reliably by using venv: https://pip.pypa.io/warnings/venv
/opt/conda/lib/python3.7/site-packages/secretstorage/dhcrypto.py:16: CryptographyDeprecationWarning: int_from_bytes is deprecated, use int.from_bytes instead
  from cryptography.utils import int_from_bytes
/opt/conda/lib/python3.7/site-packages/secretstorage/util.py:25: CryptographyDeprecationWarning: int_from_bytes is deprecated, use int.from_bytes instead
  from cryptography.utils import int_from_bytes
WARNING: Running pip as root will break packages and permissions. You should install packages reliably by using venv: https://pip.pypa.io/warnings/venv
/opt/conda/lib/python3.7/site-packages/secretstorage/dhcrypto.py:16: CryptographyDeprecationWarning: int_from_bytes is deprecated, use int.from_bytes instead
  from cryptography.utils import int_from_bytes
/opt/conda/lib/python3.7/site-packages/secretstorage/util.py:25: CryptographyDeprecationWarning: int_from_bytes is deprecated, use int.from_bytes instead
  from cryptography.utils import int_from_bytes
WARNING: Running pip as root will break packages and permissions. You should install packages reliably by using venv: https://pip.pypa.io/warnings/venv
/opt/conda/lib/python3.7/site-packages/secretstorage/dhcrypto.py:16: CryptographyDeprecationWarning: int_from_bytes is deprecated, use int.from_bytes instead
  from cryptography.utils import int_from_bytes
/opt/conda/lib/python3.7/site-packages/secretstorage/util.py:25: CryptographyDeprecationWarning: int_from_bytes is deprecated, use int.from_bytes instead
  from cryptography.utils import int_from_bytes
WARNING: Running pip as root will break packages and permissions. You should install packages reliably by using venv: https://pip.pypa.io/warnings/venv
/opt/conda/lib/python3.7/site-packages/secretstorage/dhcrypto.py:16: CryptographyDeprecationWarning: int_from_bytes is deprecated, use int.from_bytes instead
  from cryptography.utils import int_from_bytes
/opt/conda/lib/python3.7/site-packages/secretstorage/util.py:25: CryptographyDeprecationWarning: int_from_bytes is deprecated, use int.from_bytes instead
  from cryptography.utils import int_from_bytes
WARNING: Running pip as root will break packages and permissions. You should install packages reliably by using venv: https://pip.pypa.io/warnings/venv
/opt/conda/lib/python3.7/site-packages/secretstorage/dhcrypto.py:16: CryptographyDeprecationWarning: int_from_bytes is deprecated, use int.from_bytes instead
  from cryptography.utils import int_from_bytes
/opt/conda/lib/python3.7/site-packages/secretstorage/util.py:25: CryptographyDeprecationWarning: int_from_bytes is deprecated, use int.from_bytes instead
  from cryptography.utils import int_from_bytes
WARNING: Running pip as root will break packages and permissions. You should install packages reliably by using venv: https://pip.pypa.io/warnings/venv

# 1. Ingest and transform the public dataset

The dataset [Women's Clothing Reviews](https://www.kaggle.com/nicapotato/womens-ecommerce-clothing-reviews) has been chosen as the main dataset.

It is shared in a public Amazon S3 bucket, and is available as a comma-separated value (CSV) text format:

`s3://dlai-practical-data-science/data/raw/womens_clothing_ecommerce_reviews.csv`_____no_output_____
### 1.1. List the dataset files in the public S3 bucket

The [AWS Command Line Interface (CLI)](https://awscli.amazonaws.com/v2/documentation/api/latest/index.html) is a unified tool to manage your AWS services. With just one tool, you can control multiple AWS services from the command line and automate them through scripts. You will use it to list the dataset files._____no_output_____**View dataset files in CSV format**_____no_output_____```aws s3 ls [bucket_name]``` function lists all objects in the S3 bucket. Let's use it to view the reviews data files in CSV format:_____no_output_____
### Exercise 1

View the list of the files available in the public bucket `s3://dlai-practical-data-science/data/raw/`.

**Instructions**:
Use `aws s3 ls [bucket_name]` function. To run the AWS CLI command from the notebook you will need to put an exclamation mark in front of it: `!aws`. You should see the data file `womens_clothing_ecommerce_reviews.csv` in the list._____no_output_____

### BEGIN SOLUTION - DO NOT delete this comment for grading purposes
!aws s3 ls s3://dlai-practical-data-science/data/raw/. # Replace None
### END SOLUTION - DO NOT delete this comment for grading purposes

# EXPECTED OUTPUT
# ... womens_clothing_ecommerce_reviews.csv_____no_output_____

### 1.2. Copy the data locally to the notebook_____no_output_____```aws s3 cp [bucket_name/file_name] [file_name]``` function copies the file from the S3 bucket into the local environment or into another S3 bucket. Let's use it to copy the file with the dataset locally._____no_output_____

!aws s3 cp s3://dlai-practical-data-science/data/raw/womens_clothing_ecommerce_reviews.csv ./womens_clothing_ecommerce_reviews.csvdownload: s3://dlai-practical-data-science/data/raw/womens_clothing_ecommerce_reviews.csv to ./womens_clothing_ecommerce_reviews.csv

Now use the Pandas dataframe to load and preview the data._____no_output_____

import pandas as pd
import csv

df = pd.read_csv('./womens_clothing_ecommerce_reviews.csv',
                 index_col=0)

df.shape_____no_output_____df_____no_output_____

### 1.3. Transform the data
To simplify the task, you will transform the data into a comma-separated value (CSV) file that contains only a `review_body`, `product_category`, and `sentiment` derived from the original data._____no_output_____

df_transformed = df.rename(columns={'Review Text': 'review_body',
                                    'Rating': 'star_rating',
                                    'Class Name': 'product_category'})
df_transformed.drop(columns=['Clothing ID', 'Age', 'Title', 'Recommended IND', 'Positive Feedback Count', 'Division Name', 'Department Name'],
                    inplace=True)

df_transformed.dropna(inplace=True)

df_transformed.shape_____no_output_____

Now convert the `star_rating` into the `sentiment` (positive, neutral, negative), which later on will be for the prediction._____no_output_____

def to_sentiment(star_rating):
    if star_rating in {1, 2}: # negative
        return -1
    if star_rating == 3:      # neutral
        return 0
    if star_rating in {4, 5}: # positive
        return 1

# transform star_rating into the sentiment
df_transformed['sentiment'] = df_transformed['star_rating'].apply(lambda star_rating:
    to_sentiment(star_rating=star_rating)
)

# drop the star rating column
df_transformed.drop(columns=['star_rating'],
                    inplace=True)

# remove reviews for product_categories with < 10 reviews
df_transformed = df_transformed.groupby('product_category').filter(lambda reviews : len(reviews) > 10)[['sentiment', 'review_body', 'product_category']]

df_transformed.shape_____no_output_____# preview the results
df_transformed_____no_output_____

### 1.4 Write the data to a CSV file_____no_output_____

df_transformed.to_csv('./womens_clothing_ecommerce_reviews_transformed.csv',
                      index=False)_____no_output_____!head -n 5 ./womens_clothing_ecommerce_reviews_transformed.csvsentiment,review_body,product_category
1,If this product was in petite  i would get the petite. the regular is a little long on me but a tailor can do a simple fix on that.     fits nicely! i'm 5'4  130lb and pregnant so i bough t medium to grow into.     the tie can be front or back so provides for some nice flexibility on form fitting.,Blouses
1,"Love this dress!  it's sooo pretty.  i happened to find it in a store  and i'm glad i did bc i never would have ordered it online bc it's petite.  i bought a petite and am 5'8"".  i love the length on me- hits just a little below the knee.  would definitely be a true midi on someone who is truly petite.",Dresses
0,I had such high hopes for this dress and really wanted it to work for me. i initially ordered the petite small (my usual size) but i found this to be outrageously small. so small in fact that i could not zip it up! i reordered it in petite medium  which was just ok. overall  the top half was comfortable and fit nicely  but the bottom half had a very tight under layer and several somewhat cheap (net) over layers. imo  a major design flaw was the net over layer sewn directly into the zipper - it c,Dresses
1,I love  love  love this jumpsuit. it's fun  flirty  and fabulous! every time i wear it  i get nothing but great compliments!,Pants

# 2. Register the public dataset for querying and visualizing
You will register the public dataset into an S3-backed database table so you can query and visualize our dataset at scale. _____no_output_____
### 2.1. Register S3 dataset files as a table for querying
Let's import required modules.

`boto3` is the AWS SDK for Python to create, configure, and manage AWS services, such as Amazon Elastic Compute Cloud (Amazon EC2) and Amazon Simple Storage Service (Amazon S3). The SDK provides an object-oriented API as well as low-level access to AWS services.

`sagemaker` is the SageMaker Python SDK which provides several high-level abstractions for working with the Amazon SageMaker._____no_output_____

import boto3
import sagemaker
import pandas as pd
import numpy as np

sess   = sagemaker.Session()
# S3 bucket name
bucket = sess.default_bucket()
# AWS region
region = boto3.Session().region_name

# Account ID
sts = boto3.Session(region_name=region).client(service_name="sts", region_name=region)
account_id = sts.get_caller_identity()['Account']

print('S3 Bucket: {}'.format(bucket))
print('Region: {}'.format(region))
print('Account ID: {}'.format(account_id))S3 Bucket: sagemaker-us-east-1-517313198700
Region: us-east-1
Account ID: 517313198700

Review the empty bucket which was created automatically for this account.

**Instructions**:
- open the link
- click on the S3 bucket name `sagemaker-us-east-1-ACCOUNT`
- check that it is empty at this stage_____no_output_____

from IPython.core.display import display, HTML

display(HTML('Review Amazon S3 buckets'.format(region)))_____no_output_____

Copy the file into the S3 bucket._____no_output_____

!aws s3 cp ./womens_clothing_ecommerce_reviews_transformed.csv s3://$bucket/data/transformed/womens_clothing_ecommerce_reviews_transformed.csvupload: ./womens_clothing_ecommerce_reviews_transformed.csv to s3://sagemaker-us-east-1-517313198700/data/transformed/womens_clothing_ecommerce_reviews_transformed.csv

Review the bucket with the file we uploaded above.

**Instructions**:
- open the link
- check that the CSV file is located in the S3 bucket
- check the location directory structure is the same as in the CLI command above
- click on the file name and see the available information about the file (region, size, S3 URI, Amazon Resource Name (ARN))_____no_output_____

from IPython.core.display import display, HTML

display(HTML('Review Amazon S3 buckets'.format(bucket, region)))_____no_output_____

**Import AWS Data Wrangler**

[AWS Data Wrangler](https://github.com/awslabs/aws-data-wrangler) is an AWS Professional Service open source python initiative that extends the power of Pandas library to AWS connecting dataframes and AWS data related services (Amazon Redshift, AWS Glue, Amazon Athena, Amazon EMR, Amazon QuickSight, etc).

Built on top of other open-source projects like Pandas, Apache Arrow, Boto3, SQLAlchemy, Psycopg2 and PyMySQL, it offers abstracted functions to execute usual ETL tasks like load/unload data from data lakes, data warehouses and databases._____no_output_____Review the AWS Data Wrangler documentation: https://aws-data-wrangler.readthedocs.io/en/stable/_____no_output_____

import awswrangler as wr_____no_output_____

**Create AWS Glue Catalog database**_____no_output_____The data catalog features of **AWS Glue** and the inbuilt integration to Amazon S3 simplify the process of identifying data and deriving the schema definition out of the discovered data. Using AWS Glue crawlers within your data catalog, you can traverse your data stored in Amazon S3 and build out the metadata tables that are defined in your data catalog.

Here you will use `wr.catalog.create_database` function to create a database with the name `dsoaws_deep_learning` ("dsoaws" stands for "Data Science on AWS")._____no_output_____

wr.catalog.create_database(
    name='dsoaws_deep_learning',
    exist_ok=True
)_____no_output_____dbs = wr.catalog.get_databases()

for db in dbs:
    print("Database name: " + db['Name'])Database name: dsoaws_deep_learning

Review the created database in the AWS Glue Catalog.

**Instructions**:
- open the link
- on the left side panel notice that you are in the AWS Glue -> Data Catalog -> Databases
- check that the database `dsoaws_deep_learning` has been created
- click on the name of the database
- click on the `Tables in dsoaws_deep_learning` link to see that there are no tables_____no_output_____

from IPython.core.display import display, HTML

display(HTML('Review AWS Glue Databases'.format(region)))_____no_output_____

**Register CSV data with AWS Glue Catalog**_____no_output_____
### Exercise 2

Register CSV data with AWS Glue Catalog.

**Instructions**:
Use ```wr.catalog.create_csv_table``` function with the following parameters
```python
res = wr.catalog.create_csv_table(
    database='...', # AWS Glue Catalog database name
    path='s3://{}/data/transformed/'.format(bucket), # S3 object path for the data
    table='reviews', # registered table name
    columns_types={
        'sentiment': 'int',
        'review_body': 'string',
        'product_category': 'string'
    },
    mode='overwrite',
    skip_header_line_count=1,
    sep=','
)
```_____no_output_____

wr.catalog.create_csv_table(
    ### BEGIN SOLUTION - DO NOT delete this comment for grading purposes
    database='dsoaws_deep_learning', # Replace None
    ### END SOLUTION - DO NOT delete this comment for grading purposes
    path='s3://{}/data/transformed/'.format(bucket),
    table="reviews",
    columns_types={
        'sentiment': 'int',
        'review_body': 'string',
        'product_category': 'string'
    },
    mode='overwrite',
    skip_header_line_count=1,
    sep=','
)_____no_output_____

Review the registered table in the AWS Glue Catalog.

**Instructions**:
- open the link
- on the left side panel notice that you are in the AWS Glue -> Data Catalog -> Databases -> Tables
- check that you can see the table `reviews` from the database `dsoaws_deep_learning` in the list
- click on the name of the table
- explore the available information about the table (name, database, classification, location, schema etc.)_____no_output_____

from IPython.core.display import display, HTML

display(HTML('Review AWS Glue Catalog'.format(region)))_____no_output_____

Review the table shape:_____no_output_____

table = wr.catalog.table(database='dsoaws_deep_learning',
                         table='reviews')
table_____no_output_____

### 2.2. Create default S3 bucket for Amazon Athena

Amazon Athena requires this S3 bucket to store temporary query results and improve performance of subsequent queries.

The contents of this bucket are mostly binary and human-unreadable. _____no_output_____

# S3 bucket name
wr.athena.create_athena_bucket()

# EXPECTED OUTPUT
# 's3://aws-athena-query-results-ACCOUNT-REGION/'_____no_output_____

# 3. Visualize data

**Reviews dataset - column descriptions**

- `sentiment`: The review's sentiment (-1, 0, 1).
- `product_category`: Broad product category that can be used to group reviews (in this case digital videos).
- `review_body`: The text of the review._____no_output_____
### 3.1. Preparation for data visualization

**Imports**_____no_output_____

import numpy as np
import seaborn as sns

import matplotlib.pyplot as plt
%matplotlib inline
%config InlineBackend.figure_format='retina'_____no_output_____

**Settings**_____no_output_____Set AWS Glue database and table name._____no_output_____

# Do not change the database and table names - they are used for grading purposes!
database_name = 'dsoaws_deep_learning'
table_name = 'reviews'_____no_output_____

Set seaborn parameters. You can review seaborn documentation following the [link](https://seaborn.pydata.org/index.html)._____no_output_____

sns.set_style = 'seaborn-whitegrid'

sns.set(rc={"font.style":"normal",
            "axes.facecolor":"white",
            'grid.color': '.8',
            'grid.linestyle': '-',
            "figure.facecolor":"white",
            "figure.titlesize":20,
            "text.color":"black",
            "xtick.color":"black",
            "ytick.color":"black",
            "axes.labelcolor":"black",
            "axes.grid":True,
            'axes.labelsize':10,
            'xtick.labelsize':10,
            'font.size':10,
            'ytick.labelsize':10})_____no_output_____

Helper code to display values on barplots:_____no_output_____**Run SQL queries using Amazon Athena**_____no_output_____**Amazon Athena** lets you query data in Amazon S3 using a standard SQL interface. It reflects the databases and tables in the AWS Glue Catalog. You can create interactive queries and perform any data manipulations required for further downstream processing._____no_output_____Standard SQL query can be saved as a string and then passed as a parameter into the Athena query. Run the following cells as an example to count the total number of reviews by sentiment. The SQL query here will take the following form:

```sql
SELECT column_name, COUNT(column_name) as new_column_name
FROM table_name
GROUP BY column_name
ORDER BY column_name
```

If you are not familiar with the SQL query statements, you can review some tutorials following the [link](https://www.w3schools.com/sql/default.asp)._____no_output_____
### 3.2. How many reviews per sentiment?_____no_output_____Set the SQL statement to find the count of sentiments:_____no_output_____

statement_count_by_sentiment = """
SELECT sentiment, COUNT(sentiment) AS count_sentiment
FROM reviews
GROUP BY sentiment
ORDER BY sentiment
"""

print(statement_count_by_sentiment)
SELECT sentiment, COUNT(sentiment) AS count_sentiment
FROM reviews
GROUP BY sentiment
ORDER BY sentiment

Query data in Amazon Athena database cluster using the prepared SQL statement:_____no_output_____

df_count_by_sentiment = wr.athena.read_sql_query(
    sql=statement_count_by_sentiment,
    database=database_name
)

print(df_count_by_sentiment)   sentiment  count_sentiment
0         -1             2370
1          0             2823
2          1            17433

Preview the results of the query:_____no_output_____

df_count_by_sentiment.plot(kind='bar', x='sentiment', y='count_sentiment', rot=0)_____no_output_____

### Exercise 3

Use Amazon Athena query with the standard SQL statement passed as a parameter, to calculate the total number of reviews per `product_category` in the table ```reviews```.

**Instructions**: Pass the SQL statement of the form

```sql
SELECT category_column, COUNT(column_name) AS new_column_name
FROM table_name
GROUP BY category_column
ORDER BY new_column_name DESC
```

as a triple quote string into the variable `statement_count_by_category`. Please use the column `sentiment` in the `COUNT` function and give it a new name `count_sentiment`._____no_output_____

# Replace all None
### BEGIN SOLUTION - DO NOT delete this comment for grading purposes
statement_count_by_category = """
SELECT product_category, COUNT(sentiment) AS count_sentiment
FROM reviews
GROUP BY product_category
ORDER BY count_sentiment DESC
"""
### END SOLUTION - DO NOT delete this comment for grading purposes
print(statement_count_by_category)
SELECT product_category, COUNT(sentiment) AS count_sentiment
FROM reviews
GROUP BY product_category
ORDER BY count_sentiment DESC

Query data in Amazon Athena database passing the prepared SQL statement:_____no_output_____

%%time
df_count_by_category = wr.athena.read_sql_query(
    sql=statement_count_by_category,
    database=database_name
)

df_count_by_category

# EXPECTED OUTPUT
# Dresses: 6145
# Knits: 4626
# Blouses: 2983
# Sweaters: 1380
# Pants: 1350
# ...CPU times: user 468 ms, sys: 40.1 ms, total: 508 ms
Wall time: 3.9 s

### 3.3. Which product categories are highest rated by average sentiment?_____no_output_____Set the SQL statement to find the average sentiment per product category, showing the results in the descending order:_____no_output_____

statement_avg_by_category = """
SELECT product_category, AVG(sentiment) AS avg_sentiment
FROM {}
GROUP BY product_category
ORDER BY avg_sentiment DESC
""".format(table_name)

print(statement_avg_by_category)
SELECT product_category, AVG(sentiment) AS avg_sentiment
FROM reviews
GROUP BY product_category
ORDER BY avg_sentiment DESC

Query data in Amazon Athena database passing the prepared SQL statement:_____no_output_____

%%time
df_avg_by_category = wr.athena.read_sql_query(
    sql=statement_avg_by_category,
    database=database_name
)CPU times: user 318 ms, sys: 19.2 ms, total: 337 ms
Wall time: 3.31 s

Preview the query results in the temporary S3 bucket:  `s3://aws-athena-query-results-ACCOUNT-REGION/`

**Instructions**:
- open the link
- check the name of the S3 bucket
- briefly check the content of it_____no_output_____

from IPython.core.display import display, HTML

display(HTML('Review Amazon S3 buckets'.format(account_id, region, region)))_____no_output_____

Preview the results of the query:_____no_output_____

df_avg_by_category_____no_output_____

**Visualization**_____no_output_____

def show_values_barplot(axs, space):
    def _show_on_plot(ax):
        for p in ax.patches:
            _x = p.get_x() + p.get_width() + float(space)
            _y = p.get_y() + p.get_height()
            value = round(float(p.get_width()),2)
            ax.text(_x, _y, value, ha="left")

    if isinstance(axs, np.ndarray):
        for idx, ax in np.ndenumerate(axs):
            _show_on_plot(ax)
    else:
        _show_on_plot(axs)_____no_output_____# Create plot
barplot = sns.barplot(
    data = df_avg_by_category,
    y='product_category',
    x='avg_sentiment',
    color="b",
    saturation=1
)

# Set the size of the figure
sns.set(rc={'figure.figsize':(15.0, 10.0)})

# Set title and x-axis ticks
plt.title('Average sentiment by product category')
#plt.xticks([-1, 0, 1], ['Negative', 'Neutral', 'Positive'])

# Helper code to show actual values afters bars
show_values_barplot(barplot, 0.1)

plt.xlabel("Average sentiment")
plt.ylabel("Product category")

plt.tight_layout()
# Do not change the figure name - it is used for grading purposes!
plt.savefig('avg_sentiment_per_category.png', dpi=300)

# Show graphic
plt.show(barplot)_____no_output_____# Upload image to S3 bucket
sess.upload_data(path='avg_sentiment_per_category.png', bucket=bucket, key_prefix="images")_____no_output_____

Review the bucket on the account.

**Instructions**:
- open the link
- click on the S3 bucket name `sagemaker-us-east-1-ACCOUNT`
- open the images folder
- check the existence of the image `avg_sentiment_per_category.png`
- if you click on the image name, you can see the information about the image file. You can also download the file with the command on the top right Object Actions -> Download / Download as
_____no_output_____

from IPython.core.display import display, HTML

display(HTML('Review Amazon S3 buckets'.format(region)))_____no_output_____

### 3.4. Which product categories have the most reviews?

Set the SQL statement to find the count of sentiment per product category, showing the results in the descending order:_____no_output_____

statement_count_by_category_desc = """
SELECT product_category, COUNT(*) AS count_reviews
FROM {}
GROUP BY product_category
ORDER BY count_reviews DESC
""".format(table_name)

print(statement_count_by_category_desc)
SELECT product_category, COUNT(*) AS count_reviews
FROM reviews
GROUP BY product_category
ORDER BY count_reviews DESC

Query data in Amazon Athena database passing the prepared SQL statement:_____no_output_____

%%time
df_count_by_category_desc = wr.athena.read_sql_query(
    sql=statement_count_by_category_desc,
    database=database_name
)CPU times: user 314 ms, sys: 27.6 ms, total: 342 ms
Wall time: 3.26 s

Store maximum number of sentiment for the visualization plot:_____no_output_____

max_sentiment = df_count_by_category_desc['count_reviews'].max()
print('Highest number of reviews (in a single category): {}'.format(max_sentiment))Highest number of reviews (in a single category): 6145

**Visualization**_____no_output_____
### Exercise 4

Use `barplot` function to plot number of reviews per product category.

**Instructions**: Use the `barplot` chart example in the previous section, passing the newly defined dataframe `df_count_by_category_desc` with the count of reviews. Here, please put the `product_category` column into the `y` argument._____no_output_____

# Create seaborn barplot
barplot = sns.barplot(
    ### BEGIN SOLUTION - DO NOT delete this comment for grading purposes
    data=df_count_by_category_desc, # Replace None
    y='product_category', # Replace None
    x='count_reviews', # Replace None
    ### END SOLUTION - DO NOT delete this comment for grading purposes
    color="b",
    saturation=1
)

# Set the size of the figure
sns.set(rc={'figure.figsize':(15.0, 10.0)})

# Set title
plt.title("Number of reviews per product category")
plt.xlabel("Number of reviews")
plt.ylabel("Product category")

plt.tight_layout()

# Do not change the figure name - it is used for grading purposes!
plt.savefig('num_reviews_per_category.png', dpi=300)

# Show the barplot
plt.show(barplot)_____no_output_____# Upload image to S3 bucket
sess.upload_data(path='num_reviews_per_category.png', bucket=bucket, key_prefix="images")_____no_output_____

### 3.5. What is the breakdown of sentiments per product category?_____no_output_____Set the SQL statement to find the count of sentiment per product category and sentiment:_____no_output_____

statement_count_by_category_and_sentiment = """
SELECT product_category,
         sentiment,
         COUNT(*) AS count_reviews
FROM {}
GROUP BY  product_category, sentiment
ORDER BY  product_category ASC, sentiment DESC, count_reviews
""".format(table_name)

print(statement_count_by_category_and_sentiment)
SELECT product_category,
         sentiment,
         COUNT(*) AS count_reviews
FROM reviews
GROUP BY  product_category, sentiment
ORDER BY  product_category ASC, sentiment DESC, count_reviews

Query data in Amazon Athena database passing the prepared SQL statement:_____no_output_____

%%time
df_count_by_category_and_sentiment = wr.athena.read_sql_query(
    sql=statement_count_by_category_and_sentiment,
    database=database_name
)CPU times: user 425 ms, sys: 23.8 ms, total: 449 ms
Wall time: 3.05 s

Prepare for stacked percentage horizontal bar plot showing proportion of sentiments per product category._____no_output_____

# Create grouped dataframes by category and by sentiment
grouped_category = df_count_by_category_and_sentiment.groupby('product_category')
grouped_star = df_count_by_category_and_sentiment.groupby('sentiment')

# Create sum of sentiments per star sentiment
df_sum = df_count_by_category_and_sentiment.groupby(['sentiment']).sum()

# Calculate total number of sentiments
total = df_sum['count_reviews'].sum()
print('Total number of reviews: {}'.format(total))Total number of reviews: 22626

Create dictionary of product categories and array of star rating distribution per category._____no_output_____

distribution = {}
count_reviews_per_star = []
i=0

for category, sentiments in grouped_category:
    count_reviews_per_star = []
    for star in sentiments['sentiment']:
        count_reviews_per_star.append(sentiments.at[i, 'count_reviews'])
        i=i+1;
    distribution[category] = count_reviews_per_star_____no_output_____

Build array per star across all categories._____no_output_____

distribution_____no_output_____df_distribution_pct = pd.DataFrame(distribution).transpose().apply(
    lambda num_sentiments: num_sentiments/sum(num_sentiments)*100, axis=1
)
df_distribution_pct.columns=['-1', '0', '1']
df_distribution_pct_____no_output_____

**Visualization**

Plot the distributions of sentiments per product category._____no_output_____

categories = df_distribution_pct.index

# Plot bars
plt.figure(figsize=(10,5))

df_distribution_pct.plot(kind="barh",
                         stacked=True,
                         edgecolor='white',
                         width=1.0,
                         color=['green',
                                'orange',
                                'blue'])

plt.title("Distribution of reviews per sentiment per category",
          fontsize='16')

plt.legend(bbox_to_anchor=(1.04,1),
           loc="upper left",
           labels=['Positive',
                   'Neutral',
                   'Negative'])

plt.xlabel("% Breakdown of sentiments", fontsize='14')
plt.gca().invert_yaxis()
plt.tight_layout()

# Do not change the figure name - it is used for grading purposes!
plt.savefig('distribution_sentiment_per_category.png', dpi=300)
plt.show()_____no_output_____# Upload image to S3 bucket
sess.upload_data(path='distribution_sentiment_per_category.png', bucket=bucket, key_prefix="images")_____no_output_____

### 3.6. Analyze the distribution of review word counts_____no_output_____Set the SQL statement to count the number of the words in each of the reviews:_____no_output_____

statement_num_words = """
    SELECT CARDINALITY(SPLIT(review_body, ' ')) as num_words
    FROM {}
""".format(table_name)

print(statement_num_words)
    SELECT CARDINALITY(SPLIT(review_body, ' ')) as num_words
    FROM reviews

Query data in Amazon Athena database passing the SQL statement:_____no_output_____

%%time
df_num_words = wr.athena.read_sql_query(
    sql=statement_num_words,
    database=database_name
)CPU times: user 324 ms, sys: 11.5 ms, total: 336 ms
Wall time: 3.41 s

Print out and analyse some descriptive statistics: _____no_output_____

summary = df_num_words["num_words"].describe(percentiles=[0.10, 0.20, 0.30, 0.40, 0.50, 0.60, 0.70, 0.80, 0.90, 1.00])
summary_____no_output_____

Plot the distribution of the words number per review:_____no_output_____

df_num_words["num_words"].plot.hist(xticks=[0, 16, 32, 64, 128, 256], bins=100, range=[0, 256]).axvline(
    x=summary["100%"], c="red"
)

plt.xlabel("Words number", fontsize='14')
plt.ylabel("Frequency", fontsize='14')
plt.savefig('distribution_num_words_per_review.png', dpi=300)
plt.show()_____no_output_____# Upload image to S3 bucket
sess.upload_data(path='distribution_num_words_per_review.png', bucket=bucket, key_prefix="images")_____no_output_____

Upload the notebook into S3 bucket for grading purposes.

**Note**: you may need to click on "Save" button before the upload._____no_output_____

!aws s3 cp ./C1_W1_Assignment.ipynb s3://$bucket/C1_W1_Assignment_Learner.ipynbupload: ./C1_W1_Assignment.ipynb to s3://sagemaker-us-east-1-517313198700/C1_W1_Assignment_Learner.ipynb

Please go to the main lab window and click on `Submit` button (see the `Finish the lab` section of the instructions)._____no_output_____


# Notebook from MahopacHS/spring2019-ditoccoa0302
Path: content/lessons/12/Watch-Me-Code/WMC3-Reading-Data.ipynb

# Watch Me Code 3: Reading Data From an API into Pandas

This example will read from the star wars api http://swapi.co/api/people/ into a Pandas DataFrame_____no_output_____

<code>
import pandas as pd
import requests_____no_output_____# function to get star wars characters
def GetStarWarsCharacters():
    response = requests.get('http://swapi.co/api/people/')
    data_dict = response.json()
    characters = data_dict['results']
    return characters_____no_output_____# let's try out our function. notice it's a list of dictionary.
GetStarWarsCharacters()_____no_output_____# example of reading the star-wars api
characters = GetStarWarsCharacters() # This function gets the people as json
character_df = pd.DataFrame(characters)
character_df_____no_output_____# Let's get character names eye_xolorand skin colors
character_df[['name','skin_color', 'eye_color']]_____no_output_____# let's do the same for the male characters only
character_df[['name','skin_color', 'eye_color']][character_df['gender'] == 'male']_____no_output_____
</code>


# Notebook from johnnyhchen/EPEtutorials
Path: notebooks/08-functions-and-modules.ipynb

# Functions and Modules_____no_output_____Materials by: [John Blischak](https://github.com/jdblischak "GitHub"), Anthony Scopatz, and other Software Carpentry instructors (Joshua R. Smith, Milad Fatenejad, Katy Huff, Tommy Guy and many more)_____no_output_____Computers are very useful for doing the same operation over and over. When you know you will be performing the same operation many times, it is best to encapsulate this similar code into a function or a method. Functions are based on mathematical functions which are used to reffer to a sequence of operations, often parameterized in some way - e.g. $f(x)$._____no_output_____# Built-in string methods

The base distribution comes with many useful functions. When a function is a propety of, or owned by, on a specific type (ints, floats, lists, strings, dictionaries, etc.) it is called a method.  First let's look at some basic string methods since they are very useful for reading data into Python._____no_output_____

# Find the start codon of a gene
dna = 'CTGTTGACATGCATTCACGCTACGCTAGCT'
dna.find('ATG')_____no_output_____

Consider what this output means. Is the output what you expected?_____no_output_____

# parsing a line from a comma-delimted file
lotto_numbers = '4,8,15,16,23,42\n'
lotto_numbers = lotto_numbers.strip() #.strip() removes white space
print(lotto_numbers.split(','))['4', '8', '15', '16', '23', '42']

Consider what the type of the above output is._____no_output_____

question = '%H%ow%z%d%@d%z%th%ez$%@p%ste%rzb%ur%nz%$%@szt%on%gue%?%'
question = question.replace('%', '')
question = question.replace('@', 'i')
question = question.replace('$', 'h')
question = question.replace('z', ' ')
print(question)How did the hipster burn his tongue?
# Since the .replace() function outputs a string, we can also string
# these functions together
answer = '=H=&!dr=a=nk!c=~ff&&!be=f~r&=!i=t!w=as!c=~~l.='
print(answer.replace('=', '').replace('&', 'e').replace('~', 'o').replace('!', ' '))He drank coffee before it was cool.

## Short Exercise: Calculate GC content of DNA_____no_output_____Because the binding strength of guanine (G) to cytosine (C) is different from the binding strength of adenine (A) to thymine (T) (and many other differences), it is often useful to know the fraction of a DNA sequence that is G's or C's. Go to the [string method section](https://docs.python.org/3.7/library/string.html) of the Python documentation and find the string method that will allow you to calculate this fraction._____no_output_____

# Calculate the fraction of G's and C's in this DNA sequence
seq1 = 'ACGTACGTAGCTAGTAGCTACGTAGCTACGTA'
gc = (seq1.count('G') + seq1.count('C')) / len(seq1)_____no_output_____

Check your work:_____no_output_____

round(gc, ndigits = 2) == .47_____no_output_____

# Creating your own functions!
When there is not an available function to perform a task, you can write your own functions.  The simplest functions have the  following format in Python:

    def <function name>():
        <function body>_____no_output_____

def do_nothing():
    s = "I don't do much"_____no_output_____

However, this often isn't very useful since we haven't returned any values from this file.  Note: that if you don't return anything from a function in Python, you implcitly have returned the special `None` singleton.  To return values that you computed locally in the function body, use the **return** keyword.

    def <function name>():
        <functiom body>
        return <local variable 1>      _____no_output_____Functions, may be defined to take parameters or **arguments**.

    def <function name>(<argument>):
        <function body>
        return <local variable 1>

The function name, arguments, and retun are jointly known as the **function signature** since the uniquely define the function's **interface**._____no_output_____

def square(x):
    sqr = x * x
    return sqr_____no_output_____

Using a function is done by placing parentheses `()` after the function name after you have defined it.  This is known as **calling** the function.  If the function requires arguments, the values for these arguments are inside of the parentheses_____no_output_____

square(2)_____no_output_____

Like mathematical funtions, you can compose a function with other functions or with itself!_____no_output_____

square(square(2))_____no_output_____

Fucntions may be defined such that they have multiple arguments or multiple return values:

    def <function name>(<arg1>, <arg2>, ...):
        <functiom body>
        return <var1> , <var2>, ..._____no_output_____

def hello(time, name):
    print('Good ' + time + ', ' + name + '!')_____no_output_____hello('afternoon', 'Software Carpentry')Good afternoon, Software Carpentry!
# return both the quotient and remainder
def quorem(a, b):
    quo = a / b
    rem = a % b
    return quo, rem_____no_output_____quorem(42, 16)_____no_output_____

Note that when you return multiple values you may unpack these into individual variables:_____no_output_____

q, r = quorem(42, 16)
print(q)
print(r)2.625
10

## Keyword Arguments

In Python, functions also support options default values for arguments.  Arguments with an associated default are called **keyword arguments**.  If this function is then called without one of these arguments being pesent the default value is used.  All keyword arguments must come after normal arguments in the function definition:

    def <function name>(<arg1>, <arg2>, <arg3>=<arg3 default>, <arg4>=<arg4 default>, ...):
        <function body>
        return <rtn>_____no_output_____

def add_space(s, t="Mom"):
    return s + " " + t_____no_output_____print(add_space("Hello"))
print(add_space("Morning", "Dad"))Hello Mom
Morning Dad

You can also call any functions with their arguments, regular and keyword, with their argument names explictly in the call.  This uses equal signs in the same way that keyword arguments are defined._____no_output_____

print(add_space(s="Hello"))
print(add_space(s="Morning", t="Dad"))Hello Mom
Morning Dad

If you have many keyword arguments, then they may be out of order in function call as long as they are explicit._____no_output_____

def f(x=1, y=2, z=3):
    return 2*x**3 + 42*y - z_____no_output_____f(y=17, z=15, x=2)_____no_output_____

**Warning:**  be careful with mutable containers and default values.  The value changes with every function call._____no_output_____

def add_to_list(val, seq=[]):
    seq.append(val)
    return seq_____no_output_____add_to_list(42)_____no_output_____add_to_list(16)_____no_output_____add_to_list(65)_____no_output_____

## Packing & Unpacking Arguments

Say that you have a list or tuple of values that you would like to call a function with.  You might be tempted to do the following:_____no_output_____

def myadd(x, y):
    return x + y_____no_output_____addme = (42, 65)
myadd(addme[0], addme[1])_____no_output_____

However, this becomes very tedious very quickly.  To solve this, Python allows you to call a function with the original list prepended by an asterisk `*`.  This expands, or **unpacks**, the values of the list into the function call._____no_output_____

myadd(*addme)_____no_output_____myadd(*[14, 18])_____no_output_____

Not just lists, but any sequence is allowed to be expanded in this way.  While this works for normal arguments, keyword arguments are more similar to dictionaries.  Therefore any mapping is also able to be expanded into keyword arguments.  To distingush keyword argument unpacking from upacking normal arguments, you must use a double asterisk `**` to unpack a dictionary._____no_output_____

def f(x=1, y=2, z=3):
    return 2*x**3 + 42*y - z_____no_output_____v = {'x': 1, 'y': 2, 'z': 3}
u = [1, 2, 3]
f(**v)
f(*u) #Also works, but bad style & can cause bugs in other cases_____no_output_____

This allow you to build up dictionaires of values with which to call function *before* calling that function.

In addition to calling functions with lists or tuples and dicts, you may also define fucntions such that any extra arguments are put into a tuple and any extra keyword arguments are placed into a dictionary.  These special catch-all conatiners must come after all other arguments and all other keyword arguments.  This is known as argument packing and uses the single and double asterisk as above, but this time in the function definition:

    def <function name>(<arg1>, <arg2>=<arg2 default>, *<other args>, **<other keyword args>):
        <function body>
        return <rtn>_____no_output_____

def what(x, y, *args):
    print("x = ",  x)
    print("y = ", y)
    print("extra argument(s) = ", args)_____no_output_____what(1, 10, 178.0, "hello", [42, 42])x =  1
y =  10
extra argument(s) =  (178.0, 'hello', [42, 42])
def classroom(teacher="Anthony", **kwargs):
    print("Teacher's name is ", teacher)
    print("Teacher has a lot to deal with:", kwargs)_____no_output_____classroom(lesson1="functions", lesson2="Python")Teacher's name is  Anthony
Teacher has a lot to deal with: {'lesson1': 'functions', 'lesson2': 'Python'}
def sweet_nothings(x, y, *args, **kwargs):
    print("Not doing anything with x, y")
    print(args)
    print(kwargs)_____no_output_____sweet_nothings(0, 0.0, 42, "Nothing", zero="zero", z="", f=True, empty="de nada")Not doing anything with x, y
(42, 'Nothing')
{'zero': 'zero', 'z': '', 'f': True, 'empty': 'de nada'}

So the most general function signature that you can define is:_____no_output_____

def func(*args, **kwargs):
    print(args)
    print(kwargs)_____no_output_____func("Some arg", 1, 2.0, k="k", one = 1)  # try me out!('Some arg', 1, 2.0)
{'k': 'k', 'one': 1}

## Recursion

One of the greatest features of functions is that they may call themselves from withing their own bodies!  This is known as **recurssion**.  _____no_output_____

def count_backwards(x):
    print(x)
    count_backwards(x-1)_____no_output_____count_backwards(10)10
9
8
7
6
5
4
3
2
1
0
-1
-2
-3
-4
-5
-6
-7
-8
-9
-10
-11
-12
-13
-14
-15
-16
-17
-18
-19
-20
-21
-22
-23
-24
-25
-26
-27
-28
-29
-30
-31
-32
-33
-34
-35
-36
-37
-38
-39
-40
-41
-42
-43
-44
-45
-46
-47
-48
-49
-50
-51
-52
-53
-54
-55
-56
-57
-58
-59
-60
-61
-62
-63
-64
-65
-66
-67
-68
-69
-70
-71
-72
-73
-74
-75
-76
-77
-78
-79
-80
-81
-82
-83
-84
-85
-86
-87
-88
-89
-90
-91
-92
-93
-94
-95
-96
-97
-98
-99
-100
-101
-102
-103
-104
-105
-106
-107
-108
-109
-110
-111
-112
-113
-114
-115
-116
-117
-118
-119
-120
-121
-122
-123
-124
-125
-126
-127
-128
-129
-130
-131
-132
-133
-134
-135
-136
-137
-138
-139
-140
-141
-142
-143
-144
-145
-146
-147
-148
-149
-150
-151
-152
-153
-154
-155
-156
-157
-158
-159
-160
-161
-162
-163
-164
-165
-166
-167
-168
-169
-170
-171
-172
-173
-174
-175
-176
-177
-178
-179
-180
-181
-182
-183
-184
-185
-186
-187
-188
-189
-190
-191
-192
-193
-194
-195
-196
-197
-198
-199
-200
-201
-202
-203
-204
-205
-206
-207
-208
-209
-210
-211
-212
-213
-214
-215
-216
-217
-218
-219
-220
-221
-222
-223
-224
-225
-226
-227
-228
-229
-230
-231
-232
-233
-234
-235
-236
-237
-238
-239
-240
-241
-242
-243
-244
-245
-246
-247
-248
-249
-250
-251
-252
-253
-254
-255
-256
-257
-258
-259
-260
-261
-262
-263
-264
-265
-266
-267
-268
-269
-270
-271
-272
-273
-274
-275
-276
-277
-278
-279
-280
-281
-282
-283
-284
-285
-286
-287
-288
-289
-290
-291
-292
-293
-294
-295
-296
-297
-298
-299
-300
-301
-302
-303
-304
-305
-306
-307
-308
-309
-310
-311
-312
-313
-314
-315
-316
-317
-318
-319
-320
-321
-322
-323
-324
-325
-326
-327
-328
-329
-330
-331
-332
-333
-334
-335
-336
-337
-338
-339
-340
-341
-342
-343
-344
-345
-346
-347
-348
-349
-350
-351
-352
-353
-354
-355
-356
-357
-358
-359
-360
-361
-362
-363
-364
-365
-366
-367
-368
-369
-370
-371
-372
-373
-374
-375
-376
-377
-378
-379
-380
-381
-382
-383
-384
-385
-386
-387
-388
-389
-390
-391
-392
-393
-394
-395
-396
-397
-398
-399
-400
-401
-402
-403
-404
-405
-406
-407
-408
-409
-410
-411
-412
-413
-414
-415
-416
-417
-418
-419
-420
-421
-422
-423
-424
-425
-426
-427
-428
-429
-430
-431
-432
-433
-434
-435
-436
-437
-438
-439
-440
-441
-442
-443
-444
-445
-446
-447
-448
-449
-450
-451
-452
-453
-454
-455
-456
-457
-458
-459
-460
-461
-462
-463
-464
-465
-466
-467
-468
-469
-470
-471
-472
-473
-474
-475
-476
-477
-478
-479
-480
-481
-482
-483
-484
-485
-486
-487
-488
-489
-490
-491
-492
-493
-494
-495
-496
-497
-498
-499
-500
-501
-502
-503
-504
-505
-506
-507
-508
-509
-510
-511
-512
-513
-514
-515
-516
-517
-518
-519
-520
-521
-522
-523
-524
-525
-526
-527
-528
-529
-530
-531
-532
-533
-534
-535
-536
-537
-538
-539
-540
-541
-542
-543
-544
-545
-546
-547
-548
-549
-550
-551
-552
-553
-554
-555
-556
-557
-558
-559
-560
-561
-562
-563
-564
-565
-566
-567
-568
-569
-570
-571
-572
-573
-574
-575
-576
-577
-578
-579
-580
-581
-582
-583
-584
-585
-586
-587
-588
-589
-590
-591
-592
-593
-594
-595
-596
-597
-598
-599
-600
-601
-602
-603
-604
-605
-606
-607
-608
-609
-610
-611
-612
-613
-614
-615
-616
-617
-618
-619
-620
-621
-622
-623
-624
-625
-626
-627
-628
-629
-630
-631
-632
-633
-634
-635
-636
-637
-638
-639
-640
-641
-642
-643
-644
-645
-646
-647
-648
-649
-650
-651
-652
-653
-654
-655
-656
-657
-658
-659
-660
-661
-662
-663
-664
-665
-666
-667
-668
-669
-670
-671
-672
-673
-674
-675
-676
-677
-678
-679
-680
-681
-682
-683
-684
-685
-686
-687
-688
-689
-690
-691
-692
-693
-694
-695
-696
-697
-698
-699
-700
-701
-702
-703
-704
-705
-706
-707
-708
-709
-710
-711
-712
-713
-714
-715
-716
-717
-718
-719
-720
-721
-722
-723
-724
-725
-726
-727
-728
-729
-730
-731
-732
-733
-734
-735
-736
-737
-738
-739
-740
-741
-742
-743
-744
-745
-746
-747
-748
-749
-750
-751
-752
-753
-754
-755
-756
-757
-758
-759
-760
-761
-762
-763
-764
-765
-766
-767
-768
-769
-770
-771
-772
-773
-774
-775
-776
-777
-778
-779
-780
-781
-782
-783
-784
-785
-786
-787
-788
-789
-790
-791
-792
-793
-794
-795
-796
-797
-798
-799
-800
-801
-802
-803
-804
-805
-806
-807
-808
-809
-810
-811
-812
-813
-814
-815
-816
-817
-818
-819
-820
-821
-822
-823
-824
-825
-826
-827
-828
-829
-830
-831
-832
-833
-834
-835
-836
-837
-838
-839
-840
-841
-842
-843
-844
-845
-846
-847
-848
-849
-850
-851
-852
-853
-854
-855
-856
-857
-858
-859
-860
-861
-862
-863
-864
-865
-866
-867
-868
-869
-870
-871
-872
-873
-874
-875
-876
-877
-878
-879
-880
-881
-882
-883
-884
-885
-886
-887
-888
-889
-890
-891
-892
-893
-894
-895
-896
-897
-898
-899
-900
-901
-902
-903
-904
-905
-906
-907
-908
-909
-910
-911
-912
-913
-914
-915
-916
-917
-918
-919
-920
-921
-922
-923
-924
-925
-926
-927
-928
-929
-930
-931
-932
-933
-934
-935
-936
-937
-938
-939
-940
-941
-942
-943
-944
-945
-946
-947
-948
-949
-950
-951
-952
-953
-954
-955
-956
-957
-958
-959
-960
-961
-962
-963
-964
-965
-966
-967
-968
-969
-970
-971
-972
-973
-974
-975
-976
-977
-978
-979
-980
-981
-982
-983
-984
-985
-986
-987
-988
-989
-990
-991
-992
-993
-994
-995
-996
-997
-998
-999
-1000
-1001
-1002
-1003
-1004
-1005
-1006
-1007
-1008
-1009
-1010
-1011
-1012
-1013
-1014
-1015
-1016
-1017
-1018
-1019
-1020
-1021
-1022
-1023
-1024
-1025
-1026
-1027
-1028
-1029
-1030
-1031
-1032
-1033
-1034
-1035
-1036
-1037
-1038
-1039
-1040
-1041
-1042
-1043
-1044
-1045
-1046
-1047
-1048
-1049
-1050
-1051
-1052
-1053
-1054
-1055
-1056
-1057
-1058
-1059
-1060
-1061
-1062
-1063
-1064
-1065
-1066
-1067
-1068
-1069
-1070
-1071
-1072
-1073
-1074
-1075
-1076
-1077
-1078
-1079
-1080
-1081
-1082
-1083
-1084
-1085
-1086
-1087
-1088
-1089
-1090
-1091
-1092
-1093
-1094
-1095
-1096
-1097
-1098
-1099
-1100
-1101
-1102
-1103
-1104
-1105
-1106
-1107
-1108
-1109
-1110
-1111
-1112
-1113
-1114
-1115
-1116
-1117
-1118
-1119
-1120
-1121
-1122
-1123
-1124
-1125
-1126
-1127
-1128
-1129
-1130
-1131
-1132
-1133
-1134
-1135
-1136
-1137
-1138
-1139
-1140
-1141
-1142
-1143
-1144
-1145
-1146
-1147
-1148
-1149
-1150
-1151
-1152
-1153
-1154
-1155
-1156
-1157
-1158
-1159
-1160
-1161
-1162
-1163
-1164
-1165
-1166
-1167
-1168
-1169
-1170
-1171
-1172
-1173
-1174
-1175
-1176
-1177
-1178
-1179
-1180
-1181
-1182
-1183
-1184
-1185
-1186
-1187
-1188
-1189
-1190
-1191
-1192
-1193
-1194
-1195
-1196
-1197
-1198
-1199
-1200
-1201
-1202
-1203
-1204
-1205
-1206
-1207
-1208
-1209
-1210
-1211
-1212
-1213
-1214
-1215
-1216
-1217
-1218
-1219
-1220
-1221
-1222
-1223
-1224
-1225
-1226
-1227
-1228
-1229
-1230
-1231
-1232
-1233
-1234
-1235
-1236
-1237
-1238
-1239
-1240
-1241
-1242
-1243
-1244
-1245
-1246
-1247
-1248
-1249
-1250
-1251
-1252
-1253
-1254
-1255
-1256
-1257
-1258
-1259
-1260
-1261
-1262
-1263
-1264
-1265
-1266
-1267
-1268
-1269
-1270
-1271
-1272
-1273
-1274
-1275
-1276
-1277
-1278
-1279
-1280
-1281
-1282
-1283
-1284
-1285
-1286
-1287
-1288
-1289
-1290
-1291
-1292
-1293
-1294
-1295
-1296
-1297
-1298
-1299
-1300
-1301
-1302
-1303
-1304
-1305
-1306
-1307
-1308
-1309
-1310
-1311
-1312
-1313
-1314
-1315
-1316
-1317
-1318
-1319
-1320
-1321
-1322
-1323
-1324
-1325
-1326
-1327
-1328
-1329
-1330
-1331
-1332
-1333
-1334
-1335
-1336
-1337
-1338
-1339
-1340
-1341
-1342
-1343
-1344
-1345
-1346
-1347
-1348
-1349
-1350
-1351
-1352
-1353
-1354
-1355
-1356
-1357
-1358
-1359
-1360
-1361
-1362
-1363
-1364
-1365
-1366
-1367
-1368
-1369
-1370
-1371
-1372
-1373
-1374
-1375
-1376
-1377
-1378
-1379
-1380
-1381
-1382
-1383
-1384
-1385
-1386
-1387
-1388
-1389
-1390
-1391
-1392
-1393
-1394
-1395
-1396
-1397
-1398
-1399
-1400
-1401
-1402
-1403
-1404
-1405
-1406
-1407
-1408
-1409
-1410
-1411
-1412
-1413
-1414
-1415
-1416
-1417
-1418
-1419
-1420
-1421
-1422
-1423
-1424
-1425
-1426
-1427
-1428
-1429
-1430
-1431
-1432
-1433
-1434
-1435
-1436
-1437
-1438
-1439
-1440
-1441
-1442
-1443
-1444
-1445
-1446
-1447
-1448
-1449
-1450
-1451
-1452
-1453
-1454
-1455
-1456
-1457
-1458
-1459
-1460
-1461
-1462
-1463
-1464
-1465
-1466
-1467
-1468
-1469
-1470
-1471
-1472
-1473
-1474
-1475
-1476
-1477
-1478
-1479
-1480
-1481
-1482
-1483
-1484
-1485
-1486
-1487
-1488
-1489
-1490
-1491
-1492
-1493
-1494
-1495
-1496
-1497
-1498
-1499
-1500
-1501
-1502
-1503
-1504
-1505
-1506
-1507
-1508
-1509
-1510
-1511
-1512
-1513
-1514
-1515
-1516
-1517
-1518
-1519
-1520
-1521
-1522
-1523
-1524
-1525
-1526
-1527
-1528
-1529
-1530
-1531
-1532
-1533
-1534
-1535
-1536
-1537
-1538
-1539
-1540
-1541
-1542
-1543
-1544
-1545
-1546
-1547
-1548
-1549
-1550
-1551
-1552
-1553
-1554
-1555
-1556
-1557
-1558
-1559
-1560
-1561
-1562
-1563
-1564
-1565
-1566
-1567
-1568
-1569
-1570
-1571
-1572
-1573
-1574
-1575
-1576
-1577
-1578
-1579
-1580
-1581
-1582
-1583
-1584
-1585
-1586
-1587
-1588
-1589
-1590
-1591
-1592
-1593
-1594
-1595
-1596
-1597
-1598
-1599
-1600
-1601
-1602
-1603
-1604
-1605
-1606
-1607
-1608
-1609
-1610
-1611
-1612
-1613
-1614
-1615
-1616
-1617
-1618
-1619
-1620
-1621
-1622
-1623
-1624
-1625
-1626
-1627
-1628
-1629
-1630
-1631
-1632
-1633
-1634
-1635
-1636
-1637
-1638
-1639
-1640
-1641
-1642
-1643
-1644
-1645
-1646
-1647
-1648
-1649
-1650
-1651
-1652
-1653
-1654
-1655
-1656
-1657
-1658
-1659
-1660
-1661
-1662
-1663
-1664
-1665
-1666
-1667
-1668
-1669
-1670
-1671
-1672
-1673
-1674
-1675
-1676
-1677
-1678
-1679
-1680
-1681
-1682
-1683
-1684
-1685
-1686
-1687
-1688
-1689
-1690
-1691
-1692
-1693
-1694
-1695
-1696
-1697
-1698
-1699
-1700
-1701
-1702
-1703
-1704
-1705
-1706
-1707
-1708
-1709
-1710
-1711
-1712
-1713
-1714
-1715
-1716
-1717
-1718
-1719
-1720
-1721
-1722
-1723
-1724
-1725
-1726
-1727
-1728
-1729
-1730
-1731
-1732
-1733
-1734
-1735
-1736
-1737
-1738
-1739
-1740
-1741
-1742
-1743
-1744
-1745
-1746
-1747
-1748
-1749
-1750
-1751
-1752
-1753
-1754
-1755
-1756
-1757
-1758
-1759
-1760
-1761
-1762
-1763
-1764
-1765
-1766
-1767
-1768
-1769
-1770
-1771
-1772
-1773
-1774
-1775
-1776
-1777
-1778
-1779
-1780
-1781
-1782
-1783
-1784
-1785
-1786
-1787
-1788
-1789
-1790
-1791
-1792
-1793
-1794
-1795
-1796
-1797
-1798
-1799
-1800
-1801
-1802
-1803
-1804
-1805
-1806
-1807
-1808
-1809
-1810
-1811
-1812
-1813
-1814
-1815
-1816
-1817
-1818
-1819
-1820
-1821
-1822
-1823
-1824
-1825
-1826
-1827
-1828
-1829
-1830
-1831
-1832
-1833
-1834
-1835
-1836
-1837
-1838
-1839
-1840
-1841
-1842
-1843
-1844
-1845
-1846
-1847
-1848
-1849
-1850
-1851
-1852
-1853
-1854
-1855
-1856
-1857
-1858
-1859
-1860
-1861
-1862
-1863
-1864
-1865
-1866
-1867
-1868
-1869
-1870
-1871
-1872
-1873
-1874
-1875
-1876
-1877
-1878
-1879
-1880
-1881
-1882
-1883
-1884
-1885
-1886
-1887
-1888
-1889
-1890
-1891
-1892
-1893
-1894
-1895
-1896
-1897
-1898
-1899
-1900
-1901
-1902
-1903

Well, that was too much.  Thus it is important to ensure that reccursive functions have some case which does not call itself.  This will terminate the recurrsion._____no_output_____

def count_backwards(x):
    print(x)
    if 0 < x:
        count_backwards(x-1)_____no_output_____count_backwards(10)10
9
8
7
6
5
4
3
2
1
0

One of the most famous recurssive sequences is the [Fibonacci sequence](http://en.wikipedia.org/wiki/Fibonacci_number).  This can be defined as a single recursive function._____no_output_____

def fib(n):
    if n == 0 or n == 1:
        return n
    else:
        return fib(n - 1) + fib(n - 2)_____no_output_____fib(10)_____no_output_____

## Decorators_____no_output_____In Python, functions are **first class objects**.  This means that anything any other variable can do, a function can also do.  This is because they are normal variables in the language.  _____no_output_____

def myfunc(x):
    return x_____no_output_____print(myfunc)<function myfunc at 0x7fde5a01a050>
yourfunc = myfunc
yourfunc("hello")_____no_output_____dir(myfunc)_____no_output_____

This allows functions be passed other functions as arguments:_____no_output_____

def just_one():
    return 1

def add_one(f):
    return f() + 1

add_one(just_one)_____no_output_____

Functions may also then define other functions and return them:_____no_output_____

def outter():
    def inner(*args, **kwargs):
        return True
    return inner_____no_output_____outter()_____no_output_____

When you define a function which takes a function and (usually) returns a function, this is called a **decorator**.  Decorators are used to wrap other functions._____no_output_____

def loud(f):
    def newfunc(*args, **kwargs):
        print("calling with:", args, kwargs)
        rtn = f(*args, **kwargs)
        print("got", rtn)
        return rtn
    return newfunc_____no_output_____def myadd(x, y):
    return x + y

loudadd = loud(myadd)
v = loudadd(2, 4)calling with: (2, 4) {}
got 6

Most of the time such functions may be used such that the original function name is preserved. **ERROR: Typo. 'may' be used not 'amy' be used**_____no_output_____

myadd = loud(myadd)
myadd(42, 65)calling with: (42, 65) {}
got 107

Python also has a shortcut for using decorators which preserve the same name.  The 'at' symbol `@` followed by the decorator name is placed on the line above the function definition._____no_output_____

@loud
def mysub(x, y):
    return x - y_____no_output_____v = mysub(42, 65)calling with: (42, 65) {}
got -23

## Lambdas

Lambdas are small, single expression functions that are **anonymous** (they have no name).  They come from functional programming languages and the [Lambda Calculus](http://en.wikipedia.org/wiki/Lambda_calculus). Since they are so small they may be written on a single line.

    lambda <args>: <expr>
    _____no_output_____

lambda x: x + 1_____no_output_____

Note that just because they are implicitly anonymous, doesn't mean that you can't name them._____no_output_____

f = lambda x, y: x + y +1_____no_output_____

This is much more useful than it might seem at first glance. Lambdas are first class functions, which means they can be passed like variables to functions. Since this is the case, lambdas are a really convenient way to write small functions like a mathematical formula or to use as a passible conditional statement. This last part is exceptionally nice for using lambdas as a simple filter. **ERROR: Extra o in 'small' functions**_____no_output_____## Short exercise: Write a function to calculate GC content of DNA
Make a function that calculate the GC content of a given DNA sequence. For the more advanced participants, make your function able to handle sequences of mixed case (see the third test case)._____no_output_____

# Calculates the GC content of DNA sequence x.
# x: a string composed only of A's, T's, G's, and C's.
def calculate_gc(x):
    return (x.lower().count('c') + x.lower().count('g')) / len(x)_____no_output_____

Check your work:_____no_output_____

print(round(calculate_gc('ATGC'), ndigits = 2) == 0.50)
print(round(calculate_gc('AGCGTCGTCAGTCGT'), ndigits = 2) == 0.60)
print(round(calculate_gc('ATaGtTCaAGcTCgATtGaATaGgTAaCt'), ndigits = 2) == 0.34)True
True
True

# Modules
Python has a lot of useful data type and functions built into the language, some of which you have already seen. For a full list, you can type `dir(__builtins__)`. However, there are even more functions stored in modules. An example is the sine function, which is stored in the math module. In order to access mathematical functions, like sin, we need to **import** the math module. Lets take a look at a simple example:_____no_output_____

print(sin(3)) # Error! Python doesn't know what sin is...yet_____no_output_____import math # Import the math module
math.sin(3)_____no_output_____dir(math) # See a list of everything in the math module_____no_output_____help(math) # Get help information for the math moduleHelp on module math:

NAME
    math

MODULE REFERENCE
    https://docs.python.org/3.7/library/math

    The following documentation is automatically generated from the Python
    source files.  It may be incomplete, incorrect or include features that
    are considered implementation detail and may vary between Python
    implementations.  When in doubt, consult the module reference at the
    location listed above.

DESCRIPTION
    This module provides access to the mathematical functions
    defined by the C standard.

FUNCTIONS
    acos(x, /)
        Return the arc cosine (measured in radians) of x.

    acosh(x, /)
        Return the inverse hyperbolic cosine of x.

    asin(x, /)
        Return the arc sine (measured in radians) of x.

    asinh(x, /)
        Return the inverse hyperbolic sine of x.

    atan(x, /)
        Return the arc tangent (measured in radians) of x.

    atan2(y, x, /)
        Return the arc tangent (measured in radians) of y/x.

        Unlike atan(y/x), the signs of both x and y are considered.

    atanh(x, /)
        Return the inverse hyperbolic tangent of x.

    ceil(x, /)
        Return the ceiling of x as an Integral.

        This is the smallest integer >= x.

    copysign(x, y, /)
        Return a float with the magnitude (absolute value) of x but the sign of y.

        On platforms that support signed zeros, copysign(1.0, -0.0)
        returns -1.0.

    cos(x, /)
        Return the cosine of x (measured in radians).

    cosh(x, /)
        Return the hyperbolic cosine of x.

    degrees(x, /)
        Convert angle x from radians to degrees.

    erf(x, /)
        Error function at x.

    erfc(x, /)
        Complementary error function at x.

    exp(x, /)
        Return e raised to the power of x.

    expm1(x, /)
        Return exp(x)-1.

        This function avoids the loss of precision involved in the direct evaluation of exp(x)-1 for small x.

    fabs(x, /)
        Return the absolute value of the float x.

    factorial(x, /)
        Find x!.

        Raise a ValueError if x is negative or non-integral.

    floor(x, /)
        Return the floor of x as an Integral.

        This is the largest integer <= x.

    fmod(x, y, /)
        Return fmod(x, y), according to platform C.

        x % y may differ.

    frexp(x, /)
        Return the mantissa and exponent of x, as pair (m, e).

        m is a float and e is an int, such that x = m * 2.**e.
        If x is 0, m and e are both 0.  Else 0.5 <= abs(m) < 1.0.

    fsum(seq, /)
        Return an accurate floating point sum of values in the iterable seq.

        Assumes IEEE-754 floating point arithmetic.

    gamma(x, /)
        Gamma function at x.

    gcd(x, y, /)
        greatest common divisor of x and y

    hypot(x, y, /)
        Return the Euclidean distance, sqrt(x*x + y*y).

    isclose(a, b, *, rel_tol=1e-09, abs_tol=0.0)
        Determine whether two floating point numbers are close in value.

          rel_tol
            maximum difference for being considered "close", relative to the
            magnitude of the input values
          abs_tol
            maximum difference for being considered "close", regardless of the
            magnitude of the input values

        Return True if a is close in value to b, and False otherwise.

        For the values to be considered close, the difference between them
        must be smaller than at least one of the tolerances.

        -inf, inf and NaN behave similarly to the IEEE 754 Standard.  That
        is, NaN is not close to anything, even itself.  inf and -inf are
        only close to themselves.

    isfinite(x, /)
        Return True if x is neither an infinity nor a NaN, and False otherwise.

    isinf(x, /)
        Return True if x is a positive or negative infinity, and False otherwise.

    isnan(x, /)
        Return True if x is a NaN (not a number), and False otherwise.

    ldexp(x, i, /)
        Return x * (2**i).

        This is essentially the inverse of frexp().

    lgamma(x, /)
        Natural logarithm of absolute value of Gamma function at x.

    log(...)
        log(x, [base=math.e])
        Return the logarithm of x to the given base.

        If the base not specified, returns the natural logarithm (base e) of x.

    log10(x, /)
        Return the base 10 logarithm of x.

    log1p(x, /)
        Return the natural logarithm of 1+x (base e).

        The result is computed in a way which is accurate for x near zero.

    log2(x, /)
        Return the base 2 logarithm of x.

    modf(x, /)
        Return the fractional and integer parts of x.

        Both results carry the sign of x and are floats.

    pow(x, y, /)
        Return x**y (x to the power of y).

    radians(x, /)
        Convert angle x from degrees to radians.

    remainder(x, y, /)
        Difference between x and the closest integer multiple of y.

        Return x - n*y where n*y is the closest integer multiple of y.
        In the case where x is exactly halfway between two multiples of
        y, the nearest even value of n is used. The result is always exact.

    sin(x, /)
        Return the sine of x (measured in radians).

    sinh(x, /)
        Return the hyperbolic sine of x.

    sqrt(x, /)
        Return the square root of x.

    tan(x, /)
        Return the tangent of x (measured in radians).

    tanh(x, /)
        Return the hyperbolic tangent of x.

    trunc(x, /)
        Truncates the Real x to the nearest Integral toward 0.

        Uses the __trunc__ magic method.

DATA
    e = 2.718281828459045
    inf = inf
    nan = nan
    pi = 3.141592653589793
    tau = 6.283185307179586

FILE
    /Users/johnny/opt/anaconda3/lib/python3.7/lib-dynload/math.cpython-37m-darwin.so

It is not very difficult to use modules - you just have to know the module name and import it. There are a few variations on the import statement that can be used to make your life easier. Lets take a look at an example:_____no_output_____

from math import *  # import everything from math into the global namespace (A BAD IDEA IN GENERAL)
print(sin(3))       # notice that we don't need to type math.sin anymore
print(tan(3))       # the tangent function was also in math, so we can use that too0.1411200080598672
-0.1425465430742778
reset # Clear everything from IPythonOnce deleted, variables cannot be recovered. Proceed (y/[n])? y
Don't know how to reset  #, please run `%reset?` for details
Don't know how to reset  clear, please run `%reset?` for details
Don't know how to reset  everything, please run `%reset?` for details
Don't know how to reset  from, please run `%reset?` for details
Don't know how to reset  ipython, please run `%reset?` for details
from math import sin  # Import just sin from the math module. This is a good idea.
print(sin(3))         # We can use sin because we just imported it
print(tan(3))        # Error: We only imported sin - not tan0.1411200080598672
-0.1425465430742778
reset                 # Clear everythingOnce deleted, variables cannot be recovered. Proceed (y/[n])? y
Don't know how to reset  #, please run `%reset?` for details
Don't know how to reset  clear, please run `%reset?` for details
Don't know how to reset  everything, please run `%reset?` for details
import math as m      # Same as import math, except we are renaming the module m
print(m.sin(3))       # This is really handy if you have module names that are long0.1411200080598672

Python comes with a *huge* number of modules available as part of the standard library (batteries included).  It has a gargantuan number of third party modules as well.  This ecosystem is what makes scientific software development in Python great!_____no_output_____## The General Problem
![xkcd](http://imgs.xkcd.com/comics/the_general_problem.png "I find that when someone's taking time to do something right in the present, they're a perfectionist with no ability to prioritize, whereas when someone took time to do something right in the past, they're a master artisan of great foresight.")
From [xkcd](http://www.xkcd.com)

Now that you can write your own functions, you too will experience the dilemma of deciding whether to spend the extra time to make your code more general, and therefore more easily reused in the future._____no_output_____# Longer exercise: Reading Cochlear implant into Python
For this exercise we will return to the cochlear implant data first introduced in the section on the shell. In order to analyse the data, we need to import the data into Python. Furthermore, since this is something that would have to be done many times, we will write a function to do this. As before, beginners should aim to complete Part 1 and more advanced participants should try to complete Part 2 and Part 3 as well._____no_output_____## Part 1: View the contents of the file from within Python
Write a function `view_cochlear` that will open the file and print out each line. The only input to the function should be the name of the file as a string. _____no_output_____

def view_cochlear(filename):
    """Prints out the contents of the file specified by filename
    """
    f = open(filename, 'r')
    print(f.read())
    f.close()_____no_output_____

Test it out:_____no_output_____

# Note that this assumes you have the tutorial downloaded to your home directory
view_cochlear('./data/alexander/data_216.DATA')#
Reported: Wed Aug 17 13:56:38 2011
Subject: madonnaStarr178
Year/month of birth: 1995/02
Sex: N
CI type: 8
Volume: 7
Range: 3
Discrimination: 5

view_cochlear('./data/Lawrence/Data0525')#
Reported: Mon Jun 27 13:46:07 2011
Subject: scaryLennon414
Year/month of birth: 1996/02
Sex: N
CI type: 18
Volume: 3
Range: 9
Discrimination: 6

## Part 2:
Adapt your function above to exclude the first line using the flow control techniques we learned in the last lesson. The first line is just `#` (but don't forget to remove the `'\n'`)._____no_output_____

def view_cochlear(filename):
    """prints out contents of filename excluding the first line
    """
    f = open(filename, 'r')
    lines = f.readlines()
    new_contents = ""
    for i in range(1, len(lines)):
        new_contents += lines[i]
    print(new_contents)_____no_output_____

Test it out:_____no_output_____

view_cochlear('./data/alexander/data_216.DATA')Reported: Wed Aug 17 13:56:38 2011
Subject: madonnaStarr178
Year/month of birth: 1995/02
Sex: N
CI type: 8
Volume: 7
Range: 3
Discrimination: 5

view_cochlear('./data/Lawrence/Data0525')Reported: Mon Jun 27 13:46:07 2011
Subject: scaryLennon414
Year/month of birth: 1996/02
Sex: N
CI type: 18
Volume: 3
Range: 9
Discrimination: 6

## Part 3:
Adapt your function above to return a dictionary containing the contents of the file. Split each line of the file by a colon followed by a space (': '). The first half of the string should be the key of the dictionary, and the second half should be the value of the dictionary._____no_output_____

def save_cochlear(filename):
    """returns a dictionary representation of the contents in the given file specified by filename
    """
    f = open(filename, 'r')
    lines = f.readlines()
    lines.pop(0)
    dict = {}
    for line in lines:
        key_value = line.split(': ')
        dict[key_value[0]] = key_value[1].strip()
    return dict_____no_output_____

Check your work:_____no_output_____

data_216 = save_cochlear("./data/alexander/data_216.DATA")
print(data_216["Subject"])madonnaStarr178
Data0525 = save_cochlear("./data/alexander/data_216.DATA")
print(Data0525["CI type"])8

# Bonus Exercise: Transcribe DNA to RNA
### Motivation:
During transcription, an enzyme called RNA Polymerase reads the DNA sequence and creates a complementary RNA sequence. Furthermore, RNA has the nucleotide uracil (U) instead of thymine (T).
### Task:
Write a function that mimics transcription. The input argument is a string that contains the letters A, T, G, and C. Create a new string following these rules:

* Convert A to U

* Convert T to A

* Convert G to C

* Convert C to G

Hint: You can iterate through a string using a for loop similary to how you loop through a list._____no_output_____

def transcribe(seq):
    """mimics transcription of DNA to RNA. takes
       input string representing DNA sequence.
       returns string representing corresponding RNA sequence.
    """
    rna = ""
    conversion = {'A': 'U', 'T': 'A', 'G': 'C', 'C': 'G'}
    for char in seq:
        rna += conversion[char]
    return rna_____no_output_____

Check your work:_____no_output_____

transcribe('ATGC') == 'UACG'_____no_output_____transcribe('ATGCAGTCAGTGCAGTCAGT') == 'UACGUCAGUCACGUCAGUCA'_____no_output_____