# Notebook from BioTuring-Notebooks/CellphoneDB
Path: notebooks/0_prepare_your_data_from_Seurat.ipynb

# Example 2
## Step 0 - prepare your data

Prepare cellphoneDB inputs starting from a seurat object object_____no_output_____

library(Seurat)
library(SeuratObject)
library(Matrix)Registered S3 methods overwritten by 'SeuratObject':
  method                     from
  $.JackStrawData            Seurat
  $.Seurat                   Seurat
  $.SeuratCommand            Seurat
  $<-.Seurat                 Seurat
  [.Assay                    Seurat
  [.DimReduc                 Seurat
  [.Seurat                   Seurat
  [.SeuratCommand            Seurat
  [.SpatialImage             Seurat
  [[.Assay                   Seurat
  [[.DimReduc                Seurat
  [[.Seurat                  Seurat
  levels<-.Seurat            Seurat
  .DollarNames.JackStrawData Seurat
  .DollarNames.Seurat        Seurat
  .DollarNames.SeuratCommand Seurat
  as.list.SeuratCommand      Seurat
  as.logical.JackStrawData   Seurat
  dim.Assay                  Seurat
  dim.DimReduc               Seurat
  dim.Neighbor               Seurat
  dim.Seurat                 Seurat
  dim.SpatialImage           Seurat
  dimnames.Assay             Seurat
  dimnames.DimReduc          Seurat
  dimnames.Seurat            Seurat
  droplevels.Seurat          Seurat
  length.DimReduc            Seurat
  levels.Seurat              Seurat
  merge.Assay                Seurat
  merge.Seurat               Seurat
  names.DimReduc             Seurat
  names.Seurat               Seurat
  print.DimReduc             Seurat
  subset.Assay               Seurat
  subset.DimReduc            Seurat
  subset.Seurat              Seurat
  subset.SpatialImage        Seurat

Attaching package: ‘SeuratObject’

The following objects are masked from ‘package:Seurat’:

    AddMetaData, as.Graph, as.Neighbor, as.Seurat, as.sparse, Assays,
    Cells, CellsByIdentities, Command, CreateAssayObject,
    CreateDimReducObject, CreateSeuratObject, DefaultAssay,
    DefaultAssay<-, Distances, Embeddings, FetchData, GetAssayData,
    GetImage, GetTissueCoordinates, HVFInfo, Idents, Idents<-, Images,
    Index, Index<-, Indices, IsGlobal, JS, JS<-, Key, Key<-, Loadings,
    Loadings<-, LogSeuratCommand, Misc, Misc<-, Neighbors, pbmc_small,
    Project, Project<-, Radius, Reductions, RenameAssays, RenameCells,
    RenameIdents, ReorderIdent, RowMergeSparseMatrices, SetAssayData,
    SetIdent, SpatiallyVariableFeatures, StashIdent, Stdev, SVFInfo,
    Tool, Tool<-, UpdateSeuratObject, VariableFeatures,
    VariableFeatures<-, WhichCells

### 1. Load seurat object

The seurat object contains counts that have been normalized (per cell) and log-transformed.
If your data are raw counts, please normalize accordingly_____no_output_____

so = readRDS('endometrium_example_counts_seurat.rds')_____no_output_____so_____no_output_____Idents(so) = so$cell_type_____no_output_____# # If your data is not normalize, do it accordinlgy
# # For example
# so <- NormalizeData(object = so)_____no_output_____

### 2. Write gene expression in mtx format
_____no_output_____

writeMM(so@assays$RNA@counts, file = 'endometrium_example_counts_mtx/matrix.mtx')
# save gene and cell names
write(x = rownames(so@assays$RNA@counts), file = "endometrium_example_counts_mtx/genes.tsv")
write(x = colnames(so@assays$RNA@counts), file = "endometrium_example_counts_mtx/barcodes.tsv")_____no_output_____

### 3. Generate your meta

In this example, our input is an anndata containing the cluster/celltype information in `metadat$'cell_type'`

The object also has `metadat$'lineage'` information wich will be used below for a hierarchical DEGs approach. _____no_output_____

table(so@meta.data$cell_type)_____no_output_____so@meta.data$Cell = rownames(so@meta.data)
df = so@meta.data[, c('Cell', 'cell_type')]
write.table(df, file ='endometrium_example_meta.tsv', sep = '\t', quote = F, row.names = F)_____no_output_____

### 4. Compute DEGs (optional)

Use Seurat `FindAllMarkers` to compute differentially expressed genes and extract the corresponding data frame `DEGs`.
Here there are three options you may be interested on:
1. Identify DEGs for each cell type (compare cell type vs rest, most likely option)
2. Identify DEGs for each cell type using a per-lineage hierarchycal approach (compare cell type vs rest in the lineage, such as in endometrium paper Garcia-Alonso et al 2021)

In the endometrium paper (Garcia-Alonso et al 2021) we're interested in the differences within the stromal and epithelial lineages, rather than the commonalities (example, what is specific of epithelials in the glands compared to epithelials in the lumen). The reason is that epithelial and stromal subtypes vary in space and type and thus we wanna extract the subtile differences within the lineage to better understand their differential location/ biological role.
_____no_output_____

## OPTION 1 - compute DEGs for all cell types
## Extract DEGs for each cell_type
# DEGs <- FindAllMarkers(so,
#                        test.use = 'LR',
#                        verbose = F,
#                        only.pos = T,
#                        random.seed = 1,
#                        logfc.threshold = 0.2,
#                        min.pct = 0.1,
#                        return.thresh = 0.05)

# OPTION 2 - optional - Re-compute  hierarchical (per lineage) DEGs for Epithelial and Stromal lineages
DEGs = c()
for( lin in c('Epithelial', 'Stromal') ){
    message('Computing DEGs within linage ', lin)
    so_in_lineage = subset(so, cells = Cells(so)[ so$lineage == lin ] )
    celltye_in_lineage = unique(so$cell_type[ so$lineage == lin ])
    DEGs_lin = FindAllMarkers(so_in_lineage,
                       verbose = F,
                       only.pos = T,
                       random.seed = 1,
                       logfc.threshold = 0,
                       min.pct = 0.1,
                       return.thresh = 0.05)
    DEGs = rbind(DEGs_lin, DEGs)
}Computing DEGs within linage Epithelial

For a more efficient implementation of the Wilcoxon Rank Sum Test,
(default method for FindMarkers) please install the limma package
--------------------------------------------
install.packages('BiocManager')
BiocManager::install('limma')
--------------------------------------------
After installation of limma, Seurat will automatically use the more
efficient implementation (no further action necessary).
This message will be shown once per session

'DKK1' %in% rownames(so@assays$RNA@counts)_____no_output_____fDEGs = subset(DEGs, p_val_adj < 0.05 & avg_logFC > 0.1)

# 1st column = cluster; 2nd column = gene
fDEGs = fDEGs[, c('cluster', 'gene', 'p_val_adj', 'p_val', 'avg_logFC', 'pct.1', 'pct.2')]
write.table(fDEGs, file ='endometrium_example_DEGs.tsv', sep = '\t', quote = F, row.names = F)_____no_output_____head(fDEGs)_____no_output_____

### 4. Run cellphoneDB_____no_output_____```
cellphonedb method degs_analysis  \
    endometrium_example_meta.tsv  \
    endometrium_example_counts_mtx  \
    endometrium_example_DEGs.tsv  \
    --microenvs endometrium_example_microenviroments.tsv  \ #optional
    --counts-data hgnc_symbol  \
    --database database/database/cellphonedb_user_2021-06-29-11_41.db \
    --threshold 0.1
```_____no_output_____

sessionInfo()_____no_output_____


# Notebook from fastscape-lem/crc-1211-short-course
Path: notebooks/FastScape_CRC_1.ipynb

# Modeling an orogenic system using FastScape: 1. the basic equations_____no_output_____![Henry Mountains](HenryMountains.jpg "Henry Mountains")_____no_output_____The source area is the mountain. This is where erosion takes place (mostly).

Two processes compete in the mountain: bedrock incision along channels and hillslope transport (or diffusion). It is commonly assumed that bedrock incision sets the pace of landscape evolution. Hillslope processes react to slope formation and amplification produced by channel incision. There are many potential feedbacks to this, including the tools provided by the hilslope processes or the perturbation to river flow by the product of hillslope processes (landslides).

We will neglect the effect of glacial processes._____no_output_____## Basic equation

The basic equation governing landscape evolution (through erosion) in a mountain or orogenic area is:

$$\frac{\partial h}{\partial t}=U-K_fA^mS^n+K_d\nabla^2h$$

where $h$ is topographic elevation or height (in m), $t$ is time (in yrs), $U$ is uplift rate (in m/yr), $A$ is drainage area in m$^2$, $S$ is slope (in m/m), and $\nabla^2 h$ is curvature (second order derivative of topography, in m/m$^2$). $K_f$ is the bedrock incision rate coefficient (in m$^{1-2m}$/yr) that depends on lithology (i.e., rock type, fracturing level, etc) and precipitation rate, $\nu$, to the power $m$. $K_d$ is hillslope transport coeeficient (or diffusivity, in m$^2$/yr). $m$ and $n$ are the area and slope exponents that have values varying between $m\in[0.3,0.8]$ and $n\in[1-2]$; the ratio $m/n$ is better constrained by the concavity of river profiles to be $\in[0.4-0.6]$._____no_output_____## FastScape algorithm

Solving this equation is not easy, especially because of the river incision term (or Stream Power Law - SPL term):

$$\frac{\partial h}{\partial t}=-K_fA^mS^n=-K_fA^m(\frac{\partial h}{\partial s})^n$$

because the slope, $S$, is in fact the spatial derivative of the topography, i.e., in the direction $s$ of water flow, in this case the steepest descent. This leads to an equation in the form of an *advection* equation, which is known to be prone to instability and inaccuracy.

The FastScape algorithm, developed by [Braun and Willett, 2013](https://www.sciencedirect.com/science/article/pii/S0169555X12004618), is unconditionally stable, highly efficient and relatively accurate, although it suffers from problems, i.e., it is not weel suited to study the propagation of knockpoints, which it tends to smooth out (see [Campforts et al, 2015](https://agupubs.onlinelibrary.wiley.com/doi/abs/10.1002/2014JF003376)).

It is based on an ordering of the nodes that is obtained after defining the receiver node to each node, from which the donor nodes to each node are computer.

![Donor-Receiver](DonorReceiver.jpg "Donor-Receiver")_____no_output_____## Solution of the diffusion equation/term

There are many different ways to solve the diffusion equaiont. If the diffusivity, $K_d$, is spatially uniform, an efficient and unconditionally stable algorithm exists: the Alternating Direction Implicit (or ADI) algorithm (see [ADI](https://en.wikipedia.org/wiki/Alternating_direction_implicit_method))._____no_output_____## FastScape Python implementation

Over the past few years, we have developed a Python package to solve the SPL and diffusion equations. It can be foound here: [FastScape on GitHub](https://github.com/fastscape-lem/fastscape). Documentation can be found here: [FastScape Docs](https://fastscape.readthedocs.io/en/latest/index.html).

FastScape was built using the *Xarray-Simlab* framework, developed at the GFZ by Benoit Bovy. Documentation can be found here: [Xarray-Simlab](https://xarray-simlab.readthedocs.io/en/latest/)._____no_output_____## Setup

To solve a Partial Differential Equation (PDE) as the one shown above, one also needs to specify the geometry of the problem and, most importantly, boundary conditions. In FastScape, the PDE is solved in a rectangular domain of dimensions $L_x\times L_y$ which is discretized using a rectangular (regular) discretization of $n_x\times n_y$ points or *nodes*.

Along the four boundaries, boundary conditions can be of three types:
1. *fixed_value*: the elevation is fixed to base level; note that rivers must "exit" through a base level, which implies that at least one of the boundaries MUST be fixed;
2. *core*: the boundary is reflective (no flux of material allowed), which means that rivers must "bounce back" from such boundaries;
3. *looped*: or cyclic boundary conditions, which means that material flowing through the boundary "reappears" automatically on the opposite boundary; note that *looped* boundaries must be specified as opposite pairs.

![DMesh](Mesh.jpg "Mesh")

FInally, FastScape also needs an initial condition, i.e.:

$$h(t=0)=h_0(x,y)$$

which is commonly assumed to be a very small white noise (i.e. of amplitude = 1 m)._____no_output_____


# Notebook from Cheongsoo/Data-8
Path: project2.ipynb

# Project 2: Inference and Capital Punishment

Welcome to Project 2!  You will investigate the relationship between murder and capital punishment (the death penalty) in the United States.  By the end of the project, you should know how to:

1. Test whether observed data appears to be a random sample from a distribution
2. Analyze a natural experiment
3. Implement and interpret a sign test
4. Create a function to run a general hypothesis test
5. Analyze visualizations and draw conclusions from them

### Logistics

**Deadline.** This project is due at 11:59pm on Thursday 10/26. You can earn an early submission bonus point by submitting your completed project by Wednesday 10/25. It's **much** better to be early than late, so start working now.

**Checkpoint.** For full credit, you must also **complete Parts 1 and 2 of the project (out of 5) and submit them by 11:59pm on Friday 10/20**. You will have some lab time to work on these questions, but we recommend that you start the project before lab and leave time to finish the checkpoint afterward.

**Partners.** You may work with one other partner. It's best to work with someone in your lab. Only one of you is required to submit the project. On [okpy.org](http://okpy.org), the person who submits should also designate their partner so that both of you receive credit.

**Rules.** Don't share your code with anybody but your partner. You are welcome to discuss questions with other students, but don't share the answers. The experience of solving the problems in this project will prepare you for exams (and life). If someone asks you for the answer, resist! Instead, you can demonstrate how you would solve a similar problem.

**Support.** You are not alone! Come to office hours, post on Piazza, and talk to your classmates. If you want to ask about the details of your solution to a problem, make a private Piazza post and the staff will respond. If you're ever feeling overwhelmed or don't know how to make progress, email your TA or tutor for help. You can find contact information for the staff on the [course website](http://data8.org/sp17/staff.html).

**Tests.** Passing the tests for a question **does not** mean that you answered the question correctly. Tests usually only check that your table has the correct column labels. However, more tests will be applied to verify the correctness of your submission in order to assign your final score, so be careful and check your work!

**Advice.** Develop your answers incrementally. To perform a complicated table manipulation, break it up into steps, perform each step on a different line, give a new name to each result, and check that each intermediate result is what you expect. You can add any additional names or functions you want to the provided cells.

All of the concepts necessary for this project are found in the textbook. If you are stuck on a particular problem, reading through the relevant textbook section often will help clarify the concept.

To get started, load `datascience`, `numpy`, `plots`, and `ok`._____no_output_____

from datascience import *
import numpy as np

%matplotlib inline
import matplotlib.pyplot as plots
plots.style.use('fivethirtyeight')

from client.api.notebook import Notebook
ok = Notebook('project2.ok')
_ = ok.auth(inline=True)=====================================================================
Assignment: Project 2: Inference and Capital Punishment
OK, version v1.12.5
=====================================================================

Open the following URL:

https://okpy.org/client/login/

After logging in, copy the code from the web page and paste it into the box.
Then press the "Enter" key on your keyboard.

Paste your code here: 6rGnZ4EqRarixFloDd7JoEYLFLAg83
Successfully logged in as kcs1992@berkeley.edu

## Overview

Punishment for crime has many [philosophical justifications](http://plato.stanford.edu/entries/punishment/#ThePun).  An important one is that fear of punishment may *deter* people from committing crimes.

In the United States, some jurisdictions execute some people who are convicted of particularly serious crimes, such as murder.  This punishment is called the *death penalty* or *capital punishment*.  The death penalty is controversial, and deterrence has been one focal point of the debate.  There are other reasons to support or oppose the death penalty, but in this project we'll focus on deterrence.

The key question about deterrence is:

> Through our exploration, does instituting a death penalty for murder actually reduce the number of murders?

You might have a strong intuition in one direction, but the evidence turns out to be surprisingly complex.  Different sides have variously argued that the death penalty has no deterrent effect and that each execution prevents 8 murders, all using statistical arguments!  We'll try to come to our own conclusion.

Here is a road map for this project:

1. In section 1, we'll investigate the main dataset we'll be using.
2. In section 2, we'll see how to test null hypotheses such as this: "For this set of U.S. states, the murder rate was equally likely to go up or down each year."
3. In section 3, we'll apply a similar test to see whether U.S. states that suddenly ended or reinstituted the death penalty were more likely to see murder rates increase than decrease.
4. In section 4, we'll run some more tests to further claims we had been developing in previous sections.
5. In section 5, we'll try to answer our question about deterrence using a visualization rather than a formal hypothesis test.

#### The data

The main data source for this project comes from a [paper](http://cjlf.org/deathpenalty/DezRubShepDeterFinal.pdf) by three researchers, Dezhbakhsh, Rubin, and Shepherd.  The dataset contains rates of various violent crimes for every year 1960-2003 (44 years) in every US state.  The researchers compiled the data from the FBI's Uniform Crime Reports.

Since crimes are committed by people, not states, we need to account for the number of people in each state when we're looking at state-level data.  Murder rates are calculated as follows:

$$\text{murder rate for state X in year Y} = \frac{\text{number of murders in state X in year Y}}{\text{population in state X in year Y}}*100000$$

(Murder is rare, so we multiply by 100,000 just to avoid dealing with tiny numbers.)_____no_output_____

a=np.array([[1,2,3],
          [4,5,6]])
a>1
_____no_output_____a=np.ones((5,5,3))
a[:,:,1]_____no_output_____murder_rates = Table.read_table('crime_rates.csv').select('State', 'Year', 'Population', 'Murder Rate')
murder_rates.set_format("Population", NumberFormatter)_____no_output_____

## 1. Murder rates_____no_output_____So far, this looks like a dataset that lends itself to an observational study.  In fact, the murder rates dataset isn't even enough to demonstrate an *association* between the existence of the death penalty in a state in a year and the murder rate in that state and year!

**Question 1.1.** What additional information will we need before we can check for that association? Assign `extra_info` to a Python list (i.e. [#] or [#, #, ...]) containing the number(s) for all of the additional facts below that we *require* in order to check for association.

1) What year(s) the death penalty was introduced in each state (if any).

2) Day to day data about when murders occurred.

3) What year(s) the death penalty was abolished in each state (if any).

4) Rates of other crimes in each state._____no_output_____

extra_info = [1,3]_____no_output______ = ok.grade("q1_1")
_ = ok.backup()~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Running tests

---------------------------------------------------------------------
Test summary
    Passed: 1
    Failed: 0
[ooooooooook] 100.0% passed

Murder rates vary over time, and different states exhibit different trends. The rates in some states change dramatically from year to year, while others are quite stable. Let's plot a couple, just to see the variety.

**Question 1.2.** Draw a line plot with years on the horizontal axis and murder rates on the
vertical axis. Include two lines: one for Alaska murder rates and one for Minnesota murder rates. Create this plot using a single call, `ak_mn.plot('Year')`.

*Hint*: To create two lines, you will need create the table `ak_mn` with two columns of murder rates, in addition to a column of years. This table will have the following structure:

| Year | Murder rate in Alaska | Murder rate in Minnesota |
|------|-----------------------|--------------------------|
| 1960 | 10.2                  | 1.2                      |
| 1961 | 11.5                  | 1                        |
| 1962 | 4.5                   | 0.9                      |

... (41 rows omitted)_____no_output_____

# The next lines are provided for you.  They create a table
# containing only the Alaska information and one containing
# only the Minnesota information.
ak = murder_rates.where('State', 'Alaska').drop('State', 'Population').relabeled(1, 'Murder rate in Alaska')
mn = murder_rates.where('State', 'Minnesota').drop('State', 'Population').relabeled(1, 'Murder rate in Minnesota')

# Fill in this line to make a table like the one pictured above.
ak_mn = ak.join('Year', mn, 'Year')
ak_mn.plot('Year')_____no_output______ = ok.grade("q1_2")
_ = ok.backup()~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Running tests

---------------------------------------------------------------------
Test summary
    Passed: 1
    Failed: 0
[ooooooooook] 100.0% passed

Now what about the murder rates of other states? Say, for example, California and New York? Fill in the cell below to plot the murder rates of different pairs of states. **Note:** this should use similar code to question 1.2, with only the variable names changed. The cell below will not be graded, but it creates a cool interactive module!_____no_output_____

# Compare the murder rates of any two states by filling in the blanks below

from ipywidgets import interact, interactive, fixed, interact_manual
import ipywidgets as widgets

def state(state1, state2):
    state1_table = murder_rates.where('State', state1).drop('State', 'Population').relabeled(1, 'Murder rate in {}'.format(state1))
    state2_table = murder_rates.where('State', state2).drop('State', 'Population').relabeled(1, 'Murder rate in {}'.format(state2))
    s1_s2 = state1_table.join('Year', state2_table, 'Year')
    s1_s2.plot('Year')
    plots.show()

states_array = murder_rates.group('State').column('State')

_ = interact(state,
             state1=widgets.Dropdown(options=list(states_array),value='California'),
             state2=widgets.Dropdown(options=list(states_array),value='New York')
            )_____no_output_____

##### A reminder about tests
The automated tests check for basic errors (like the number of rows in your `ak_mn` table, or whether you defined a function named `most_murderous` for the next question), but they **aren't comprehensive**.

If you're not sure that your answer is correct, think about how you can check it.  For example, if a table has the right number of rows and columns, and a few randomly-selected values from each column are correct, then you can be somewhat confident you've computed it correctly.  For the previous question, try checking some of the values in `ak_mn` manually, by searching through the `murder_rates` table._____no_output_____**Question 1.3.** Implement the function `most_murderous`, which takes a year (an integer) as its argument. It does two things:
1. It draws a horizontal bar chart of the 5 states that had the highest murder rate in that year.
2. It returns an array of the names of these states in order of *increasing* murder rate.

Assume that the argument is a year in `murder_rates`. You do not need to check that it is._____no_output_____

# most = murder_rates.where('State', year).sort('Murder Rate', descending = False)
# most
murder_rates.where('Year', 1990).sort('Murder Rate', descending = True).take(np.arange(0,5)).sort('Murder Rate', descending = False)_____no_output_____def most_murderous(year):
    # Assign most to a table of the most murderous states this year in ascending order.
    most = murder_rates.where('Year', year).sort('Murder Rate', descending = True).take(np.arange(0,5)).sort('Murder Rate', descending = False)
    most.barh('State', 'Murder Rate')
    return most.column('State')

most_murderous(1990) # California, Mississippi, ..., _____no_output______ = ok.grade("q1_3")
_ = ok.backup()~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Running tests

---------------------------------------------------------------------
Test summary
    Passed: 1
    Failed: 0
[ooooooooook] 100.0% passed

**Question 1.4.** How many more people were murdered in California in 1988 than in 1975? Assign `ca_change` to the answer.

*Hint*: Make sure you understand how murder rate is calculated. Recall the formula given at the beginning of the project:

$$\text{murder rate for state X in year Y} = \frac{\text{number of murders in state X in year Y}}{\text{population in state X in year Y}}*100000$$_____no_output_____Certain mistakes would make your answer to the previous question way too small or way too big, and the automatic tests don't check that.  Make sure your answer looks reasonable before moving on._____no_output_____

ca = murder_rates.where('State', are.equal_to('California'))
ca.where('Year', 1988).column(3) * ca.where('Year', 1988).column(2)/100000 - ca.where('Year', 1975).column(3) * ca.where('Year', 1975).column(2)/100000

_____no_output_____ca = murder_rates.where('State', are.equal_to('California'))
ca_change = ca.where('Year', 1988).column(3) * ca.where('Year', 1988).column(2)/100000 - ca.where('Year', 1975).column(3) * ca.where('Year', 1975).column(2)/100000
ca_change = int(ca_change)
np.round(ca_change)_____no_output______ = ok.grade('q1_4')
_ = ok.backup()~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Running tests

---------------------------------------------------------------------
Test summary
    Passed: 1
    Failed: 0
[ooooooooook] 100.0% passed

# 2. Changes in Murder Rates

In this section, we'll see how to test null hypotheses such as this: "For this set of U.S. states, the murder rate was equally likely to go up or down each year."

Murder rates vary widely across states and years, presumably due to the vast array of differences among states and across US history. Rather than attempting to analyze rates themselves, here we will restrict our analysis to whether or not murder rates increased or decreased over certain time spans. **We will not concern ourselves with how much rates increased or decreased; only the direction of the change** - whether they increased or decreased._____no_output_____The `np.diff` function takes an array of values and computes the differences between adjacent items of a list or array as such:

    [item 1 - item 0 , item 2 - item 1 , item 3 - item 2, ...]

Instead, we may wish to compute the difference between items that are two positions apart. For example, given a 5-element array, we may want:

    [item 2 - item 0 , item 3 - item 1 , item 4 - item 2]

The `diff_n` function below computes this result. Don't worry if the implementation uses unfamiliar features of Python, as long as you understand its behavior._____no_output_____

def diff_n(values, n):
    return np.array(values)[n:] - np.array(values)[:-n]

diff_n(make_array(1, 10, 100, 1000, 10000), 2)_____no_output_____

**Question 2.1.** Implement the function `two_year_changes` that takes an array of murder rates for a state, ordered by increasing year. For all two-year periods (e.g., from 1960 to 1962), it computes and returns **the number of increases minus the number of decreases.**

For example, the array `r = make_array(10, 7, 12, 9, 13, 9, 11)` contains 3 increases (10 to 12, 7 to 9, and 12 to 13), 1 decrease (13 to 11), and 1 change that is neither an increase or decrease (9 to 9). Therefore, `two_year_changes(r)` would return 2, the difference between 3 increases and 1 decrease.

*Hint*: Consider using the `diff_n` function._____no_output_____

def two_year_changes(rates):
    "Return the number of increases minus the number of decreases after two years."
    return np.count_nonzero(diff_n(rates,2)>0)-np.count_nonzero(diff_n(rates,2)<0)

print('Alaska:',    two_year_changes(ak.column('Murder rate in Alaska')))
print('Minnesota:', two_year_changes(mn.column('Murder rate in Minnesota')))Alaska: -5
Minnesota: 6
_ = ok.grade("q2_1")
_ = ok.backup()~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Running tests

---------------------------------------------------------------------
Test summary
    Passed: 1
    Failed: 0
[ooooooooook] 100.0% passed

We can use `two_year_changes` to summarize whether rates are mostly increasing or decreasing over time for some state or group of states. Let's see how it varies across the 50 US states.

**Question 2.2.** Assign `changes_by_state` to a table with one row per state that has two columns: the `State` name and the `Murder Rate two_year_changes` statistic computed across all years in our data set for that state.  Its first 2 rows should look like this:

|State|Murder Rate two_year_changes|
|-|-|
|Alabama|-6|
|Alaska|-5||

... (48 rows omitted)
_____no_output_____

#Table().with_column('State', murder_rates.group('State').column('State'))
#murder_rates.group('State').select('State')
murder_rates.group('State', two_year_changes).drop(1).drop(1)_____no_output_____changes_by_state = murder_rates.group('State', two_year_changes).drop(1).drop(1)

# Here is a histogram of the two-year changes for the states.
# Since there are 50 states, each state contributes 2% to one
# bar.
changes_by_state.hist("Murder Rate two_year_changes", bins=np.arange(-11, 12, 2))_____no_output______ = ok.grade("q2_2")
_ = ok.backup()~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Running tests

---------------------------------------------------------------------
Test summary
    Passed: 1
    Failed: 0
[ooooooooook] 100.0% passed

Some states have more increases than decreases (a positive number), while some have more decreases than increases (a negative number).

**Question 2.3.** Assign `total_changes` to the total increases minus the total decreases for all two-year periods and all states in our data set. For example, if the murder rate in Ohio went up 23 times and fell 17 times, the total change for Ohio would be 6. We want the total value for all the states together._____no_output_____

a = changes_by_state.where('Murder Rate two_year_changes', are.above(0)).column(1).sum()
b = changes_by_state.where('Murder Rate two_year_changes', are.below(0)).column(1).sum()
a + b

_____no_output_____a = changes_by_state.where('Murder Rate two_year_changes', are.above(0)).column(1).sum()
b = changes_by_state.where('Murder Rate two_year_changes', are.below(0)).column(1).sum()

total_changes = a + b

print('Total increases minus total decreases, across all states and years:', total_changes)Total increases minus total decreases, across all states and years: 45
_ = ok.grade("q2_3")
_ = ok.backup()~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Running tests

---------------------------------------------------------------------
Test summary
    Passed: 1
    Failed: 0
[ooooooooook] 100.0% passed

"More increases than decreases," one student exclaims, "Murder rates tend to go up across two-year periods. What dire times we live in."

"Not so fast," another student replies, "Even if murder rates just moved up and down uniformly at random, there would be some difference between the increases and decreases. There were a lot of states and a lot of years, so there were many chances for changes to happen. If state murder rates increase and decrease at random with equal probability, perhaps this difference was simply due to chance!"

**Question 2.4.** Set `num_changes` to the number of distinct two-year periods in the entire data set that could result in a change of a state's murder rate for all states. Include both those periods where a change occurred and the periods where a state's rate happened to stay the same.

For example, Alaska during 1968 to 1970 would count as one distinct two-year period - how many of these periods are there total? _____no_output_____

murder_rates.group('State', two_year_changes)_____no_output_____
num_changes = 42*50
num_changes_____no_output______ = ok.grade("q2_4")
_ = ok.backup()~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Running tests

---------------------------------------------------------------------
Test summary
    Passed: 1
    Failed: 0
[ooooooooook] 100.0% passed

We now have enough information to perform a hypothesis test.

> **Null Hypothesis**: State murder rates increase and decrease over two-year periods as if
"increase" or "decrease" were sampled at random from a uniform distribution, like a fair coin flip.

Murder rates can be more likely to go up or more likely to go down. Since we observed 45 more increases than decreases for all two year periods in our dataset, we formulate an alternative hypothesis in accordance with our suspicion:

> **Alternative Hypothesis**: State murder rates are more likely to increase over two-year periods.

If we had observed more decreases than increases, our alternative hypothesis would have been defined accordingly (that state murder rates are more likely to *decrease*). This is typical in statistical testing - we first observe a trend in the data and then run a hypothesis test to confirm or reject that trend.

*Technical note*: These changes in murder rates are not random samples from any population. They describe all murders in all states over all recent years. However, we can imagine that history could have been different, and that the observed changes are the values observed in only one possible world: the one that happened to occur. In this sense, we can evaluate whether the observed "total increases minus total decreases" is consistent with a hypothesis that increases and decreases are drawn at random from a uniform distribution.

*Important requirements for our test statistic:* We want to choose a test statistic for which large positive values are evidence in favor of the alternative hypothesis, and other values are evidence in favor of the null hypothesis. This is because once we've determined the direction of our alternative hypothesis, we only care about the tail in that direction. If, for example, our p-value cutoff was 5%, we'd check to see if our observed test statistic fell within the largest 5% of values in our null hypothesis distribution.

Our test statistic should depend only on whether murder rates increased or decreased, not on the size of any change. Thus we choose:

> **Test Statistic**: The number of increases minus the number of decreases_____no_output_____The cell below samples increases and decreases at random from a uniform distribution 100 times. The final column of the resulting table gives the number of increases and decreases that resulted from sampling in this way. **Using `sample_from_distribution` is faster than using `sample` followed by `group` to compute the same result.**_____no_output_____

uniform = Table().with_columns(
    "Change", make_array('Increase', 'Decrease'),
    "Chance", make_array(0.5,        0.5))
uniform.sample_from_distribution('Chance', 100)_____no_output_____

**Question 2.5.** Complete the simulation below, which samples `num_changes` increases/decreases at random many times and forms an empirical distribution of your test statistic under the null hypothesis.  Your job is to
* fill in the function `simulate_under_null`, which simulates a single sample under the null hypothesis, and
* fill in its argument when it's called below._____no_output_____

np.count_nonzero(np.random.choice(uniform.column('Change'),5)=="Increase") -\
np.count_nonzero(np.random.choice(uniform.column('Change'),5)=="Decrease") _____no_output_____def simulate_under_null(num_chances_to_change):
    """Simulates some number changing several times, with an equal
    chance to increase or decrease.  Returns the value of our
    test statistic for these simulated changes.

    num_chances_to_change is the number of times the number changes.
    """
    sample = uniform.sample_from_distribution('Chance', num_chances_to_change)
    #np.random.choice(uniform.column(0),num_chances_to_change).\
    return sample.column(2).item(0) - sample.column(2).item(1)
uniform_samples = make_array()
for i in np.arange(5000):
    uniform_samples = np.append(uniform_samples, simulate_under_null(num_changes))

Table().with_column('Test statistic under null', uniform_samples).hist(0, bins=np.arange(-100, 400+25, 25))_____no_output_____

**Question 2.6.** Looking at this histogram, draw a conclusion about whether murder rates basically increase as often as they decrease. (Remember that we're only concerned with the *postive direction* because it supports our alternative hypothesis.) You **do not** need to compute a P-value for this question.

First, set `which_side` to `"Right"` or `"Left"` depending on which side of the histogram you need to look at to make your conclusion.

Then, set `reject_null` to `True` if rates increase more than they decrease, and we can reject the null hypothesis. Set `reject_null` to `False` if they do not systematically increase more than they decrease._____no_output_____

which_side = "Right"
reject_null = False_____no_output______ = ok.grade("q2_6")
_ = ok.backup()~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Running tests

---------------------------------------------------------------------
Test summary
    Passed: 2
    Failed: 0
[ooooooooook] 100.0% passed

### Congratulations: Checkpoint Reached

You have reached the project checkpoint. Please submit now in order to record your progress. If you go back and revise your answers in the section above after the checkpoint is due, that's ok. Your revised answers will be graded. **However, you will only get credit for your checkpoint submission if you have passed the tests provided for every question above.**

If you are working with a partner, only one of you needs to submit. For both of you to receive credit, the person who submits must invite the other to be their partner on [okpy.org](http://okpy.org). Please invite your partner now and tell them to accept the invitation **before** the checkpoint deadline!_____no_output_____

_ = ok.submit()_____no_output_____

# 3. The death penalty

Some US states have the death penalty, and others don't, and laws have changed over time. In addition to changes in murder rates, we will also consider whether the death penalty was in force in each state and each year.

Using this information, we would like to investigate how the death penalty affects the murder rate of a state._____no_output_____**Question 3.1.** Describe this investigation in terms of an experiment. What population are we studying? What is the control group? What is the treatment group? What outcome are we measuring? Be precise!_____no_output_____*Write your answers below.*

- Population: People from United States.
- Control Group: those who don't receceive death penalty
- Treatment Group: those who receive death penalty
- Outcome: change in murder rates_____no_output_____**Question 3.2.** We want to know whether the death penalty *causes* a change in the murder rate.  Why is it not sufficient to compare murder rates in places and times when the death penalty was in force with places and times when it wasn't?_____no_output_____*Thre could be other factors that would affect the murder rates such as patroling places where people commit crime that could possibly lead to murder.*_____no_output_____### A Natural Experiment

In order to attempt to investigate the causal relationship between the death penalty and murder rates, we're going to take advantage of a *natural experiment*.  A natural experiment happens when something other than experimental design applies a treatment to one group and not to another (control) group, and we have some hope that the treatment and control groups don't have any other systematic differences.

Our natural experiment is this: in 1972, a Supreme Court decision called *Furman v. Georgia* banned the death penalty throughout the US.  Suddenly, many states went from having the death penalty to not having the death penalty.

As a first step, let's see how murder rates changed before and after the court decision.  We'll define the test as follows:

> **Population:** All the states that had the death penalty before the 1972 abolition.  (There is no control group for the states that already lacked the death penalty in 1972, so we must omit them.)  This includes all US states **except** Alaska, Hawaii, Maine, Michigan, Wisconsin, and Minnesota.

> **Treatment group:** The states in that population, in the year after 1972.

> **Control group:** The states in that population, in the year before 1972.

> **Null hypothesis:** Each state's murder rate was equally likely to be higher or lower in the treatment period than in the control period.  (Whether the murder rate increased or decreased in each state was like the flip of a fair coin.)

> **Alternative hypothesis:** The murder rate was more likely to increase.

Our alternative hypothesis is in keeping with our suspicion that murder rates increase when the death penalty is eliminated.

*Technical Note:* It's not clear that the murder rates were a "sample" from any larger population.  Again, it's useful to imagine that our data could have come out differently and to test the null hypothesis that the murder rates were equally likely to move up or down.

The `death_penalty` table below describes whether each state allowed the death penalty in 1971._____no_output_____

non_death_penalty_states = make_array('Alaska', 'Hawaii', 'Maine', 'Michigan', 'Wisconsin', 'Minnesota')
def had_death_penalty_in_1971(state):
    """Returns True if the argument is the name of a state that had the death penalty in 1971."""
    # The implementation of this function uses a bit of syntax
    # we haven't seen before.  Just trust that it behaves as its
    # documentation claims.
    return state not in non_death_penalty_states

states = murder_rates.group('State').select('State')
death_penalty = states.with_column('Death Penalty', states.apply(had_death_penalty_in_1971, 0))
death_penalty_____no_output_____num_death_penalty_states = death_penalty.where("Death Penalty", are.equal_to(True)).num_rows
num_death_penalty_states_____no_output_____

**Question 3.3.** Assign `death_penalty_murder_rates` to a table with the same columns and data as `murder_rates`, but that has only the rows for states that had the death penalty in 1971.

The first 2 rows of your table should look like this:

|State|Year|Population|Murder Rate|
|-----|----|----------|-----------|
|Alabama|1960|3266740|12.4|
|Alabama|1961|3302000|12.9|_____no_output_____

death_penalty_murder_rates = death_penalty.where("Death Penalty", True).join('State', murder_rates, 'State').drop('Death Penalty')

death_penalty_murder_rates_____no_output_____

The null hypothesis doesn't specify *how* the murder rate changes; it only talks about increasing or decreasing.  So, we will use the same test statistic we defined in section 2._____no_output_____**Question 3.4.** Assign `test_stat_72` to the value of the test statistic for the years 1971 to 1973 and the states in `death_penalty_murder_rates`. As before, the test statistic is, "the number of increases minus the number of decreases."_____no_output_____

test_stat=death_penalty_murder_rates.where('Year', are.between_or_equal_to(
    1971,1973)).select('Murder Rate','State').group('State', two_year_changes)
test_stat_____no_output_____test_stat=death_penalty_murder_rates.where('Year', are.between_or_equal_to(
    1971,1973)).select('Murder Rate','State').group('State', two_year_changes)
#before = death_penalty_murder_rates.where('Year', 1971).select('Murder Rate').sum()
test_stat_72 = test_stat.column(1).sum()
print('Test statistic from 1971 to 1973:', test_stat_72)Test statistic from 1971 to 1973: 22

Look at the data (or perhaps a random sample) to verify that your answer is correct._____no_output_____

new_changes=death_penalty_murder_rates.group('State', two_year_changes).drop(1).drop(1)
new_changes_____no_output_____c = new_changes.where('Murder Rate two_year_changes', are.above(0)).column(1).sum()
d = new_changes.where('Murder Rate two_year_changes', are.below(0)).column(1).sum()
new_total = c+d
death_penalty_murder_rates.group('State', two_year_changes)
new_num=42*44
_____no_output_____

**Question 3.5.**: Draw an empirical histogram of the statistic under the null hypothesis by simulating the test statistic 10,000 times.

*Hint:* In Part 2 of this project, you have already written a function that runs such a simulation once._____no_output_____

samples = make_array()
for i in np.arange(10000):
    samples = np.append(samples, simulate_under_null(num_death_penalty_states))
Table().with_column('Test statistic under null', samples).hist(bins=np.arange(-4, 28+2, 2))_____no_output_____

### Conclusion
**Question 3.6.** Complete the analysis as follows:
1. Compute a P-value.
2. Draw a conclusion about the null and alternative hypotheses.
3. Describe your findings using simple, non-technical language.  Be careful not to claim that the statistical analysis has established more than it really has._____no_output_____**P-value:0.007**

**Conclusion about the hypotheses:reject null**

**Findings: There exists more factors to affect the murder rate than just chance.** _____no_output_____

# Use this cell to compute the P-value, if you wish (ungraded)
#np.count nonzero>- len(samples)?_____no_output_____

# 4. Further evidence

So far, we have discovered evidence that when executions were outlawed, the murder rate increased in many more states than we would expect from random chance. We have also seen that across all states and all recent years, the murder rate goes up about as much as it goes down over two-year periods.

These discoveries seem to support the claim that eliminating the death penalty increases the murder rate. Should we be convinced? Let's conduct some more tests to strengthen our claim.

Conducting a test for this data set requires the following steps:

1. Select a table containing murder rates for certain states and all years,
2. Choose two years and compute the observed value of the test statistic,
3. Simulate the test statistic under the null hypothesis that increases and decreases are drawn uniformly at random, then
4. Compare the observed difference to the empirical distribution to compute a P-value._____no_output_____This entire process can be expressed in a single function, called `run_test`.

**Question 4.1.** Implement `run_test`, which takes the following arguments:

- A table of murder `rates` for certain states, sorted by state and year like `murder_rates`, and
- the year when the analysis starts.  (The comparison group is two years later.)

It prints out the observed test statistic and returns the P-value for this statistic under the null hypothesis.

Keep in mind that `run_test` should work **regardless of the direction of association in your alternative hypothesis.** Therefore, you'll need to account for if `observed_test_statistic` is negative or positive.

*Hint 1*: You can complete most of this question by copying code you wrote earlier.

*Hint 2*: This problem might seem daunting.  Start by writing out the different steps involved in running a test._____no_output_____

death_penalty_murder_rates_____no_output_____# def run_test(rates, start_year):
#     """Prints out the observed test statistic and returns a P-value for this statistic under the null hypothesis
#     Parameters
#     ----------
#     rates : Table
#        A table of murder rates for certain states, sorted by state and year like murder_rates
#     start_year : int
#        The year when the analysis starts
#     """
#     end_year = start_year + 2
#     observed_test_statistic = sum(rates.where('Year',are.between(start_year, end_year +1)).select('State','Murder Rate').group('State', two_year_changes).column('Murder Rate two_year_changes')
#     num_states = rates.group('State').num_rows

#     print('Test statistic', start_year, 'to', end_year, ':', observed_test_statistic)
#     samples = make_array()
#     for i in np.arange(5000):
#         samples = np.append(samples, simulate_under_null(num_states))

#     if observed_test_statistic > 0:
#         return np.count_nonzero(samples >= observed_test_statistic/len(samples))

#     elif observed_test_statistic < 0:
#         return np.count_nonzero(samples <= observed_test_statistic/len(samples))

# run_test(death_penalty_murder_rates, 1971)_____no_output_____def run_test(rates, start_year):

    end_year = start_year + 2
    observed_test_statistic = sum(rates.where('Year',are.between(start_year, end_year +1)).select('State','Murder Rate').group(
    'State', two_year_changes).column('Murder Rate two_year_changes'))
    num_states = rates.group('State').num_rows
    print('Test statistic', start_year, 'to', end_year, ':', observed_test_statistic)
    samples = make_array()
    for i in np.arange(5000):
        samples = np.append(samples,simulate_under_null(num_states))

    if observed_test_statistic > 0:
        return np.count_nonzero(samples >= observed_test_statistic)/len(samples)
    elif observed_test_statistic < 0:
        return np.count_nonzero(samples <= observed_test_statistic)/len(samples)

run_test(death_penalty_murder_rates, 1971)Test statistic 1971 to 1973 : 22
_ = ok.grade("q4_1")
_ = ok.backup()~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Running tests

---------------------------------------------------------------------
Test summary
    Passed: 1
    Failed: 0
[ooooooooook] 100.0% passed

### The rest of the states

We found a dramatic increase in murder rates for those states affected by the 1972 Supreme Court ruling, but what about the rest of the states? There were six states that had already outlawed execution at the time of the ruling.

**Question 4.2.** Create a table called `non_death_penalty_murder_rates` with the same columns as `murder_rates` but only containing rows for the six states without the death penalty in 1971. Perform the same test on this table. **Then**, set reject_null_2 to whether their murder rates were also more likely to increase from 1971 to 1973._____no_output_____

death_penalty.where("Death Penalty", False).join('State', murder_rates, 'State').drop('Death Penalty')
_____no_output_____non_death_penalty_murder_rates = death_penalty.where("Death Penalty", are.equal_to(False)).join('State', murder_rates, 'State').drop('Death Penalty')

run_test(non_death_penalty_murder_rates, 1971)Test statistic 1971 to 1973 : 1
reject_null_2 = False_____no_output______ = ok.grade("q4_2")
_ = ok.backup()~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Running tests

---------------------------------------------------------------------
Test summary
    Passed: 1
    Failed: 0
[ooooooooook] 100.0% passed

### The death penalty reinstated

In 1976, the Supreme Court repealed its ban on the death penalty in its rulings on [a series of cases including Gregg v. Georgia](https://en.wikipedia.org/wiki/Gregg_v._Georgia), so the death penalty was reinstated where it was previously banned.  This generated a second natural experiment.  To the extent that the death penalty deters murder, reinstating it should decrease murder rates, just as banning it should increase them. Let's see what happened._____no_output_____

print("Increases minus decreases from 1975 to 1977 (when the death penalty was reinstated) among death penalty states:",
      sum(death_penalty_murder_rates.where('Year', are.between_or_equal_to(1975, 1977))
                                    .group('State', two_year_changes)
                                    .column("Murder Rate two_year_changes")))
run_test(death_penalty_murder_rates, 1975)Increases minus decreases from 1975 to 1977 (when the death penalty was reinstated) among death penalty states: -18
Test statistic 1975 to 1977 : -18

*Hint:* To sanity check your results, figure out what your test statistic should be when there are 18 more decreases than increases, and verify that that's the test statistic that was printed.  Also, you should have found a P-value near 0.005. If your P-value is very different, go back and inspect your `run_test` implementation to make sure that it correctly produces low P-values when there are many more decreases than increases._____no_output_____**Question 4.3.** Now we've analyzed states where the death penalty went away and came back, as well as states where the death penalty was outlawed all along.  What do you conclude from  the results of the tests we have conducted so far? Does all the evidence consistently point toward one conclusion, or is there a contradiction?

1) Our results point toward the conclusion that the death penalty moratorium increased murder rates.

2) Our results point toward the conclusion that the death penalty moratorium increased murder rates, but we have not accounted for time as a confounding factor.

3) Our results don't allow us to make any conclusion about murder rates and death penalties.

4) Our results point toward the conclusion that the death penalty moratorium didn't influence murder rates.

5) None of these conclusions are valid, or multiple of these conclusions are valid

Below, set we_conclude to a single number, corresponding to your answer._____no_output_____

we_conclude = 2_____no_output______ = ok.grade("q4_3")
_ = ok.backup()~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Running tests

---------------------------------------------------------------------
Test summary
    Passed: 1
    Failed: 0
[ooooooooook] 100.0% passed

## 5. Visualization

While our analysis appears to support the conclusion that the death penalty deters murder, [a 2006 Stanford Law Review paper](http://users.nber.org/~jwolfers/papers/DeathPenalty%28SLR%29.pdf) argues the opposite: that historical murder rates do **not** provide evidence that the death penalty deters murderers.

To understand their argument, we will draw a picture.  In fact, we've gone at this whole analysis rather backward; typically we should draw a picture first and ask precise statistical questions later!

What plot should we draw?

We know that we want to compare murder rates of states with and without the death penalty.  We know we should focus on the period around the two natural experiments of 1972 and 1976, and we want to understand the evolution of murder rates over time for those groups of states.  It might be useful to look at other time periods, so let's plot them all for good measure._____no_output_____**Question 5.1.** Create a table called `average_murder_rates` with 1 row for each year in `murder_rates`.  It should have 3 columns:
* `Year`, the year,
* `Death penalty states`, the average murder rate of the states that had the death penalty in 1971, and
* `No death penalty states`, the average murder rate of the other states.

`average_murder_rates` should be sorted in increasing order by year.  Its first three rows should look like:

|Year|Death penalty states|No death penalty states|
|-|-|-|
|1960| | |
|1961| | |
|1962| | ||

*Hint:*  It's easiest to use `pivot`. We don't expect you to account for differences in population._____no_output_____

no_penalty = death_penalty.where("Death Penalty", False).join('State', murder_rates, 'State').select('Year','Murder Rate')
penalty = death_penalty.where("Death Penalty", True).join('State', murder_rates, 'State').select('Year','Murder Rate')
penalty.group('Year', np.mean)
no_penalty.group('Year', np.mean)
Table().with_column('Deat', penalty.group('Year', np.mean).column(1))
Table().with_columns('Year', penalty.group('Year').column(0),
                    'Death penalty states', penalty.group('Year', np.mean).column(1),
                    'No death penalty states', no_penalty.group('Year', np.mean).column(1))

                      _____no_output_____# For reference, our solution used 5 method calls
no_penalty = death_penalty.where("Death Penalty", False).join('State', murder_rates, 'State').select('Year','Murder Rate')
penalty = death_penalty.where("Death Penalty", True).join('State', murder_rates, 'State').select('Year','Murder Rate')
penalty.group('Year', np.mean)
no_penalty.group('Year', np.mean)
Table().with_column('Deat', penalty.group('Year', np.mean).column(1))
Table().with_columns('Year', penalty.group('Year').column(0),
                    'Death penalty states', penalty.group('Year', np.mean).column(1),
                    'No death penalty states', no_penalty.group('Year', np.mean).column(1))

average_murder_rates = Table().with_columns('Year', penalty.group('Year').column(0),
                    'Death penalty states', penalty.group('Year', np.mean).column(1),
                    'No death penalty states', no_penalty.group('Year', np.mean).column(1))

average_murder_rates_____no_output______ = ok.grade('q5_1')
_ = ok.backup()~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Running tests

---------------------------------------------------------------------
Test summary
    Passed: 1
    Failed: 0
[ooooooooook] 100.0% passed

**Question 5.2.** Describe in **one short sentence** a high-level takeaway from the line plot below. Are the murder rates in these two groups of states related?_____no_output_____

average_murder_rates.plot('Year')_____no_output_____

*The shape of two plots are roughly the same so they're related.*_____no_output_____Let's bring in another source of information: Canada._____no_output_____

canada = Table.read_table('canada.csv')
murder_rates_with_canada = average_murder_rates.join("Year", canada.select("Year", "Homicide").relabeled("Homicide", "Canada"))
murder_rates_with_canada.plot('Year')_____no_output_____

The line plot we generated above is similar to a figure from the [paper](http://users.nber.org/~jwolfers/papers/DeathPenalty%28SLR%29.pdf).

Canada has not executed a criminal since 1962. Since 1967, the only crime that can be punished by execution in Canada is the murder of on-duty law enforcement personnel. The paper states, "The most striking finding is that the homicide rate in Canada has moved in
virtual lockstep with the rate in the United States."_____no_output_____**Question 5.3.** Complete their argument in 2-3 sentences; what features of these plots indicate that the death penalty is not an important factor in determining the murder rate? (If you're stuck, read the [paper](http://users.nber.org/~jwolfers/papers/DeathPenalty%28SLR%29.pdf).)_____no_output_____*Murder rate in the U.S and Canada both increased durng abolished. The graph between U.S and Canada looks roughly same when U.S reestablisehd death peanlty, whereas Canada is still abolished the death penalty. *_____no_output_____**Conclusion**. The authors that created this visualization argue that even though murder rates increased when the death penalty was taken away, and decreased when it was reintroduced, these changes were probably not caused by the death penalty itself. Based on your analysis, what conclusion can you make? Address the steps you took throughout the project in your answer as well._____no_output_____*I first compare the murder rate between the states by two_year changes and performed the hypothesis testing to see that State murder rates increase and decrease like a fair coin flip. It turns out that i rejected the null and decided that State murder rates are more likely to increase rather than a random chance. I also checked the natural experiment between the treatment and control group to compare the states that had the death penalty before and after 1972. This test also indecates that murder rate was more likely to increase due to the death penalty. Thus, i conclude that death penaly does somewhat affect the murder rate.  *_____no_output_____**You're done! Congratulations.**_____no_output_____

# Submit the project!
_ = ok.submit()_____no_output_____# For your convenience, you can run this cell to run all the tests at once!
import os
print("Running all tests...")
_ = [ok.grade(q[:-3]) for q in os.listdir("tests") if q.startswith('q')]
print("Finished running all tests.")_____no_output_____


# Notebook from wanZzz6/Modules-Learn
Path: OpenCV/图像 上的算术运算、抠图.ipynb

cd D:\OneDrive - business\git_clone\OpenCV-Python-Tutorial-masterD:\OneDrive - business\git_clone\OpenCV-Python-Tutorial-master
import cv2
import numpy as np
from matplotlib import pyplot as plt
%matplotlib inline_____no_output_____

# 目标

- 学习图像上的算术?算?加法?减法?位?算等。
- 我们将?学习的函数与有：cv2.add()，cv2.addWeighted() 等。_____no_output_____## 图像加法

你可以使用函数 `cv2.add()` 将两幅图像??加法?算?当然也可以直接使
用 numpy, `res=img1+img`。两幅图像的大小，类型必须一致，或者第二个
图像可以使一个简单的标量值。

**注意?OpenCV 中的加法与 Numpy 的加法是有所不同的。OpenCV 的加法
是一种饱和操作，而 Numpy 的加法是一种模操作**

例如下面的两个例子：_____no_output_____

x = np.uint8([250])
y = np.uint8([10])
print(cv2.add(x,y)) # 250+10 = 260 => 255
print(x+y) # 250+10 = 260 % 256 = 4[[255]]
[4]

这种差别在你对两幅图像进行加法时会更加明显。OpenCV 的结果会更好
一点。所以我们尽量使用 OpenCV 中的函数。_____no_output_____## 图像混合

这其实也是加法，但是不同的是两幅图像的权重不同，这就会给人一种混合或者透明的感觉。图像混合的计算公式如下：

g(x)=(1-a)f0(x)+af1(x)

通过修改a的值（0→1），可以实现非常酷的混合。
现在我们把两幅图混合在一起。第一幅图的权重是0.7，第二幅图的权重是0.3。函数`cV2.addWeighted()`可以按下面的公式对图片进行混合操作。_____no_output_____

# 图像混合
img1 = cv2.imread('./data/ml.png')
img2 = cv2.imread('./data/star.png')
img1.shape, img2.shape_____no_output_____# 注意图像尺寸
dst = cv2.addWeighted(img1[:190, :200] , 0.7, img2[:190,: 200], 0.3, 0)  # 第一幅图的权重是 0.7 第二幅图的权重是 0.3

cv2.imshow('src1', img1[:190, :200])
cv2.imshow('src2', img2[:190,: 200])
cv2.imshow('dst', dst)
cv2.waitKey(0)
cv2.destroyAllWindows()_____no_output_____

## 粘贴
_____no_output_____

a = np.ones((500, 500, 3), dtype=np.uint8)
a[:, : ] = (123, 0, 234)

b = np.ones((200, 200, 3), dtype=np.uint8)
b[:, : ] = (0, 0, 0)

def paste(src, crop, position):
    """
    position: left_top :(x, y)
    """
    crop_shape = list(crop.shape)

    if position[0] + crop.shape[1] > src.shape[1]:
        crop_shape[1] -= position[0] + crop.shape[1] - src.shape[1]

    if position[1] + crop.shape[0] > src.shape[0]:
        crop_shape[0] -= position[1] + crop.shape[0] - src.shape[0]
    print(crop_shape)
    src[position[1]: b.shape[0] + position[1], position[0]: crop.shape[1] + position[0]] = crop[:crop_shape[0], :crop_shape[1]]
    return src

cv2.imshow('dst', paste(a, b, (200, 100)))
cv2.waitKey(0)
cv2.destroyAllWindows()[200, 200, 3]

# 按位运算

这里包括的按位操作有：AND，OR，NOT，XOR等。当我们提取图像的一部分，**选择非矩形ROI时这些操作会很有用**（下一章你就会明白）。下面的例子就是教给我们如何改变一幅图的特定区域。

我想把OpenCV的标志放到另一幅图像上。如果我使用加法，颜色会改变，如果使用混合，会得到透明效果，但是我不想要透明。如果他是矩形我可以象上一章那样使用ROI。但是他不是矩形。但是我们可以通过下面的按位运算实现：

## bitwise_and_____no_output_____

# Load two images
img1 = cv2.imread('./data/messi5.jpg')
img2 = cv2.imread('./data/opencv_logo.png')
# logo 原图太大了，必须比目标图像尺寸小
img2 = cv2.pyrDown(img2)

plt.subplot(1, 3, 1)
plt.imshow(img1)
plt.subplot(1, 3, 2)
plt.imshow(img2)
plt.show()_____no_output_____# I want to put logo on top-left corner, So I create a ROI
rows, cols, channels = img2.shape
roi = img1[0:rows, 0:cols]
plt.subplot(1, 1, 1)
plt.imshow(roi)
plt.show()_____no_output_____roi.shape,img2.shape, img1.shape, _____no_output_____# Now create a mask of logo and create its inverse mask also
img2gray = cv2.cvtColor(img2, cv2.COLOR_BGR2GRAY)
ret, mask = cv2.threshold(img2gray, 175, 255, cv2.THRESH_BINARY)
mask_inv = cv2.bitwise_not(mask)
plt.subplot(1, 3, 1)
plt.imshow(img2gray, cmap='gray')
plt.subplot(1, 3, 2)
plt.imshow(mask, cmap='gray')
plt.subplot(1, 3, 3)
plt.imshow(mask_inv, cmap='gray')
plt.show()_____no_output_____# Now black-out the area of logo in ROI
img1_bg = cv2.bitwise_and(roi, roi, mask=mask)
plt.subplot(1, 3, 1)
plt.imshow(img1_bg)
plt.show()_____no_output_____img2_fg = cv2.bitwise_and(img2, img2, mask=mask_inv)
plt.subplot(1, 3, 1)
plt.imshow(img2_fg)
plt.show()_____no_output_____# Put logo in ROI and modify the main image
dst = cv2.add(img1_bg, img2_fg)
plt.subplot(1, 3, 1)
plt.imshow(dst)
plt.show()_____no_output_____img1[0:rows, 0:cols] = dst
plt.subplot(1, 3, 1)
plt.imshow(img1)
plt.show()_____no_output_____

## 图像相减1_____no_output_____

cd ch10-图像上的算术运算/D:\OneDrive - business\git_clone\OpenCV-Python-Tutorial-master\ch10-图像上的算术运算
import cv2
import numpy as np

# img1=cv2.imread('subtract1.jpg')
img1 = cv2.imread('subtract1.jpg', 0)  #灰度图
# img2=cv2.imread('subtract2.jpg')
img2 = cv2.imread('subtract2.jpg', 0)

plt.subplot(1, 2, 1)
plt.imshow(img1, cmap='gray')
plt.subplot(1, 2, 2)
plt.imshow(img2, cmap='gray')
plt.show()_____no_output_____st=img2-img1
# st=img1-img2#相反
plt.subplot(1, 2, 2)
plt.imshow(st, cmap='gray')
plt.show()_____no_output_____#效果好一点
# ret,threshold=cv2.threshold(st,0, 127, cv2.THRESH_BINARY)
ret, threshold = cv2.threshold(st, 50, 255, cv2.THRESH_BINARY)
plt.subplot(1, 2, 2)
plt.imshow(threshold, cmap='gray')
plt.show()_____no_output_____


# Notebook from doganlab/scmags
Path: docs/source/visualization.ipynb

# Visualize Markers_____no_output_____* This tutorial shows how to visualize selected markers with the **scmags** package.

First, the markers need to be selected.
Let's start with the marker selection process._____no_output_____

import scmags as mg
baron_h1 = mg.datasets.baron_h1()
baron_h1.filter_genes()
baron_h1.sel_clust_marker()-> Eliminating low expression genes
-> Selecting cluster-specific candidate marker genes
-> Selecting  markers for each cluster

After making a marker selection, you can visualize markers in 3 ways. You can also perform k-NN classification with selected markers and visualize the normalized confusion matrix.\
Let's start with the heatmap_____no_output_____## Heatmap

In this heatmap, the selected markers for each cluster are visualized sequentially.\
On the right, the cluster labels are colored so that it can be seen how distinctive the markers are._____no_output_____

baron_h1.markers_heatmap()_____no_output_____

## t-SNE

The t-SNE plot shown below is plotted as a result of the t-SNE calculated from the selected markers. If the selected markers are dissociative, their clusters are expected to be differentiated in the t-SNE plot._____no_output_____

baron_h1.markers_tSNE()_____no_output_____

## Dotplot

It was prepared using the dotplot [scanpy](https://github.com/theislab/scanpy) library.\
In dotplot, on the other hand, it is desired that the markers selected for the cluster are in dark color within the cluster, which means that the expression within the cluster has a high mean. In addition, the drawn rings should still be larger within the cluster, with larger rings representing higher expression._____no_output_____

baron_h1.dot_plot(largest_dot=50)_____no_output_____

## Confusion Matrix

Here, we first perform k-NN clustering and visualize the results with a normalized confusion matrix.
_____no_output_____

baron_h1.knn_classifier()_____no_output_____


# Notebook from jupyrdf/ipykgb
Path: notebooks/CoRef_Evaluation.ipynb

# Coreference Resolution_____no_output_____In this notebook, we investigate methods for coreference resolution. In NLP, coreference resolution analyzes references of text that point to the same entity. For example, given a sentence such as \
**'My brother loves to play basketball. He plays as a point guard'**, \
this sentence will get resolved as \
**'My brother loves to play basketball. My brother plays as a point guard'**.\
\
This reduces the complexity in speech that is evident in a wide variety of text sources._____no_output_____

starwars_text = 'Darth Vader, also known by his birth name Anakin Skywalker, is a fictional character in the Star Wars franchise. Darth Vader appears in the original film trilogy as a pivotal antagonist whose actions drive the plot, while his past as Anakin Skywalker and the story of his corruption are central to the narrative of the prequel trilogy. The character was created by George Lucas and has been portrayed by numerous actors. His appearances span the first six Star Wars films, as well as Rogue One, and his character is heavily referenced in Star Wars: The Force Awakens. He is also an important character in the Star Wars expanded universe of television series, video games, novels, literature and comic books. Originally a Jedi who was prophesied to bring balance to the Force, he falls to the dark side of the Force and serves the evil Galactic Empire at the right hand of his Sith master, Emperor Palpatine (also known as Darth Sidious).'
starwars_text_____no_output_____import spacy
from stanfordcorenlp import StanfordCoreNLP
import json
from collections import defaultdict
import nltk_____no_output_____

# Stanford CoreNLP_____no_output_____

nlp = StanfordCoreNLP("../stanford-corenlp-4.2.0", quiet=False)
annotated = nlp.annotate(starwars_text, properties={'annotators': 'coref', 'pipelineLanguage': 'en', 'coref.algorithm' : 'statistical'})
result = json.loads(annotated)
corefs = result['corefs']_____no_output_____import sys
sys.path.append('../')
from text_to_graph import process_NER, process_corefs, process_dependency_matching_____no_output_____# Perform Named Entity Recognition with spaCy
ner_dict = process_NER(text=starwars_text)
print('***** Completed NER *****')

# Generate Coreferences and Dependencies with CoreNLP
corefs = process_corefs(text=starwars_text, corenlp_path='../stanford-corenlp-4.2.0')
print("Coreferences found: ", len(corefs))

# Perform Replacement with Named Entities and Dependencies
resolved_text = process_dependency_matching(text=starwars_text, ner_dict=ner_dict, corefs=corefs)
print('***** Completed Coreference Resolution *****')***** Completed NER *****
Coreferences found:  3
***** Completed Coreference Resolution *****
starwars_text_____no_output_____resolved_text_____no_output_____

# Neural Coref_____no_output_____

import neuralcoref
import spacy
nlp = spacy.load("en_core_web_lg")_____no_output_____raw_text = 'Hello, world. Here are two sentences.'
# nlp = English()
nlp.add_pipe(nlp.create_pipe('sentencizer'))
doc = nlp(raw_text)
sentences = [sent.string.strip() for sent in doc.sents]
print(sentences)['Hello, world.', 'Here are two sentences.']

neuralcoref.add_to_pipe(nlp)

doc = nlp('My sister has a dog. She loves him.')

print(doc._.has_coref)
print(doc._.coref_clusters)
print(doc._.coref_resolved)True
[My sister: [My sister, She], a dog: [a dog, him]]
My sister has a dog. My sister loves a dog.

### Large Text Example_____no_output_____

doc = nlp(starwars_text)
doc._.has_coref_____no_output_____doc._.coref_clusters_____no_output_____doc._.coref_resolved_____no_output_____! pip install neuralcorefRequirement already satisfied: neuralcoref in /Users/rhythmsyed/miniconda3/envs/nlp/lib/python3.6/site-packages (4.0)
Requirement already satisfied: requests<3.0.0,>=2.13.0 in /Users/rhythmsyed/miniconda3/envs/nlp/lib/python3.6/site-packages (from neuralcoref) (2.25.1)
Requirement already satisfied: numpy>=1.15.0 in /Users/rhythmsyed/miniconda3/envs/nlp/lib/python3.6/site-packages (from neuralcoref) (1.19.5)
Requirement already satisfied: spacy>=2.1.0 in /Users/rhythmsyed/miniconda3/envs/nlp/lib/python3.6/site-packages (from neuralcoref) (2.2.4)
Requirement already satisfied: boto3 in /Users/rhythmsyed/miniconda3/envs/nlp/lib/python3.6/site-packages (from neuralcoref) (1.17.25)
Requirement already satisfied: certifi>=2017.4.17 in /Users/rhythmsyed/miniconda3/envs/nlp/lib/python3.6/site-packages (from requests<3.0.0,>=2.13.0->neuralcoref) (2020.12.5)
Requirement already satisfied: urllib3<1.27,>=1.21.1 in /Users/rhythmsyed/miniconda3/envs/nlp/lib/python3.6/site-packages (from requests<3.0.0,>=2.13.0->neuralcoref) (1.26.3)
Requirement already satisfied: chardet<5,>=3.0.2 in /Users/rhythmsyed/miniconda3/envs/nlp/lib/python3.6/site-packages (from requests<3.0.0,>=2.13.0->neuralcoref) (4.0.0)
Requirement already satisfied: idna<3,>=2.5 in /Users/rhythmsyed/miniconda3/envs/nlp/lib/python3.6/site-packages (from requests<3.0.0,>=2.13.0->neuralcoref) (2.10)
Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /Users/rhythmsyed/miniconda3/envs/nlp/lib/python3.6/site-packages (from spacy>=2.1.0->neuralcoref) (2.0.5)
Requirement already satisfied: blis<0.5.0,>=0.4.0 in /Users/rhythmsyed/miniconda3/envs/nlp/lib/python3.6/site-packages (from spacy>=2.1.0->neuralcoref) (0.4.1)
Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /Users/rhythmsyed/miniconda3/envs/nlp/lib/python3.6/site-packages (from spacy>=2.1.0->neuralcoref) (4.57.0)
Requirement already satisfied: catalogue<1.1.0,>=0.0.7 in /Users/rhythmsyed/miniconda3/envs/nlp/lib/python3.6/site-packages (from spacy>=2.1.0->neuralcoref) (1.0.0)
Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /Users/rhythmsyed/miniconda3/envs/nlp/lib/python3.6/site-packages (from spacy>=2.1.0->neuralcoref) (1.0.5)
Requirement already satisfied: setuptools in /Users/rhythmsyed/miniconda3/envs/nlp/lib/python3.6/site-packages (from spacy>=2.1.0->neuralcoref) (52.0.0.post20210125)
Requirement already satisfied: wasabi<1.1.0,>=0.4.0 in /Users/rhythmsyed/miniconda3/envs/nlp/lib/python3.6/site-packages (from spacy>=2.1.0->neuralcoref) (0.8.2)
Requirement already satisfied: plac<1.2.0,>=0.9.6 in /Users/rhythmsyed/miniconda3/envs/nlp/lib/python3.6/site-packages (from spacy>=2.1.0->neuralcoref) (1.1.3)
Requirement already satisfied: srsly<1.1.0,>=1.0.2 in /Users/rhythmsyed/miniconda3/envs/nlp/lib/python3.6/site-packages (from spacy>=2.1.0->neuralcoref) (1.0.5)
Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /Users/rhythmsyed/miniconda3/envs/nlp/lib/python3.6/site-packages (from spacy>=2.1.0->neuralcoref) (3.0.5)
Requirement already satisfied: thinc==7.4.0 in /Users/rhythmsyed/miniconda3/envs/nlp/lib/python3.6/site-packages (from spacy>=2.1.0->neuralcoref) (7.4.0)
Requirement already satisfied: importlib-metadata>=0.20 in /Users/rhythmsyed/miniconda3/envs/nlp/lib/python3.6/site-packages (from catalogue<1.1.0,>=0.0.7->spacy>=2.1.0->neuralcoref) (2.0.0)
Requirement already satisfied: zipp>=0.5 in /Users/rhythmsyed/miniconda3/envs/nlp/lib/python3.6/site-packages (from importlib-metadata>=0.20->catalogue<1.1.0,>=0.0.7->spacy>=2.1.0->neuralcoref) (3.4.0)
Requirement already satisfied: s3transfer<0.4.0,>=0.3.0 in /Users/rhythmsyed/miniconda3/envs/nlp/lib/python3.6/site-packages (from boto3->neuralcoref) (0.3.4)
Requirement already satisfied: botocore<1.21.0,>=1.20.25 in /Users/rhythmsyed/miniconda3/envs/nlp/lib/python3.6/site-packages (from boto3->neuralcoref) (1.20.25)
Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /Users/rhythmsyed/miniconda3/envs/nlp/lib/python3.6/site-packages (from boto3->neuralcoref) (0.10.0)
Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /Users/rhythmsyed/miniconda3/envs/nlp/lib/python3.6/site-packages (from botocore<1.21.0,>=1.20.25->boto3->neuralcoref) (2.8.1)
Requirement already satisfied: six>=1.5 in /Users/rhythmsyed/miniconda3/envs/nlp/lib/python3.6/site-packages (from python-dateutil<3.0.0,>=2.1->botocore<1.21.0,>=1.20.25->boto3->neuralcoref) (1.15.0)
import spacy
import neuralcoref

nlp = spacy.load('en')
neuralcoref.add_to_pipe(nlp)
doc1 = nlp('My sister has a dog. She loves him.')
print(doc1._.coref_clusters)

doc2 = nlp('Angela lives in Boston. She is quite happy in that city.')
for ent in doc2.ents:
    print(ent._.coref_cluster)_____no_output_____import spacy
import neuralcoref

nlp = spacy.load('en')/Users/rhythmsyed/miniconda3/envs/entitylink/lib/python3.7/importlib/_bootstrap.py:219: RuntimeWarning: spacy.morphology.Morphology size changed, may indicate binary incompatibility. Expected 104 from C header, got 112 from PyObject
  return f(*args, **kwds)
/Users/rhythmsyed/miniconda3/envs/entitylink/lib/python3.7/importlib/_bootstrap.py:219: RuntimeWarning: spacy.vocab.Vocab size changed, may indicate binary incompatibility. Expected 96 from C header, got 104 from PyObject
  return f(*args, **kwds)
/Users/rhythmsyed/miniconda3/envs/entitylink/lib/python3.7/importlib/_bootstrap.py:219: RuntimeWarning: spacy.tokens.span.Span size changed, may indicate binary incompatibility. Expected 72 from C header, got 80 from PyObject
  return f(*args, **kwds)
neuralcoref.add_to_pipe(nlp)_____no_output_____doc1 = nlp('My sister has a dog. She loves him.')_____no_output_____


# Notebook from xup5/Computational-Neuroscience-Class
Path: Lab 7 Convolutional neural network/cnn_fashion_keras.ipynb

_____no_output_____# **Convolutional neural network (CNN) for Fashion MNIST dataset**

by Md Nasir Uddin Laskar;
Edited 2022: Odelia Schwartz and Xu Pan

Change colab runtime type to GPU to accelerate: "Runtime" --> "Change runtime type"--> under "Hardware accelerator" select "GPU".

-  We will implement a simple CNN model with 2 convolution and 2 fully connected layers as the following figure. **conv1 --> relu1 --> pool1 ----> conv2 --> relu2 --> pool2 ----> fc1 --> fc2**

![picture](https://nasirml.files.wordpress.com/2019/01/simple_convnet-3.png?w=580&h=182)

## **Common deep learning steps**

* Step 1: Generate training and test data (and preprocess)
* Step 2: Initialize the network parameters
* Step 3: Forward propagation
* Step 4: Compute the cost/loss
* Step 5: Backpropagation or create an optimizer to minimize the cost from step 4
* Step 6: Evaluate the model with your test set
_____no_output_____

# imports
import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
from tensorflow import keras
import os   # to save the checkpoint
#sess = tf.InteractiveSession()_____no_output_____

# **Load the dataset**
* **Fashion-MNIST** is a dataset of article images—consisting of a training set of 60,000 examples and a test set of 10,000 examples. Each example is a 28x28 grayscale image, associated with a label from 10 classes. It shares the same image size and structure of training and testing splits of MNIST.

* The images are 28x28 NumPy arrays, with pixel values ranging between 0 and 255. The labels are an array of integers, ranging from 0 to 9. These correspond to the class of clothing the image represents.

* MNIST is very easy. Classic machine learning algorithms can also achieve 97% easily as you have see in the last lab. With convolutional nets, we can achieve 99.7% on MNIST. That is why we are trying a more serious dataset which a bit challenging but still gives us the fexibility and ease of MNIST.

* Sample images from the Fashion MNIST data in the following figure (each class takes three rows).

_____no_output_____

    Figure 1. Fashion-MNIST samples (by Zalando, MIT License).

_____no_output_____Loading the dataset returns four NumPy arrays:

* The `x_train` and `y_train` arrays are the *training set*—the data the model uses to learn.
* The model is tested against the *test set*, the `x_test`, and `y_test` arrays.

The images are 28x28 NumPy arrays, with pixel values ranging between 0 and 255. The *labels* are an array of integers, ranging from 0 to 9. These correspond to the *class* of clothing the image represents:

    Label
    Class

    0
    T-shirt/top

    1
    Trouser

    2
    Pullover

    3
    Dress

    4
    Coat

    5
    Sandal

    6
    Shirt

    7
    Sneaker

    8
    Bag

    9
    Ankle boot

* Each image is mapped to a single label. Since the *class names* are not included with the dataset, store them here to use later when plotting the images:

* **loading the dataset is very easy as it is already included in the TensorFlow/Keras library. We just have to include `tf.keras.datasets.fashion_mnist` and call the `load_data()` function.**
* It returns training and test datasets including the labels.
_____no_output_____

def read_dataset():
  '''
  - read fashion mnist dataset
  -
  '''
  fashion_mnist = tf.keras.datasets.fashion_mnist
  (x_train, y_train), (x_test, y_test) = fashion_mnist.load_data()

  return x_train, y_train, x_test, y_test
_____no_output_____# call the function to read fashion mnist
x_train, y_train, x_test, y_test = read_dataset()_____no_output_____

# **Look at the data**

* **Training data** — used for training the model.

* **Test data** — are the images not seen the network yet. A totally new set to test the generalization capability of the network.

_____no_output_____

print('Training data size:' + str(x_train.shape))
print('Training labels size: ' + str(y_train.shape))

print('Test data size: ' + str(x_test.shape))
print('Test label size: ' + str(y_test.shape))_____no_output_____fashion_mnist_labels = ['T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat',
                        'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle boot']
print(fashion_mnist_labels)
print(y_train[100], fashion_mnist_labels[y_train[100]])_____no_output_____# look at the first 10 images
for i in range(10):
    plt.subplot(2, 5, i+1); plt.axis('off')
    plt.imshow(np.reshape(x_train[i], [28, 28]), cmap='gray')
    plt.title(fashion_mnist_labels[y_train[i]])
plt.show()_____no_output_____

# **Preprocess data**
* **One-hot encoding.**
* Reshape to `28 x 28x 1`
_____no_output_____

# normalize the images
#x_train = x_train.astype('float32')/255.0  # sometimes standard to have pixel values in the range [0,1]
#x_test = x_test.astype('float32')/255.0

# Convert the labels to one-hot vectors
print(y_train[0:5])
y_train = tf.keras.utils.to_categorical(y_train, 10)  # 10 classes
y_test = tf.keras.utils.to_categorical(y_test, 10)
print(y_train[0:5])_____no_output_____# Reshape input data from (28,28) to (28, 28, 1), to 3D
# height x width x channel. we have gray scale images so channel = 1
print(np.shape(x_test))
w, h = 28, 28
x_train = x_train.reshape(x_train.shape[0], w, h, 1)
x_test = x_test.reshape(x_test.shape[0], w, h, 1)
print(np.shape(x_test))_____no_output_____

# **Build the CNN model**
- Make sure to define the input shape in the first layer of the neural network.
- The rest of the layers can adjust the input and output shapes automatically

---

![picture](https://nasirml.files.wordpress.com/2019/01/simple_convnet-3.png?w=580&h=182)

*   Convolutional layer (conv):
*   ReLU layer:
*   Pooling layer:
*   Dropout layer:
*   Fully connected layer (FC):

_____no_output_____
* We use `model.compile()`  to configure the learning process before training the model.

* This is where we define the type of loss function, optimizer and the metrics evaluated by the model during training and testing.

* **Important:** before we apply the fully-connected layers, we need to flatten the the current output._____no_output_____

def create_model():
  model = tf.keras.Sequential()

  model.add(tf.keras.layers.Conv2D(filters=64, kernel_size=3, padding='same', activation='relu', input_shape=(28,28,1)))
  model.add(tf.keras.layers.MaxPooling2D(pool_size=2))

  model.add(tf.keras.layers.Conv2D(filters=32, kernel_size=3, padding='same', activation='relu'))
  model.add(tf.keras.layers.MaxPooling2D(pool_size=2))

  model.add(tf.keras.layers.Flatten())
  model.add(tf.keras.layers.Dense(256, activation='relu')) # 1568*256+256 = 401664 params total
  model.add(tf.keras.layers.Dropout(0.5))
  model.add(tf.keras.layers.Dense(10, activation='softmax'))

  model.compile(loss='categorical_crossentropy',
             optimizer='adam',
             metrics=['accuracy'])

  return model_____no_output_____# create the model

model = create_model()_____no_output_____

## **Model summary**

* `conv2d` means the convolution in 2D (using 3x3 filters).
* Number of parameters 640 comes from `3*3*64 + 64`
_____no_output_____

# Take a look at the model summary
model.summary()
_____no_output_____

# **Now Train our model CNN**

*  Train the model and also save the checkpoint so that we do not have to train everytime

*   Shuffle is TRUE by default. Still I include it to tell that this is very important.

*  Takes around 20 minutes for 10 epochs/iterations.

* Loss decreases and accuracy increases

_____no_output_____

'''
- more params
  - validation_split=0.1 means 10% of the training data will be used for validation set
  -
'''
n_iteration = 10
history = model.fit(x_train, y_train, batch_size=64, epochs=n_iteration, shuffle=True)
_____no_output_____

# **See how the training has gone**

* Lets evaluate our model on the test set.
* Looks like our CNN model doing a pretty good job at classifying the unknown images with almost 90% accuracy with 3 iterations and  and 92% with 10 epochs!_____no_output_____

test_loss, test_acc = model.evaluate(x_test, y_test)
print('Test accuracy: %f' %test_acc)_____no_output_____

## **See the loss**

* Loss should decrease over time.
* If you see that the loss in your program does not decrease over time, most likely, something went wrong and you should check with your input and network initialization and hyper-parameters.
_____no_output_____

# history. all losses
all_loss = history.history['loss']
print(len(all_loss))

# show the training errors
plt.plot(np.squeeze(all_loss))
plt.xlabel('iterations');plt.ylabel('cost')
#plt.title('Learning rate %f' %learning_rate)
plt.show()

_____no_output_____

## **Visualize some predictions**

* Let's randomly pick some **test** images and visualize the prediction using the model we just trained. First we get the predictions with the model from the test data. Then we print out some images from the test data set, and set the titles with the prediction (and the groud truth label). If the prediction matches the true label, the title will be green; otherwise it's displayed in red.

* As we know the accuracy is around 90%, we see that the network occasionaly makes mistakes to predict a class.

* You can convince yourself that the mistakes are somewhat reasonable._____no_output_____

# find the probability of the 10 classes and then assign the class label with
# the higher probability.  model.predict(x_test) gives directly the probabilities
# predicted for each of the class labels.
y_hat = model.predict(x_test)
print(y_hat.shape)
print (y_hat[500])
print (np.round(y_hat[500],2))

# Plot a random sample of 16 test images, their predicted labels and ground truth
figure = plt.figure(figsize=(20, 8))
for i, index in enumerate(np.random.choice(x_test.shape[0], size=16, replace=False)):
    ax = figure.add_subplot(4, 4, i + 1, xticks=[], yticks=[])
    # Display each image
    ax.imshow(np.squeeze(x_test[index]),cmap='gray')
    predict_index = np.argmax(y_hat[index])                 # The fashion label number that is predicted
    true_index = np.argmax(y_test[index])                   # The true label number
    # Set the title for each image
    ax.set_title("{} ({})".format(fashion_mnist_labels[predict_index],
                                  fashion_mnist_labels[true_index]),
                                  color=("green" if predict_index == true_index else "red"))_____no_output_____

## **TODO**

1. **Add one convolution layer** in the network and train again. See if it increases the accuracy. Optional: You could further try changing other aspects of the network archirecture and computations to see if you can improve the result.

2. Find the **Confusion Matrix** and see how many samples in each class the network can predict correctly and how many are not predicted correctly. To do so, create a 10 by 10 matrix initialized to zeros, with the X axis corresponding to the true index and the Y axis to the predicted index. Loop over the 10,000 samples and each time add 1 to the matrix position asscoiated with the true index on the X axis and predicted index on the Y axis. Use plt.imshow to display your resulting matrix. What fashion items are most confused? What would the confusion matrix look like for a perfect network that always predicts correctly?

3. Try training the regular mnist database (replace fashion_mnist with mnist). How do the test accuracy and confusion matrices compare to the fashion_mnist?

_____no_output_____