# Notebook from git-nmf/Orange_County_Voting_Trends
Path: 31_model_logreg_extended.ipynb

## This page uses compares logistic regression and other classification models for modeling several economic variables from census data to understand the drivers of voting results.

Classification models:
- Logistic regression_____no_output_____

# imports
import pandas as pd
import numpy as np
import pickle as pkl
import matplotlib.pyplot as plt

import seaborn as sns
import warnings_____no_output_____# pd.set_option('display.max_columns', None)
# pd.set_option('display.max_rows', None)
np.random.seed(42)_____no_output_____# modeling imports
from scipy import stats
from sklearn.metrics import mean_squared_error
from sklearn.linear_model import LogisticRegression
from sklearn.experimental import enable_hist_gradient_boosting
from sklearn.ensemble import HistGradientBoostingClassifier

from sklearn.model_selection import train_test_split, cross_val_score, GridSearchCV
from sklearn.preprocessing import StandardScaler, PolynomialFeatures
from sklearn.pipeline import make_pipeline
from sklearn.feature_selection import SelectKBest
from sklearn.svm import SVC
from sklearn.tree import DecisionTreeClassifier
from sklearn.metrics import f1_score
from sklearn.metrics import confusion_matrix, plot_confusion_matrix_____no_output_____

### Logistic Regression classifier, to learn feature importance.

Y = outcome of congressional vote:
- 0 = REP, because has traditionally been the baseline.
- 1 = DEM, because this is a new trend._____no_output_____#### Bring in transformed data set, which combines voting and census data for each precinct_____no_output_____

df = pd.read_pickle('./data/combined18_extended.pkl')
df.shape_____no_output_____df.head()_____no_output_____df.info()<class 'pandas.core.frame.DataFrame'>
Int64Index: 1334 entries, 0 to 1333
Data columns (total 37 columns):
 #   Column                    Non-Null Count  Dtype
---  ------                    --------------  -----
 0   srprec_orig               1334 non-null   float64
 1   subindex                  1334 non-null   float64
 2   pctsrprec_tract           1334 non-null   float64
 3   employed_wgt              1334 non-null   float64
 4   unemployed_wgt            1334 non-null   float64
 5   empl_military_wgt         1334 non-null   float64
 6   not_inlaborforce_wgt      1334 non-null   float64
 7   working_women_wgt         1334 non-null   float64
 8   parents_work_under6_wgt   1334 non-null   float64
 9   parents_work_0617_wgt     1334 non-null   float64
 10  occ_mgmt_sci_art_wgt      1334 non-null   float64
 11  occ_service_sector_wgt    1334 non-null   float64
 12  occ_sales_gen_office_wgt  1334 non-null   float64
 13  occ_constr_maintc_wgt     1334 non-null   float64
 14  occ_manuf_transpo_wgt     1334 non-null   float64
 15  hh_med_income_wgt         1334 non-null   float64
 16  hlthins_priv_wgt          1334 non-null   float64
 17  hlthins_public_wgt        1334 non-null   float64
 18  hlthins_none_wgt          1334 non-null   float64
 19  county                    1334 non-null   int64
 20  srprec                    1334 non-null   int64
 21  cddist                    1334 non-null   int64
 22  TOTREG                    1334 non-null   int64
 23  TOTVOTE                   1334 non-null   int64
 24  CNGDEM01                  1334 non-null   int64
 25  CNGREP01                  1334 non-null   int64
 26  election                  1334 non-null   object
 27  totreg_r                  1334 non-null   float64
 28  dem                       1334 non-null   float64
 29  rep                       1334 non-null   float64
 30  dcl                       1334 non-null   float64
 31  male                      1334 non-null   float64
 32  female                    1334 non-null   float64
 33  hispdem                   1334 non-null   float64
 34  hisprep                   1334 non-null   float64
 35  hispdcl                   1334 non-null   float64
 36  hispoth                   1334 non-null   float64
dtypes: float64(29), int64(7), object(1)
memory usage: 396.0+ KB
df['srprec_orig'] = df['srprec_orig'].astype('int64')_____no_output_____df['cddist'].value_counts()_____no_output_____

### Define X and y from voter data (can enhance with census data later on)_____no_output_____#### define y_____no_output_____

# create a categorical column where CNGREP01 < CNGDEM01  (where Democrat wins, since Dems were victorious in 2018)_____no_output_____y_raw18 = combo[['CNGDEM01', 'CNGREP01']]
y_raw18.head()_____no_output_____y_raw18.describe()_____no_output_____y_raw18['dem_win'] = combo['CNGDEM01'] > combo['CNGREP01']
# will return a slice-of-dataframe warning, OK to proceed<ipython-input-49-fc75eef615fc>:1: SettingWithCopyWarning:
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  y_raw18['dem_win'] = combo['CNGDEM01'] > combo['CNGREP01']
y_raw18_____no_output_____y_raw18['dem_win'].value_counts(normalize=True)
#slightly unbalanced 60/40, but not horrible_____no_output_____y_raw18.info()<class 'pandas.core.frame.DataFrame'>
Int64Index: 1334 entries, 0 to 1333
Data columns (total 3 columns):
 #   Column    Non-Null Count  Dtype
---  ------    --------------  -----
 0   CNGDEM01  1334 non-null   int64
 1   CNGREP01  1334 non-null   int64
 2   dem_win   1334 non-null   bool
dtypes: bool(1), int64(2)
memory usage: 32.6 KB
y_raw18['dem_win'] = y_raw18['dem_win'].astype(int)
# will return a slice-of-dataframe warning, OK to proceed<ipython-input-53-8b3119850fea>:1: SettingWithCopyWarning:
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  y_raw18['dem_win'] = y_raw18['dem_win'].astype(int)
y_raw18.info()<class 'pandas.core.frame.DataFrame'>
Int64Index: 1334 entries, 0 to 1333
Data columns (total 3 columns):
 #   Column    Non-Null Count  Dtype
---  ------    --------------  -----
 0   CNGDEM01  1334 non-null   int64
 1   CNGREP01  1334 non-null   int64
 2   dem_win   1334 non-null   int64
dtypes: int64(3)
memory usage: 41.7 KB
y = y_raw18['dem_win']
y.head()_____no_output_____y.to_pickle('./data/y.pkl')_____no_output_____df.columns.tolist()_____no_output_____X = df[['employed_wgt',
 'unemployed_wgt',
 'empl_military_wgt',
 'not_inlaborforce_wgt',
 'working_women_wgt',
 'parents_work_under6_wgt',
 'parents_work_0617_wgt',
 'occ_mgmt_sci_art_wgt',
 'occ_service_sector_wgt',
 'occ_sales_gen_office_wgt',
 'occ_constr_maintc_wgt',
 'occ_manuf_transpo_wgt',
 'hh_med_income_wgt',
 'hlthins_priv_wgt',
 'hlthins_public_wgt',
 'hlthins_none_wgt',
#  'TOTREG',
#  'TOTVOTE',
#  'dem',
#  'rep',
 'dcl',
 'male',
 'female',
 'hispdem',
 'hisprep',
 'hispdcl',
 'hispoth']]
_____no_output_____X.head()_____no_output_____X.to_pickle('./data/X.pkl')_____no_output_____

#### run the extended dataset through LogReg, no scaling
_____no_output_____

X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42)
X_train.head(3)_____no_output_____import sklearn_____no_output_____logreg = LogisticRegression(n_jobs=-1, max_iter=10_000)_____no_output_____logreg.fit(X_train, y_train)_____no_output_____logreg.score(X_train, y_train)_____no_output_____logreg.score(X_test, y_test)_____no_output_____f1_train = f1_score(y_train, logreg.predict(X_train))
f1_test = f1_score(y_test, logreg.predict(X_test))
overfit = f1_train > f1_test
round(f1_train, 4), round(f1_test, 4), overfit_____no_output_____# vs. null model?
y_test.value_counts(normalize=True)

# our logreg score (65%) is slightly better than null model (60%), but not great.  _____no_output_____

#### Examine feature importance:_____no_output_____

logreg.coef______no_output_____np.exp(logreg.coef_)_____no_output_____features = X.columns.to_list()
features_____no_output_____logreg_coef = pd.DataFrame({'columns':features, 'coef':logreg.coef_[0]})
_____no_output_____viz = logreg_coef.sort_values('coef')
viz_____no_output_____viz['columns'].tolist()_____no_output_____viz['features'] = ['private health ins',
                   'occup: management',
                   'male',
                   'empl: military',
                   'household median income',
                   'female',
                   'Hispanic GOP registered',
                   'Hispanic Other registered',
                   'Hispanic no party affil',
                   'No party affiliation',
                   'Both parents work, children under 6',
                   'Unemployed',
                   'occup: Sales, Office',
                   'Both parents work, children 6-17 yo',
                   'Hispanic DEM registered',
                   'occup: construction',
                   'not in laborforce',
                   'working women',
                   'occup: transport',
                   'occup: service sector',
                   'no health ins',
                   'employed',
                   'public health ins']
_____no_output_____viz_____no_output_____viz.drop(columns='columns', inplace=True)_____no_output_____viz.rename(columns={'coef': 'importance'}, inplace=True)_____no_output_____vizviz = viz[['features', 'importance']].loc[viz['importance'] > 4.880033e-05]_____no_output_____vizviz.sort_values('importance')_____no_output_____vizviz.plot(kind='barh', x='features', y='importance').legend(
    loc='best', ncol=3)
plt.title("Relative importance of economic factors and Latinx")
plt.xlabel("relative importance")
plt.ylabel("feature")
# plt.figure(figsize=(25, 10))
# plt.yticks(viz['columns'].tolist());_____no_output_____# model evaluation scores are below._____no_output_____

#### Conclusions:
1. overfits
2. lower model scores (approx 60%) vs voting data only (approx 90%) (party affiliation, gender, etc.)._____no_output_____#### scale the data because they are not on the same scale:
1. `hh_med_income_wgt` is in dollars
2. other fields are expressed as population numbers, however may be of different scales. best to standardize the scale of the data_____no_output_____

ss = StandardScaler()_____no_output_____#estimators
logreg2 = LogisticRegression()
dtree = DecisionTreeClassifier()
hist = HistGradientBoostingClassifier()_____no_output_____ss_logreg2 = make_pipeline(ss, logreg2)
ss_dtree = make_pipeline(ss, dtree)
ss_hist = make_pipeline(ss, hist)
for est in (ss_logreg2, ss_dtree, ss_hist):
    est.fit(X_train, y_train)
    print(f'The Score on the train data is: {est.score(X_train, y_train)}')
    print(f'The Score on the test data is: {est.score(X_test, y_test)}')
The Score on the train data is: 0.845
The Score on the test data is: 0.8532934131736527
The Score on the train data is: 1.0
The Score on the test data is: 0.7365269461077845
The Score on the train data is: 0.999
The Score on the test data is: 0.8502994011976048

**OBSERVATIONS**
- Gradient Boost also has decent results on test data:  85%
- For LogReg, scaling the data greatly improved the scores, from approx 65% to 85%_____no_output_____### Further tuning via feature selection (Polynomial Features, Select K Best, Decision Tree)_____no_output_____#### Polynomial features with logistic regression
**RESULT** Overfits the data, by a lot_____no_output_____

pf = PolynomialFeatures()
logreg3 = LogisticRegression(max_iter=10_000)_____no_output_____pf_logreg3 = make_pipeline(ss, pf, logreg3)

pf_logreg3.fit(X_train, y_train)
print(f'The Score on the train data is: {est.score(X_train, y_train)}')
print(f'The Score on the test data is: {est.score(X_test, y_test)}')The Score on the train data is: 0.999
The Score on the test data is: 0.8502994011976048
# DataFrame of coefficients
# pd.DataFrame(list(zip(X.columns, logreg3.coef_)))_____no_output_____

#### Select K best features
Grid Search over a few options_____no_output_____

ss4 = StandardScaler()
pf4 = PolynomialFeatures()
select = SelectKBest()
logreg4 = LogisticRegression()_____no_output_____pf_logreg4 = make_pipeline(ss4, pf4, select, logreg4)_____no_output_____params = {
    'selectkbest__k': range(5,10,15),
    'logisticregression__penalty': ['none', 'l2'],
    'logisticregression__max_iter':[100000]
}_____no_output_____gs = GridSearchCV(pf_logreg4, param_grid=params)
gs.fit(X_train, y_train)

#returns a warning regarding ("Features %s are constant.")
# very little data in X_train['empl_military_wgt'] ... OK to ignore/Users/nevochka/opt/anaconda3/lib/python3.8/site-packages/sklearn/feature_selection/_univariate_selection.py:114: UserWarning: Features [0] are constant.
  warnings.warn("Features %s are constant." % constant_features_idx,
/Users/nevochka/opt/anaconda3/lib/python3.8/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide
  f = msb / msw
/Users/nevochka/opt/anaconda3/lib/python3.8/site-packages/sklearn/feature_selection/_univariate_selection.py:114: UserWarning: Features [0] are constant.
  warnings.warn("Features %s are constant." % constant_features_idx,
/Users/nevochka/opt/anaconda3/lib/python3.8/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide
  f = msb / msw
/Users/nevochka/opt/anaconda3/lib/python3.8/site-packages/sklearn/feature_selection/_univariate_selection.py:114: UserWarning: Features [0] are constant.
  warnings.warn("Features %s are constant." % constant_features_idx,
/Users/nevochka/opt/anaconda3/lib/python3.8/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide
  f = msb / msw
/Users/nevochka/opt/anaconda3/lib/python3.8/site-packages/sklearn/feature_selection/_univariate_selection.py:114: UserWarning: Features [0] are constant.
  warnings.warn("Features %s are constant." % constant_features_idx,
/Users/nevochka/opt/anaconda3/lib/python3.8/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide
  f = msb / msw
/Users/nevochka/opt/anaconda3/lib/python3.8/site-packages/sklearn/feature_selection/_univariate_selection.py:114: UserWarning: Features [0] are constant.
  warnings.warn("Features %s are constant." % constant_features_idx,
/Users/nevochka/opt/anaconda3/lib/python3.8/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide
  f = msb / msw
/Users/nevochka/opt/anaconda3/lib/python3.8/site-packages/sklearn/feature_selection/_univariate_selection.py:114: UserWarning: Features [0] are constant.
  warnings.warn("Features %s are constant." % constant_features_idx,
/Users/nevochka/opt/anaconda3/lib/python3.8/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide
  f = msb / msw
/Users/nevochka/opt/anaconda3/lib/python3.8/site-packages/sklearn/feature_selection/_univariate_selection.py:114: UserWarning: Features [0] are constant.
  warnings.warn("Features %s are constant." % constant_features_idx,
/Users/nevochka/opt/anaconda3/lib/python3.8/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide
  f = msb / msw
/Users/nevochka/opt/anaconda3/lib/python3.8/site-packages/sklearn/feature_selection/_univariate_selection.py:114: UserWarning: Features [0] are constant.
  warnings.warn("Features %s are constant." % constant_features_idx,
/Users/nevochka/opt/anaconda3/lib/python3.8/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide
  f = msb / msw
/Users/nevochka/opt/anaconda3/lib/python3.8/site-packages/sklearn/feature_selection/_univariate_selection.py:114: UserWarning: Features [0] are constant.
  warnings.warn("Features %s are constant." % constant_features_idx,
/Users/nevochka/opt/anaconda3/lib/python3.8/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide
  f = msb / msw
/Users/nevochka/opt/anaconda3/lib/python3.8/site-packages/sklearn/feature_selection/_univariate_selection.py:114: UserWarning: Features [0] are constant.
  warnings.warn("Features %s are constant." % constant_features_idx,
/Users/nevochka/opt/anaconda3/lib/python3.8/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide
  f = msb / msw
/Users/nevochka/opt/anaconda3/lib/python3.8/site-packages/sklearn/feature_selection/_univariate_selection.py:114: UserWarning: Features [0] are constant.
  warnings.warn("Features %s are constant." % constant_features_idx,
/Users/nevochka/opt/anaconda3/lib/python3.8/site-packages/sklearn/feature_selection/_univariate_selection.py:116: RuntimeWarning: invalid value encountered in true_divide
  f = msb / msw
X_train['empl_military_wgt'].value_counts()_____no_output_____print('Train Best', gs.best_score_)
print('Test Score', gs.score(X_test, y_test))
print(gs.best_params_)
print(gs.best_estimator_)
Train Best 0.716
Test Score 0.7335329341317365
{'logisticregression__max_iter': 100000, 'logisticregression__penalty': 'none', 'selectkbest__k': 5}
Pipeline(steps=[('standardscaler', StandardScaler()),
                ('polynomialfeatures', PolynomialFeatures()),
                ('selectkbest', SelectKBest(k=5)),
                ('logisticregression',
                 LogisticRegression(max_iter=100000, penalty='none'))])

### Decision Tree to highlight important features_____no_output_____

# Instantiate model with random_state = 42.
dtree5 = DecisionTreeClassifier(random_state=42)_____no_output_____ss5 = StandardScaler()_____no_output_____#scale X
X_train_ss5 = ss5.fit_transform(X_train)
X_test_ss5 = ss5.transform(X_test)_____no_output_____dtree5.fit(X_train_ss5, y_train)_____no_output_____params = {'max_depth': [2,3,5,7],
          'min_samples_split': [5, 10, 15, 20],
          'min_samples_leaf': [2,3,4,5,6],
          'ccp_alpha': [0, 0.001, 0.01, 0.1, 1, 10]}_____no_output_____grid5 = GridSearchCV(dtree5, param_grid=params)
grid5.fit(X_train_ss5, y_train)_____no_output_____grid5.best_params______no_output_____# # DataFrame of coefficients
# pd.DataFrame(list(zip(X.columns, grid5.best_estimator_.coef_)))_____no_output_____dtree5best = DecisionTreeClassifier(random_state=42, ccp_alpha=0.001, max_depth=7, min_samples_leaf=4, min_samples_split=5)_____no_output_____dtree5best.fit(X_train_ss5, y_train)_____no_output_____dtree5best.feature_importances______no_output_____viz_dtree5best = pd.DataFrame({'features': X_train.columns,
              'importances':dtree5best.feature_importances_}).sort_values('importances')
viz_dtree5best_____no_output_____vizdtree = viz_dtree5best.loc[viz_dtree5best['importances'] > 0.01]_____no_output_____fig, ax = plt.subplots(figsize=(10,6))
vizdtree.plot(kind='barh', x='features', y='importances', ax=ax).legend(
    loc='best', ncol=3)
plt.title("Relative importance of economic factors and Latinx (dtree)")
plt.xlabel("relative importance")
plt.ylabel("feature")
# plt.yticks(viz['columns'].tolist());_____no_output_____# Import export_text from sklearn.tree module.
from sklearn.tree import export_text

# Print out tree in plaintext.
print(export_text(grid5.best_estimator_,
                  list(X_train.columns)))|--- hh_med_income_wgt <= 88656.00
|   |--- hlthins_none_wgt <= 220.50
|   |   |--- parents_work_under6_wgt <= 140.00
|   |   |   |--- hlthins_priv_wgt <= 2927.00
|   |   |   |   |--- occ_service_sector_wgt <= 170.00
|   |   |   |   |   |--- class: 1
|   |   |   |   |--- occ_service_sector_wgt >  170.00
|   |   |   |   |   |--- class: 1
|   |   |   |--- hlthins_priv_wgt >  2927.00
|   |   |   |   |--- class: 0
|   |   |--- parents_work_under6_wgt >  140.00
|   |   |   |--- class: 0
|   |--- hlthins_none_wgt >  220.50
|   |   |--- hispdcl <= 1.50
|   |   |   |--- hispdem <= 7.50
|   |   |   |   |--- unemployed_wgt <= 155.00
|   |   |   |   |   |--- class: 0
|   |   |   |   |--- unemployed_wgt >  155.00
|   |   |   |   |   |--- employed_wgt <= 3275.50
|   |   |   |   |   |   |--- class: 1
|   |   |   |   |   |--- employed_wgt >  3275.50
|   |   |   |   |   |   |--- occ_service_sector_wgt <= 566.00
|   |   |   |   |   |   |   |--- class: 1
|   |   |   |   |   |   |--- occ_service_sector_wgt >  566.00
|   |   |   |   |   |   |   |--- class: 0
|   |   |   |--- hispdem >  7.50
|   |   |   |   |--- class: 1
|   |   |--- hispdcl >  1.50
|   |   |   |--- hispdem <= 45.50
|   |   |   |   |--- hisprep <= 11.50
|   |   |   |   |   |--- hispdcl <= 4.50
|   |   |   |   |   |   |--- male <= 88.00
|   |   |   |   |   |   |   |--- class: 1
|   |   |   |   |   |   |--- male >  88.00
|   |   |   |   |   |   |   |--- class: 0
|   |   |   |   |   |--- hispdcl >  4.50
|   |   |   |   |   |   |--- class: 1
|   |   |   |   |--- hisprep >  11.50
|   |   |   |   |   |--- employed_wgt <= 3678.50
|   |   |   |   |   |   |--- occ_constr_maintc_wgt <= 243.50
|   |   |   |   |   |   |   |--- class: 1
|   |   |   |   |   |   |--- occ_constr_maintc_wgt >  243.50
|   |   |   |   |   |   |   |--- class: 0
|   |   |   |   |   |--- employed_wgt >  3678.50
|   |   |   |   |   |   |--- class: 1
|   |   |   |--- hispdem >  45.50
|   |   |   |   |--- occ_manuf_transpo_wgt <= 293.00
|   |   |   |   |   |--- hisprep <= 65.50
|   |   |   |   |   |   |--- class: 1
|   |   |   |   |   |--- hisprep >  65.50
|   |   |   |   |   |   |--- class: 0
|   |   |   |   |--- occ_manuf_transpo_wgt >  293.00
|   |   |   |   |   |--- class: 1
|--- hh_med_income_wgt >  88656.00
|   |--- hh_med_income_wgt <= 105036.50
|   |   |--- hispdcl <= 20.50
|   |   |   |--- occ_constr_maintc_wgt <= 13.00
|   |   |   |   |--- class: 1
|   |   |   |--- occ_constr_maintc_wgt >  13.00
|   |   |   |   |--- occ_constr_maintc_wgt <= 178.50
|   |   |   |   |   |--- hispdem <= 15.50
|   |   |   |   |   |   |--- class: 0
|   |   |   |   |   |--- hispdem >  15.50
|   |   |   |   |   |   |--- hisprep <= 17.00
|   |   |   |   |   |   |   |--- class: 1
|   |   |   |   |   |   |--- hisprep >  17.00
|   |   |   |   |   |   |   |--- class: 0
|   |   |   |   |--- occ_constr_maintc_wgt >  178.50
|   |   |   |   |   |--- male <= 187.00
|   |   |   |   |   |   |--- class: 1
|   |   |   |   |   |--- male >  187.00
|   |   |   |   |   |   |--- occ_constr_maintc_wgt <= 228.50
|   |   |   |   |   |   |   |--- class: 1
|   |   |   |   |   |   |--- occ_constr_maintc_wgt >  228.50
|   |   |   |   |   |   |   |--- class: 0
|   |   |--- hispdcl >  20.50
|   |   |   |--- hisprep <= 72.00
|   |   |   |   |--- male <= 449.00
|   |   |   |   |   |--- hispdem <= 88.00
|   |   |   |   |   |   |--- hisprep <= 24.00
|   |   |   |   |   |   |   |--- class: 1
|   |   |   |   |   |   |--- hisprep >  24.00
|   |   |   |   |   |   |   |--- class: 0
|   |   |   |   |   |--- hispdem >  88.00
|   |   |   |   |   |   |--- class: 1
|   |   |   |   |--- male >  449.00
|   |   |   |   |   |--- female <= 901.50
|   |   |   |   |   |   |--- hlthins_none_wgt <= 241.00
|   |   |   |   |   |   |   |--- class: 1
|   |   |   |   |   |   |--- hlthins_none_wgt >  241.00
|   |   |   |   |   |   |   |--- class: 1
|   |   |   |   |   |--- female >  901.50
|   |   |   |   |   |   |--- class: 0
|   |   |   |--- hisprep >  72.00
|   |   |   |   |--- hlthins_priv_wgt <= 3678.00
|   |   |   |   |   |--- class: 0
|   |   |   |   |--- hlthins_priv_wgt >  3678.00
|   |   |   |   |   |--- class: 0
|   |--- hh_med_income_wgt >  105036.50
|   |   |--- working_women_wgt <= 3000.50
|   |   |   |--- hispdem <= 90.00
|   |   |   |   |--- hisprep <= 33.50
|   |   |   |   |   |--- dcl <= 403.50
|   |   |   |   |   |   |--- not_inlaborforce_wgt <= 1634.50
|   |   |   |   |   |   |   |--- class: 0
|   |   |   |   |   |   |--- not_inlaborforce_wgt >  1634.50
|   |   |   |   |   |   |   |--- class: 0
|   |   |   |   |   |--- dcl >  403.50
|   |   |   |   |   |   |--- class: 1
|   |   |   |   |--- hisprep >  33.50
|   |   |   |   |   |--- not_inlaborforce_wgt <= 1574.50
|   |   |   |   |   |   |--- unemployed_wgt <= 159.50
|   |   |   |   |   |   |   |--- class: 0
|   |   |   |   |   |   |--- unemployed_wgt >  159.50
|   |   |   |   |   |   |   |--- class: 1
|   |   |   |   |   |--- not_inlaborforce_wgt >  1574.50
|   |   |   |   |   |   |--- class: 0
|   |   |   |--- hispdem >  90.00
|   |   |   |   |--- hisprep <= 55.00
|   |   |   |   |   |--- class: 1
|   |   |   |   |--- hisprep >  55.00
|   |   |   |   |   |--- class: 0
|   |   |--- working_women_wgt >  3000.50
|   |   |   |--- class: 1

## Model evaluation_____no_output_____#### LogReg2, `accuracy = 0.85`
- logistic regression with scaled data
- standard scaler applied within a pipeline
- scores are generated from pipeline_____no_output_____

# Instantiate and fit a logisitic regression model
#above

# Check accuracy score on training data
print(f'The score on the train data is {ss_logreg2.score(X_train, y_train)}')
print(f'The score on the test data is {ss_logreg2.score(X_test, y_test)}')
# score = ss_logreg2.score(X_train, y_train)
# scoreThe score on the train data is 0.845
The score on the test data is 0.8532934131736527
# Make predictions on testing data
logreg2_preds = ss_logreg2.predict(X_test)
logreg2_preds[:5]_____no_output_____# Generate confusion matrix
cm_logreg2 = confusion_matrix(y_test, logreg2_preds)
cm_logreg2.flatten()_____no_output_____# Extract tn, fp, fn, tp
tn, fp, fn, tp = cm_logreg2.ravel()

# Convert the values to a confusion matrix (multiple methods will work)
pd.DataFrame(cm_logreg2, columns=['Pred Neg', 'Pred Pos'], index=['Act Neg', 'Act Pos'])

def class_metrics(TP, TN, FP, FN):
    acc = (TP + TN) / (TP + FP + TN + FN)
    mis = 1 - acc
    sen = TP / (TP + FN)
    spec = TN / (TN + FP)
    prec = TP / (TP + FP)
    results = [acc, mis, sen, spec, prec]
    ## create dictionary for easy display of results
    names = ['accuracy', 'misclassification', 'sensitivity', 'specificity', 'precision']
    res_dict = dict(zip(names, results))
    return res_dict

class_metrics(tp, tn, fp, fn)
_____no_output_____plot_confusion_matrix(ss_logreg2, X_test, y_test, display_labels=['GOP win', 'Dem win'], normalize='true');
_____no_output_____

#### dtree, `accuracy = 0.736`
- Decision Tree with scaled data
- standard scaler applied within a pipeline
- scores are generated from pipeline
- the intent of running this model is more for feature importance, but scoring the model neverthless
- Decision Tree will yield very high scores, but are typically very overfit, and cannot handle new data._____no_output_____

# Instantiate and fit a decision tree model
#above

# Check accuracy score on training data
print(f'The score on the train data is {ss_dtree.score(X_train, y_train)}')
print(f'The score on the test data is {ss_dtree.score(X_test, y_test)}')The score on the train data is 1.0
The score on the test data is 0.7365269461077845
# Make predictions on testing data
dtree_preds = ss_dtree.predict(X_test)
dtree_preds[:5]_____no_output_____# Generate confusion matrix
cm_dtree = confusion_matrix(y_test, dtree_preds)
cm_dtree.flatten()_____no_output_____# Extract tn, fp, fn, tp
tn, fp, fn, tp = cm_dtree.ravel()

# Convert the values to a confusion matrix (multiple methods will work)
pd.DataFrame(cm_dtree, columns=['Pred Neg', 'Pred Pos'], index=['Act Neg', 'Act Pos'])

def class_metrics(TP, TN, FP, FN):
    acc = (TP + TN) / (TP + FP + TN + FN)
    mis = 1 - acc
    sen = TP / (TP + FN)
    spec = TN / (TN + FP)
    prec = TP / (TP + FP)
    results = [acc, mis, sen, spec, prec]
    ## create dictionary for easy display of results
    names = ['accuracy', 'misclassification', 'sensitivity', 'specificity', 'precision']
    res_dict = dict(zip(names, results))
    return res_dict

class_metrics(tp, tn, fp, fn)_____no_output_____plot_confusion_matrix(ss_dtree, X_test, y_test, display_labels=['GOP win', 'Dem win'], normalize='true');_____no_output_____

#### HistGradientBoost, `accuracy = 0.850`
- Gradient Boost classifier with scaled data
- standard scaler applied within a pipeline
- scores are generated from pipeline
- similar accuracy to Logistic Regression logreg2_____no_output_____

# Instantiate and fit a Gradient Boost model
#above

# Check accuracy score on training data
print(f'The score on the train data is {ss_hist.score(X_train, y_train)}')
print(f'The score on the test data is {ss_hist.score(X_test, y_test)}')The score on the train data is 0.999
The score on the test data is 0.8502994011976048
# Make predictions on testing data
hist_preds = ss_hist.predict(X_test)
hist_preds[:5]_____no_output_____# Generate confusion matrix
cm_hist = confusion_matrix(y_test, hist_preds)
cm_hist.flatten()_____no_output_____# Extract tn, fp, fn, tp
tn, fp, fn, tp = cm_hist.ravel()

# Convert the values to a confusion matrix (multiple methods will work)
pd.DataFrame(cm_hist, columns=['Pred Neg', 'Pred Pos'], index=['Act Neg', 'Act Pos'])

def class_metrics(TP, TN, FP, FN):
    acc = (TP + TN) / (TP + FP + TN + FN)
    mis = 1 - acc
    sen = TP / (TP + FN)
    spec = TN / (TN + FP)
    prec = TP / (TP + FP)
    results = [acc, mis, sen, spec, prec]
    ## create dictionary for easy display of results
    names = ['accuracy', 'misclassification', 'sensitivity', 'specificity', 'precision']
    res_dict = dict(zip(names, results))
    return res_dict

class_metrics(tp, tn, fp, fn)_____no_output_____plot_confusion_matrix(ss_hist, X_test, y_test, display_labels=['GOP win', 'Dem win'], normalize='true');_____no_output_____

#### logreg3, accuracy = 0.811
- Logistic Regression with scaled data and polynomial features
- standard scaler and poly-features applied within a pipeline
- scores are generated from pipeline
- poly-features accuracy score is less than without poly-features (logreg2, above)_____no_output_____

# Instantiate and fit a Logistic Regression model
#above

# Check accuracy score on training data
print(f'The score on the train data is {pf_logreg3.score(X_train, y_train)}')
print(f'The score on the test data is {pf_logreg3.score(X_test, y_test)}')The score on the train data is 0.906
The score on the test data is 0.811377245508982
# Make predictions on testing data
logreg3_preds = pf_logreg3.predict(X_test)
logreg3_preds[:5]_____no_output_____# Generate confusion matrix
cm_logreg3 = confusion_matrix(y_test, logreg3_preds)
cm_logreg3.flatten()_____no_output_____# Extract tn, fp, fn, tp
tn, fp, fn, tp = cm_logreg3.ravel()

# Convert the values to a confusion matrix (multiple methods will work)
pd.DataFrame(cm_logreg3, columns=['Pred Neg', 'Pred Pos'], index=['Act Neg', 'Act Pos'])

def class_metrics(TP, TN, FP, FN):
    acc = (TP + TN) / (TP + FP + TN + FN)
    mis = 1 - acc
    sen = TP / (TP + FN)
    spec = TN / (TN + FP)
    prec = TP / (TP + FP)
    results = [acc, mis, sen, spec, prec]
    ## create dictionary for easy display of results
    names = ['accuracy', 'misclassification', 'sensitivity', 'specificity', 'precision']
    res_dict = dict(zip(names, results))
    return res_dict

class_metrics(tp, tn, fp, fn)_____no_output_____plot_confusion_matrix(pf_logreg3, X_test, y_test, display_labels=['GOP win', 'Dem win'], normalize='true');_____no_output_____

#### logreg4, `accuracy = 0.734`
- Logistic Regression with scaled data and polynomial features with a gridsearch for Select K-Best number of features
- standard scaler, poly-features, and select k-best applied within a pipeline
- then we do a grid search for different parameters for Select K-best features
- scores are generated from grid
- these scores are even lower than logistic regression alone_____no_output_____

# Instantiate and fit a Logistic Regression model and Select K Best
#above

# Check accuracy score on training data
print(f'The score on the train data is {gs.score(X_train, y_train)}')
print(f'The score on the test data is {gs.score(X_test, y_test)}')The score on the train data is 0.725
The score on the test data is 0.7335329341317365
# Make predictions on testing data
logreg4_preds = gs.predict(X_test)
logreg4_preds[:5]_____no_output_____# Generate confusion matrix
cm_logreg4 = confusion_matrix(y_test, logreg4_preds)
cm_logreg4.flatten()_____no_output_____# Extract tn, fp, fn, tp
tn, fp, fn, tp = cm_logreg4.ravel()

# Convert the values to a confusion matrix (multiple methods will work)
pd.DataFrame(cm_logreg4, columns=['Pred Neg', 'Pred Pos'], index=['Act Neg', 'Act Pos'])

def class_metrics(TP, TN, FP, FN):
    acc = (TP + TN) / (TP + FP + TN + FN)
    mis = 1 - acc
    sen = TP / (TP + FN)
    spec = TN / (TN + FP)
    prec = TP / (TP + FP)
    results = [acc, mis, sen, spec, prec]
    ## create dictionary for easy display of results
    names = ['accuracy', 'misclassification', 'sensitivity', 'specificity', 'precision']
    res_dict = dict(zip(names, results))
    return res_dict

class_metrics(tp, tn, fp, fn)_____no_output_____plot_confusion_matrix(gs, X_test, y_test, display_labels=['GOP win', 'Dem win'], normalize='true');_____no_output_____

#### dtree5, accuracy = 0.___
- Decision Tree with gridsearch for best params to create a rank-order list of important features
- use these high-importance features to reduce the number of input features, to see if we get a better prediction result with Logistic Regression
- standard scaler, poly-features, and select k-best applied within a pipeline
- then we do a grid search for different parameters for Select K-best features
- scores are generated from grid
- these scores are even lower than logistic regression alone_____no_output_____

# Check accuracy score on training data
print(f'The score on the train data is {dtree5best.score(X_train, y_train)}')
print(f'The score on the test data is {dtree5best.score(X_test, y_test)}')The score on the train data is 0.607
The score on the test data is 0.5988023952095808
# Make predictions on testing data
dtree5best_preds = dtree5best.predict(X_test)
dtree5best_preds[:5]_____no_output_____dtree5best_preds.sum()_____no_output_____len(dtree5best_preds)_____no_output_____
# Instantiate and fit a logisitic regression model (solver='liblinear')
log = LogisticRegression()
log.fit(X_train_ss, y_train)

# Check accuracy score on training data
score = log.score(X_train_ss, y_train)
score

# Make predictions on testing data
y_preds = log.predict(X_test_ss)
y_preds[:5]

# Generate confusion matrix
cm = confusion_matrix(y_test, y_preds)

cm.flatten()

# Extract tn, fp, fn, tp
tn, fp, fn, tp = cm.ravel()

# Convert the values to a confusion matrix (multiple methods will work)
pd.DataFrame(cm, columns=['Pred Neg', 'Pred Pos'], index=['Act Neg', 'Act Pos'])

def class_metrics(TP, TN, FP, FN):
    acc = (TP + TN) / (TP + FP + TN + FN)
    mis = 1 - acc
    sen = TP / (TP + FN)
    spec = TN / (TN + FP)
    prec = TP / (TP + FP)
    results = [acc, mis, sen, spec, prec]
    ## create dictionary for easy display of results
    names = ['accuracy', 'misclassification', 'sensitivity', 'specificity', 'precision']
    res_dict = dict(zip(names, results))
    return res_dict

class_metrics(tp, tn, fp, fn)

_____no_output_____preds = grid_mnb.predict(X_test)
preds

cmat_grid_mnb = confusion_matrix(y_test, preds)
cmat_grid_mnb

tn, fp, fn, tp = cmat_grid_mnb.ravel()
cmat_grid_mnb.ravel()

plot_confusion_matrix(grid_mnb, X_test, y_test, display_labels=['Star Wars', 'Game of Thrones'], normalize='true');
_____no_output_____

#### model-only rmse_____no_output_____

def rmse_model(model):

    # calculate rmse scores
    rmse_train = np.sqrt(mean_squared_error(y_train, model.predict(X_train)))
    rmse_test = np.sqrt(mean_squared_error(y_test, model.predict(X_test)))
    rmse_overfit = rmse_train < rmse_test

    # return rmse scores
    return f'for {model} rmse_train = {rmse_train}, and rmse_test = {rmse_test}.  Overfit = {rmse_overfit}'_____no_output_____# rmse_model(logreg2)
# # winner!  logistic regression w/ scaling
# # while overfit, it is not by that much_____no_output_____rmse_model(dtree)_____no_output_____rmse_model(hist)_____no_output_____rmse_model(logreg)_____no_output_____def rmse_pipe(pipe):

    # calculate rmse scores
    rmse_train = np.sqrt(mean_squared_error(y_train, pipe.predict(X_train)))
    rmse_test = np.sqrt(mean_squared_error(y_test, pipe.predict(X_test)))
    rmse_overfit = rmse_train < rmse_test

    # return rmse scores
    return f'for {pipe} rmse_train = {rmse_train}, and rmse_test = {rmse_test}.  Overfit = {rmse_overfit}'_____no_output_____rmse_pipe(ss_logreg2)
# logreg2 still a winner!  _____no_output_____rmse_pipe(ss_dtree)_____no_output_____rmse_pipe(ss_hist)_____no_output_____rmse_pipe(pf_logreg3)_____no_output_____def rmse_gs(grid):

    # calculate rmse scores
    rmse_train = np.sqrt(mean_squared_error(y_train, grid.predict(X_train)))
    rmse_test = np.sqrt(mean_squared_error(y_test, grid.predict(X_test)))
    rmse_overfit = rmse_train < rmse_test

    # return rmse scores
    return f'for {grid} rmse_train = {rmse_train}, and rmse_test = {rmse_test}.  Overfit = {rmse_overfit}'_____no_output_____rmse_gs(gs)_____no_output_____rmse_gs(grid5)_____no_output_____

**CONCLUSION**

Loading all of these factors into a gridsearch removes the overfitting, however RMSE is much higher (52% vs 30% for logreg + standard scaler)_____no_output_____


# Notebook from danich1/annorxiver
Path: biorxiv/pca_association_experiment/03_biorxiv_pca_category_bootstrap.ipynb

# Determine Word to PCA Associations_____no_output_____This notebook is designed to run PCA over the document embeddings and calculate category-pca associations with each principal component._____no_output_____

from pathlib import Path
import subprocess

import numpy as np
import pandas as pd
import plotnine as p9
from scipy.spatial.distance import cdist
from sklearn.decomposition import PCA
from tqdm import tqdm_notebook_____no_output_____journal_map_df = (
    pd.read_csv(
        Path("..")
        / Path("exploratory_data_analysis")
        / Path("output/biorxiv_article_metadata.tsv"),
        sep="\t",
    )
    .groupby("doi")
    .agg(
        {
            "doi": "last",
            "document": "first",
            "category": "last",
        }
    )
)
journal_map_df.head()_____no_output_____

# PCA the Documents_____no_output_____Run PCA over the documents. Generates 50 principal components, but can generate more or less._____no_output_____

n_components = 50
random_state = 100_____no_output_____biorxiv_articles_df = pd.read_csv(
    Path("..")
    / Path("word_vector_experiment")
    / Path("output/word2vec_output")
    / Path("biorxiv_all_articles_300.tsv.xz"),
    sep="\t",
)
biorxiv_articles_df = biorxiv_articles_df.dropna()
biorxiv_articles_df.head()_____no_output_____reducer = PCA(n_components=n_components, random_state=random_state)

reducer.fit(biorxiv_articles_df[[f"feat_{idx}" for idx in range(300)]].values)_____no_output_____document_categories_df = journal_map_df[["document", "category"]].merge(
    biorxiv_articles_df, on="document"
)
document_categories_df.head()_____no_output_____

# Bootstrap 95% Confidence Intervals Cosine sim_____no_output_____

boostraped_iterations = 10000
pc_1 = []
pc_2 = []

# The for loop performs the following algorithm to estimate 95% confidence intervals:
#
# 1. Group document embeddings by category
# 2. Randomly sample a document from each category
# 3. Calculate the similarity scores between the documents and the first two principal components
# 4. Repeat the above steps for 10000 iterations.
# 5. Finally take the 25th percentile and the 97.5th percentile to make up the interval

for iteration in tqdm_notebook(range(boostraped_iterations)):
    sampled_df = (
        document_categories_df.groupby("category")
        .apply(lambda x: x.sample(1, random_state=iteration))
        .reset_index(drop=True)
        .sort_values("category")
    )

    document_distance = 1 - cdist(
        (sampled_df.drop(["document", "category"], axis=1).values),
        reducer.components_[0:2],
        "cosine",
    )

    pc_1.append(document_distance[:, 0])
    pc_2.append(document_distance[:, 1])

pc_1 = np.stack(pc_1)
pc_2 = np.stack(pc_2)_____no_output_____mean_similarity_df = pd.DataFrame(
    (
        1
        - cdist(
            document_categories_df.drop(["document", "category"], axis=1).values,
            reducer.components_[0:2],
            "cosine",
        )
    ),
    columns=["pca1_cossim", "pca2_cossim"],
).assign(
    category=document_categories_df.category.tolist(),
    document=document_categories_df.document.tolist(),
)
mean_similarity_df.head()_____no_output_____category_sim_df = (
    mean_similarity_df.groupby("category")
    .agg({"pca1_cossim": "mean", "pca2_cossim": "mean"})
    .assign(
        pca1_cossim_upper=np.percentile(pc_1, 97.5, axis=0),
        pca1_cossim_lower=np.percentile(pc_1, 25, axis=0),
        pca2_cossim_upper=np.percentile(pc_2, 97.5, axis=0),
        pca2_cossim_lower=np.percentile(pc_2, 25, axis=0),
    )
    .reset_index()[
        [
            "category",
            "pca1_cossim_lower",
            "pca1_cossim",
            "pca1_cossim_upper",
            "pca2_cossim_lower",
            "pca2_cossim",
            "pca2_cossim_upper",
        ]
    ]
)
category_sim_df.head()_____no_output_____category_sim_df.to_csv("output/category_cossim_95_ci.tsv", sep="\t", index=False)_____no_output_____g = (
    p9.ggplot(category_sim_df)
    + p9.aes(
        x="category",
        y="pca1_cossim",
        ymin="pca1_cossim_lower",
        ymax="pca1_cossim_upper",
    )
    + p9.geom_pointrange()
    + p9.coord_flip()
    + p9.theme_bw()
    + p9.scale_x_discrete(limits=category_sim_df.category.tolist()[::-1])
    + p9.theme(
        figure_size=(11, 8.5),
        text=p9.element_text(size=12),
        panel_grid_major_y=p9.element_blank(),
    )
    + p9.labs(y="PC1 Cosine Similarity")
)
g.save("output/pca_plots/figures/category_pca1_95_ci.png", dpi=500)
print(g)/home/danich1/anaconda3/envs/annorxiver/lib/python3.7/site-packages/plotnine/ggplot.py:729: PlotnineWarning: Saving 11 x 7 in image.
/home/danich1/anaconda3/envs/annorxiver/lib/python3.7/site-packages/plotnine/ggplot.py:730: PlotnineWarning: Filename: output/pca_plots/figures/category_pca1_95_ci.svg
/home/danich1/anaconda3/envs/annorxiver/lib/python3.7/site-packages/plotnine/ggplot.py:729: PlotnineWarning: Saving 11 x 7 in image.
/home/danich1/anaconda3/envs/annorxiver/lib/python3.7/site-packages/plotnine/ggplot.py:730: PlotnineWarning: Filename: output/pca_plots/figures/category_pca1_95_ci.png
g = (
    p9.ggplot(category_sim_df)
    + p9.aes(
        x="category",
        y="pca2_cossim",
        ymax="pca2_cossim_upper",
        ymin="pca2_cossim_lower",
    )
    + p9.geom_pointrange()
    + p9.coord_flip()
    + p9.theme_bw()
    + p9.scale_x_discrete(limits=category_sim_df.category.tolist()[::-1])
    + p9.theme(
        figure_size=(11, 8.5),
        text=p9.element_text(size=12),
        panel_grid_major_y=p9.element_blank(),
    )
    + p9.labs(y="PC2 Cosine Similarity")
)
g.save("output/pca_plots/figures/category_pca2_95_ci.png", dpi=500)
print(g)/home/danich1/anaconda3/envs/annorxiver/lib/python3.7/site-packages/plotnine/ggplot.py:729: PlotnineWarning: Saving 11 x 8.5 in image.
/home/danich1/anaconda3/envs/annorxiver/lib/python3.7/site-packages/plotnine/ggplot.py:730: PlotnineWarning: Filename: output/pca_plots/figures/category_pca2_95_ci.svg
/home/danich1/anaconda3/envs/annorxiver/lib/python3.7/site-packages/plotnine/ggplot.py:729: PlotnineWarning: Saving 11 x 8.5 in image.
/home/danich1/anaconda3/envs/annorxiver/lib/python3.7/site-packages/plotnine/ggplot.py:730: PlotnineWarning: Filename: output/pca_plots/figures/category_pca2_95_ci.png

# Plot Documents Projected on PCs Grouped by Category_____no_output_____

projected_documents = reducer.transform(
    document_categories_df[[f"feat_{idx}" for idx in range(300)]]
)
projected_documents.shape_____no_output_____projected_documents_df = pd.DataFrame(
    projected_documents, columns=[f"PC_{dim+1}" for dim in range(n_components)]
).assign(
    category=document_categories_df.category.tolist(),
    document=document_categories_df.document.tolist(),
)
projected_documents_df_____no_output_____g = (
    p9.ggplot(projected_documents_df)
    + p9.aes(x="factor(category)", y="PC_1")
    + p9.geom_boxplot(
        fill="#a6cee3",
        outlier_size=1,
        outlier_alpha=0.65,
        fatten=1.5,
    )
    + p9.coord_flip()
    + p9.scale_x_discrete(
        limits=(
            projected_documents_df.groupby("category")
            .agg({"PC_1": "median"})
            .sort_values("PC_1", ascending=False)
            .reset_index()
            .category.tolist()[::-1]
        )
    )
    + p9.labs(x="Article Category", y="PC1")
    + p9.theme_seaborn(context="paper", style="ticks", font="Arial", font_scale=2)
    + p9.theme(figure_size=(11, 8.5))
)
g.save("output/pca_plots/figures/category_box_plot_pc1.png", dpi=250)
g.save(
    "output/pca_plots/svg_files/category_box_plot/category_box_plot_pc1.svg", dpi=250
)
print(g)/home/danich1/anaconda3/envs/annorxiver/lib/python3.7/site-packages/plotnine/ggplot.py:729: PlotnineWarning: Saving 11 x 8.5 in image.
/home/danich1/anaconda3/envs/annorxiver/lib/python3.7/site-packages/plotnine/ggplot.py:730: PlotnineWarning: Filename: output/pca_plots/figures/category_box_plot_pc1.png
/home/danich1/anaconda3/envs/annorxiver/lib/python3.7/site-packages/plotnine/ggplot.py:729: PlotnineWarning: Saving 11 x 8.5 in image.
/home/danich1/anaconda3/envs/annorxiver/lib/python3.7/site-packages/plotnine/ggplot.py:730: PlotnineWarning: Filename: output/pca_plots/svg_files/category_box_plot/category_box_plot_pc1.svg
g = (
    p9.ggplot(projected_documents_df)
    + p9.aes(x="factor(category)", y="PC_2")
    + p9.geom_boxplot(
        fill="#a6cee3",
        outlier_size=1,
        outlier_alpha=0.65,
        fatten=1.5,
    )
    + p9.coord_flip()
    + p9.scale_x_discrete(
        limits=(
            projected_documents_df.groupby("category")
            .agg({"PC_2": "median"})
            .sort_values("PC_2", ascending=False)
            .reset_index()
            .category.tolist()[::-1]
        )
    )
    + p9.labs(x="Article Category", y="PC2")
    + p9.theme_seaborn(context="paper", style="ticks", font="Arial", font_scale=2)
    + p9.theme(figure_size=(11, 8.5))
)
g.save("output/pca_plots/figures/category_box_plot_pc2.png", dpi=250)
g.save(
    "output/pca_plots/svg_files/category_box_plot/category_box_plot_pc2.svg", dpi=250
)
print(g)/home/danich1/anaconda3/envs/annorxiver/lib/python3.7/site-packages/plotnine/ggplot.py:729: PlotnineWarning: Saving 11 x 8.5 in image.
/home/danich1/anaconda3/envs/annorxiver/lib/python3.7/site-packages/plotnine/ggplot.py:730: PlotnineWarning: Filename: output/pca_plots/figures/category_box_plot_pc2.png
/home/danich1/anaconda3/envs/annorxiver/lib/python3.7/site-packages/plotnine/ggplot.py:729: PlotnineWarning: Saving 11 x 8.5 in image.
/home/danich1/anaconda3/envs/annorxiver/lib/python3.7/site-packages/plotnine/ggplot.py:730: PlotnineWarning: Filename: output/pca_plots/svg_files/category_box_plot/category_box_plot_pc2.svg

# Tables with Figures_____no_output_____

def dump_figures_and_table(
    table_df, figure_selector, output_path="output/table", column_value="PC_1"
):

    # Output figures to folder
    for idx, row in table_df.iterrows():
        subprocess.Popen(
            [
                "unzip",
                "-j",
                "-o",
                Path("..") / Path(row["hash"]),
                Path("content") / Path(figure_selector[row["document"]]),
                "-d",
                Path("output/table_figures"),
            ]
        )

    # Output table in Manubot format
    # to make incorporating this figure much easier in text
    # http://greenelab.github.io/annorxiver_manuscript
    (
        table_df[["document", "doi", column_value]]
        # doi - a manubot formatted doi link. Manubot creaes citations via [@doi:doi_link]
        # figure - a markdown formmated link to show figure images within a given table.
        .assign(
            doi=lambda x: x.doi.apply(lambda link: f"[@doi:{link}]"),
            figure=lambda x: x.document.apply(
                lambda doc: f"![](table_figures/{document_figure_selector[doc]})"
            ),
        )
        .drop("document", axis=1)
        .to_csv(f"{output_path}.tsv", sep="\t", index=False)
    )_____no_output_____document_hash_df = pd.read_csv("../biorxiv_doc_hash_mapper.tsv", sep="\t")
document_hash_df.head()_____no_output_____

## Top PC1_____no_output_____

# 035014_v1.xml does not have a figure.
# Use replacement document instead
top_pc1_documents = (
    journal_map_df[["document", "doi"]]
    .merge(
        projected_documents_df.query("category=='systems biology'")
        .sort_values("PC_1", ascending=False)
        .head(5),
        on="document",
    )
    .merge(document_hash_df, left_on="document", right_on="doc_number")
)
top_pc1_documents_____no_output_____document_figure_selector = {
    "044818_v1.xml": "044818_fig1.tif",
    "107250_v1.xml": "107250_fig1.tif",
    "197400_v1.xml": "197400_fig1.tif",
    "769299_v1.xml": "769299v1_fig1.tif",
    "266775_v1.xml": "266775_fig1.tif",
}_____no_output_____dump_figures_and_table(
    top_pc1_documents,
    document_figure_selector,
    output_path="output/tables/top_pc1_table",
    column_value="PC_1",
)_____no_output_____

## Bottom PC1_____no_output_____

bottom_pc1_documents = (
    journal_map_df[["document", "doi"]]
    .merge(
        projected_documents_df.query("category=='systems biology'")
        .sort_values("PC_1", ascending=True)
        .head(),
        on="document",
    )
    .merge(document_hash_df, left_on="document", right_on="doc_number")
)
bottom_pc1_documents_____no_output_____document_figure_selector = {
    "872887_v1.xml": "872887v1_fig1.tif",
    "455048_v1.xml": "455048_fig1.tif",
    "733162_v1.xml": "733162v1_fig1.tif",
    "745943_v1.xml": "745943v1_fig1.tif",
    "754572_v1.xml": "754572v1_fig1.tif",
}_____no_output_____dump_figures_and_table(
    bottom_pc1_documents,
    document_figure_selector,
    output_path="output/tables/bottom_pc1_table",
    column_value="PC_1",
)_____no_output_____

## Top PC2_____no_output_____

top_pc2_documents = (
    journal_map_df[["document", "doi"]]
    .merge(
        projected_documents_df.query("category=='systems biology'")
        .sort_values("PC_2", ascending=False)
        .head(),
        on="document",
    )
    .merge(document_hash_df, left_on="document", right_on="doc_number")
)
top_pc2_documents_____no_output_____document_figure_selector = {
    "220152_v1.xml": "220152_fig3.tif",
    "328591_v1.xml": "328591_fig1.tif",
    "595819_v1.xml": "595819_fig1.tif",
    "484204_v2.xml": "484204v2_fig1.tif",
    "781328_v1.xml": "781328v1_fig3.tif",
}_____no_output_____dump_figures_and_table(
    top_pc2_documents,
    document_figure_selector,
    output_path="output/tables/top_pc2_table",
    column_value="PC_2",
)_____no_output_____

## Bottom PC2_____no_output_____

bottom_pc2_documents = (
    journal_map_df[["document", "doi"]]
    .merge(
        projected_documents_df.query("category=='systems biology'")
        .sort_values("PC_2", ascending=True)
        .head(),
        on="document",
    )
    .merge(document_hash_df, left_on="document", right_on="doc_number")
)
bottom_pc2_documents_____no_output_____document_figure_selector = {
    "019687_v1.xml": "019687_fig1.tif",
    "301051_v1.xml": "301051_fig3.tif",
    "357939_v1.xml": "357939_fig4.tif",
    "386367_v3.xml": "386367v3_fig1.tif",
    "840280_v1.xml": "840280v1_fig1.tif",
}_____no_output_____dump_figures_and_table(
    bottom_pc2_documents,
    document_figure_selector,
    output_path="output/tables/bottom_pc2_table",
    column_value="PC_2",
)_____no_output_____


# Notebook from lompabo/aiiti-04-2021
Path: 3. Component Wear Anomalies.ipynb

## Notebook Setup

Let's start by downloading the dataset (if not already done) and seting up the notebook:_____no_output_____

!wget https://github.com/lompabo/aiiti-04-2021/releases/download/data/data.zip
!unzip -o data.zip
!ls .Archive:  data/hpc.csv.zip
  inflating: data/hpc.csv
Archive:  data/vegashrinker.csv.zip
  inflating: data/vegashrinker.csv
# ============================================================
# Notebook setup
# ============================================================

# Control figure size
interactive_figures = False
if interactive_figures:
    # Normal behavior
    %matplotlib widget
    figsize=(9, 3)
else:
    # PDF export behavior
    figsize=(14, 3)

from sklearn.neighbors import KernelDensity
from sklearn.model_selection import GridSearchCV
from util import nn
import numpy as np
from matplotlib import pyplot as plt
import pandas as pd
from tensorflow import keras
from tensorflow.keras.layers import Dense
from tensorflow.keras.callbacks import EarlyStopping

# Load data
data_folder = 'data'
vs = pd.read_csv(data_folder+ '/vegashrinker.csv')

# Identify input columns
vs_in = vs.columns[6:]

# Prepare for caching
cache_file = '/app/data/vegashrinker.h5'_____no_output_____

# Component Wear Anomalies_____no_output_____## OCME Vega Shrinker

**Let's consider the Vega skinwrapper family of packaging machines by [OCME](https://www.ocme.com/en?_ptc=4)**

* They work by wrapping products (bottles) in a _plastic film_
* ...Which is _cut and heated_, so that the film shrinks and stabilizes the content

_____no_output_____## OCME Vega Shrinker

**A public dataset about one of their machines is [publicly available from Kaggle](https://www.kaggle.com/inIT-OWL/one-year-industrial-component-degradation)**

* The dataset contains a _run-to-failure experiment_
* I.e. the machine was left running until one of it components became unserviceable
  - Specificallly, it was the blade for cutting the film

**This is an example of anomaly _due to component wear_**

* It's a common type of anomaly
* ...And run-to-failure experiments are a typical way to investigate them

**All problems in this class share a few _properties_**

* The behavior becomes _more and more distant_ from normal over time
* There is a critical anomaly _at the end of the experiment_

**We will try to tackle the problem using the techniques we know**_____no_output_____## The Dataset

**Let's have a first look at the dataset**_____no_output_____

print(f'Number of examples: {len(vs)}, number of inputs: {len(vs_in)}')
vs.head()Number of examples: 1062912, number of inputs: 8

* There aren't many columns, but there are _many examples_!
* The data refers to different measurement intervals (or "segments")
* Each segment contains data sampled _every 4ms______no_output_____## The Dataset

**Let's check some statistics**_____no_output_____

vs.describe()_____no_output_____

* The data is neither normalized nor standardized_____no_output_____## The Dataset

**Let's check for missing values**_____no_output_____

vs[vs_in].isnull().any()_____no_output_____

* There are none_____no_output_____## The Dataset

**And let's check the length of each segment**_____no_output_____

vs.groupby('segment')['mode'].count().describe()_____no_output_____

* There are 519 segments overall
* ...Each with 2048 samples_____no_output_____## The Dataset

**The machine has multiple operating modes**_____no_output_____

nn.plot_series(vs['mode'], figsize=figsize)_____no_output_____

* The mode is _a controlled parameter_ and does not change in the middle of a segment
* Intuitively, the mode _has an impact_ on the machine behavior_____no_output_____## The Dataset

**Let's have a look at all the sensor readings**_____no_output_____

vss = vs.copy()
vss[vs_in] = (vss[vs_in] - vss[vs_in].mean()) / vss[vs_in].std()
nn.plot_dataframe(vss[vs_in], figsize=figsize)_____no_output_____

# Data Preparation_____no_output_____## Binning

**This dataset contain _high-frequency data_ (4ms sampling period)**

* In this situation, feeding the raw data to a model does not usually make sense
* Instead, we reduce the frequency of the data via a process called _binning_

**A binning approach typically works as follows**

We apply a sliding window, but so that its consecutive applications _do not overlap_

* Each window application is called a _bin_
* ...From which we extract one or more _features_

**The result is series that contains a _smaller number of samples_**

...But typically a _larger number of features______no_output_____## Binning

**There are two broad classes of features that are usually extracted**

* _Time-domain_ features (e.g. mean, standard deviation)
* _Frequency-domain_ features (e.g. specific FFT amplitudes)

For this case, we will focus on time-domain features

**As a first step, we defined _which features_ we are going to extract**_____no_output_____

functions = ['mean', 'std', 'skew', lambda x: x.kurtosis()]
aggmap = {a: functions for a in vs_in}
aggmap['mode'] = 'first'
str(aggmap)_____no_output_____

## Binning

**Then we define bin numbers, and extract the features via a `groupby` operation**

We need to take care so that _no bin crosses between different segments______no_output_____

%%time
binsize = 128
bins = []
for sname, sdata in vs.groupby('segment'):
    # Build the bin numbers
    sdata['bin'] = sdata.index // binsize
    # Apply the aggregation functions
    tmp = sdata.groupby('bin').agg(aggmap)
    bins.append(tmp)
vsb = pd.concat(bins)CPU times: user 43.6 s, sys: 0 ns, total: 43.6 s
Wall time: 43.6 s

* This can be a relatively slow operation
* Bin numbers are usually easy to define using positional indexes and an integer division_____no_output_____## Binning

**If we choose the bin size correctly, we can speed up the operation**

* In particular, if all segments have the same length...
* ...And we choose bin size that is a submultiple of the segment length

...Then we can avoid processing each segment separately:_____no_output_____

%%time
# Build the bin numbers
binsize = 128
vsb = vs.copy()
vsb['bin'] = vs.index // binsize
vsb = vsb.groupby('bin').agg(aggmap)_____no_output_____

* This kind of approach is significantly faster_____no_output_____## Binning

**The resulting dataframe has a hierarchical column index**_____no_output_____

vsb.iloc[:1]_____no_output_____

**It may be worth _flattening_, so as to simplify access:**_____no_output_____

if isinstance(vsb.columns, pd.MultiIndex):
    vsb.columns = ['::'.join(c) for c in vsb.columns]
vsb.iloc[:1]_____no_output_____

## Standardization

**Before we can train any model, we need some preparation**

We will _standardize sensor inputs_ (all except the mode) using the first third of the series _____no_output_____

sep = int(np.round(len(vsb) * 0.34))
vsb_in = vsb.columns[:-1]
vsbs = vsb.copy()
tmp = vsbs[vsb_in].iloc[:sep]
vsbs[vsb_in] = (vsbs[vsb_in] - tmp.mean()) / tmp.std()
vsbs.iloc[:3]_____no_output_____

## Categorical Mode

**We will also adopt a _categorical encoding_ for the operating mode**

This is critical for neural network approaches in particular_____no_output_____

from tensorflow.keras.utils import to_categorical
cmode = to_categorical(vsbs['mode::first'])
cols = [f'm{i}' for i in range(cmode.shape[1])]
cmode = pd.DataFrame(index=vsbs.index, data=cmode, columns=cols)
vsbs[cols] = cmode
vsbs.head()_____no_output_____

# KDE Approach_____no_output_____## KDE Approach

**Now, let's try anomaly detection via KDE**

First we estimate the optimal bandwidth:_____no_output_____

%%time
vsbs_tr = vsbs.iloc[:sep]

params = {'bandwidth': np.linspace(0.2, 0.8, 10)}
opt = GridSearchCV(KernelDensity(kernel='gaussian'), params, cv=5)
opt.fit(vsbs_tr)
best_params = pd.Series(index=opt.best_params_.keys(), data=opt.best_params_.values())
print(best_params)    bandwidth    0.333333
dtype: float64
CPU times: user 8.04 s, sys: 8.95 ms, total: 8.05 s
Wall time: 8.07 s

## KDE Approach

**Then we can train an estimator**_____no_output_____

h = opt.best_params_['bandwidth']
kde = KernelDensity(bandwidth=h)
kde.fit(vsbs_tr)_____no_output_____

**...And we can generate the alarm signal**_____no_output_____

%%time
ldens = kde.score_samples(vsbs)
signal_kde = pd.Series(index=vsbs.index, data=-ldens)CPU times: user 3.02 s, sys: 4.31 ms, total: 3.03 s
Wall time: 3.02 s

## KDE Approach

**Let's plot the signal**_____no_output_____

nn.plot_signal(signal_kde, figsize=figsize)_____no_output_____

* A peak at mid run makes the signal difficult to read_____no_output_____## KDE Approach

**Since the anomaly arises slowly, it makes sense to _clip_ and _smooth_ the signal**_____no_output_____

nn.plot_signal(signal_kde.clip(upper=25).ewm(512).mean(), figsize=figsize)_____no_output_____

* The smoothed signal has a tendency to grow
* There is a plateau in the middle that is difficult to explain_____no_output_____# Autoencoder Approach_____no_output_____## Autoencoder Approach

**Let's repeat our analysis using an autoencoder**

First we define its structure:_____no_output_____

input_shape = vsbs.shape[1]
output_shape = len(vsb_in)
ae_x = keras.Input(shape=input_shape, dtype='float32')
ae_z = Dense(16, activation='relu')(ae_x)
ae_y = Dense(output_shape, activation='linear')(ae_z)
ae = keras.Model(ae_x, ae_y)_____no_output_____

* The input includes the operating mode
* ...But _the output does not_!

We have _no interest in reconstructing controlled parameters______no_output_____## Autoencoder Approach

**Now we can perform training**_____no_output_____

%%time
ae.compile(optimizer='RMSProp', loss='mse')
callbacks = [EarlyStopping(monitor='val_loss', patience=10, restore_best_weights=True)]
history = ae.fit(vsbs_tr, vsbs_tr[vsb_in], validation_split=0.2, callbacks=callbacks,
                 batch_size=32, epochs=50, verbose=2)Epoch 1/50
71/71 - 0s - loss: 0.9992 - val_loss: 0.6601
Epoch 2/50
71/71 - 0s - loss: 0.6836 - val_loss: 0.4621
Epoch 3/50
71/71 - 0s - loss: 0.4980 - val_loss: 0.3478
Epoch 4/50
71/71 - 0s - loss: 0.4001 - val_loss: 0.2919
Epoch 5/50
71/71 - 0s - loss: 0.3441 - val_loss: 0.2556
Epoch 6/50
71/71 - 0s - loss: 0.3057 - val_loss: 0.2313
Epoch 7/50
71/71 - 0s - loss: 0.2766 - val_loss: 0.2095
Epoch 8/50
71/71 - 0s - loss: 0.2532 - val_loss: 0.1921
Epoch 9/50
71/71 - 0s - loss: 0.2331 - val_loss: 0.1740
Epoch 10/50
71/71 - 0s - loss: 0.2159 - val_loss: 0.1620
Epoch 11/50
71/71 - 0s - loss: 0.2011 - val_loss: 0.1505
Epoch 12/50
71/71 - 0s - loss: 0.1882 - val_loss: 0.1386
Epoch 13/50
71/71 - 0s - loss: 0.1767 - val_loss: 0.1309
Epoch 14/50
71/71 - 0s - loss: 0.1676 - val_loss: 0.1240
Epoch 15/50
71/71 - 0s - loss: 0.1600 - val_loss: 0.1174
Epoch 16/50
71/71 - 0s - loss: 0.1531 - val_loss: 0.1119
Epoch 17/50
71/71 - 0s - loss: 0.1474 - val_loss: 0.1079
Epoch 18/50
71/71 - 0s - loss: 0.1416 - val_loss: 0.1059
Epoch 19/50
71/71 - 0s - loss: 0.1372 - val_loss: 0.1036
Epoch 20/50
71/71 - 0s - loss: 0.1326 - val_loss: 0.0990
Epoch 21/50
71/71 - 0s - loss: 0.1292 - val_loss: 0.0982
Epoch 22/50
71/71 - 0s - loss: 0.1259 - val_loss: 0.0949
Epoch 23/50
71/71 - 0s - loss: 0.1226 - val_loss: 0.0945
Epoch 24/50
71/71 - 0s - loss: 0.1202 - val_loss: 0.0922
Epoch 25/50
71/71 - 0s - loss: 0.1183 - val_loss: 0.0916
Epoch 26/50
71/71 - 0s - loss: 0.1162 - val_loss: 0.0893
Epoch 27/50
71/71 - 0s - loss: 0.1141 - val_loss: 0.0891
Epoch 28/50
71/71 - 0s - loss: 0.1113 - val_loss: 0.0895
Epoch 29/50
71/71 - 0s - loss: 0.1099 - val_loss: 0.0864
Epoch 30/50
71/71 - 0s - loss: 0.1078 - val_loss: 0.0874
Epoch 31/50
71/71 - 0s - loss: 0.1062 - val_loss: 0.0861
Epoch 32/50
71/71 - 0s - loss: 0.1047 - val_loss: 0.0833
Epoch 33/50
71/71 - 0s - loss: 0.1031 - val_loss: 0.0829
Epoch 34/50
71/71 - 0s - loss: 0.1019 - val_loss: 0.0811
Epoch 35/50
71/71 - 0s - loss: 0.0996 - val_loss: 0.0837
Epoch 36/50
71/71 - 0s - loss: 0.0986 - val_loss: 0.0829
Epoch 37/50
71/71 - 0s - loss: 0.0976 - val_loss: 0.0796
Epoch 38/50
71/71 - 0s - loss: 0.0962 - val_loss: 0.0809
Epoch 39/50
71/71 - 0s - loss: 0.0949 - val_loss: 0.0814
Epoch 40/50
71/71 - 0s - loss: 0.0940 - val_loss: 0.0807
Epoch 41/50
71/71 - 0s - loss: 0.0930 - val_loss: 0.0789
Epoch 42/50
71/71 - 0s - loss: 0.0914 - val_loss: 0.0792
Epoch 43/50
71/71 - 0s - loss: 0.0913 - val_loss: 0.0781
Epoch 44/50
71/71 - 0s - loss: 0.0904 - val_loss: 0.0779
Epoch 45/50
71/71 - 0s - loss: 0.0896 - val_loss: 0.0782
Epoch 46/50
71/71 - 0s - loss: 0.0887 - val_loss: 0.0800
Epoch 47/50
71/71 - 0s - loss: 0.0872 - val_loss: 0.0775
Epoch 48/50
71/71 - 0s - loss: 0.0871 - val_loss: 0.0778
Epoch 49/50
71/71 - 0s - loss: 0.0855 - val_loss: 0.0799
Epoch 50/50
71/71 - 0s - loss: 0.0852 - val_loss: 0.0780
CPU times: user 7.11 s, sys: 3.49 s, total: 10.6 s
Wall time: 6.94 s

## Autoencoder Approach

**Let's see the loss evolution over time**_____no_output_____

nn.plot_training_history(history, figsize=figsize)_____no_output_____

## Autoencoder Approach

**Then we obtain our predictions**_____no_output_____

%%time
preds = ae.predict(vsbs)
preds = pd.DataFrame(index=vsbs.index, columns=vsb_in, data=preds)CPU times: user 263 ms, sys: 149 ms, total: 413 ms
Wall time: 291 ms

...And we generate the alarm signal:_____no_output_____

se = np.square(preds - vsbs[vsb_in])
sse = np.sum(se, axis=1)
signal_ae = pd.Series(index=vsbs.index, data=sse)_____no_output_____

* We stored also the individual errors `se` for a later analysis_____no_output_____## Autoencoder Approach

**We can now plot the alarm signal**_____no_output_____

nn.plot_signal(signal_ae, figsize=figsize)_____no_output_____

* Once again, the signal is difficult to read due to the peak_____no_output_____## Autoencoder Approach

**Let's apply clipping and smoothing**_____no_output_____

nn.plot_signal(signal_ae.clip(upper=10).ewm(512).mean(), figsize=figsize)_____no_output_____

* Again, a tendency to grow (weaker than before)
* ...But this time _we can try  to explain the peak_!_____no_output_____## Mutiple Signal Analysis

**Let's investigate the situation**_____no_output_____

signals_ae = pd.DataFrame(index=vsbs.index, columns=vsb_in, data=se)
nn.plot_dataframe(signals_ae, vmin=-1, vmax=1, figsize=figsize)_____no_output_____

* As expected, errors are concentrated on a few features_____no_output_____## Multiple Signal Analysis

**Let's focus on the peak at mid run and check the largest errors**_____no_output_____

tmp = se.iloc[3500:4000].mean().sort_values(ascending=False)[:10]
nn.plot_bars(tmp, figsize=figsize, tick_gap=-1)_____no_output_____print(f'The largest error is on {tmp.index[0]}')The largest error is on pSvolFilm::CTRL_Position_controller::Actual_position::<lambda_0>

## Multiple Signal Analysis

**Let's what was going on with the original series**_____no_output_____

nn.plot_series(vs['pCut::CTRL_Position_controller::Actual_position'], figsize=figsize)_____no_output_____

* Indeed, _there is an unusual oscillation_! A domain expert may make sense of that_____no_output_____# RNVP Approach_____no_output_____## RNVP Approach

**Let's make an attempt with Real NVP**_____no_output_____

%%time
vs_rnvp = nn.RealNVP(input_shape=vsbs.shape[1],
                num_coupling=6, units_coupling=32, depth_coupling=1, reg_coupling=0.01)
vs_rnvp.compile(optimizer='Adam')
X = vsbs_tr.astype(np.float32).values
cb = [EarlyStopping(monitor='loss', patience=10, min_delta=0.001, restore_best_weights=True)]
history = vs_rnvp.fit(X, batch_size=256, epochs=300, verbose=1, callbacks=cb)Epoch 1/300
12/12 [==============================] - 0s 6ms/step - loss: 219.0808
Epoch 2/300
12/12 [==============================] - 0s 5ms/step - loss: 102.5352
Epoch 3/300
12/12 [==============================] - 0s 6ms/step - loss: 77.7634
Epoch 4/300
12/12 [==============================] - 0s 6ms/step - loss: 68.9585
Epoch 5/300
12/12 [==============================] - 0s 6ms/step - loss: 66.0839
Epoch 6/300
12/12 [==============================] - 0s 6ms/step - loss: 63.3582
Epoch 7/300
12/12 [==============================] - 0s 6ms/step - loss: 61.7042
Epoch 8/300
12/12 [==============================] - 0s 6ms/step - loss: 59.6041
Epoch 9/300
12/12 [==============================] - 0s 6ms/step - loss: 58.0961
Epoch 10/300
12/12 [==============================] - 0s 6ms/step - loss: 56.0438
Epoch 11/300
12/12 [==============================] - 0s 6ms/step - loss: 54.8829
Epoch 12/300
12/12 [==============================] - 0s 6ms/step - loss: 53.0074
Epoch 13/300
12/12 [==============================] - 0s 6ms/step - loss: 51.5139
Epoch 14/300
12/12 [==============================] - 0s 6ms/step - loss: 49.4959
Epoch 15/300
12/12 [==============================] - 0s 6ms/step - loss: 48.4653
Epoch 16/300
12/12 [==============================] - 0s 6ms/step - loss: 46.9011
Epoch 17/300
12/12 [==============================] - 0s 6ms/step - loss: 45.1730
Epoch 18/300
12/12 [==============================] - 0s 6ms/step - loss: 44.0730
Epoch 19/300
12/12 [==============================] - 0s 6ms/step - loss: 42.2174
Epoch 20/300
12/12 [==============================] - 0s 5ms/step - loss: 41.3083
Epoch 21/300
12/12 [==============================] - 0s 6ms/step - loss: 39.7833
Epoch 22/300
12/12 [==============================] - 0s 6ms/step - loss: 38.0073
Epoch 23/300
12/12 [==============================] - 0s 5ms/step - loss: 36.6685
Epoch 24/300
12/12 [==============================] - 0s 6ms/step - loss: 36.4102
Epoch 25/300
12/12 [==============================] - 0s 6ms/step - loss: 34.7182
Epoch 26/300
12/12 [==============================] - 0s 6ms/step - loss: 33.0557
Epoch 27/300
12/12 [==============================] - 0s 6ms/step - loss: 32.0614
Epoch 28/300
12/12 [==============================] - 0s 6ms/step - loss: 30.6622
Epoch 29/300
12/12 [==============================] - 0s 5ms/step - loss: 30.8519
Epoch 30/300
12/12 [==============================] - 0s 6ms/step - loss: 28.4374
Epoch 31/300
12/12 [==============================] - 0s 5ms/step - loss: 26.7517
Epoch 32/300
12/12 [==============================] - 0s 6ms/step - loss: 25.9339
Epoch 33/300
12/12 [==============================] - 0s 6ms/step - loss: 25.8942
Epoch 34/300
12/12 [==============================] - 0s 6ms/step - loss: 24.5311
Epoch 35/300
12/12 [==============================] - 0s 6ms/step - loss: 23.5868
Epoch 36/300
12/12 [==============================] - 0s 5ms/step - loss: 21.6661
Epoch 37/300
12/12 [==============================] - 0s 6ms/step - loss: 21.5126
Epoch 38/300
12/12 [==============================] - 0s 6ms/step - loss: 20.6197
Epoch 39/300
12/12 [==============================] - 0s 6ms/step - loss: 19.0592
Epoch 40/300
12/12 [==============================] - 0s 6ms/step - loss: 18.5783
Epoch 41/300
12/12 [==============================] - 0s 6ms/step - loss: 17.2298
Epoch 42/300
12/12 [==============================] - 0s 7ms/step - loss: 17.1351
Epoch 43/300
12/12 [==============================] - 0s 6ms/step - loss: 15.8378
Epoch 44/300
12/12 [==============================] - 0s 5ms/step - loss: 14.2928
Epoch 45/300
12/12 [==============================] - 0s 5ms/step - loss: 14.3107
Epoch 46/300
12/12 [==============================] - 0s 6ms/step - loss: 13.0894
Epoch 47/300
12/12 [==============================] - 0s 6ms/step - loss: 12.8905
Epoch 48/300
12/12 [==============================] - 0s 6ms/step - loss: 11.5461
Epoch 49/300
12/12 [==============================] - 0s 6ms/step - loss: 11.1805
Epoch 50/300
12/12 [==============================] - 0s 5ms/step - loss: 10.3422
Epoch 51/300
12/12 [==============================] - 0s 6ms/step - loss: 9.5980
Epoch 52/300
12/12 [==============================] - 0s 6ms/step - loss: 9.0789
Epoch 53/300
12/12 [==============================] - 0s 5ms/step - loss: 7.3922
Epoch 54/300
12/12 [==============================] - 0s 6ms/step - loss: 7.2703
Epoch 55/300
12/12 [==============================] - 0s 5ms/step - loss: 6.2726
Epoch 56/300
12/12 [==============================] - 0s 6ms/step - loss: 5.9323
Epoch 57/300
12/12 [==============================] - 0s 5ms/step - loss: 6.8783
Epoch 58/300
12/12 [==============================] - 0s 6ms/step - loss: 5.1018
Epoch 59/300
12/12 [==============================] - 0s 6ms/step - loss: 4.8963
Epoch 60/300
12/12 [==============================] - 0s 5ms/step - loss: 4.6755
Epoch 61/300
12/12 [==============================] - 0s 6ms/step - loss: 3.6234
Epoch 62/300
12/12 [==============================] - 0s 5ms/step - loss: 3.1792
Epoch 63/300
12/12 [==============================] - 0s 5ms/step - loss: 3.4293
Epoch 64/300
12/12 [==============================] - 0s 5ms/step - loss: 2.6820
Epoch 65/300
12/12 [==============================] - 0s 6ms/step - loss: 3.0232
Epoch 66/300
12/12 [==============================] - 0s 5ms/step - loss: 1.4771
Epoch 67/300
12/12 [==============================] - 0s 6ms/step - loss: 1.6644
Epoch 68/300
12/12 [==============================] - 0s 6ms/step - loss: 1.2278
Epoch 69/300
12/12 [==============================] - 0s 6ms/step - loss: -0.6657
Epoch 70/300
12/12 [==============================] - 0s 5ms/step - loss: -0.0771
Epoch 71/300
12/12 [==============================] - 0s 6ms/step - loss: -1.1926
Epoch 72/300
12/12 [==============================] - 0s 6ms/step - loss: -1.1881
Epoch 73/300
12/12 [==============================] - 0s 5ms/step - loss: -2.1953
Epoch 74/300
12/12 [==============================] - 0s 6ms/step - loss: -2.2644
Epoch 75/300
12/12 [==============================] - 0s 6ms/step - loss: -2.7564
Epoch 76/300
12/12 [==============================] - 0s 5ms/step - loss: -1.8401
Epoch 77/300
12/12 [==============================] - 0s 6ms/step - loss: -3.0556
Epoch 78/300
12/12 [==============================] - 0s 6ms/step - loss: -3.7993
Epoch 79/300
12/12 [==============================] - 0s 6ms/step - loss: -3.4182
Epoch 80/300
12/12 [==============================] - 0s 5ms/step - loss: -3.1349
Epoch 81/300
12/12 [==============================] - 0s 6ms/step - loss: -4.9013
Epoch 82/300
12/12 [==============================] - 0s 6ms/step - loss: -4.8442
Epoch 83/300
12/12 [==============================] - 0s 6ms/step - loss: -5.9123
Epoch 84/300
12/12 [==============================] - 0s 6ms/step - loss: -5.8474
Epoch 85/300
12/12 [==============================] - 0s 5ms/step - loss: -6.4380
Epoch 86/300
12/12 [==============================] - 0s 6ms/step - loss: -6.6160
Epoch 87/300
12/12 [==============================] - 0s 5ms/step - loss: -7.4917
Epoch 88/300
12/12 [==============================] - 0s 5ms/step - loss: -6.7260
Epoch 89/300
12/12 [==============================] - 0s 5ms/step - loss: -7.5400
Epoch 90/300
12/12 [==============================] - 0s 6ms/step - loss: -8.0798
Epoch 91/300
12/12 [==============================] - 0s 5ms/step - loss: -7.1802
Epoch 92/300
12/12 [==============================] - 0s 5ms/step - loss: -7.6470
Epoch 93/300
12/12 [==============================] - 0s 6ms/step - loss: -8.8107
Epoch 94/300
12/12 [==============================] - 0s 6ms/step - loss: -9.3837
Epoch 95/300
12/12 [==============================] - 0s 5ms/step - loss: -8.8574
Epoch 96/300
12/12 [==============================] - 0s 5ms/step - loss: -9.1089
Epoch 97/300
12/12 [==============================] - 0s 6ms/step - loss: -9.6635
Epoch 98/300
12/12 [==============================] - 0s 6ms/step - loss: -10.4566
Epoch 99/300
12/12 [==============================] - 0s 5ms/step - loss: -9.5484
Epoch 100/300
12/12 [==============================] - 0s 5ms/step - loss: -10.1051
Epoch 101/300
12/12 [==============================] - 0s 5ms/step - loss: -9.9481
Epoch 102/300
12/12 [==============================] - 0s 6ms/step - loss: -10.5164
Epoch 103/300
12/12 [==============================] - 0s 5ms/step - loss: -11.1741
Epoch 104/300
12/12 [==============================] - 0s 5ms/step - loss: -11.0538
Epoch 105/300
12/12 [==============================] - 0s 5ms/step - loss: -11.2741
Epoch 106/300
12/12 [==============================] - 0s 5ms/step - loss: -12.1662
Epoch 107/300
12/12 [==============================] - 0s 5ms/step - loss: -12.7626
Epoch 108/300
12/12 [==============================] - 0s 5ms/step - loss: -13.3599
Epoch 109/300
12/12 [==============================] - 0s 5ms/step - loss: -13.5170
Epoch 110/300
12/12 [==============================] - 0s 5ms/step - loss: -13.6442
Epoch 111/300
12/12 [==============================] - 0s 5ms/step - loss: -12.4917
Epoch 112/300
12/12 [==============================] - 0s 5ms/step - loss: -12.5180
Epoch 113/300
12/12 [==============================] - 0s 5ms/step - loss: -12.8183
Epoch 114/300
12/12 [==============================] - 0s 5ms/step - loss: -13.1459
Epoch 115/300
12/12 [==============================] - 0s 4ms/step - loss: -13.4275
Epoch 116/300
12/12 [==============================] - 0s 6ms/step - loss: -14.5674
Epoch 117/300
12/12 [==============================] - 0s 5ms/step - loss: -14.2125
Epoch 118/300
12/12 [==============================] - 0s 5ms/step - loss: -15.0486
Epoch 119/300
12/12 [==============================] - 0s 5ms/step - loss: -14.9357
Epoch 120/300
12/12 [==============================] - 0s 6ms/step - loss: -15.1186
Epoch 121/300
12/12 [==============================] - 0s 5ms/step - loss: -14.5618
Epoch 122/300
12/12 [==============================] - 0s 6ms/step - loss: -15.8116
Epoch 123/300
12/12 [==============================] - 0s 6ms/step - loss: -16.4995
Epoch 124/300
12/12 [==============================] - 0s 5ms/step - loss: -15.0127
Epoch 125/300
12/12 [==============================] - 0s 6ms/step - loss: -16.0360
Epoch 126/300
12/12 [==============================] - 0s 5ms/step - loss: -15.8846
Epoch 127/300
12/12 [==============================] - 0s 6ms/step - loss: -17.0661
Epoch 128/300
12/12 [==============================] - 0s 6ms/step - loss: -17.2013
Epoch 129/300
12/12 [==============================] - 0s 6ms/step - loss: -17.3317
Epoch 130/300
12/12 [==============================] - 0s 5ms/step - loss: -14.6197
Epoch 131/300
12/12 [==============================] - 0s 5ms/step - loss: -15.3839
Epoch 132/300
12/12 [==============================] - 0s 5ms/step - loss: -17.6289
Epoch 133/300
12/12 [==============================] - 0s 5ms/step - loss: -17.1079
Epoch 134/300
12/12 [==============================] - 0s 6ms/step - loss: -17.5051
Epoch 135/300
12/12 [==============================] - 0s 5ms/step - loss: -17.3079
Epoch 136/300
12/12 [==============================] - 0s 6ms/step - loss: -18.0315
Epoch 137/300
12/12 [==============================] - 0s 6ms/step - loss: -18.3939
Epoch 138/300
12/12 [==============================] - 0s 5ms/step - loss: -19.1197
Epoch 139/300
12/12 [==============================] - 0s 5ms/step - loss: -19.4268
Epoch 140/300
12/12 [==============================] - 0s 5ms/step - loss: -19.1550
Epoch 141/300
12/12 [==============================] - 0s 5ms/step - loss: -19.4506
Epoch 142/300
12/12 [==============================] - 0s 6ms/step - loss: -18.4763
Epoch 143/300
12/12 [==============================] - 0s 5ms/step - loss: -19.0793
Epoch 144/300
12/12 [==============================] - 0s 6ms/step - loss: -19.5101
Epoch 145/300
12/12 [==============================] - 0s 5ms/step - loss: -19.7879
Epoch 146/300
12/12 [==============================] - 0s 5ms/step - loss: -18.8700
Epoch 147/300
12/12 [==============================] - 0s 6ms/step - loss: -19.6665
Epoch 148/300
12/12 [==============================] - 0s 6ms/step - loss: -20.2469
Epoch 149/300
12/12 [==============================] - 0s 6ms/step - loss: -21.1253
Epoch 150/300
12/12 [==============================] - 0s 5ms/step - loss: -20.1028
Epoch 151/300
12/12 [==============================] - 0s 5ms/step - loss: -20.1170
Epoch 152/300
12/12 [==============================] - 0s 5ms/step - loss: -21.4135
Epoch 153/300
12/12 [==============================] - 0s 6ms/step - loss: -21.1128
Epoch 154/300
12/12 [==============================] - 0s 5ms/step - loss: -21.3679
Epoch 155/300
12/12 [==============================] - 0s 6ms/step - loss: -21.6237
Epoch 156/300
12/12 [==============================] - 0s 7ms/step - loss: -21.6824
Epoch 157/300
12/12 [==============================] - 0s 6ms/step - loss: -21.9478
Epoch 158/300
12/12 [==============================] - 0s 6ms/step - loss: -22.5938
Epoch 159/300
12/12 [==============================] - 0s 6ms/step - loss: -21.5247
Epoch 160/300
12/12 [==============================] - 0s 6ms/step - loss: -21.9027
Epoch 161/300
12/12 [==============================] - 0s 5ms/step - loss: -21.5008
Epoch 162/300
12/12 [==============================] - 0s 5ms/step - loss: -21.5450
Epoch 163/300
12/12 [==============================] - 0s 6ms/step - loss: -22.7133
Epoch 164/300
12/12 [==============================] - 0s 5ms/step - loss: -22.2807
Epoch 165/300
12/12 [==============================] - 0s 6ms/step - loss: -23.3369
Epoch 166/300
12/12 [==============================] - 0s 5ms/step - loss: -22.2364
Epoch 167/300
12/12 [==============================] - 0s 5ms/step - loss: -22.7833
Epoch 168/300
12/12 [==============================] - 0s 5ms/step - loss: -23.3356
Epoch 169/300
12/12 [==============================] - 0s 5ms/step - loss: -23.7397
Epoch 170/300
12/12 [==============================] - 0s 5ms/step - loss: -23.8302
Epoch 171/300
12/12 [==============================] - 0s 5ms/step - loss: -22.7387
Epoch 172/300
12/12 [==============================] - 0s 5ms/step - loss: -23.2083
Epoch 173/300
12/12 [==============================] - 0s 5ms/step - loss: -23.6722
Epoch 174/300
12/12 [==============================] - 0s 5ms/step - loss: -24.4671
Epoch 175/300
12/12 [==============================] - 0s 5ms/step - loss: -24.2075
Epoch 176/300
12/12 [==============================] - 0s 5ms/step - loss: -23.5264
Epoch 177/300
12/12 [==============================] - 0s 5ms/step - loss: -23.6589
Epoch 178/300
12/12 [==============================] - 0s 5ms/step - loss: -22.8957
Epoch 179/300
12/12 [==============================] - 0s 5ms/step - loss: -24.2849
Epoch 180/300
12/12 [==============================] - 0s 5ms/step - loss: -23.7460
Epoch 181/300
12/12 [==============================] - 0s 6ms/step - loss: -24.5254
Epoch 182/300
12/12 [==============================] - 0s 5ms/step - loss: -25.4004
Epoch 183/300
12/12 [==============================] - 0s 5ms/step - loss: -25.4806
Epoch 184/300
12/12 [==============================] - 0s 5ms/step - loss: -25.1498
Epoch 185/300
12/12 [==============================] - 0s 5ms/step - loss: -24.4989
Epoch 186/300
12/12 [==============================] - 0s 5ms/step - loss: -25.5472
Epoch 187/300
12/12 [==============================] - 0s 5ms/step - loss: -23.8039
Epoch 188/300
12/12 [==============================] - 0s 6ms/step - loss: -24.5722
Epoch 189/300
12/12 [==============================] - 0s 6ms/step - loss: -25.9971
Epoch 190/300
12/12 [==============================] - 0s 5ms/step - loss: -25.7359
Epoch 191/300
12/12 [==============================] - 0s 6ms/step - loss: -26.7237
Epoch 192/300
12/12 [==============================] - 0s 5ms/step - loss: -26.2259
Epoch 193/300
12/12 [==============================] - 0s 5ms/step - loss: -25.9594
Epoch 194/300
12/12 [==============================] - 0s 5ms/step - loss: -26.0848
Epoch 195/300
12/12 [==============================] - 0s 6ms/step - loss: -25.5031
Epoch 196/300
12/12 [==============================] - 0s 5ms/step - loss: -26.7001
Epoch 197/300
12/12 [==============================] - 0s 5ms/step - loss: -25.9709
Epoch 198/300

## RNVP Approach

**Let's have a look at the loss evolution over time**_____no_output_____

nn.plot_training_history(history, figsize=figsize)_____no_output_____

## RNVP Approach

**Now we can generate and plot the signal**_____no_output_____

ldens = vs_rnvp.score_samples(vsbs.astype(np.float32).values)
signal_vs = pd.Series(index=vsbs.index, data=-ldens)
nn.plot_signal(signal_vs, figsize=figsize)_____no_output_____

## RNVP Approach

**Finally, we apply clipping and smoothing**_____no_output_____

nn.plot_signal(signal_vs.clip(upper=10).ewm(512).mean(), figsize=figsize)_____no_output_____

* This is the signal with _the clearest trend so far_
* Intuitively, component wear should grow _progressively_ over time_____no_output_____


# Notebook from carsen-stringer/course-content-dl
Path: tutorials/student/intro.ipynb

_____no_output_____# Introduction

Welcome to the Neuromatch computational neuroscience course!_____no_output_____**Our 2021 Sponsors, including Presenting Sponsor Facebook Reality Labs**

_____no_output_____## Welcome Video_____no_output_____

# @markdown
from ipywidgets import widgets

out2 = widgets.Output()
with out2:
  from IPython.display import IFrame
  class BiliVideo(IFrame):
      def __init__(self, id, page=1, width=400, height=300, **kwargs):
          self.id=id
          src = 'https://player.bilibili.com/player.html?bvid={0}&page={1}'.format(id, page)
          super(BiliVideo, self).__init__(src, width, height, **kwargs)

  video = BiliVideo(id="", width=854, height=480, fs=1)
  print('Video available at https://www.bilibili.com/video/{0}'.format(video.id))
  display(video)

out1 = widgets.Output()
with out1:
  from IPython.display import YouTubeVideo
  video = YouTubeVideo(id="", width=854, height=480, fs=1, rel=0)
  print('Video available at https://youtube.com/watch?v=' + video.id)
  display(video)

out = widgets.Tab([out1, out2])
out.set_title(0, 'Youtube')
out.set_title(1, 'Bilibili')

display(out)_____no_output_____

## Curriculum overview

Welcome to the deep learning course!_____no_output_____