# Notebook from timtyree/bgmc
Path: python/analyzing annihilation rates from the oscillatory model.ipynb

# analyzing Annihilation Rates from the Oscillatory Model
Tim Tyree

1.10.2022_____no_output_____

# TODO: simplify initialization
from scipy.optimize import minimize
from lib.my_initialization import *
from lib import *
from lib.lib_care.measure.level_sets import comp_longest_level_set_and_smooth
%load_ext autoreload
%autoreload 2

import matplotlib.pyplot as plt, numpy as np
from mpltools import annotation
import random,scipy
from scipy.interpolate import LinearNDInterpolator
from scipy.interpolate import CloughTocher2DInterpolator
import matplotlib as mpl #for colorbar
from scipy import stats

#for particle params to MSR generating functional
from scipy.signal import savgol_filter
import dask.bag as db
from lib.lib_care.measure.bootstrap import bin_and_bootstrap_xy_values_parallel

darkmode=False
if darkmode:
    # For darkmode plots
    from jupyterthemes import jtplot
    jtplot.style(theme='monokai', context='notebook', ticks=True, grid=False)

import warnings
warnings.simplefilter("ignore", UserWarning)
%load_ext autoreload
%autoreload 2

#optional
import cupy as cp
cp.random.seed(42)
np.random.seed(42)The autoreload extension is already loaded. To reload it, use:
  %reload_ext autoreload
darkmode=False
if darkmode:
    # For darkmode plots
    from jupyterthemes import jtplot
    jtplot.style(theme='monokai', context='notebook', ticks=True, grid=False)_____no_output_____

# visualize annihilation rates when using the noninteracting model for phase evolution, to show the correct exponents were not observed_____no_output_____

wjr=recall_powerlaw_fits_to_full_models()
print(*wjr)fk_pbc lr_pbc fk_ncbc lr_ncbc
data_dir="/home/timothytyree/Documents/GitHub/bgmc/python/data/osg_output/run_21_all.csv"
df=pd.read_csv(data_dir)
# df.head()

print(list(df.columns))

assert (not (df.CollRate<0).any())['x0', 'varkappa', 'set_second', 'reflect', 'r', 'no_repulsion', 'no_attraction', 'niter', 'neighbor', 'kappa', 'force_code', 'dt', 'N', 'L', 'Dt', 'D', 'CollRate']
#derived values
# df['CollRate']=1./df['CollTime']
df['A']=df['L']**2
df['q']=df['N']/df['A'] #number of tips per square centimeter
df['w']=df['CollRate']/df['A'] #[mHz?]/cm^2

# df=df[df.niter==250].copy()
#extract column values
r_values=np.array(sorted(set(df.r.values)))#cm
D_values=np.array(sorted(set(df.D.values)))#cm^2/s
L_values=np.array(sorted(set(df.L.values)))#cm
A_values=L_values**2#cm^2
kappa_values=np.array(sorted(set(df.kappa.values)))#1/s
varkappa_values=np.array(sorted(set(df.varkappa.values)))#1/s
x0_values=np.array(sorted(set(df.x0.values)))#1/s
set_second_values=np.array(sorted(set(df.set_second.values)))
reflect_values=np.array(sorted(set(df.reflect.values)))
no_repulsion_values=np.array(sorted(set(df.no_repulsion.values)))
no_attraction_values=np.array(sorted(set(df.no_attraction.values)))
neighbor_values=np.array(sorted(set(df.neighbor.values)))
force_code_values=np.array(sorted(set(df.force_code.values)))

#make test for whether there is one input parameter present in an input DataFrame
print(f"parameters:")
print(f"r~{r_values}")
print(f"D~{D_values}")
print(f"L~{L_values}")
print(f"kappa~{kappa_values}")
print(f"a~{varkappa_values}")
print(f"x0~{x0_values}")
print(f"set_second~{set_second_values}")
print(f"reflect~{reflect_values}")
print(f"no_repulsion~{no_repulsion_values}")
print(f"no_attraction~{no_attraction_values}")
print(f"neighbor~{neighbor_values}")
print(f"force_code~{force_code_values}")parameters:
r~[0.3  0.35 0.4  0.41 0.45 0.5 ]
D~[1.76795]
L~[10]
kappa~[ 300  400  500  600  700  800  900 1000 1100 1200 1300 1400 1500]
a~[3.5359]
x0~[0.06 0.1  0.12 0.5 ]
set_second~[0]
reflect~[0]
no_repulsion~[0]
no_attraction~[0]
neighbor~[0 1]
force_code~[2]
#DONE: what settings finished?
#DONE: ask self which ~half of settings did not finish.  LR finished. FK was not attempted.  i'd bet the low kappa*r**2 trials failed to finish
#TODO(later): add minimum r**2*kappa threshold to gen_run_22.py
#DONE: did any powerlaw fits have a reasonably low RMSE_full? No.
#DONE: visualize individual settings_____no_output_____

Note: Run 21 is missing trials from the neighbor=0 case.

I should consider why the neighbor=0 trials didn't finish if I try inplementing neighbor-neighbor phase interactions._____no_output_____

#Warning: computed powerlaw fits for  >5000 trials successfully.
#what do the annihilation rates look like?
#for FK model
#query the control
varkappa=varkappa_values[0] #cm^2/s
D=D_values[0]
r=r_values[0]
L=L_values[0]
x0=x0_values[-2] #s

set_second=0
no_repulsion=0
no_attraction=0
reflect=0
neighbor=0
force_code=force_code_values[0]

#query the DataFrame
query =(df.set_second==set_second)&(df.reflect==reflect)
query&=df.r==r
query&=df.D==D
query&=df.L==L
query&=df.varkappa==varkappa
query&=df.x0==x0
query&=(df.no_repulsion==no_repulsion)&(df.no_attraction==no_attraction)
print(df[query].size)
query&=(df.neighbor==neighbor)
print(df[query].size)
query&=(df.force_code==force_code)
print(df[query].size)
dg=df[query]
kappa_values=np.array(sorted(set(dg.kappa.values)))

21060
21060
21060
kappa=kappa_values[0]
dh=dg[dg.kappa==kappa]
x_values=dh.q.values
y_values=dh.w.values
x_values_control=x_values.copy()
y_values_control=y_values.copy()

x0=x0_values[0]        #cm
#query the DataFrame
query =(df.set_second==set_second)&(df.reflect==reflect)
query&=df.r==r
query&=df.D==D
query&=df.L==L
query&=df.varkappa==varkappa
query&=df.x0==x0
query&=(df.no_repulsion==no_repulsion)&(df.no_attraction==no_attraction)
query&=(df.neighbor==neighbor)&(df.force_code==force_code)
dg=df[query]

# kappa_values=np.array(sorted(set(dg.kappa.values)))
# kappa=kappa_values[1]
dh=dg[dg.kappa==kappa]
x_values=dh.q.values
y_values=dh.w.values
x_values_test=x_values.copy()
y_values_test=y_values.copy()
print((r,D,L,kappa,varkappa,x0))(0.3, 1.76795, 10, 300, 3.5359, 0.06)
#TODO(later?): dev averaging step for computation of powerlaw fits.
# Option 2: map run_21_all to run_21_all_merged using the averaging filter
# Option 1: see if the run_21_all.csv already did all of the merging for me, in which case I can connect run_21_all.csv to the bootstrapping method already in use._____no_output_____#plot fits for full model
m_fk=1.945;#+-0.030; B_fk=2.441+-0.051
m_lr=1.544;#+-0.034; B_lr=5.870+-0.137
M_fk=5.67;#+-0.39 Hz*cm^{2(m-1)}
M_lr=15.37;#+-1.57 Hz*cm^{2(m-1)}
# RMSE_fk=0.1252 Hz/cm^2
# RMSE_lr=0.0974 Hz/cm^2
# R^2=0.997 (FK)
# R^2=0.994 (LR)
# yscale=10**3
xv=np.arange(0.1,1.,.05)
yv_fk=M_fk*(xv)**m_fk
yv_lr=M_lr*(xv)**m_lr

fontsize=16
# plt.xlim([0.1,1])
# plt.ylim([1e-1,15])
plt.yscale('log')
plt.xscale('log')
# plt.plot(xv,yv_fk,label='FK power law fit',zorder=3,lw=4)
plt.plot(xv,yv_lr,label='LR power law fit',zorder=3,lw=4,color='C1')

fontsize=18
# plt.plot(x_values_control,y_values_control,'-',c='k',alpha=.7,label=r'approx. infinite basin',lw=3)
plt.plot(x_values_test,y_values_test,'-',c='k',alpha=.7,label=f'Oscillatory Model\n(no phase interactions)',lw=3)

# plt.plot(x_values,y_values,c='C2',alpha=.7,label='simulation',lw=6)
# plt.plot(x_values_force,y_values_force,c='C3',alpha=.7,label='simulation with forces',lw=6)
# plt.plot(x_values,y_values,c=c_values,alpha=0.4,cmap='bwr')
# plt.title(u'comparison to simulation\nwith two hybrid modes',fontsize=fontsize)
plt.xlabel(r'q (cm$^{-2}$)',fontsize=fontsize)
plt.ylabel(r'w (Hz cm$^{-2}$)', fontsize=fontsize)
plt.tick_params(axis='both', which='major', labelsize=fontsize)
plt.tick_params(axis='both', which='minor', labelsize=0)
plt.legend(fontsize=fontsize-5)
# plt.xlim([0.08,1])
# print(f'varkappa={varkappa}  Hz');print(f'      x0={x0} cm')
# plt.title(r'$\varkappa=$'+f'{varkappa}  Hz, '+r'$x_0=$'+f'{x0} cm\nforce_code={force_code}, neighbors={neighbor}\n',fontsize=fontsize)
plt.title(r'$T=$'+f'{x0} sec, '+r'$a=$'+f'{varkappa}, '+r'$\kappa=$'+f'{kappa:.1f}\nforce_code={force_code}, neighbors={neighbor}\nr={r},D={D},L={L}\n',fontsize=fontsize)
plt.show()_____no_output_____# from generation of powerfits .ipynb
print(f"powerfit_dir='/home/timothytyree/Documents/GitHub/bgmc/python/data/osg_output/run_21_all_powerlaw_fits.csv'")powerfit_dir='/home/timothytyree/Documents/GitHub/bgmc/python/data/osg_output/run_21_all_powerlaw_fits.csv'
powerfit_dir='/home/timothytyree/Documents/GitHub/bgmc/python/data/osg_output/run_21_all_powerlaw_fits.csv'_____no_output_____df_powerfits=pd.read_csv(powerfit_dir)
df_powerfits.head()_____no_output_____nu_values=df_powerfits['m'].values
yv=np.linspace(0,1,10)
plt.plot(0.*yv+m_lr,yv,'C1--',label='Luo-Rudy Model')
plt.plot(0.*yv+m_fk,yv,'C0--',label='Fenton-Karma Model')
plt.hist(nu_values,color='k',alpha=0.8,label=f'Oscillatory Model\n(3126 settings)\n(no phase interactions)',density=True)
format_plot(ax=plt.gca(),xlabel='Exponent Value',ylabel='PDF')
plt.legend(fontsize=12)
plt.show()_____no_output_____#TODO: how many neighbor=0 trials finished? how many neighbor=1 trials finished
#TODO: make ^this into a pie chart, because the same number of settings were attempted for neighbor=0 versus 1_____no_output_____num_neighor_settings=df[df.neighbor==1].w.values.shape[0]
num_non_neighor_settings=df[df.neighbor==0].w.values.shape[0]
print(f"{100*num_neighor_settings/(num_neighor_settings+num_non_neighor_settings):.2f}% percent of settings that terminated used neighbor-only forces.")1.89% percent of settings that terminated used neighbor-only forces.
yv=np.linspace(0,1,10)
plt.plot(0.*yv+m_lr,yv,'C1--',label='Luo-Rudy Model')
plt.plot(0.*yv+m_fk,yv,'C0--',label='Fenton-Karma Model')
nu_values=df_powerfits[df.neighbor==1]['m'].values
plt.hist(nu_values,color='k',alpha=0.8,label=f'Neighbor-Forces',density=True)
nu_values=df_powerfits[df.neighbor==0]['m'].values
plt.hist(nu_values,color='g',alpha=0.8,label=f'Vector-Summed-Forces',density=True)
format_plot(ax=plt.gca(),xlabel='Exponent Value',ylabel='PDF')
plt.legend(fontsize=12)
plt.show()_____no_output_____


# Notebook from pcsingh/Predicting-Repo-popularity
Path: Notebooks/data_extraction.ipynb

This `ipynb` file contains the codes for the extraction of the data that is used for the training of the model which will predict the popularity of the model.

For the extraction of data you need the GitHub token which you can generate from [here](https://github.com/settings/tokens) and replace `PASTE_YOUR_GITHUB_TOKEN` with the generated token._____no_output_____

# Importing neccessary libraries
import requests
import json
import pandas as pd
import re
from bs4 import BeautifulSoup _____no_output_____header={'Accept':'application/vnd.github.mercy-preview+json',
'visibility':'PUBLIC',
"Authorization": "token PASTE_YOUR_GITHUB_TOKEN"
}_____no_output_____

### Scraping

The Scraping part begins from here._____no_output_____

def extract(repo_name, user, contri):
    """
    Function extract the data from the repository web page

    Args:
        repo_name (String): Name of the repository
        user (String): Username of the owner
        contri (Int): Number of Contributors

    Returns:
        info (List): Contains the extracted data from the repository web page
    """
    # print('https://github.com/'+user+"/"+repo_name)

    info=[]
    r = requests.get('https://github.com/'+user+"/"+repo_name)
    # file=requests.get('https://api.github.com/repos/'+repo_name,headers=header)

    soup = BeautifulSoup(r.content)
    info.append(repo_name)
    al=soup.find_all('a',class_="social-count")

    info.append(al[1].text.replace("\n","").strip(" "))
    #info.append(al[2].text.replace("\n","").strip(" "))
    info.append(al[0].text.replace("\n","").strip(" "))
    try:
        info.append(soup.find_all('a', class_='js-selected-navigation-item UnderlineNav-item hx_underlinenav-item no-wrap js-responsive-underlinenav-item')[0].find_all('span',class_='Counter')[0].text)
    except:
        info.append([])
    # a=(soup.find_all('a',class_='js-selected-navigation-item UnderlineNav-item hx_underlinenav-item no-wrapjs-responsive-underlinenav-item').find_all('span',class_='Counter')[0].text)

    try:
        tag=soup.find_all('a',class_='topic-tag topic-tag-link')
        tags=[]
        for t in tag:
            tags.append(t.text.replace("\n"," ").strip(" "))
        info.append(tags)
    except:
        info.append([])

    try:
        info.append(soup.find_all('p',class_='f4 mt-3')[0].text.replace("\n","").strip(" "))
    except:
        info.append([])

    try:
        info.append(soup.find_all('a',class_='link-gray-dark no-underline')[-1].find_all('span',class_='Counter')[0].text)
    except:
        info.append(contri)

    try:
        info.append(soup.find_all('a',{'class':'muted-link'})[0].text.replace("\n","").replace("License","").strip(" "))
    except:
        info.append([])

    info.append('https://github.com/'+user+repo_name)

    try:
        info.append(soup.find_all('span',{'class':'text-gray-dark text-bold mr-1'})[0].text)
    except:
        info.append([])

    return info_____no_output_____df=[]
names=[]
response=requests.get('https://api.github.com/repositories',headers=header)
data=response.json()
print(len(data))

for i in data:
    info=[]

    owner_name=i['owner']['login']
    if owner_name not in names:
        names.append(owner_name)
        # print(owner_name)
        url_owner_repos='https://api.github.com/users/'+owner_name+'/repos'
        list_repo=requests.get(url_owner_repos,headers=header)
        repo_list=list_repo.json()

        for file in repo_list:
            contri=0
            repo_name=file['name']
            # print(repo_name)
            contri_url='https://api.github.com/repos/'+repo_name+'/contributors'
            forks_url=requests.get(contri_url,headers=header)

            contributers=len(forks_url.json())
            h=extract(repo_name, owner_name, contributers)
            #     for c in forks_url.json():
            #         contri+=c['contributions']
            #     df.append([repo_name,forks,tag,language,about,watchers,contributors,contri,popularity,licens,issue,url])
            # except:
            df.append(h)
gist_inliner
https://github.com/cristibalan/gist_inliner
i18n-js
https://github.com/cristibalan/i18n-js
kbdhax
https://github.com/cristibalan/kbdhax
macvim
https://github.com/cristibalan/macvim
mailinho
https://github.com/cristibalan/mailinho
rails
https://github.com/cristibalan/rails
re-frisk
https://github.com/cristibalan/re-frisk
reportable
https://github.com/cristibalan/reportable
skittlish
https://github.com/cristibalan/skittlish
Sonny
https://github.com/cristibalan/Sonny
thorax
https://github.com/cristibalan/thorax
simonjefford
apibutler
https://github.com/simonjefford/apibutler
atlas-terraform-github-tutorial
https://github.com/simonjefford/atlas-terraform-github-tutorial
autotest_example
https://github.com/simonjefford/autotest_example
beats
https://github.com/simonjefford/beats
beats-forwarder
https://github.com/simonjefford/beats-forwarder
betabuilder
https://github.com/simonjefford/betabuilder
bucketwise
https://github.com/simonjefford/bucketwise
bundler
https://github.com/simonjefford/bundler
caddy
https://github.com/simonjefford/caddy
delicious_adder
https://github.com/simonjefford/delicious_adder
docker
https://github.com/simonjefford/docker
docprinter
https://github.com/simonjefford/docprinter
doozerd
https://github.com/simonjefford/doozerd
dotnet-docker
https://github.com/simonjefford/dotnet-docker
emacs.d
https://github.com/simonjefford/emacs.d
emacsconfig
https://github.com/simonjefford/emacsconfig
emacsconfig-old
https://github.com/simonjefford/emacsconfig-old
ember-cli
https://github.com/simonjefford/ember-cli
ember-cli-materialize
https://github.com/simonjefford/ember-cli-materialize
ember-github-issues
https://github.com/simonjefford/ember-github-issues
emberfire
https://github.com/simonjefford/emberfire
feedparser
https://github.com/simonjefford/feedparser
fourthbot
https://github.com/simonjefford/fourthbot
gftest
https://github.com/simonjefford/gftest
GildedRose
https://github.com/simonjefford/GildedRose
git-talk
https://github.com/simonjefford/git-talk
go-exercism
https://github.com/simonjefford/go-exercism
Go-Redis
https://github.com/simonjefford/Go-Redis
gobyexample
https://github.com/simonjefford/gobyexample
gooddata.github.com
https://github.com/simonjefford/gooddata.github.com
leahneukirchen
5x13
https://github.com/leahneukirchen/5x13
absurd
https://github.com/leahneukirchen/absurd
actions-archive-all
https://github.com/leahneukirchen/actions-archive-all
adventofcode2015
https://github.com/leahneukirchen/adventofcode2015
adventofcode2016
https://github.com/leahneukirchen/adventofcode2016
adventofcode2017
https://github.com/leahneukirchen/adventofcode2017
adventofcode2018
https://github.com/leahneukirchen/adventofcode2018
adventofcode2019
https://github.com/leahneukirchen/adventofcode2019
adventofcode2020
https://github.com/leahneukirchen/adventofcode2020
amok
https://github.com/leahneukirchen/amok
arr
https://github.com/leahneukirchen/arr
ast
https://github.com/leahneukirchen/ast
atreus-leah2
https://github.com/leahneukirchen/atreus-leah2
atxec
https://github.com/leahneukirchen/atxec
bacon
https://github.com/leahneukirchen/bacon
bcachefs-tools
https://github.com/leahneukirchen/bcachefs-tools
borschtsch
https://github.com/leahneukirchen/borschtsch
btac
https://github.com/leahneukirchen/btac
CampFahrplan
https://github.com/leahneukirchen/CampFahrplan
ceph-ansible
https://github.com/leahneukirchen/ceph-ansible
challis
https://github.com/leahneukirchen/challis
clitter
https://github.com/leahneukirchen/clitter
colfm
https://github.com/leahneukirchen/colfm
coma
https://github.com/leahneukirchen/coma
coset-mirror
https://github.com/leahneukirchen/coset-mirror
cwm
https://github.com/leahneukirchen/cwm
daiquiri
https://github.com/leahneukirchen/daiquiri
database_cleaner-active_record
https://github.com/leahneukirchen/database_cleaner-active_record
docker-lab-bgp
https://github.com/leahneukirchen/docker-lab-bgp
dwm-chris2
https://github.com/leahneukirchen/dwm-chris2
engineyard
active_model_serializers
https://github.com/engineyard/active_model_serializers
addressable
https://github.com/engineyard/addressable
angular-wizard
https://github.com/engineyard/angular-wizard
appcloud-for-chrome-webstore
https://github.com/engineyard/appcloud-for-chrome-webstore
ardm
https://github.com/engineyard/ardm
ar_properties
https://github.com/engineyard/ar_properties
async
https://github.com/engineyard/async
aws-s3
https://github.com/engineyard/aws-s3
azure-sdk-for-python
https://github.com/engineyard/azure-sdk-for-python
backup
https://github.com/engineyard/backup
basic-chat
https://github.com/engineyard/basic-chat
bcrypt-ruby
https://github.com/engineyard/bcrypt-ruby
bundler
https://github.com/engineyard/bundler
capybara-screenshot
https://github.com/engineyard/capybara-screenshot
chat
https://github.com/engineyard/chat
chef-deploy
https://github.com/engineyard/chef-deploy
chef-mongodb
https://github.com/engineyard/chef-mongodb
chef-percona
https://github.com/engineyard/chef-percona
chef-postgresql
https://github.com/engineyard/chef-postgresql
chef-vault
https://github.com/engineyard/chef-vault
chronatog
https://github.com/engineyard/chronatog
cistern
https://github.com/engineyard/cistern
cliff
https://github.com/engineyard/cliff
cloudex
https://github.com/engineyard/cloudex
container-services-examples
https://github.com/engineyard/container-services-examples
core-client-rb
https://github.com/engineyard/core-client-rb
coveralls-ruby
https://github.com/engineyard/coveralls-ruby
cucumber
https://github.com/engineyard/cucumber
database_cleaner
https://github.com/engineyard/database_cleaner
datafabric-example
https://github.com/engineyard/datafabric-example
kevinclark
AdventOfCode2020
https://github.com/kevinclark/AdventOfCode2020
ale
https://github.com/kevinclark/ale
aletheia
https://github.com/kevinclark/aletheia
aoc2019
https://github.com/kevinclark/aoc2019
chordal
https://github.com/kevinclark/chordal
climate-clock-kit
https://github.com/kevinclark/climate-clock-kit
docker-registry
https://github.com/kevinclark/docker-registry
dotfiles
https://github.com/kevinclark/dotfiles
dust
https://github.com/kevinclark/dust
git-tutorial-8033
https://github.com/kevinclark/git-tutorial-8033
glu.ttono.us
https://github.com/kevinclark/glu.ttono.us
god
https://github.com/kevinclark/god
gq-4x4
https://github.com/kevinclark/gq-4x4
indexer
https://github.com/kevinclark/indexer
inmate-scraping
https://github.com/kevinclark/inmate-scraping
iospec2
https://github.com/kevinclark/iospec2
janus
https://github.com/kevinclark/janus
kata
https://github.com/kevinclark/kata
Lesson-Plans
https://github.com/kevinclark/Lesson-Plans
metrics
https://github.com/kevinclark/metrics
neovim
https://github.com/kevinclark/neovim
netty
https://github.com/kevinclark/netty
nitpick
https://github.com/kevinclark/nitpick
project_euler
https://github.com/kevinclark/project_euler
quicly
https://github.com/kevinclark/quicly
redis-py
https://github.com/kevinclark/redis-py
reqwest
https://github.com/kevinclark/reqwest
robot-army
https://github.com/kevinclark/robot-army
rsearch
https://github.com/kevinclark/rsearch
ruby-kqueue
https://github.com/kevinclark/ruby-kqueue
technomancy
3rd-party
https://github.com/technomancy/3rd-party
atreus
https://github.com/technomancy/atreus
atreus-firmware
https://github.com/technomancy/atreus-firmware
better-defaults
https://github.com/technomancy/better-defaults
bludgeon
https://github.com/technomancy/bludgeon
bus-scheme
https://github.com/technomancy/bus-scheme
bussard
https://github.com/technomancy/bussard
calandria
https://github.com/technomancy/calandria
chopstx
https://github.com/technomancy/chopstx
chortles
https://github.com/technomancy/chortles
circleci.el
https://github.com/technomancy/circleci.el
clojars-verify
https://github.com/technomancy/clojars-verify
clojars-web
https://github.com/technomancy/clojars-web
clojure
https://github.com/technomancy/clojure
clojure-gem
https://github.com/technomancy/clojure-gem
clojure-http-client
https://github.com/technomancy/clojure-http-client
clojurewest2012-slides
https://github.com/technomancy/clojurewest2012-slides
clucy
https://github.com/technomancy/clucy
commodore-night-vision
https://github.com/technomancy/commodore-night-vision
concourse
https://github.com/technomancy/concourse
conspire
https://github.com/technomancy/conspire
cooper
https://github.com/technomancy/cooper
corkscrew
https://github.com/technomancy/corkscrew
die-roboter
https://github.com/technomancy/die-roboter
diginet
https://github.com/technomancy/diginet
drainbot
https://github.com/technomancy/drainbot
durendal
https://github.com/technomancy/durendal
edgestow
https://github.com/technomancy/edgestow
el-get
https://github.com/technomancy/el-get
emacs-starter-kit
https://github.com/technomancy/emacs-starter-kit
sevenwire
active_presenter
https://github.com/sevenwire/active_presenter
alphadecimal
https://github.com/sevenwire/alphadecimal
best_in_place
https://github.com/sevenwire/best_in_place
bootstrapper
https://github.com/sevenwire/bootstrapper
db_branch
https://github.com/sevenwire/db_branch
ezcrypto
https://github.com/sevenwire/ezcrypto
forgery
https://github.com/sevenwire/forgery
gotogether.co
https://github.com/sevenwire/gotogether.co
in_place_editing
https://github.com/sevenwire/in_place_editing
multipass
https://github.com/sevenwire/multipass
sambal
https://github.com/sevenwire/sambal
sevenwire-rubocop
https://github.com/sevenwire/sevenwire-rubocop
sevenwire.com
https://github.com/sevenwire/sevenwire.com
userstamp
https://github.com/sevenwire/userstamp
technicalpickles
4tehlulz.biz
https://github.com/technicalpickles/4tehlulz.biz
ack.vim
https://github.com/technicalpickles/ack.vim
acnh
https://github.com/technicalpickles/acnh
adopt-a-garden
https://github.com/technicalpickles/adopt-a-garden
ale
https://github.com/technicalpickles/ale
amazon_associate
https://github.com/technicalpickles/amazon_associate
ambitious-sphinx
https://github.com/technicalpickles/ambitious-sphinx
appraisal
https://github.com/technicalpickles/appraisal
authlogic_facebook_connect
https://github.com/technicalpickles/authlogic_facebook_connect
autochronic
https://github.com/technicalpickles/autochronic
BabyParse
https://github.com/technicalpickles/BabyParse
bcb4-attendees
https://github.com/technicalpickles/bcb4-attendees
bento
https://github.com/technicalpickles/bento
blueprint-layout
https://github.com/technicalpickles/blueprint-layout
bootstrap-generators
https://github.com/technicalpickles/bootstrap-generators
bootstrapper
https://github.com/technicalpickles/bootstrapper
bootswatch
https://github.com/technicalpickles/bootswatch
bostonrb
https://github.com/technicalpickles/bostonrb
botkit-helper-slack
https://github.com/technicalpickles/botkit-helper-slack
bridge_troll
https://github.com/technicalpickles/bridge_troll
brigade
https://github.com/technicalpickles/brigade
bringing-vim-to-the-people
https://github.com/technicalpickles/bringing-vim-to-the-people
bundler
https://github.com/technicalpickles/bundler
calais
https://github.com/technicalpickles/calais
call_for_proposals_2013
https://github.com/technicalpickles/call_for_proposals_2013
can_flag
https://github.com/technicalpickles/can_flag
capistrano
https://github.com/technicalpickles/capistrano
capistrano-benchmark
https://github.com/technicalpickles/capistrano-benchmark
capistrano-campfire
https://github.com/technicalpickles/capistrano-campfire
capistrano-gitflow
https://github.com/technicalpickles/capistrano-gitflow
lazyatom
.doom.d
https://github.com/lazyatom/.doom.d
a2_printer
https://github.com/lazyatom/a2_printer
activejob-lock
https://github.com/lazyatom/activejob-lock
acts_as_hasselhoff
https://github.com/lazyatom/acts_as_hasselhoff
aemeredith.com
https://github.com/lazyatom/aemeredith.com
backchat
https://github.com/lazyatom/backchat
boomerang-rails
https://github.com/lazyatom/boomerang-rails
cmmid.github.io
https://github.com/lazyatom/cmmid.github.io
contacts
https://github.com/lazyatom/contacts
docker-ruby-2.0
https://github.com/lazyatom/docker-ruby-2.0
docker-ruby-build
https://github.com/lazyatom/docker-ruby-build
docker-test-rails-app
https://github.com/lazyatom/docker-test-rails-app
docker-ubuntu
https://github.com/lazyatom/docker-ubuntu
dockerfiles
https://github.com/lazyatom/dockerfiles
dokku
https://github.com/lazyatom/dokku
dokku-buildpack-jekyll3-nginx
https://github.com/lazyatom/dokku-buildpack-jekyll3-nginx
dokku-chrome
https://github.com/lazyatom/dokku-chrome
doom-emacs
https://github.com/lazyatom/doom-emacs
dotfiles
https://github.com/lazyatom/dotfiles
engines
https://github.com/lazyatom/engines
freeagent-widget
https://github.com/lazyatom/freeagent-widget
gem-this
https://github.com/lazyatom/gem-this
ghost-harmonia-theme
https://github.com/lazyatom/ghost-harmonia-theme
gitnode
https://github.com/lazyatom/gitnode
golf-scores
https://github.com/lazyatom/golf-scores
harmonia-graphs
https://github.com/lazyatom/harmonia-graphs
hashblue.app
https://github.com/lazyatom/hashblue.app
heroku-buildpack-apt
https://github.com/lazyatom/heroku-buildpack-apt
homebrew-cask
https://github.com/lazyatom/homebrew-cask
hostess
https://github.com/lazyatom/hostess
josh
Aware
https://github.com/josh/Aware
brew
https://github.com/josh/brew
certbot-tickerd
https://github.com/josh/certbot-tickerd
css-explain
https://github.com/josh/css-explain
csv2json
https://github.com/josh/csv2json
displayrcd
https://github.com/josh/displayrcd
docker-ondemand
https://github.com/josh/docker-ondemand
dotfiles
https://github.com/josh/dotfiles
emacs.d
https://github.com/josh/emacs.d
google-domains-ddns
https://github.com/josh/google-domains-ddns
hktwiliocallbox
https://github.com/josh/hktwiliocallbox
homebrew-core
https://github.com/josh/homebrew-core
homebrew-tap
https://github.com/josh/homebrew-tap
icloud-backup-utils
https://github.com/josh/icloud-backup-utils
imdb-trakt-sync
https://github.com/josh/imdb-trakt-sync
itunes-trakt
https://github.com/josh/itunes-trakt
josh
https://github.com/josh/josh
jquery-selector-set
https://github.com/josh/jquery-selector-set
launchdns
https://github.com/josh/launchdns
list-unsubscribe
https://github.com/josh/list-unsubscribe
nack
https://github.com/josh/nack
offlineimap-gmail
https://github.com/josh/offlineimap-gmail
overcast-sonos
https://github.com/josh/overcast-sonos
pdftotext
https://github.com/josh/pdftotext
python-tickerd
https://github.com/josh/python-tickerd
rack-openid
https://github.com/josh/rack-openid
rails
https://github.com/josh/rails
rails-behaviors
https://github.com/josh/rails-behaviors
scroll-anchoring
https://github.com/josh/scroll-anchoring
selector-observer
https://github.com/josh/selector-observer
cdcarter
artfully_ose
https://github.com/cdcarter/artfully_ose
backpacking
https://github.com/cdcarter/backpacking
bitstarter
https://github.com/cdcarter/bitstarter
bootstrap
https://github.com/cdcarter/bootstrap
bootstrap-visualforce-101
https://github.com/cdcarter/bootstrap-visualforce-101
cdcarter.github.com
https://github.com/cdcarter/cdcarter.github.com
chicago-atlas
https://github.com/cdcarter/chicago-atlas
chill
https://github.com/cdcarter/chill
commonjs-date-formatting
https://github.com/cdcarter/commonjs-date-formatting
cookiecutter-django
https://github.com/cdcarter/cookiecutter-django
cookiecutter-django-rest
https://github.com/cdcarter/cookiecutter-django-rest
couchapp
https://github.com/cdcarter/couchapp
crowdbuilder
https://github.com/cdcarter/crowdbuilder
CSV-GitHub-import-export
https://github.com/cdcarter/CSV-GitHub-import-export
Cumulus
https://github.com/cdcarter/Cumulus
CumulusCI
https://github.com/cdcarter/CumulusCI
CumulusCI-OAuth-Tool
https://github.com/cdcarter/CumulusCI-OAuth-Tool
CumulusCI-Test
https://github.com/cdcarter/CumulusCI-Test
declarative-autonaming
https://github.com/cdcarter/declarative-autonaming
declarative-lookup-rollup-summaries
https://github.com/cdcarter/declarative-lookup-rollup-summaries
design-system
https://github.com/cdcarter/design-system
design-system-starter-kit
https://github.com/cdcarter/design-system-starter-kit
dino
https://github.com/cdcarter/dino
dj-request-correlation
https://github.com/cdcarter/dj-request-correlation
django-admin2
https://github.com/cdcarter/django-admin2
django-allauth
https://github.com/cdcarter/django-allauth
django-log-request-id
https://github.com/cdcarter/django-log-request-id
django-object-actions
https://github.com/cdcarter/django-object-actions
django-request-logging
https://github.com/cdcarter/django-request-logging
django-rest-framework
https://github.com/cdcarter/django-rest-framework
bs
2021-usa-roadtrip
https://github.com/bs/2021-usa-roadtrip
AlarmServer
https://github.com/bs/AlarmServer
alfred-github-workflow
https://github.com/bs/alfred-github-workflow
alfred-pinboard-rs
https://github.com/bs/alfred-pinboard-rs
awesome-go
https://github.com/bs/awesome-go
browser-laptop
https://github.com/bs/browser-laptop
cocktail-party-problem
https://github.com/bs/cocktail-party-problem
dendron
https://github.com/bs/dendron
dendron-template
https://github.com/bs/dendron-template
dotfiles
https://github.com/bs/dotfiles
forkmysite
https://github.com/bs/forkmysite
from_zero_to_DSP
https://github.com/bs/from_zero_to_DSP
Glass-Dome
https://github.com/bs/Glass-Dome
hello-github-actions
https://github.com/bs/hello-github-actions
homebridge-nest
https://github.com/bs/homebridge-nest
jekyll-compose
https://github.com/bs/jekyll-compose
jschesstest
https://github.com/bs/jschesstest
machine-learning-for-software-engineers
https://github.com/bs/machine-learning-for-software-engineers
markdown-portfolio
https://github.com/bs/markdown-portfolio
my-mac-os
https://github.com/bs/my-mac-os
oss-enterprise
https://github.com/bs/oss-enterprise
p5.js-sound
https://github.com/bs/p5.js-sound
pleasejobme
https://github.com/bs/pleasejobme
shields
https://github.com/bs/shields
signal-muon
https://github.com/bs/signal-muon
SmartThingsPublic
https://github.com/bs/SmartThingsPublic
starling
https://github.com/bs/starling
TiddlyWiki5
https://github.com/bs/TiddlyWiki5
vscode
https://github.com/bs/vscode
wikish
https://github.com/bs/wikish
collectiveidea
actionview-fields_for_with_template
https://github.com/collectiveidea/actionview-fields_for_with_template
action_mailer_optional_tls
https://github.com/collectiveidea/action_mailer_optional_tls
active_merchant
https://github.com/collectiveidea/active_merchant
active_resource_throttle
https://github.com/collectiveidea/active_resource_throttle
acts-as-taggable-on
https://github.com/collectiveidea/acts-as-taggable-on
acts_as_audited
https://github.com/collectiveidea/acts_as_audited
acts_as_geocodable
https://github.com/collectiveidea/acts_as_geocodable
acts_as_money
https://github.com/collectiveidea/acts_as_money
adapter-simpledb
https://github.com/collectiveidea/adapter-simpledb
address_book
https://github.com/collectiveidea/address_book
apnotic
https://github.com/collectiveidea/apnotic
ARPlaneDetector
https://github.com/collectiveidea/ARPlaneDetector
ARtistry
https://github.com/collectiveidea/ARtistry
attribute_fu
https://github.com/collectiveidea/attribute_fu
audit-demo
https://github.com/collectiveidea/audit-demo
audited
https://github.com/collectiveidea/audited
awesome-backup
https://github.com/collectiveidea/awesome-backup
awesomeness
https://github.com/collectiveidea/awesomeness
awesome_nested_set
https://github.com/collectiveidea/awesome_nested_set
bender
https://github.com/collectiveidea/bender
BTExample
https://github.com/collectiveidea/BTExample
build-test
https://github.com/collectiveidea/build-test
buildlight
https://github.com/collectiveidea/buildlight
caldav
https://github.com/collectiveidea/caldav
calendar_builder
https://github.com/collectiveidea/calendar_builder
capybara-ui
https://github.com/collectiveidea/capybara-ui
chat
https://github.com/collectiveidea/chat
clear_empty_attributes
https://github.com/collectiveidea/clear_empty_attributes
cms
https://github.com/collectiveidea/cms
coderay
https://github.com/collectiveidea/coderay

Rest Api_____no_output_____

df=[]
names=[]
response=requests.get(' https://api.github.com/repositories',headers=header)
data=response.json()
print(len(data))

for i in data:
    info=[]

    owner_name=i['owner']['login']
    if owner_name not in names:
        names.append(owner_name)
        # print(owner_name)
        url_owner_repos='https://api.github.com/users/'+owner_name+'/repos'
        list_repo=requests.get(url_owner_repos,headers=header)
        repo_list=list_repo.json()

        for file in repo_list:
            contri=0
            repo=file['full_name']
            # r = requests.get()

            repo_name, forks, tag, language, about, watchers, popularity, licens, issue, url = map(file.get, ['full_name','forks','topics', 'language','description','watchers','stargazers_count','license','open_issues','clone_url'])

            contri_url='https://api.github.com/repos/'+repo_name+'/contributors'
            forks_url=requests.get(contri_url, headers=header)
            contributors=0
            try:
                contributors=len(forks_url.json())
                for c in forks_url.json():
                    contri += c['contributions']
                df.append([repo_name, popularity, forks, watchers, issue, tag, about, contributors, licens['key'], url, language])
            except:
                df.append([repo_name, popularity, forks, watchers, issue, tag, about, contributors, licens, url, language])100
mojombo
wycats
rubinius
vanpelt
defunkt
technoweenie
caged
anotherjesse
macournoyer
jamesgolick
KirinDave
bmizerany
jnewland
ruby-git
ezmobius
uggedal
mmower
abhay
benburkert
sr
queso
drnic
danwrong
wayneeseguin
mattetti
grempe
peterc
hornbeck
up_the_irons
atmos
brosner
jnicklas
cristibalan
simonjefford
leahneukirchen
engineyard
kevinclark
technomancy
sevenwire
technicalpickles
lazyatom
josh
cdcarter
bs
collectiveidea
api_data = pd.DataFrame(df, columns=['repo_name','star','fork','watch','issue','tags','description','contributers','license','repo_url','most_used_lang'])_____no_output_____##getting the name of repo
api_data['repo_name'] = api_data['repo_name'].apply(lambda x: x.split('/')[-1]) _____no_output_____# Saving the scraped data into github_api.csv file
api_data.to_csv('github_api.csv')_____no_output_____


# Notebook from jabrusce/DC-Public-Schools
Path: 02_notebooks/03_Modeling/c_regression_models_2018-2019.ipynb

# Regression Models (Star Rating 2018-2019)_____no_output_____

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns

from sklearn.ensemble import RandomForestRegressor, BaggingRegressor, AdaBoostRegressor
from sklearn.feature_selection import RFE
from sklearn.linear_model import LinearRegression, Lasso, Ridge
from sklearn.metrics import mean_squared_error, r2_score
from sklearn.model_selection import train_test_split, GridSearchCV
from sklearn.neighbors import KNeighborsRegressor
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler, PolynomialFeatures
from sklearn.svm import SVR
from sklearn.tree import DecisionTreeRegressor
from category_encoders import OneHotEncoder _____no_output_____data = pd.read_csv('../../01_data/cleaned_data/school_df_v6.csv')_____no_output_____data.info()<class 'pandas.core.frame.DataFrame'>
RangeIndex: 844 entries, 0 to 843
Data columns (total 56 columns):
 #   Column                       Non-Null Count  Dtype
---  ------                       --------------  -----
 0   code                         109 non-null    float64
 1   name                         109 non-null    object
 2   grade_band                   109 non-null    object
 3   enrollment_SY1718            109 non-null    float64
 4   enrollment_SY1819            109 non-null    float64
 5   star_score_SY1718            109 non-null    float64
 6   star_score_SY1819            109 non-null    float64
 7   star_rating_SY1718           109 non-null    float64
 8   star_rating_SY1819           109 non-null    float64
 9   capacity_SY1718              109 non-null    float64
 10  capacity_SY1819              109 non-null    float64
 11  latitude                     109 non-null    float64
 12  longitude                    109 non-null    float64
 13  cluster                      109 non-null    float64
 14  ward                         109 non-null    float64
 15  count_any_SY1819             109 non-null    float64
 16  count_0_SY1819               109 non-null    float64
 17  pct_0_SY1819                 109 non-null    float64
 18  count_1-5_SY1819             107 non-null    float64
 19  pct_1-5_SY1819               109 non-null    float64
 20  count_6-10_SY1819            107 non-null    float64
 21  pct_6-10_SY1819              109 non-null    float64
 22  count_11-20_SY1819           107 non-null    float64
 23  pct_11-20_SY1819             109 non-null    float64
 24  count_20+_SY1819             108 non-null    float64
 25  pct_20+_SY1819               109 non-null    float64
 26  count_any_SY1718             109 non-null    float64
 27  count_0_SY1718               109 non-null    float64
 28  pct_0_SY1718                 109 non-null    float64
 29  count_1-5_SY1718             107 non-null    float64
 30  pct_1-5_SY1718               109 non-null    float64
 31  count_6-10_SY1718            107 non-null    float64
 32  pct_6-10__SY1718             109 non-null    float64
 33  count_11-20_SY1718           107 non-null    float64
 34  pct_11-20_SY1718             109 non-null    float64
 35  count_20+_SY1718             107 non-null    float64
 36  pct_20+_SY1718               109 non-null    float64
 37  budgeted_amount_FY16         109 non-null    float64
 38  budgeted_enrollment_FY16     109 non-null    float64
 39  budgeted_amount_FY17         109 non-null    float64
 40  budgeted_enrollment_FY17     109 non-null    float64
 41  pct_meet_exceed_math_SY1718  109 non-null    float64
 42  pct_meet_exceed_ela_SY1718   109 non-null    float64
 43  pct_meet_exceed_math_SY1819  109 non-null    float64
 44  pct_meet_exceed_ela_SY1819   109 non-null    float64
 45  Unnamed: 45                  0 non-null      float64
 46  Unnamed: 46                  0 non-null      float64
 47  Unnamed: 47                  0 non-null      float64
 48  Unnamed: 48                  0 non-null      float64
 49  Unnamed: 49                  0 non-null      float64
 50  Unnamed: 50                  0 non-null      float64
 51  Unnamed: 51                  0 non-null      float64
 52  Unnamed: 52                  0 non-null      float64
 53  Unnamed: 53                  0 non-null      float64
 54  Unnamed: 54                  0 non-null      float64
 55  Unnamed: 55                  0 non-null      float64
dtypes: float64(54), object(2)
memory usage: 369.4+ KB
# drop unnamed columns and "bad import" full-nan rows
data = data.iloc[:,:45]
data.dropna(how='all', inplace=True)_____no_output_____# drop 2 schools that have no attendance data
data.drop(data[data['code'] == 201].index, inplace = True) # Oyster Adams Bilingual School (Adams) has no attendance data
data.drop(data[data['code'] == 347].index, inplace = True) # Brookland Middle School has no attendance data_____no_output_____# drop columns looking at "count" of absences (non-relatable across schools of different sizes)
X = data.drop(columns = data.filter(regex='^count',axis=1)).drop(columns='name')_____no_output_____# drop school_code
X.drop('code', axis = 1, inplace = True)_____no_output_____

## Tailor DF to SY 1819 data_____no_output_____

# drop columns looking at school year 17-18 and FY 16
X = X.drop(columns = X.filter(regex='SY1718$',axis=1)).drop(columns = data.filter(regex='FY16$',axis=1))_____no_output_____# OHE grade_band column
X_ohe = OneHotEncoder(cols=['grade_band']).fit_transform(X)c:\programdata\miniconda3\lib\site-packages\category_encoders\utils.py:21: FutureWarning: is_categorical is deprecated and will be removed in a future version.  Use is_categorical_dtype instead
  elif pd.api.types.is_categorical(cols):

# Create no_nan_df
### Only looks at 82 rows
* Need a better solution for dealing with np.nan (originally cast as -1)_____no_output_____

no_nan_df = X_ohe.replace(-1,np.nan)_____no_output_____no_nan_df.dropna(how='any',inplace=True)_____no_output_____no_nan_y = no_nan_df['star_rating_SY1819']
no_nan_X = no_nan_df.drop(columns=['star_score_SY1819','star_rating_SY1819'])_____no_output_____X_train, X_test, y_train, y_test = train_test_split(no_nan_X, no_nan_y)_____no_output_____

# Regression Models_____no_output_____

def print_results(grid, X_train, X_test, y_train, y_test):
    #print(f'Estimator : {grid.get_estimator_}')

    print('*** TRAIN set ***')
    print(f'Score : {grid.score(X_train, y_train)}')
    print(f'RMSE: {mean_squared_error(y_train, grid.predict(X_train), squared = False)}')
    print(f'MSE: {mean_squared_error(y_train, grid.predict(X_train))}')
    print()
    print('*** TEST set ***')
    print(f'Score : {grid.score(X_test, y_test)}')
    print(f'RMSE: {mean_squared_error(y_test, grid.predict(X_test), squared = False)}')
    print(f'MSE: {mean_squared_error(y_test, grid.predict(X_test))}')
    print()
    print(f'Best Params : {grid.best_params_}')

    y_pred = grid.predict(X_test)
    fig, ax = plt.subplots(figsize=(4, 4))
    plt.scatter(y_test, y_pred)
    ax.plot([0,1], [0,1], transform=ax.transAxes, ls="--", c="orange")
    #plt.title('')
    plt.xlabel('Star Rating')
    plt.ylabel('Model Predictions');_____no_output_____

_____no_output_____### Multiple Linear Regression Model_____no_output_____

pipe = Pipeline([('poly', PolynomialFeatures()), ('scaler', StandardScaler()), ('lr', LinearRegression())])
param = [
    {'lr__normalize': ['True','False']}
]
grid = GridSearchCV(pipe, param, n_jobs = -1)
grid.fit(X_train, y_train)
print_results(grid, X_train, X_test, y_train, y_test)*** TRAIN set ***
Score : 1.0
RMSE: 6.705366596777549e-15
MSE: 4.496194119718013e-29

*** TEST set ***
Score : -7.921706688930369
RMSE: 3.2386008310654866
MSE: 10.48853534297806

Best Params : {'lr__normalize': 'True'}

_____no_output_____### Ridge Model_____no_output_____

pipe1 = Pipeline([('poly', PolynomialFeatures()), ('scaler', StandardScaler()), ('ridge', Ridge())])
param1 = [
    {'ridge__alpha': [.01, .1, 1, 10, 100]}
]
grid1 = GridSearchCV(pipe1, param1, n_jobs = -1)
grid1.fit(X_train, y_train)
print_results(grid1, X_train, X_test, y_train, y_test)*** TRAIN set ***
Score : 0.7566822959099055
RMSE: 0.5862364964869728
MSE: 0.34367322981332055

*** TEST set ***
Score : 0.43889302287304444
RMSE: 0.812187473250503
MSE: 0.6596484917050366

Best Params : {'ridge__alpha': 100}

_____no_output_____### KNeighbors Model_____no_output_____

pipe2 = Pipeline([('poly', PolynomialFeatures()), ('scaler', StandardScaler()), ('knearest', KNeighborsRegressor())])
param2 = [
    {'knearest__n_neighbors': [3,7,11,25,35]}
]
grid2 = GridSearchCV(pipe2, param2, n_jobs = -1)
grid2.fit(X_train, y_train)
print_results(grid2, X_train, X_test, y_train, y_test)*** TRAIN set ***
Score : 0.6219030377723986
RMSE: 0.7307815787633797
MSE: 0.5340417158598977

*** TEST set ***
Score : 0.4069340150183737
RMSE: 0.834997086962981
MSE: 0.6972201352366642

Best Params : {'knearest__n_neighbors': 11}

_____no_output_____### Decision Tree Model_____no_output_____

pipe3 = Pipeline([('poly', PolynomialFeatures()), ('scaler', StandardScaler()), ('dtree', DecisionTreeRegressor())])
param3 = [
    {'dtree__max_depth':[3,5,7,9],
    'dtree__min_samples_split':[3,5,7,9],
    'dtree__min_samples_leaf':[3,5,7,9],
    'dtree__ccp_alpha':[.01, .1, 1, 10, 100]}
]
grid3 = GridSearchCV(pipe3, param3, n_jobs = -1)
grid3.fit(X_train, y_train)
print_results(grid3, X_train, X_test, y_train, y_test)*** TRAIN set ***
Score : 0.8304138423118088
RMSE: 0.48941941941733036
MSE: 0.2395313681027967

*** TEST set ***
Score : 0.5059064781936989
RMSE: 0.7621457501276072
MSE: 0.580866144437573

Best Params : {'dtree__ccp_alpha': 0.01, 'dtree__max_depth': 3, 'dtree__min_samples_leaf': 9, 'dtree__min_samples_split': 5}

_____no_output_____### Bagged Decision Tree Model_____no_output_____

pipe4 = Pipeline([('poly', PolynomialFeatures()), ('scaler', StandardScaler()), ('btree', BaggingRegressor())])
param4 = [
    {'btree__n_estimators':[50, 100, 500]}
]
grid4 = GridSearchCV(pipe4, param4, n_jobs = -1)
grid4.fit(X_train, y_train)
print_results(grid4, X_train, X_test, y_train, y_test)*** TRAIN set ***
Score : 0.9509032886193365
RMSE: 0.2633371910507063
MSE: 0.06934647619047618

*** TEST set ***
Score : 0.6223831845342708
RMSE: 0.6662835869071203
MSE: 0.4439338181818181

Best Params : {'btree__n_estimators': 500}

_____no_output_____### Random Forest Model_____no_output_____

pipe5 = Pipeline([('poly', PolynomialFeatures()), ('scaler', StandardScaler()), ('randforest', RandomForestRegressor())])
param5 = [
    {'randforest__max_depth':[3,5,7,9],
    'randforest__min_samples_split':[3,5,7,9],
    'randforest__min_samples_leaf':[3,5,7,9],
    'randforest__n_estimators':[50, 100, 500]}
]
grid5 = GridSearchCV(pipe5, param5, n_jobs = -1)
grid5.fit(X_train, y_train)
print_results(grid5, X_train, X_test, y_train, y_test)*** TRAIN set ***
Score : 0.9147940059215071
RMSE: 0.3469133965596169
MSE: 0.12034890471253001

*** TEST set ***
Score : 0.5844144678502763
RMSE: 0.6989782504585065
MSE: 0.4885705946140347

Best Params : {'randforest__max_depth': 7, 'randforest__min_samples_leaf': 3, 'randforest__min_samples_split': 3, 'randforest__n_estimators': 50}

_____no_output_____### Adaboost Model_____no_output_____

pipe6 = Pipeline([('poly', PolynomialFeatures()), ('scaler', StandardScaler()), ('ada', AdaBoostRegressor())])
param6 = [
    {'ada__n_estimators':[50, 100, 500]}
]
grid6 = GridSearchCV(pipe6, param6, n_jobs = -1)
grid6.fit(X_train, y_train)
print_results(grid6, X_train, X_test, y_train, y_test)*** TRAIN set ***
Score : 0.9930523556129573
RMSE: 0.09906147445032458
MSE: 0.00981317572027231

*** TEST set ***
Score : 0.5192283003308753
RMSE: 0.751801001661091
MSE: 0.5652047460986197

Best Params : {'ada__n_estimators': 50}

_____no_output_____### Support Vector Regressor Model_____no_output_____#### Was only able to get a very limited SVM to ever execute fully._____no_output_____

pipe7 = Pipeline([('poly', PolynomialFeatures()), ('scaler', StandardScaler()), ('svr', SVR())])
param7 = [
    {'svr__kernel': ['linear'],
     'svr__C': [.01, 1, 10],
     'svr__gamma': np.logspace(-2, 2, 20)
    }
#     {'svr__kernel': ['linear', 'poly'],
#      'svr__C': [.01, 1, 100],
#      'svr__gamma': np.logspace(-5, 2, 20)
#     }
]
grid7 = GridSearchCV(pipe7, param7, n_jobs = -1)
grid7.fit(X_train, y_train)
print_results(grid7, X_train, X_test, y_train, y_test)*** TRAIN set ***
Score : 0.7683970167008943
RMSE: 0.5719500099671585
MSE: 0.3271268139014327

*** TEST set ***
Score : 0.37206503254322354
RMSE: 0.8591931113845315
MSE: 0.7382128026506319

Best Params : {'svr__C': 0.01, 'svr__gamma': 0.01, 'svr__kernel': 'linear'}

_____no_output_____# Regression Models (Star Score 2018-2019)_____no_output_____

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns

from sklearn.ensemble import RandomForestRegressor, BaggingRegressor, AdaBoostRegressor
from sklearn.feature_selection import RFE
from sklearn.linear_model import LinearRegression, Lasso, Ridge
from sklearn.metrics import mean_squared_error, r2_score
from sklearn.model_selection import train_test_split, GridSearchCV
from sklearn.neighbors import KNeighborsRegressor
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler, PolynomialFeatures
from sklearn.svm import SVR
from sklearn.tree import DecisionTreeRegressor
from category_encoders import OneHotEncoder _____no_output_____data = pd.read_csv('../../01_data/cleaned_data/school_df_v6.csv')_____no_output_____data.info()<class 'pandas.core.frame.DataFrame'>
RangeIndex: 844 entries, 0 to 843
Data columns (total 56 columns):
 #   Column                       Non-Null Count  Dtype
---  ------                       --------------  -----
 0   code                         109 non-null    float64
 1   name                         109 non-null    object
 2   grade_band                   109 non-null    object
 3   enrollment_SY1718            109 non-null    float64
 4   enrollment_SY1819            109 non-null    float64
 5   star_score_SY1718            109 non-null    float64
 6   star_score_SY1819            109 non-null    float64
 7   star_rating_SY1718           109 non-null    float64
 8   star_rating_SY1819           109 non-null    float64
 9   capacity_SY1718              109 non-null    float64
 10  capacity_SY1819              109 non-null    float64
 11  latitude                     109 non-null    float64
 12  longitude                    109 non-null    float64
 13  cluster                      109 non-null    float64
 14  ward                         109 non-null    float64
 15  count_any_SY1819             109 non-null    float64
 16  count_0_SY1819               109 non-null    float64
 17  pct_0_SY1819                 109 non-null    float64
 18  count_1-5_SY1819             107 non-null    float64
 19  pct_1-5_SY1819               109 non-null    float64
 20  count_6-10_SY1819            107 non-null    float64
 21  pct_6-10_SY1819              109 non-null    float64
 22  count_11-20_SY1819           107 non-null    float64
 23  pct_11-20_SY1819             109 non-null    float64
 24  count_20+_SY1819             108 non-null    float64
 25  pct_20+_SY1819               109 non-null    float64
 26  count_any_SY1718             109 non-null    float64
 27  count_0_SY1718               109 non-null    float64
 28  pct_0_SY1718                 109 non-null    float64
 29  count_1-5_SY1718             107 non-null    float64
 30  pct_1-5_SY1718               109 non-null    float64
 31  count_6-10_SY1718            107 non-null    float64
 32  pct_6-10__SY1718             109 non-null    float64
 33  count_11-20_SY1718           107 non-null    float64
 34  pct_11-20_SY1718             109 non-null    float64
 35  count_20+_SY1718             107 non-null    float64
 36  pct_20+_SY1718               109 non-null    float64
 37  budgeted_amount_FY16         109 non-null    float64
 38  budgeted_enrollment_FY16     109 non-null    float64
 39  budgeted_amount_FY17         109 non-null    float64
 40  budgeted_enrollment_FY17     109 non-null    float64
 41  pct_meet_exceed_math_SY1718  109 non-null    float64
 42  pct_meet_exceed_ela_SY1718   109 non-null    float64
 43  pct_meet_exceed_math_SY1819  109 non-null    float64
 44  pct_meet_exceed_ela_SY1819   109 non-null    float64
 45  Unnamed: 45                  0 non-null      float64
 46  Unnamed: 46                  0 non-null      float64
 47  Unnamed: 47                  0 non-null      float64
 48  Unnamed: 48                  0 non-null      float64
 49  Unnamed: 49                  0 non-null      float64
 50  Unnamed: 50                  0 non-null      float64
 51  Unnamed: 51                  0 non-null      float64
 52  Unnamed: 52                  0 non-null      float64
 53  Unnamed: 53                  0 non-null      float64
 54  Unnamed: 54                  0 non-null      float64
 55  Unnamed: 55                  0 non-null      float64
dtypes: float64(54), object(2)
memory usage: 369.4+ KB
# drop unnamed columns and "bad import" full-nan rows
data = data.iloc[:,:45]
data.dropna(how='all', inplace=True)_____no_output_____# drop 2 schools that have no attendance data
data.drop(data[data['code'] == 201].index, inplace = True) # Oyster Adams Bilingual School (Adams) has no attendance data
data.drop(data[data['code'] == 347].index, inplace = True) # Brookland Middle School has no attendance data_____no_output_____# drop columns looking at "count" of absences (non-relatable across schools of different sizes)
X = data.drop(columns = data.filter(regex='^count',axis=1)).drop(columns='name')_____no_output_____# drop school_code
X.drop('code', axis = 1, inplace = True)_____no_output_____

## Tailor DF to SY 1819 data_____no_output_____

# drop columns looking at school year 17-18 and FY 16
X = X.drop(columns = X.filter(regex='SY1718$',axis=1)).drop(columns = data.filter(regex='FY16$',axis=1))_____no_output_____# OHE grade_band column
X_ohe = OneHotEncoder(cols=['grade_band']).fit_transform(X)c:\programdata\miniconda3\lib\site-packages\category_encoders\utils.py:21: FutureWarning: is_categorical is deprecated and will be removed in a future version.  Use is_categorical_dtype instead
  elif pd.api.types.is_categorical(cols):

# Create no_nan_df
### Only looks at 82 rows
* Need a better solution for dealing with np.nan (originally cast as -1)_____no_output_____

no_nan_df = X_ohe.replace(-1,np.nan)_____no_output_____no_nan_df.dropna(how='any',inplace=True)_____no_output_____no_nan_y = no_nan_df['star_score_SY1819']
no_nan_X = no_nan_df.drop(columns=['star_score_SY1819','star_rating_SY1819'])_____no_output_____X_train, X_test, y_train, y_test = train_test_split(no_nan_X, no_nan_y)_____no_output_____

# Regression Models_____no_output_____

def print_results(grid, X_train, X_test, y_train, y_test):
    #print(f'Estimator : {grid.get_estimator_}')

    print('*** TRAIN set ***')
    print(f'Score : {grid.score(X_train, y_train)}')
    print(f'RMSE: {mean_squared_error(y_train, grid.predict(X_train), squared = False)}')
    print(f'MSE: {mean_squared_error(y_train, grid.predict(X_train))}')
    print()
    print('*** TEST set ***')
    print(f'Score : {grid.score(X_test, y_test)}')
    print(f'RMSE: {mean_squared_error(y_test, grid.predict(X_test), squared = False)}')
    print(f'MSE: {mean_squared_error(y_test, grid.predict(X_test))}')
    print()
    print(f'Best Params : {grid.best_params_}')

    y_pred = grid.predict(X_test)
    fig, ax = plt.subplots(figsize=(4, 4))
    plt.scatter(y_test, y_pred)
    ax.plot([0,1], [0,1], transform=ax.transAxes, ls="--", c="orange")
    #plt.title('')
    plt.xlabel('Star Rating')
    plt.ylabel('Model Predictions');_____no_output_____

_____no_output_____### Multiple Linear Regression Model_____no_output_____

pipe = Pipeline([('poly', PolynomialFeatures()), ('scaler', StandardScaler()), ('lr', LinearRegression())])
param = [
    {'lr__normalize': ['True','False']}
]
grid = GridSearchCV(pipe, param, n_jobs = -1)
grid.fit(X_train, y_train)
print_results(grid, X_train, X_test, y_train, y_test)*** TRAIN set ***
Score : 1.0
RMSE: 6.047119569474466e-14
MSE: 3.6567655087521046e-27

*** TEST set ***
Score : -19.71083704458932
RMSE: 113.69463450940489
MSE: 12926.469916227163

Best Params : {'lr__normalize': 'True'}

_____no_output_____### Ridge Model_____no_output_____

pipe1 = Pipeline([('poly', PolynomialFeatures()), ('scaler', StandardScaler()), ('ridge', Ridge())])
param1 = [
    {'ridge__alpha': [.01, .1, 1, 10, 100]}
]
grid1 = GridSearchCV(pipe1, param1, n_jobs = -1)
grid1.fit(X_train, y_train)
print_results(grid1, X_train, X_test, y_train, y_test)*** TRAIN set ***
Score : 0.7930741844827779
RMSE: 9.842967178695208
MSE: 96.8840028808711

*** TEST set ***
Score : 0.6049134744616741
RMSE: 15.703167140660412
MSE: 246.58945824751692

Best Params : {'ridge__alpha': 100}

_____no_output_____### KNeighbors Model_____no_output_____

pipe2 = Pipeline([('poly', PolynomialFeatures()), ('scaler', StandardScaler()), ('knearest', KNeighborsRegressor())])
param2 = [
    {'knearest__n_neighbors': [3,7,11,25,35]}
]
grid2 = GridSearchCV(pipe2, param2, n_jobs = -1)
grid2.fit(X_train, y_train)
print_results(grid2, X_train, X_test, y_train, y_test)*** TRAIN set ***
Score : 0.6003521013479312
RMSE: 13.679098162935544
MSE: 187.11772655122655

*** TEST set ***
Score : 0.5957546609473147
RMSE: 15.884138116024921
MSE: 252.30584368895572

Best Params : {'knearest__n_neighbors': 11}

_____no_output_____### Decision Tree Model_____no_output_____

pipe3 = Pipeline([('poly', PolynomialFeatures()), ('scaler', StandardScaler()), ('dtree', DecisionTreeRegressor())])
param3 = [
    {'dtree__max_depth':[3,5,7,9],
    'dtree__min_samples_split':[3,5,7,9],
    'dtree__min_samples_leaf':[3,5,7,9],
    'dtree__ccp_alpha':[.01, .1, 1, 10, 100]}
]
grid3 = GridSearchCV(pipe3, param3, n_jobs = -1)
grid3.fit(X_train, y_train)
print_results(grid3, X_train, X_test, y_train, y_test)*** TRAIN set ***
Score : 0.667709540172579
RMSE: 12.473192806473772
MSE: 155.58053878746907

*** TEST set ***
Score : 0.5705587440929484
RMSE: 16.371671663826376
MSE: 268.03163306813553

Best Params : {'dtree__ccp_alpha': 100, 'dtree__max_depth': 3, 'dtree__min_samples_leaf': 3, 'dtree__min_samples_split': 5}

_____no_output_____### Bagged Decision Tree Model_____no_output_____

pipe4 = Pipeline([('poly', PolynomialFeatures()), ('scaler', StandardScaler()), ('btree', BaggingRegressor())])
param4 = [
    {'btree__n_estimators':[50, 100, 500]}
]
grid4 = GridSearchCV(pipe4, param4, n_jobs = -1)
grid4.fit(X_train, y_train)
print_results(grid4, X_train, X_test, y_train, y_test)*** TRAIN set ***
Score : 0.9544891037293212
RMSE: 4.616112594342857
MSE: 21.30849548365074

*** TEST set ***
Score : 0.7107895682172081
RMSE: 13.435323284857573
MSE: 180.5079117686361

Best Params : {'btree__n_estimators': 100}

_____no_output_____### Random Forest Model_____no_output_____

pipe5 = Pipeline([('poly', PolynomialFeatures()), ('scaler', StandardScaler()), ('randforest', RandomForestRegressor())])
param5 = [
    {'randforest__max_depth':[3,5,7,9],
    'randforest__min_samples_split':[3,5,7,9],
    'randforest__min_samples_leaf':[3,5,7,9],
    'randforest__n_estimators':[50, 100, 500]}
]
grid5 = GridSearchCV(pipe5, param5, n_jobs = -1)
grid5.fit(X_train, y_train)
print_results(grid5, X_train, X_test, y_train, y_test)*** TRAIN set ***
Score : 0.8564360368342397
RMSE: 8.198632477561112
MSE: 67.21757450211986

*** TEST set ***
Score : 0.7578313700767675
RMSE: 12.294194694899735
MSE: 151.14722319610078

Best Params : {'randforest__max_depth': 7, 'randforest__min_samples_leaf': 7, 'randforest__min_samples_split': 3, 'randforest__n_estimators': 50}

_____no_output_____### Adaboost Model_____no_output_____

pipe6 = Pipeline([('poly', PolynomialFeatures()), ('scaler', StandardScaler()), ('ada', AdaBoostRegressor())])
param6 = [
    {'ada__n_estimators':[50, 100, 500]}
]
grid6 = GridSearchCV(pipe6, param6, n_jobs = -1)
grid6.fit(X_train, y_train)
print_results(grid6, X_train, X_test, y_train, y_test)*** TRAIN set ***
Score : 0.9690578776094588
RMSE: 3.8062187912577
MSE: 14.48730148692323

*** TEST set ***
Score : 0.7360782526851932
RMSE: 12.834493460004019
MSE: 164.72422237488595

Best Params : {'ada__n_estimators': 500}

_____no_output_____### Support Vector Regressor Model_____no_output_____#### Was only able to get a very limited SVM to ever execute fully._____no_output_____

pipe7 = Pipeline([('poly', PolynomialFeatures()), ('scaler', StandardScaler()), ('svr', SVR())])
param7 = [
    {'svr__kernel': ['linear'],
     'svr__C': [.01, 1, 10],
     'svr__gamma': np.logspace(-2, 2, 20)
    }
#     {'svr__kernel': ['linear', 'poly'],
#      'svr__C': [.01, 1, 100],
#      'svr__gamma': np.logspace(-5, 2, 20)
#     }
]
grid7 = GridSearchCV(pipe7, param7, n_jobs = -1)
grid7.fit(X_train, y_train)
print_results(grid7, X_train, X_test, y_train, y_test)*** TRAIN set ***
Score : 0.6326903142610244
RMSE: 13.113991248568135
MSE: 171.97676646752166

*** TEST set ***
Score : 0.5820837807701998
RMSE: 16.1504920104094
MSE: 260.83839217829785

Best Params : {'svr__C': 0.01, 'svr__gamma': 0.01, 'svr__kernel': 'linear'}


# Notebook from temuller/kaggle_competitions
Path: Cat_in_the_dat_II/cat_in_the_dat_ii.ipynb

# **Categorical Feature Encoding Challenge II (or Cat in the dat II)**
**Binary classification, with every feature a categorical (and interactions!)**

I read lots of discussions from the Discussion section, so this is not all mine. I took ideas from too may different people to actually put all the references here, but if you check the discussion section, you will be able to tell where I took the ideas or pieces of code from. Except for the neural network part which I did it myself with what I have learnt from previous competitions I have tried and from what I have read on the Internet._____no_output_____

import numpy as np
import matplotlib.pyplot as plt
import pandas as pd

# visualization
import seaborn as sns
%config InlineBackend.figure_format = 'retina' #set 'png' here when working on notebook
sns.set_context("talk")
sns.set_palette('husl')
sns.set()

from sklearn.impute import SimpleImputer
from sklearn.preprocessing import LabelEncoder, OneHotEncoder_____no_output_____test = pd.read_csv('test.csv')
train = pd.read_csv('train.csv')

all_data = pd.concat((train.drop('target', axis=1), test))
all_data.drop('id', axis=1, inplace=True)

all_data.head(3)_____no_output_____all_data.info()<class 'pandas.core.frame.DataFrame'>
Int64Index: 1000000 entries, 0 to 399999
Data columns (total 24 columns):
id       1000000 non-null int64
bin_0    970205 non-null float64
bin_1    969959 non-null float64
bin_2    970098 non-null float64
bin_3    970035 non-null object
bin_4    970002 non-null object
nom_0    969686 non-null object
nom_1    969897 non-null object
nom_2    969786 non-null object
nom_3    969703 non-null object
nom_4    969972 non-null object
nom_5    970310 non-null object
nom_6    969857 non-null object
nom_7    969994 non-null object
nom_8    970289 non-null object
nom_9    969867 non-null object
ord_0    969819 non-null float64
ord_1    969792 non-null object
ord_2    969820 non-null object
ord_3    970031 non-null object
ord_4    970137 non-null object
ord_5    970240 non-null object
day      970023 non-null float64
month    970028 non-null float64
dtypes: float64(6), int64(1), object(17)
memory usage: 190.7+ MB
y = train['target']
sns.distplot(y, kde=False);_____no_output_____

## Pre-Analysis

We have a million column and lots of missing data. However, the amount is still low compare to the total number of rows. We could drop all the rows with missing data, but if we do this, we drop have of our dataset, so we would need to use imputers.

The targets are inbalanced, so we will need to deal with this, but let's skip it for now._____no_output_____

numeric_index = all_data.dtypes[all_data.dtypes != "object"].index
categ_index = all_data.dtypes[all_data.dtypes == "object"].index_____no_output_____sns.pairplot(train[numeric_index].loc[:100]);_____no_output_____for col in all_data.columns:
    print(col, all_data[col].unique())id [     0      1      2 ... 999997 999998 999999]
bin_0 [ 0.  1. nan]
bin_1 [ 0.  1. nan]
bin_2 [ 0.  1. nan]
bin_3 ['F' 'T' nan]
bin_4 ['N' 'Y' nan]
nom_0 ['Red' 'Blue' 'Green' nan]
nom_1 ['Trapezoid' 'Star' nan 'Circle' 'Triangle' 'Polygon' 'Square']
nom_2 ['Hamster' 'Axolotl' 'Lion' 'Dog' 'Cat' 'Snake' nan]
nom_3 ['Russia' nan 'Canada' 'Finland' 'Costa Rica' 'China' 'India']
nom_4 ['Bassoon' 'Theremin' nan 'Oboe' 'Piano']
nom_5 ['de4c57ee2' '2bb3c3e5c' 'b574c9841' ... 'b3ad70fcb' 'd6bb2181a'
 '30019ce8a']
nom_6 ['a64bc7ddf' '3a3a936e8' '708248125' ... 'a93b49b8c' 'd80c01ec8'
 'a885aacec']
nom_7 ['598080a91' '1dddb8473' '5ddc9a726' '3a33ef960' 'bc9cc2a94' '8c30b9b0b'
 '1cba571fa' '81d67e1bb' '429114096' '56d35c774' '93360bbc9' '088aace2d'
 '12370b989' 'ba9faf5b1' 'fdebf0648' '90a2cba07' '5de0b3fa7' 'e5b00a0e0'
 '1d2ee04de' nan '69403e18c' 'cf84cbe77' 'fba315672' '6e29d3f38'
 'c3ad85934' '6b1d5295a' 'a306b1f4e' '432ee0053' '9a9e7bf71' 'b2d0012ea'
 '9d52e9089' 'bb63fcef7' 'dd4d69481' 'b0ab8b134' '2c649f60e' '42f607baa'
 '7e1c4cff1' '482233c7f' '7f31f4d2b' '11409e362' '4ae48e857' '3f63ecb96'
 '057ab40ea' '4b13d99ca' '69238b544' '5622ee17f' '1c756c04a' 'afac5238e'
 'e4340c326' '018382adf' '8dbc78da5' '3c415c2b7' 'e61c53286' '83bdea3a5'
 '25eb75323' '1eb783b97' '964fd0850' '99cb99a50' 'c79d2197d' '3efdc7987'
 'ffe61f9f6' '768bc4738' 'debb0f7ec' '51e1131b4' '83c994142' 'f74f0b894'
 '68b2de491' '3afd3697d' '28547ea20' 'f14f5e4a5' '41901460a' '942593b74'
 '623908296' '31a3c86df' '9b35d8994' 'd40ca0718' '16fa03dee' 'e60e65034'
 '20b56638c' '5e17197af' '8bc849561' '40bdb3373' 'b4678a59e' '0ffdb2c15'
 '545ad7c80' '1bd74ace9' 'a7059911d' '8a382f14d' 'e59dd3e27' '833d88c6c'
 '0764632bc' 'a8cfb0795' '1ba4e8ff1' '3599d0255' '00a7f9a05' '8402256f6'
 '74a864a6b' '0e4d3318a' '567934479' '857cbf8c7' 'ead914edc' '8157480dd'
 '6c7aa4dc4' '27560c3a3' '934155e05' '9294ac66b' '216083343' '362507ea5'
 '3d449d3e8' 'ba16219f4' '8fc780921' '199d01a7f' '67a8d4ebb' '9d4a94d03'
 '3d9d8f526' '0eb5e9781' '580231f65' '3ef3018d3' '3600c6e91' '0da2378c8'
 '7fcdbac1d' '5627228e9' '4617418f2' 'c5f7d2efd' '97bb3e7fa' '91208fa55'
 '728490578' '7b4b2eabf' '8c9b654fe' 'b97010abc' '909943002' '51a4ac0be'
 'a0730843f' 'c9aaf28b7' 'adbdefba3' '7684d9eb7' '6b56b4325' '86ec768cd'
 '6bc753dd6' '0304a1b60' '5d1ac7760' '82d196aa1' '63ab45432' 'cd9feb5c6'
 '8781919c2' '688888fbe' '2786e24d2' '21955ee26' 'cb681246b' 'fff8dea99'
 '2aeb8ae84' 'f313d1e52' '2b2108e9e' 'd2ecbd100' '8a528cb08' '303f4e39a'
 'ece4087ec' 'e06abf51f' '7f6fad56e' '6536122f4' '84b44fe42' '94d34d287'
 '7bf938847' '230229e51' 'cb47f665d' '01103c3e4' '2c4f9614e' '443cdbd48'
 'b4d7d7c94' 'd7f06353a' '1bf2f7824' 'b4eb50264' 'f12d5e8c0' '85fb6971c'
 'd8c3cfd78' '8d857a0a1' '4be936718' '4f0a50007' '22a1b223f' '2506ce93d'
 '114b1dbf3' '41afb892d' 'b2151e2cd' '7b21ee165' '6602f7bf5' '190522cfb'
 'a42fb9969' '42c7572b4' '082e2b9ae' '237fdf58b' '8da514edb' '81b479cc3'
 'ac365406b' 'b519f5dff' '4b41ba86c' '54f62d537' '77706a5ee' '0c208bfa0'
 'ec4b54637' 'd3d8f55e0' '3f2e87b23' 'ab85a7b9c' '2fa636740' 'f94d9babd'
 '5d5b262b1' '75d0e3ef8' 'e371063e2' '9289b1aab' '58a27d7e7' '49ae7bdc0'
 '9fed600dc' '7ab87e58e' 'e3342b01a' '16e07c05c' '3d2d50707' 'c273b5ace'
 '1828818ab' 'deec583dd' '3065f26f4' 'e44e2dea4' 'e9c57c4aa' '8046bea1a'
 'b39008216']
nom_8 ['0256c7a4b' '52ead350c' '745b909d1' 'bdaa56dd1' nan '690411ac0'
 'b8e63cace' 'bd9643a20' '94c5fd40c' '0279391c5' '28869f8aa' '74a092964'
 '5bdcf7324' 'e6873698e' 'a1d71618d' 'd48e8b7af' '587840371' 'dda9ca69e'
 '165e81a00' 'e0c7234cf' 'd8e02864a' '574575c14' '23dc2e63d' '25c087d05'
 'ca9ad1d4b' '484c96b82' 'd0ab86574' '53efabb84' '1df855856' '63dde2492'
 'a8009baf5' '679b63eff' 'd1c6bb70d' '60c7c84ee' '45121db5e' 'd8bad19d0'
 '79c79ecbf' 'bccfd92a2' '5e348d538' 'dcccc6c19' '43e0ac7a1' '7c77c52f3'
 '210039295' 'b279e09c3' '95414620e' '10fd52d10' 'c9bbbb717' '4164322bd'
 'ee0a32458' 'a5eef5d8b' 'abce980f5' '5a3040de6' '7e7ddd679' '7ff3ee36e'
 '5859a8a06' '90e9e961e' '8ce034bb6' 'f7b3f1d13' '9dd6a02eb' 'b4815ac52'
 '9789d425f' '7a66737e5' '94a6512e2' 'd6c8281a8' '56dbcd7de' '1ce5cf721'
 'b33edbeb2' '9fa5832d8' '777ce63ea' '77d41330d' 'da7e5f6fa' 'f635e7f7f'
 '28c042258' '153864851' '4918ca307' '9f6ed3f30' 'b72c0bbc2' '154aacc1c'
 '060a21580' 'fe070240a' '954bb9489' 'fa49d4e04' 'ef752c107' '9fcf4fab4'
 '158183c63' '32412c81c' '573c92006' '529b40fe8' '9863487ba' '590d9c11d'
 'de6d6e3d8' '8109ba950' '46a005ddf' '0f94eb834' 'afb0fafb0' '8dc88fa15'
 'cb0d7da76' '2218d9dfe' '7d7c02c57' '7f06ae9e8' '15f03b1f4' '48982830e'
 'a625ea144' '9417fa5d7' 'd7e75499d' '43a73e5ef' '2db5a9486' '4d31e3bf2'
 'd5a47d599' '22e0a764d' 'd8729dd2d' '1d88b0a79' 'e9fde8fa8' '0f269cbc8'
 '1e722bf45' '6f853fd13' 'f51683d93' '91b32f31b' '32b33a4b4' 'af22df36a'
 '750cb6661' '964a66013' 'e6061cbe7' 'cccbca824' '862d2fbff' '8d13bd3e9'
 'a89e1c1dc' 'cf30695f0' '32ff72b97' 'b42763706' 'd9148f8e7' '53419bedc'
 'ff3ba3bbc' '8aeac2495' '3d70f650d' '2061350b9' '5c1c7fa53' '67d372885'
 '0602256df' 'fbbafaadc' '4abb8589e' 'ad5be5be4' '729cafb68' '704871e3b'
 'fca8ae0cf' 'b10ef207b' 'dd674992f' '4f727aeee' '5a7ade619' 'f81a5366b'
 '6d3db6c57' '31db9b004' 'ceab144aa' '798ccea46' '6ff91d4e8' '7d7394cbd'
 '75a957c6d' '7e3a861d6' '6ab87efb1' '3776bb1ae' 'ee3c7aa8f' '2b34f5aca'
 '8e38a5c1b' '44f8a6061' 'e40c451d4' 'c2036219b' '1fe15a999' '007bff22a'
 '0e5249dbe' 'd06a44f23' '1a59581be' '7861df0a8' 'cf233e69e' 'ee474d72d'
 '1f2cd223e' '8df93e6c9' '220190c9e' '634c43a77' '35309e01b' 'a408ea8d6'
 'cadc0cc42' '2694748e3' 'f41dce3cc' 'f8510b161' '1c4931f19' 'fbea9bb4a'
 '475f992f0' '4a5423b60' '607c26084' '7c89e4b36' '6eb21874b' '7217196aa'
 '2c58293c4' 'f338368af' '3fdf92cf7' '3210cd7c1' '88eef0278' 'cc5495ab3'
 '9c63b4d79' '3e8a577e3' '83fb0260e' 'c1a4acfaf' '1653ceb33' '99bb7aedf'
 '563651ed8' '68f9b1267' '9416a52f9' '1b6796452' '8637fc534' '9cc561d1b'
 '3d0ea8194' '5f254ada1' '1195f603e' 'e4a142d80' '21595d82e' '8d31d1ab3'
 '29964d969' '4584d6fcd' '82cc2cb4f' 'e11794ad3' '4348381e1' '6492aecc3'
 '115d9fd8b']
nom_9 ['02e7c8990' 'f37df64af' nan ... 'd1e6704ed' 'b5f9881ba' '1538d82e9']
ord_0 [ 3.  1.  2. nan]
ord_1 ['Contributor' 'Grandmaster' nan 'Novice' 'Expert' 'Master']
ord_2 ['Hot' 'Warm' 'Freezing' 'Lava Hot' 'Cold' 'Boiling Hot' nan]
ord_3 ['c' 'e' 'n' 'a' 'h' 'b' 'd' 'k' 'g' 'o' 'i' 'm' nan 'f' 'l' 'j']
ord_4 ['U' 'X' 'P' 'C' 'Q' 'R' 'Y' 'N' 'I' 'O' 'M' 'E' 'V' 'K' 'G' 'B' 'H' nan
 'T' 'W' 'A' 'F' 'D' 'S' 'J' 'L' 'Z']
ord_5 ['Pw' 'pE' 'eN' nan 'OZ' 'wa' 'rg' 'PS' 'mX' 'hG' 'xF' 'RV' 'Nh' 'dp' 'SS'
 'OM' 'Rv' 'HF' 'iS' 'HK' 'Ib' 'AW' 'mi' 'jf' 'Tz' 'ok' 'nS' 'Sz' 'sf'
 'Fl' 'fV' 'iv' 'US' 'dh' 'Io' 'RD' 'oI' 'DN' 'IS' 'oh' 'YJ' 'vq' 'us'
 'sY' 'th' 'Ey' 'AI' 'Mg' 'oU' 'Dj' 'TH' 'CN' 'aA' 'tT' 'Wr' 'TZ' 'mo'
 'rM' 'Pk' 'UV' 'RT' 'DT' 'Hk' 'Dn' 'nj' 'gL' 'cy' 'JT' 'XC' 'lA' 'TL'
 'zp' 'XM' 'AG' 'WW' 'GZ' 'MV' 'Bx' 'oJ' 'ze' 'Rj' 'LY' 'Iq' 'gc' 'LS'
 'hx' 'JQ' 'fO' 'vx' 'MX' 'ly' 'BX' 'Yr' 'XR' 'yE' 'Sk' 'aE' 'wU' 'rA'
 'Ji' 'qN' 'nn' 'kv' 'pB' 'sc' 'EC' 'kT' 'be' 'uQ' 'DI' 'hT' 'pl' 'tn'
 'xG' 'NT' 'fe' 'lS' 'Mq' 'Yi' 'BL' 'AU' 'ht' 'Ro' 'al' 'mD' 'ur' 'DR'
 'Re' 'XI' 'sF' 'Rz' 'Lo' 'In' 'Kq' 'cY' 'uZ' 'rl' 'zc' 'XU' 'CU' 'lR'
 'xB' 'pT' 'gj' 'Ox' 'yK' 'vw' 'Ay' 'HO' 'TP' 'ep' 'ne' 'Oe' 'mP' 'cR'
 'IA' 'QV' 'Ty' 'qP' 'gK' 'wJ' 'cg' 'zf' 'WC' 'NS' 'Yb' 'kB' 'MU' 'kP'
 'FH' 'fF' 'jp' 'gt' 'Qm' 'Cn' 'pZ' 'nf' 'ja' 'Tg' 'uP' 'uW' 'FB' 'uI'
 'MF' 'RB' 'SL' 'PG' 'vQ' 'eA' 'gV' 'Zv']
day [ 6.  7.  5.  3.  1.  2. nan  4.]
month [ 3.  7.  9. 12.  4.  6.  1.  8.  5. 11. nan 10.  2.]

We can see that numerical columns have discrete values, so no transformation is needed. We will also need to impute the data with mode as the mean will make no sense in this case._____no_output_____

all_data = pd.concat((train.drop('target', axis=1), test))
all_data.drop('id', axis=1, inplace=True)_____no_output_____# impute data (in a more eficient way than using sklearn imputers as this is a big dataset)
for col in all_data.columns:
    all_data[col].fillna(all_data[col].mode()[0], inplace=True)_____no_output_____# deal with cyclical data
all_data['day_sin'] = np.sin(all_data.day*(2.*np.pi/24))
all_data['day_cos'] = np.cos(all_data.day*(2.*np.pi/24))
all_data['month_sin'] = np.sin((all_data.month-1)*(2.*np.pi/12))
all_data['month_cos'] = np.cos((all_data.month-1)*(2.*np.pi/12))
all_data.drop(['day', 'month'], axis=1, inplace=True)_____no_output_____

Check [this](https://www.kaggle.com/shahules/an-overview-of-encoding-techniques) for an encoding overview._____no_output_____

# Encoders
dtransf = {"ord_1":{'Novice':0, 'Contributor':1, 'Expert':2, 'Master':3, 'Grandmaster':4},
           "ord_2":{'Freezing':0, 'Cold':1, 'Warm':2, 'Hot':3, 'Boiling Hot':4, 'Lava Hot':5}
          }
for col in dtransf.keys():
    all_data[col] = [dtransf[col][cat_feat] for cat_feat in all_data[col]]

ohe = True
if ohe:
    # One-Hot Encoder
    encoder = OneHotEncoder()
    all_data = encoder.fit_transform(all_data)

else:
    # Label Encoder
    encoder = LabelEncoder()
    all_data = all_data.apply(encoder.fit_transform)_____no_output_____X_train = all_data[:train.shape[0]].tocsr()
X_test = all_data[train.shape[0]:].tocsr()
y = train['target']_____no_output_____

___
## Regression_____no_output_____

from sklearn.linear_model import LogisticRegression, LogisticRegressionCV, Ridge, RidgeCV, Lasso, LassoCV, ElasticNet, ElasticNetCV
from sklearn.metrics import roc_auc_score
from sklearn.model_selection import cross_val_score, KFold_____no_output_____model = LogisticRegression(solver='liblinear', C=0.1, max_iter=100)

kf = KFold(n_splits=5)
fold_splits = kf.split(X_train, y)
cv_scores = []

for i, (tr_index, val_index) in enumerate(fold_splits):
    print(f'K-fold number {i+1}:')

    tr_X, val_X = X_train[tr_index], X_train[val_index]  # tocsr() from above is used to handle indexes here
    tr_y, val_y = y[tr_index], y[val_index]

    model.fit(tr_X, tr_y)
    pred_val_y = model.predict_proba(val_X)[:,1]
    cv_score = roc_auc_score(val_y, pred_val_y)
    cv_scores.append(cv_score)
    print(f'CV score: {cv_score:.3f}\n')

cv_scores = np.array(cv_scores)
print(f'Final CV score: {cv_scores.mean():.3f} +/- {cv_scores.std():.3f}')K-fold number 1:
CV score: 0.784

K-fold number 2:
CV score: 0.785

K-fold number 3:
CV score: 0.785

K-fold number 4:
CV score: 0.786

K-fold number 5:
CV score: 0.784

Final CV score: 0.785 +/- 0.001
predictions = model.predict(X_test)_____no_output_____submission_dict = {'id': test.id.values,
                   'target': predictions}
submission = pd.DataFrame.from_dict(submission_dict)
submission.to_csv("submission.csv", index=False)_____no_output_____

____
## **Neural Networks**_____no_output_____

from keras.models import Sequential
from keras.layers import Dense, Dropout, Conv2D, MaxPool2D, Flatten
from keras.utils import to_categorical_____no_output_____test = pd.read_csv('test.csv')
train = pd.read_csv('train.csv')

all_data = pd.concat((train.drop('target', axis=1), test))
all_data.drop('id', axis=1, inplace=True)_____no_output_____# impute data (in a more eficient way than using sklearn imputers as this is a big dataset)
for col in all_data.columns:
    all_data[col].fillna(all_data[col].mode()[0], inplace=True)_____no_output_____# deal with cyclical data
all_data['day_sin'] = np.sin(all_data.day*(2.*np.pi/24))
all_data['day_cos'] = np.cos(all_data.day*(2.*np.pi/24))
all_data['month_sin'] = np.sin((all_data.month-1)*(2.*np.pi/12))
all_data['month_cos'] = np.cos((all_data.month-1)*(2.*np.pi/12))
all_data.drop(['day', 'month'], axis=1, inplace=True)_____no_output_____# Encoders
dtransf = {"ord_1":{'Novice':0, 'Contributor':1, 'Expert':2, 'Master':3, 'Grandmaster':4},
           "ord_2":{'Freezing':0, 'Cold':1, 'Warm':2, 'Hot':3, 'Boiling Hot':4, 'Lava Hot':5}
          }
for col in dtransf.keys():
    all_data[col] = [dtransf[col][cat_feat] for cat_feat in all_data[col]]

ohe = True
if ohe:
    # One-Hot Encoder
    encoder = OneHotEncoder()
    all_data = encoder.fit_transform(all_data)

else:
    # Label Encoder
    encoder = LabelEncoder()
    all_data = all_data.apply(encoder.fit_transform)_____no_output_____X_train = all_data[:train.shape[0]].tocsr()
X_test = all_data[train.shape[0]:].tocsr()
y = train['target']_____no_output_____model = Sequential()

n_cols = X_train.shape[1]
target = to_categorical(y)

model.add(Dense(5, activation='relu', input_shape=(n_cols, )))
model.add(Dropout(0.2))
model.add(Dense(5, activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(2, activation='softmax'))  _____no_output_____model.compile(optimizer='adam',
              loss='categorical_crossentropy',
              metrics=['accuracy']) _____no_output_____history = model.fit(X_train, target,
                    epochs=5,
                    batch_size=1000,
                    validation_split=0.2,
                    workers=6,
                    use_multiprocessing=True)Train on 480000 samples, validate on 120000 samples
Epoch 1/5
480000/480000 [==============================] - 33s 68us/step - loss: 0.4127 - accuracy: 0.8165 - val_loss: 0.4021 - val_accuracy: 0.8193
Epoch 2/5
480000/480000 [==============================] - 34s 72us/step - loss: 0.4071 - accuracy: 0.8205 - val_loss: 0.4014 - val_accuracy: 0.8216
Epoch 3/5
480000/480000 [==============================] - 32s 67us/step - loss: 0.4043 - accuracy: 0.8220 - val_loss: 0.4013 - val_accuracy: 0.8216
Epoch 4/5
480000/480000 [==============================] - 34s 70us/step - loss: 0.4028 - accuracy: 0.8234 - val_loss: 0.4023 - val_accuracy: 0.8215
Epoch 5/5
480000/480000 [==============================] - 34s 70us/step - loss: 0.4013 - accuracy: 0.8236 - val_loss: 0.4016 - val_accuracy: 0.8220
# set up figure
f = plt.figure(figsize=(12,5))
f.add_subplot(1,2, 1)

# plot accuracy as a function of epoch
plt.plot(history.history['accuracy'])
plt.plot(history.history['val_accuracy'])
plt.title('model accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.legend(['training', 'validation'], loc='best')

# plot loss as a function of epoch
f.add_subplot(1,2, 2)
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.title('model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['training', 'validation'], loc='best')
plt.show(block=True)_____no_output_____predictions = model.predict(X_test)_____no_output_____submission_dict = {'id': test.id.values,
                   'target': predictions[:, 1]}
submission = pd.DataFrame.from_dict(submission_dict)
submission.to_csv("submission.csv", index=False)_____no_output_____


# Notebook from erteich/wikinet
Path: preparation/simulate-models.ipynb

# Initialization_____no_output_____

%reload_ext autoreload
%autoreload 2
%reload_ext cython
%reload_ext line_profiler
import os,sys
sys.path.insert(1, os.path.join(sys.path[0], '..', 'module'))
import wiki
import numpy as np
import pandas as pd
import networkx as nx
import scipy as sp
import seaborn as sns
import cufflinks as cf
import matplotlib.pyplot as plt_____no_output_____topics = ['anatomy', 'biochemistry', 'cognitive science', 'evolutionary biology',
          'genetics', 'immunology', 'molecular biology', 'chemistry', 'biophysics',
          'energy', 'optics', 'earth science', 'geology', 'meteorology',
          'philosophy of language', 'philosophy of law', 'philosophy of mind',
          'philosophy of science', 'economics', 'accounting', 'education',
          'linguistics', 'law', 'psychology', 'sociology', 'electronics',
          'software engineering', 'robotics',
          'calculus', 'geometry', 'abstract algebra',
          'Boolean algebra', 'commutative algebra', 'group theory', 'linear algebra',
          'number theory', 'dynamical systems and differential equations']_____no_output_____topics = ['biochemistry']_____no_output_____path_saved = '/Users/harangju/Developer/data/wiki/graphs/dated/'_____no_output_____networks = {}
for topic in topics:
    print(topic, end=' ')
    networks[topic] = wiki.Net()
    networks[topic].load_graph(path_saved + topic + '.pickle')_____no_output_____import pickle
dct = pickle.load(open('/Users/harangju/Developer/data/wiki/models/' + 'dict.model','rb'))_____no_output_____

# Auxiliary methods_____no_output_____

%%cython -f

import numpy as np
cimport numpy as np
from cython cimport floating,boundscheck,wraparound
from cython.parallel import prange

from libc.math cimport fabs

np.import_array()

@boundscheck(False)  # Deactivate bounds checking
@wraparound(False)
def cython_manhattan(floating[::1] X_data, int[:] X_indices, int[:] X_indptr,
                     floating[::1] Y_data, int[:] Y_indices, int[:] Y_indptr,
                     double[:, ::1] D):
    """Pairwise L1 distances for CSR matrices.
    Usage:
    >>> D = np.zeros(X.shape[0], Y.shape[0])
    >>> cython_manhattan(X.data, X.indices, X.indptr,
    ...                  Y.data, Y.indices, Y.indptr,
    ...                  D)
    """
    cdef np.npy_intp px, py, i, j, ix, iy
    cdef double d = 0.0

    cdef int m = D.shape[0]
    cdef int n = D.shape[1]

    with nogil:
        for px in prange(m):
            for py in range(n):
                i = X_indptr[px]
                j = Y_indptr[py]
                d = 0.0
                while i < X_indptr[px+1] and j < Y_indptr[py+1]:
                    if i < X_indptr[px+1]: ix = X_indices[i]
                    if j < Y_indptr[py+1]: iy = Y_indices[j]

                    if ix==iy:
                        d = d+fabs(X_data[i]-Y_data[j])
                        i = i+1
                        j = j+1

                    elif ix<iy:
                        d = d+fabs(X_data[i])
                        i = i+1
                    else:
                        d = d+fabs(Y_data[j])
                        j = j+1

                if i== X_indptr[px+1]:
                    while j < Y_indptr[py+1]:
                        iy = Y_indices[j]
                        d = d+fabs(Y_data[j])
                        j = j+1
                else:
                    while i < X_indptr[px+1]:
                        ix = X_indices[i]
                        d = d+fabs(X_data[i])
                        i = i+1

                D[px,py] = d_____no_output_____import sklearn.preprocessing as skp
import sklearn.metrics.pairwise as smp

def year_diffs(graph):
    return [graph.nodes[node]['year'] - graph.nodes[neighbor]['year']
            for node in graph.nodes
            for neighbor in list(graph.successors(node))]

def neighbor_similarity(graph, tfidf):
    nodes = list(graph.nodes)
    return [smp.cosine_similarity(tfidf[:,nodes.index(node)].transpose(),
                                  tfidf[:,nodes.index(neighbor)].transpose())[0,0]
            for node in nodes
            for neighbor in list(graph.successors(node))]

def sparse_manhattan(X,Y=None):
    X, Y = smp.check_pairwise_arrays(X, Y)
    X = sp.sparse.csr_matrix(X, copy=False)
    Y = sp.sparse.csr_matrix(Y, copy=False)
    res = np.empty(shape=(X.shape[0],Y.shape[0]))
    cython_manhattan(X.data,X.indices,X.indptr,
                     Y.data,Y.indices,Y.indptr,
                             res)
    return res

def word_diffs(graph, tfidf):
    dists = sparse_manhattan(X=skp.binarize(tfidf).transpose())
    nodes = list(graph.nodes)
    return [dists[nodes.index(node), nodes.index(neighbor)]
            for node in nodes
            for neighbor in list(graph.successors(node))]

def sum_abs_weight_differences(graph, tfidf):
    nodes = list(graph.nodes)
    diff = []
    for node in nodes:
        for neighbor in graph.successors(node):
            v1 = tfidf[:,nodes.index(node)]
            v2 = tfidf[:,nodes.index(neighbor)]
            idx = np.concatenate([v1.indices, v2.indices])
            diff.append( np.sum(np.absolute(v1[idx]-v2[idx])) )
    return diff

def sum_weight_differences(graph, tfidf):
    nodes = list(graph.nodes)
    diff = []
    for node in nodes:
        for neighbor in graph.successors(node):
            v1 = tfidf[:,nodes.index(node)]
            v2 = tfidf[:,nodes.index(neighbor)]
            idx = np.concatenate([v1.indices, v2.indices])
            diff.append( np.sum(v1[idx]-v2[idx]) )
    return diff

def bin_distribution(data, steps=30, scale='log'):
    if scale=='log':
        bins = np.logspace(np.log10(np.min(data)), np.log10(np.max(data)), steps)
    elif scale=='linear':
        bins = np.linspace(np.min(data), np.max(data), num=steps)
    hist, edges = np.histogram(data, bins=bins)
    return hist, edges, bins

def plot_distribution(data):
    hist, edges, bins = bin_distribution(data)
#     hist_norm = hist/(bins[1:] - bins[:-1])
    fig = go.Figure()
    fig.add_trace(go.Scatter(x=bins[:-1],
                             y=hist/len(data),
                             mode='markers'))
    fig.update_layout(template='plotly_white',
                      xaxis={'type': 'log',
                             'title': 'x'},
                      yaxis={'type': 'log',
                             'title': 'P(x)'})
    fig.show()
    return fig_____no_output_____

# Priors - example_____no_output_____

import os
import numpy as np
import plotly.express as px
import plotly.graph_objects as go
import pandas as pd
path_fig = '/Users/harangju/Box Sync/Research/my papers/wikipedia/results/'
save_fig = False_____no_output_____import os

topic = 'anatomy'
path_plot = '3 model ex prior'

if not os.path.exists(os.path.join(path_fig, path_plot, topic)):
    os.mkdir(os.path.join(path_fig, path_plot, topic))_____no_output_____fig = plot_distribution(networks[topic].graph.graph['tfidf'].data)
if save_fig:
    fig.write_image(os.path.join(path_fig, path_plot, topic, 'tf_idf_distribution.pdf'))_____no_output_____yd = year_diffs(networks[topic].graph)
wd = word_diffs(networks[topic].graph, networks[topic].graph.graph['tfidf'])
a, b, fit_r, p, stderr = sp.stats.linregress(np.abs(yd), wd)
fig = go.Figure()
x = np.linspace(0, max(yd), 100)
fig.add_trace(go.Scatter(x=np.abs(yd), y=wd,
                         mode='markers',
                         marker={'size': 3},
                         name='edges'))
fig.add_trace(go.Scatter(x=x, y=np.multiply(a, x) + b,
                         name=f"y = {a:.1f} x + {b:.1f}"))
fig.update_layout(template='plotly_white',
                  title=f"{topic} (r = {fit_r:.2f}, p = {p:.1e})",
                  xaxis={'title': 'Δyear'},
                  yaxis={'title': 'manhattan distance'})
fig.show()
if save_fig:
    fig.write_image(os.path.join(path_fig, path_plot, topic, 'manhattan.pdf'))_____no_output_____sum_weight_diffs = sum_weight_differences(networks[topic].graph,
                                          networks[topic].graph.graph['tfidf'])
a, b, fit_r_sum_weight, p, stderr = sp.stats.linregress(np.abs(yd), sum_weight_diffs)
fig = go.Figure()
fig.add_trace(go.Scatter(x=np.abs(yd),
                         y=sum_weight_diffs,
                         mode='markers',
                         marker={'size': 3},
                         name='edges'))
x = np.linspace(0, max(yd), 100)
fig.add_trace(go.Scatter(x=x, y=np.multiply(a, x) + b,
                         name=f"y = {a:.1e} x + {b:.1f}\n"))
fig.update_layout(template='plotly_white',
                  title=f"{topic} (r = {fit_r_sum_weight:.2f}; p = {p:.1e})",
                  xaxis={'title': 'Δyear'},
                  yaxis={'title': 'Σ Δw_i'})
fig.show()
if save_fig:
    fig.write_image(os.path.join(path_fig, path_plot, topic, 'sum_diff_weights.pdf'))_____no_output_____

The distribution of word weights are not changing significantly across time._____no_output_____

import plotly.figure_factory as ff
a, b, fit_r_sum_weight, p, stderr = sp.stats.linregress(np.abs(yd), sum_weight_diffs)
mu_swd, std_swd = np.mean(sum_weight_diffs), np.std(sum_weight_diffs)
fig = ff.create_distplot([sum_weight_diffs], ['edges'], bin_size=1)
x = np.linspace(min(sum_weight_diffs), max(sum_weight_diffs), 100)
fig.add_trace(go.Scatter(x=x, y=sp.stats.norm.pdf(x, mu_swd, std_swd),
                         name='normal fit'))
fig.update_layout(template='plotly_white', title=topic,
                  xaxis={'title': 'Σ Δw_i'})
fig.show()
if save_fig:
    fig.write_image(os.path.join(path_fig, path_plot, topic, 'sum_diff_weights_dstr.pdf'))_____no_output_____sum_abs_weight_diffs = sum_abs_weight_differences(networks[topic].graph,
                                                  networks[topic].graph.graph['tfidf'])
a, b, fit_r_sum_abs_weight, p, stderr = sp.stats.linregress(np.abs(yd), sum_abs_weight_diffs)
fig = go.Figure()
fig.add_trace(go.Scatter(x=np.abs(yd),
                         y=sum_abs_weight_diffs,
                         mode='markers',
                         marker={'size': 3},
                         name='edges'))
x = np.linspace(0, max(yd), 100)
fig.add_trace(go.Scatter(x=x, y=np.multiply(a, x) + b,
                         name=f"y = {a:.1e} x + {b:.1f}\n"))
fig.update_layout(template='plotly_white',
                  title=f"{topic} (r = {fit_r_sum_abs_weight:.2f}; p = {p:.1e})",
                  xaxis={'title': 'Δyear'},
                  yaxis={'title': 'Σ |Δw_i|'})
fig.show()
if save_fig:
    fig.write_image(os.path.join(path_fig, path_plot, topic, 'sum_abs_diff_weights.pdf'))_____no_output_____neighbors = neighbor_similarity(networks[topic].graph, networks[topic].graph.graph['tfidf'])
mu_n, std_n = sp.stats.norm.fit(neighbors)
mu_n, std_n_____no_output_____import plotly.figure_factory as ff
fig = ff.create_distplot([neighbors], ['edges'], bin_size=.05)
x = np.linspace(min(neighbors), max(neighbors), 100)
fig.add_trace(go.Scatter(x=x, y=sp.stats.norm.pdf(x, mu_n, std_n),
                         name='normal fit'))
fig.update_layout(template='plotly_white', title=topic,
                  xaxis={'title': 'cosine similarity'})
fig.show()
if save_fig:
    fig.write_image(os.path.join(path_fig, path_plot, topic, 'cosine_distribution.pdf'))_____no_output_____from gensim.parsing.preprocessing import remove_stopwords
stoplist=set('for a of the and to in'.split())
nodes = []
words = []
graph = networks[topic].graph
tfidf = networks[topic].graph.graph['tfidf']
for i in range(tfidf.shape[1]):
    node = list(graph.nodes)[i]
    if tfidf[:,i].data.size == 0:
#         print(node, tfidf[:,i].data)
        continue
    top_words, idx = wiki.Model.find_top_words(tfidf[:,i], dct, top_n=5)
    nodes += [node]
    words += [top_words]
pd.DataFrame(data={'Node': nodes, 'Top words': words})_____no_output_____n=10
k=4
x = np.sum(np.abs(np.random.randint(0,n,(k,100000))-np.random.randint(0,n,(k,100000))), axis=0)
# np.mean(x), k*2*np.sum(np.arange(1,n)*np.flip(np.arange(1,n))) * ((1/n)**2)
np.mean(x), k * np.sum( ((1/n)**2) * np.sum(np.abs(np.array([np.arange(n)]).transpose()-np.arange(n)),
                                            axis=0) )
# k * Σ_i P(x_i) * Σ_j |x_i-x_j|_____no_output_____tfidf = networks[topic].graph.graph['tfidf']
rvs = lambda n: tfidf.data[np.random.choice(tfidf.data.size, size=n)]
emp = np.mean(np.sum(np.abs(rvs((1,100000))-rvs((1,100000))), axis=0))_____no_output_____h,e,_ = bin_distribution(tfidf.data, 100, 'linear')
p_x = h/len(tfidf.data)
x = np.array([np.average([e[:-1],e[1:]], axis=0)])
the = np.sum(p_x * np.sum(np.abs(p_x*x.transpose() - x), axis=0))
emp, the_____no_output_____

# Run models_____no_output_____## Initialize_____no_output_____

from IPython.display import display, HTML
import os
import dill
import datetime_____no_output_____# first_n_nodes = 10
n_seeds = 2
n_models = 3
# start_condition = lambda m: [n for n in m.graph_parent.nodes
#                              if m.graph_parent.nodes[n]['year'] <=\
#                                  sorted(list(nx.get_node_attributes(m.graph_parent, 'year')\
#                                                .values()))[first_n_nodes]]
year_start = 0
start_condition = lambda m: [
    n for n in m.graph_parent.nodes
    if m.graph_parent.nodes[n]['year'] <= year_start
]
end_condition = lambda m: (len(m.graph.nodes) >= len(m.graph_parent.nodes)) or \
                          (m.year > 2200)_____no_output_____base_dir = os.path.join('/','Users','harangju','Developer','data','wiki','simulations')
save_models = True
base_dir_____no_output_____

## Run models_____no_output_____

now = datetime.datetime.now().strftime('%Y%m%d_%H%M')
os.mkdir(os.path.join(base_dir, now))_____no_output______networks = {'earth science': networks['earth science']}_____no_output_____stats = pd.DataFrame()_____no_output_____for topic, network in networks.items():
    print(topic)
    print('Analyzing priors...')
    tfidf = network.graph.graph['tfidf']
    yd = year_diffs(network.graph)
    md = word_diffs(network.graph, tfidf)
    a_md, b_md, r_md, p_md, stderr = sp.stats.linregress(np.abs(yd), md)
    swd = sum_abs_weight_differences(network.graph, tfidf)
    a_swd, b_swd, r_swd, p_swd, stderr = sp.stats.linregress(np.abs(yd), swd)
    rvs = lambda n: tfidf.data[np.random.choice(tfidf.data.size, size=n)]
    mu_sawd = np.mean(np.sum(np.abs(rvs((1,100000))-rvs((1,100000))), axis=0))
    nb = neighbor_similarity(network.graph, tfidf)
    mu_nb, std_nb = sp.stats.norm.fit(nb)
    p_point, p_insert, p_delete = a_swd/mu_sawd, a_md/2, a_md/2
    new_stats = pd.DataFrame([[p_point,p_insert,p_delete,
                               a_md,b_md,r_md,p_md,
                               a_swd,b_swd,r_swd,p_swd,
                               mu_sawd,mu_nb,std_nb]],
                             columns=['p_pt', 'p_in', 'p_de',
                                      'a (man)', 'b (man)', 'r (man)', 'p (man)',
                                      'a (swd)', 'b (swd)', 'r (swd)', 'p (swd)',
                                      'mu (sawd)', 'mu (nei)', 'std (nei)'
                                     ])
    display(HTML(new_stats.to_html()))
    stats = pd.concat([stats, new_stats], ignore_index=True)
    for i in range(n_models):
        print(f"Running model {i}...")
        model = wiki.Model(graph_parent=network.graph,
                           vectors_parent=tfidf,
#                            year_start=sorted(list(nx.get_node_attributes(network.graph, 'year')\
#                                                     .values()))[first_n_nodes],
                           year_start=year_start,
                           start_nodes=start_condition,
                           n_seeds=n_seeds,
                           dct=dct,
                           point=(1, p_point),
                           insert=(1, p_insert, list(set(tfidf.indices))),
                           delete=(1, p_delete),
                           rvs=rvs,
                           create=lambda n: np.random.normal(loc=mu_nb, scale=std_nb, size=n))
        model.evolve(until=end_condition)
        if save_models:
            dill.dump(model,
                      open(os.path.join(base_dir, now, f"model_{topic}_{i}.pickle"), 'wb'))
    print('')
pickle.dump(stats, open(os.path.join(base_dir, now, 'stats.pickle'), 'wb'))_____no_output_____

# Network stats_____no_output_____

import dill_____no_output_____simulation = '20200422_1318'
base_dir = os.path.join('/','Users','harangju','Developer','data','wiki','simulations')_____no_output_____session_dir = os.path.join(base_dir, simulation)_____no_output_____filenames = sorted(os.listdir(session_dir))
model_topics = list(set([filename.split('_')[1] for filename in filenames[:-1]]))
model_paths = {topic: [os.path.join(session_dir, filename)
                       for filename in filenames[:-1]
                       if filename.split('_')[1]==topic]
               for topic in model_topics}_____no_output_____import bct
bct.__file_______no_output_____base_dir = os.path.join('/','Users','harangju','Developer','data','wiki','simulations')_____no_output_____for topic in topics:
    print(topic)
    for i, model_path in enumerate(model_paths[topic]):
        print(i, end=' ')
        model = dill.load(open(model_path, 'rb'))
        wiki.Net.assign_core_periphery(model.graph)
        wiki.Net.assign_communities(model.graph)
        print(model.graph.graph['coreness_be'], model.graph.graph['modularity'])
        dill.dump(model, open(model_path, 'wb'))
    print('')_____no_output_____