# Notebook from JMerrittSci/IdeaIronDemo
Path: JupyterNotebooks/steam_summary_stats_testing.ipynb

import requests
import shutil
import os
import io
import json
import time
import datetime
import dateutil
import re
import numpy as np
import pandas as pd
from IPython.display import clear_output
from pprint import pprint
import matplotlib.pyplot as plt

parent_folder='steam_data'

csv_folder=os.path.join(parent_folder,"bulk","csv_no_text","early_access_standardized")
csv_english_folder=os.path.join(parent_folder,"bulk","csv_english","early_access_standardized")
testing_folder=os.path.join(parent_folder,"bulk","testing")_____no_output_____with open(os.path.join(testing_folder,'still_in_early_access_FINAL'),'r') as file:
    current_ea_games=sorted(list(set([int(x) for x in file.read().split('\n')[:-1]])))
with open(os.path.join(testing_folder,'out_of_early_access_FINAL'),'r') as file:
    former_ea_games=sorted(list(set([int(x) for x in file.read().split('\n')[:-1]])))
full_games_list=sorted(list(set(current_ea_games+former_ea_games)))
with open(os.path.join(parent_folder,'official_appid_json_list'),'r') as file:
    temp_json=json.load(file)['applist']['apps']
official_app_list={}
for x in temp_json:
    official_app_list[x['appid']]=x['name']
print(len(full_games_list))4935
num_reviews={}_____no_output_____csv_folder=os.path.join(parent_folder,"bulk","csv_no_text","early_access_standardized")
out_folder=os.path.join(parent_folder,"bulk","csv_no_text","time_series_by_month")
if not os.path.exists(out_folder):
    os.makedirs(out_folder)

for i,appid in enumerate(current_ea_games,1):
    temp_time_series=[]
    df=pd.read_csv(os.path.join(csv_folder,str(appid)+'_reviews_all.csv'))
    num_reviews[int(appid)]=len(df)
    earliest_time=df['timestamp_updated'].min()
    latest_time=df['timestamp_updated'].max()
    num_months=int(np.ceil(float(latest_time-earliest_time)/(30.0*24.0*60.0*60.0)))
    for j in range(num_months):
        num_pos=float(len(df[(df['timestamp_updated']>=earliest_time+(j)*30.0*24.0*60.0*60.0) &
                    (df['timestamp_updated']<earliest_time+(j+1)*30.0*24.0*60.0*60.0) &
                      (df['voted_up']==True)]))
        num_neg=float(len(df[(df['timestamp_updated']>=earliest_time+(j)*30.0*24.0*60.0*60.0) &
                    (df['timestamp_updated']<earliest_time+(j+1)*30.0*24.0*60.0*60.0) &
                      (df['voted_up']==False)]))
        if num_neg>0 or num_pos>0:
            temp_time_series.append({'early_access':True,'month':j,'reviews':int(num_pos+num_neg),'ratio':num_pos/(num_neg+num_pos)})
    pd.DataFrame(temp_time_series).to_csv(os.path.join(out_folder,str(appid)+'_month_time_series.csv'))
    clear_output(wait=True)
    print(str(i)+"/"+str(len(current_ea_games)))2939/2939
csv_folder=os.path.join(parent_folder,"bulk","csv_no_text","early_access_standardized")
out_folder=os.path.join(parent_folder,"bulk","csv_no_text","time_series_by_month")
if not os.path.exists(out_folder):
    os.makedirs(out_folder)

early_access_exit_time={}
normal_review_start_time={}
time_midpoint={}

for i,appid in enumerate(former_ea_games,1):
    temp_time_series=[]
    df=pd.read_csv(os.path.join(csv_folder,str(appid)+'_reviews_all.csv'))
    if len(df[df['written_during_early_access']==True])>0:
        num_reviews[int(appid)]=len(df)
        early_access_exit_time[int(appid)]=df[df['written_during_early_access']==True]['timestamp_updated'].max()
        earliest_time=df['timestamp_updated'].min()
        latest_time=df['timestamp_updated'].max()
        #print(len(df[df['written_during_early_access']==True]))
        if(len(df[df['written_during_early_access']==False]))>0:
            normal_review_start_time[int(appid)]=df[df['written_during_early_access']==False]['timestamp_updated'].min()
            midpoint=(early_access_exit_time[int(appid)]+normal_review_start_time[int(appid)])/2
            num_months_after=int(np.ceil(float(latest_time-midpoint)/(30.0*24.0*60.0*60.0)))
        else:
            midpoint=early_access_exit_time[int(appid)]+60
            num_months_after=0
        time_midpoint[int(appid)]=midpoint
        #print(appid)
        #print(float(early_access_exit_time[int(appid)]))
        #print(float(normal_review_start_time[int(appid)]))
        num_months_before=int(np.ceil(float(time_midpoint[int(appid)]-earliest_time)/(30.0*24.0*60.0*60.0)))
        for j in range(num_months_before):
            num_pos=float(len(df[(df['timestamp_updated']>=midpoint-(num_months_before-j)*30.0*24.0*60.0*60.0) &
                        (df['timestamp_updated']<midpoint-(num_months_before-(j+1))*30.0*24.0*60.0*60.0) &
                          (df['voted_up']==True)]))
            num_neg=float(len(df[(df['timestamp_updated']>=midpoint-(num_months_before-j)*30.0*24.0*60.0*60.0) &
                        (df['timestamp_updated']<midpoint-(num_months_before-(j+1))*30.0*24.0*60.0*60.0) &
                          (df['voted_up']==False)]))
            if num_neg>0 or num_pos>0:
                temp_time_series.append({'early_access':True,'month':j,'reviews':int(num_pos+num_neg),'ratio':num_pos/(num_neg+num_pos)})
        for j in range(num_months_after):
            num_pos=float(len(df[(df['timestamp_updated']>=midpoint+(j)*30.0*24.0*60.0*60.0) &
                        (df['timestamp_updated']<midpoint+(j+1)*30.0*24.0*60.0*60.0) &
                          (df['voted_up']==True)]))
            num_neg=float(len(df[(df['timestamp_updated']>=midpoint+(j)*30.0*24.0*60.0*60.0) &
                        (df['timestamp_updated']<midpoint+(j+1)*30.0*24.0*60.0*60.0) &
                          (df['voted_up']==False)]))
            if num_neg>0 or num_pos>0:
                temp_time_series.append({'early_access':False,'month':num_months_before+j,'reviews':int(num_pos+num_neg),'ratio':num_pos/(num_neg+num_pos)})
        pd.DataFrame(temp_time_series).to_csv(os.path.join(out_folder,str(appid)+'_month_time_series.csv'))
    clear_output(wait=True)
    print(str(i)+"/"+str(len(former_ea_games)))1996/1996
csv_folder=os.path.join(parent_folder,"bulk","csv_no_text","early_access_standardized")
if not os.path.exists(out_folder):
    os.makedirs(out_folder)

useful_examples={}

for i,appid in enumerate(former_ea_games,1):
    temp_time_series=[]
    df=pd.read_csv(os.path.join(csv_folder,str(appid)+'_reviews_all.csv'))
    if len(df[df['written_during_early_access']==True])>0 and len(df[df['written_during_early_access']==False])>0:
            useful_examples[appid]={}
            useful_examples[appid]['num_edited_from_early_access']=df['edited_from_early_access'].sum()
            if len(df[df['edited_from_early_access']==True])>0:
                useful_examples[appid]['ratio_edited_positive']=df[df['edited_from_early_access']==True]['voted_up'].sum()/float(len(df[df['edited_from_early_access']==True]))
            else:
                useful_examples[appid]['ratio_edited_positive']=0.5
            midpoint=time_midpoint[appid]
            for j in range(12):
                num_pos=float(len(df[(df['timestamp_updated']>=midpoint+(j)*30.0*24.0*60.0*60.0) &
                            (df['timestamp_updated']<midpoint+(j+1)*30.0*24.0*60.0*60.0) &
                              (df['voted_up']==True)]))
                num_neg=float(len(df[(df['timestamp_updated']>=midpoint+(j)*30.0*24.0*60.0*60.0) &
                            (df['timestamp_updated']<midpoint+(j+1)*30.0*24.0*60.0*60.0) &
                              (df['voted_up']==False)]))
                if num_pos+num_neg>10:
                    break
            useful_examples[appid]['num_reviews_soon_after_early_access']=num_neg+num_pos
            if num_pos>0 or num_neg>0:
                useful_examples[appid]['ratio_soon_after_positive']=float(num_pos)/float(num_pos+num_neg)
            else:
                useful_examples[appid]['ratio_soon_after_positive']=0.5
    clear_output(wait=True)
    print(str(i)+"/"+str(len(former_ea_games)))
1996/1996
print(len(useful_examples.keys()))
%matplotlib inline
x=[]
y=[]
size1=[]
size2=[]
for appid in useful_examples:
    x.append(useful_examples[appid]['ratio_edited_positive'])
    y.append(useful_examples[appid]['ratio_soon_after_positive'])
    size1.append(useful_examples[appid]['num_reviews_soon_after_early_access']+1)
    size2.append(useful_examples[appid]['num_edited_from_early_access']+1)

for i in range(len(x)):
    plt.plot(x[i],y[i],'o',ms=np.log(size1[i]+1))
plt.show()
for i in range(len(x)):
    plt.plot(x[i],y[i],'o',ms=np.log(size2[i]+1))
plt.show()1930
print(len(full_games_list))
print(len(num_reviews))4935
4925

mscores=[]
dct_list=[]
for i,appid in enumerate(full_games_list,1):
    if int(appid) in num_reviews:
        dct={}
        dct['appid']=int(appid)
        dct['name']=official_app_list[int(appid)]
        dct['num_reviews']=num_reviews[int(appid)]
        dct['failed']=None
        with open(os.path.join(parent_folder,'official_appdata',str(appid)),'r') as file:
            temp_json=json.load(file)[str(appid)]['data']
        if 'achievements' in temp_json:
            dct['has_achievements']=temp_json['achievements']['total']>0
            dct['num_achievements']=temp_json['achievements']['total']
        else:
            dct['has_achievements']=False
            dct['num_achievements']=0
        if 'controller_support' in temp_json:
            dct['controller_support']=True
        else:
            dct['controller_support']=False
        if 'developers' in temp_json:
            dct['developers']=[]
            for name in temp_json['developers']:
                if len(name.strip())>0:
                    dct['developers'].append(name.strip())
        if 'developers' not in temp_json or len(dct['developers'])==0:
            dct['developers']=[str(appid)]
        if 'publishers' in temp_json:
            dct['publishers']=[]
            for name in temp_json['publishers']:
                if len(name.strip())>0:
                    dct['publishers'].append(name.strip())
        if 'publishers' not in temp_json or len(dct['publishers'])==0:
            dct['publishers']=[str(appid)]
        if 'screenshots' in temp_json:
            dct['num_screenshots']=len(temp_json['screenshots'])
        else:
            dct['num_screenshots']=0
        if 'movies' in temp_json:
            dct['num_movies']=len(temp_json['movies'])
        else:
            dct['num_movies']=0
        if 'metacritic' in temp_json:
            dct['has_metacritic']=True
            dct['metacritic_score']=temp_json['metacritic']['score']
            if(temp_json['metacritic']['score'])<72.68-9.32:
                dct['failed']=True
            else:
                dct['failed']=False
            mscores.append(temp_json['metacritic']['score'])
        else:
            dct['has_metacritic']=False
            dct['metacritic_score']=72.68
        if 'supported_languages' in temp_json:
            dct['num_languages']=len(temp_json['supported_languages'].split(','))
        else:
            dct['num_languages']=1
        if 'categories' in temp_json:
            dct['categories']=[x['description'] for x in temp_json['categories']]
        else:
            dct['categories']=[]
        if 'genres' in temp_json:
            dct['genres']=[x['description'] for x in temp_json['genres']]
        else:
            dct['genres']=[]
        if 'price_overview' not in temp_json:
            if temp_json['is_free']:
                dct['is_free']=True
            else:
                dct['is_free']=False
                dct['failed']=True
        else:
            dct['is_free']=False
        if 'platforms' in temp_json:
            dct['platforms']=0
            for platform in temp_json['platforms']:
                if temp_json['platforms'][platform]:
                    dct['platforms']+=1
        if 'platforms' not in dct or dct['platforms']==0:
            dct['platforms']=1
        if 'packages' in temp_json and len(temp_json['packages'])>1:
            dct['multiple_packages']=True
        else:
            dct['multiple_packages']=False
        if temp_json['content_descriptors']['notes'] is not None:
            dct['content_warning']=True
        else:
            dct['content_warning']=False
        if int(appid) in current_ea_games:
            dct['still_in_early_access']=True
        else:
            dct['still_in_early_access']=False
        dct_list.append(dct)
    clear_output(wait=True)
    print(str(i)+"/"+str(len(former_ea_games)))

#with open(os.path.join(parent_folder,'official_appdata',str(218980)),'r') as file:
#    temp_json=json.load(file)

#pprint(temp_json[str(218980)]['data'])
'''appid=264710
with open(os.path.join(parent_folder,'official_appdata',str(264710)),'r') as file:
    temp_json=json.load(file)[str(appid)]['data']
temp_json.pop('developers')
temp_json.pop('publishers')
temp_json.pop('controller_support')
temp_json.pop('screenshots')
temp_json.pop('achievements')
temp_json.pop('name')
temp_json.pop('package_groups')
temp_json.pop('pc_requirements')
temp_json.pop('recommendations')
temp_json.pop('release_date')
temp_json.pop('short_description')
temp_json.pop('support_info')
temp_json.pop('steam_appid')
temp_json.pop('type')
temp_json.pop('website')
temp_json.pop('mac_requirements')
temp_json.pop('linux_requirements')
temp_json.pop('header_image')
temp_json.pop('about_the_game')
temp_json.pop('background')
temp_json.pop('detailed_description')
temp_json.pop('metacritic')
temp_json.pop('movies')
temp_json.pop('supported_languages')
temp_json.pop('legal_notice')
temp_json.pop('categories')
temp_json.pop('genres')
temp_json.pop('price_overview')
temp_json.pop('is_free')
temp_json.pop('required_age')
temp_json.pop('platforms')
temp_json.pop('packages')
pprint(temp_json)''';

#print(str(len(mscores))+"/"+str(len(full_games_list)))
#print(min(mscores))
#print(np.std(mscores))
#print(np.mean(mscores))

#

#
Mature Content Description

4935/1996
temp_df=pd.DataFrame(dct_list)
temp_df[temp_df['still_in_early_access']==True].head()_____no_output_____all_categories=set()
all_genres=set()
all_developers={}
all_publishers={}

for i,row in temp_df.iterrows():
    all_categories=all_categories | set(row['categories'])
    all_genres=all_categories | set(row['genres'])
    for developer in row['developers']:
        if developer not in all_developers:
            all_developers[developer]=1
        else:
            all_developers[developer]+=1
    for publisher in row['publishers']:
        if publisher not in all_publishers:
            all_publishers[publisher]=1
        else:
            all_publishers[publisher]+=1
all_genres.remove('Early Access')
print(len(all_categories))
print(len(all_genres))
#for developer in all_developers:
#    if all_developers[developer]>1:
#        print(developer+": "+str(all_developers[developer]))29
30
Eclipse Games: 2
Carbon Games: 3
Bohemia Interactive: 3
Stardock Entertainment: 2
Croteam: 2
Larian Studios: 2
22cans: 2
Nival: 2
Little Green Men Games: 2
Pathea Games: 2
inXile Entertainment: 2
M2H: 3
Blackmill Games: 2
Klei Entertainment: 3
Kunos Simulazioni: 2
Digitalmindsoft: 2
Keen Software House: 2
Double Fine Productions: 3
Witch Beam: 2
RuneStorm: 2
Brace Yourself Games: 2
Hashbang Games: 2
KillHouse Games: 2
Blue Bottle Games: 2
Facepunch Studios: 2
Reverie World Studios: 2
Final Boss Entertainment: 2
Iteration 11: 2
Xaviant: 2
Krome Studios: 4
Krealit: 2
Elder Games: 2
PIXOWL INC.: 2
Wastelands Interactive: 2
SmashGames: 3
Midgar Studio: 2
Spicy Tails: 2
Ranmantaru Games: 3
HB Studios: 2
Osumia Games: 2
Cryptic Sea: 2
Free Lives: 3
Spilt Milk Studios Ltd: 2
Team Jolly Roger: 2
Factus Games: 2
WRF Studios: 2
Nvizzio Creations: 3
Bit Planet Games, LLC: 2
RageSquid: 2
Schell Games: 2
BadFly Interactive, a.s.: 3
Alientrap: 4
Crafty Studios: 2
AMPLITUDE Studios: 2
Bitbox Ltd.: 2
Panzer Gaming Studios: 2
Scorpius Games: 2
Daybreak Game Company: 2
Intravenous Software: 2
Numantian Games: 2
Asteristic Game Studio: 2
Black Forest Games: 2
Zachtronics: 5
Freejam: 2
BonusXP: 2
Game-Labs: 3
Milkstone Studios: 4
Fabio Ferrara: 2
NeocoreGames: 2
Mighty Rabbit Studios: 2
Section Studios: 2
Hyper Hippo Games: 4
Flat Earth Games: 2
Cherry Pop Games: 3
Rake in Grass: 2
Pixelz Games: 2
Cybertime System: 2
Ghost Machine: 3
GameArt Studio GmbH: 3
Axyos Games: 4
BlueEagle Productions: 2
James Mearman: 2
Pub Games: 2
Supra Games: 2
Fire Hose Games: 2
Bossa Studios: 2
Question: 2
ISOTX: 2
Polygon Art: 2
Copybugpaste: 2
Metaware Limited, LLC: 2
Tasharen Entertainment Inc.: 2
kChamp Games: 2
LuGus Studios: 2
Hesketh Studios Ltd: 2
Endless Loop Studios: 5
New Reality Games: 3
Composer-Motorin Dmitry (Djinotan): 2
Eforb: 2
Eggcode: 2
LucasGame: 2
ZeroBit Games: 4
Psytec Games Ltd: 2
Studio Wildcard: 2
Instinct Games: 3
Efecto Studios: 2
Virtual Basement LLC: 3
Xiotex Studios Ltd: 2
Red River Studio: 2
Alice & Smith: 3
UnexEvo: 2
Free Reign Entertainment: 3
Odin Game Studio: 2
Alexey Davydov: 2
Sergey Dvoynikov: 2
Timofey Shargorodskiy: 2
Polywickstudio: 2
Vostok Games: 2
Sebastian Nigro: 4
KaleidoGames: 2
ColloseusX: 3
Andrii Vintsevych: 3
Gamious: 2
Megastorm Games: 2
Mint Age Studios: 2
Ilex Games: 2
UNIT9: 2
Corrosive Studios LLC: 4
Orangepixel: 2
SIEIDI Ltd: 2
Kobra Studio: 4
Kilted Camel: 2
Behaviour Interactive Inc.: 2
Greendev.: 4
CPsoft: 2
Kazakov Oleg: 4
MuHa Games: 2
Skinny Jean Death Studios: 2
Strange Fire: 3
Batholith Entertainment: 3
Deceptive Games: 3
Stray Fawn Studio: 2
Saibot Studios: 2
Peach Pie Productions: 2
Triverske: 2
Retro Army Limited: 5
The Secret Games Company: 2
Dragom: 2
Catstudio: 2
Recreat4: 2
Hanged Bunny Studio: 2
PolyDigital: 2
PixelFade Inc: 2
One Man Left Studios: 2
FobTi interactive: 2
Robot Loves Kitty: 2
StormCube Games: 2
Prismic Studios LLC: 2
FutureVisionsGames: 2
Antonio Renna: 3
Dragon's Games: 2
The Tambourine: 2
Raptor Claw Games: 2
Soldak Entertainment: 2
Oreol: 2
FAT BOMB STUDIOS, LLC: 2
Evgeny Shcherbakov: 2
EfimovMax: 3
Ultima Games: 2
Advent RPG: 2
Pixel Federation: 2
Tall Tail Studios: 2
VisualTech48: 2
Lion's Shade: 2
Stained Glass Llama: 2
Night Node Software: 2
Evil Tortilla Games: 2
U-Play Online: 3
Goblinz Studio: 2
IndieGala: 2
Room710Games: 2
Pixeljam: 3
Industry Games: 2
Skobbejak Games: 2
Racing Bros: 5
Steve Gal: 3
Brian Lindenhof: 2
Tryzna83: 2
Silver Lemur Games: 2
VooFoo Studios: 2
Nomad Games: 2
davevillz: 2
Ed Farage: 2
Nuked Games: 2
Talentplace: 2
Anatoliy Loginovskikh: 2
Jaison Robson Gusava: 2
SC Jogos: 4
RocketWerkz: 2
Seize Studios: 2
Brainwashing Games: 2
Anea_Duo_Dev: 2
Pirate Software: 2
Acuze Interactives: 2
Jollyfish Games: 2
GK: 2
Animate Objects: 2
Lorenzo Games: 2
Michael L. Fiorentino Gutierrez: 2
Rootify: 3
Warfare Studios: 2
Blackturn Ltd: 2
Thylacine Studios: 2
Croteam VR: 2
Camex Games: 2
Skyjaz Games: 2
Dynamoid: 2
Games by Brundle: 2
GaelDK: 2
Fabien Spiess (Sogaki): 2
Antoine Rigitano: 2
Serge Sentis (Aerosteak): 2
Stas Shostak: 2
Stonepunk Studios: 2
Arcen Games, LLC: 2
raptor lab: 5
Stranga: 2
Indie Game Group: 2
Evoluta Studio: 2
EJRGames: 2
10Ants Hill: 2
Red Matter Studio: 2
MT Worlds: 2
hOSHI: 5
ciwolsey: 2
Riviysky: 2
Fast Forward Games: 2
Matt Glanville: 2
RealityRig: 2
Kedexa: 3
Juvty Worlds Ltd.: 3
Lasso Games, LLC: 2
ma_mutte: 2
Wenkly Studio Sp. z o.o.: 3
Stunlock Studios: 2
Invictus Games Ltd.: 2
HeroCraft: 2
Nicholas Rizzo: 3
Artur Rezende: 3
RLTY CHK: 2
SOLIDMESH: 2
GamersHype Productions: 3
StarworkGC: 2
ALTER-BOY: 2
Egor Magurin: 3
Snail Games USA: 3
Deceptive Games Ltd.: 3
D.W.S.: 4
疯王子: 3
CrackedGhostGames: 2
PanGuoJun: 2
Overflow: 2
Wargaming Group Limited: 2
fletcher.richman: 2
owockitalki: 2
Ivanovich Games: 3
Badim: 2
DRUNKEN APES: 4
KishMish Games: 2
Fyn Games: 2
L. Stotch: 2
Portal Studios: 2
Rafael Farias: 2
Douglas Liang: 2
Tungsten Games: 2
A Sweet Studio: 2
XREAL Games: 2
Galaxy Game Studio: 2
Tero Lunkka: 2
Linden Lab: 2
Luden.io: 2
Catland: 2
Devdan Games: 2
Zolden Games: 2
Satur Entertainment: 2
Game Cooks: 3
GuraShop: 2
VRGEN: 2
EvilCoGames: 2
Idea Cabin: 2
Surreal Games: 2
GexagonVR: 2
Cold Beam Games: 2
crowgames UG (haftungsbeschränkt): 2
Immergity: 2
LIV Inc: 2
Dnovel: 6
Rotech Digital Ltd: 2
LastnOni: 2
Dudarev Alexandr Vladimirovich: 2
Buce Studios LLC: 2
Virtual World Arcade: 2
2-Volt Games: 2
The Commander: 2
Yiming: 2
Vox Games: 2
Nincent: 2
Subdream Studios: 2
Hannes Delbeke: 2
Jakub Klementewicz: 2
Z Games: 2
MindRend Technologies: 2
Anton Shatalov: 2
Xefier Games Inc.: 3
momoStorm Entertainment: 2
MAMMOSSIX Co., Ltd.: 2
Moped Games: 2
Random Dragon Games: 2
hede: 7
Somnium Space Ltd.: 2
Benjamin Rommel Games: 2
FossettGaming: 2
Boogygames Studios: 2
MC Games: 2
Christopher Anselmo: 2
Fabio Cunha: 7
Endwise: 3
Magic VR: 2
Microlith Games: 2
Fullmetal Developer: 2
Eat All The Games: 2
Thunder Devs.: 3
Brian Macumber: 2
Steppe Hare Studio: 4
HandyGames: 2
Heero: 2
Mortis Games: 2
ELECTROSTALIN ENTERTAINMENT: 2
Juan Pedraza: 2
Shotgun with Glitters: 2
Wonderbox Games: 5
Playsnak: 2
Gamers for Good: 2
RewindApp: 25
F-Games: 3
Red13Indie: 2
Magitech Games: 3
Chris Antoni: 2
ChillFun: 2
Eloquence entertainment: 2
WoksOnStudios: 2
Battle of Kings Team: 2
Team 21 Studio: 2
Simulators Live: 2
KicklessGame: 4
Appnori Inc.: 6
Gray Wolf Interactive LLC: 2
Odooh Games: 2
Hamsters Gaming: 2
Casual.US: 5
5th VR: 2
CentVire: 2
Winturn Games: 2
NEOWIZ: 2
Star Vault AB: 2
Gamenesis: 2
Only Voxel Games: 2
Unrea: 2
Challenge Yourself Gaming: 2
IVR: 2
自由愉悦小黄猫(Free pleasure little yellow cat): 2
Angus Seung: 2
Dreambakers: 4
Softwaves: 3
LLC Blini Games: 2
Cool and Normal Entertainment: 2
Waifu Games: 3
Anime Games: 2
XWidgetSoft: 3
Caffeine: 2
print(len(temp_df))
print(len(temp_df[temp_df['num_reviews']>1]))
print(temp_df[temp_df['metacritic_score']<60][['name','appid','metacritic_score']])4925
4796
                                        name   appid  metacritic_score
24                         Clockwork Empires  224740              58.0
34                                     Dream  229580              59.0
39                   Akaneiro: Demon Hunters  230310              53.0
60                             Vector Thrust  242130              57.0
75                            Spacebase DF-9  246090              49.0
88                                    Nether  247730              57.0
136                                   Bedlam  261490              54.0
152                               Son of Nor  264000              54.0
161                      Meridian: New World  265120              43.0
167                          Worlds of Magic  265970              52.0
222                                  LA Cops  278810              45.0
227                               Soul Axiom  279900              57.0
237            Woolfe - The Red Hood Diaries  281940              55.0
239               RollerCoaster Tycoon World  282560              43.0
264                              Dead Effect  286040              59.0
460                        theHunter: Primal  322920              49.0
485   Shroud of the Avatar: Forsaken Virtues  326160              58.0
505                             The Deer God  328940              59.0
726                          Time Machine VR  356180              58.0
953                             Pylon: Rogue  386100              50.0
988                           Formula Fusion  389670              58.0
1029                           Dead Effect 2  394760              53.0
1130                 Krai Mira: Extended Cut  408920              50.0
print(temp_df.isna().sum())categories               0
content_warning          0
controller_support       0
developers               0
failed                4552
genres                   0
has_achievements         0
has_metacritic           0
is_free                  0
metacritic_score         0
multiple_packages        0
num_achievements         0
num_languages            0
num_movies               0
num_reviews              0
num_screenshots          0
platforms                0
publishers               0
dtype: int64
testing_folder=os.path.join(parent_folder,"bulk","testing")
temp_df.to_csv(os.path.join(testing_folder,'20190124_games.csv'),index=False)_____no_output_____print(len(useful_examples.keys()))
%matplotlib inline
x1=[]
y1=[]
x2=[]
y2=[]
size=[]
for appid in useful_examples:
    if appid not in temp_df[temp_df['has_metacritic']==True].appid.unique():
        x1.append(useful_examples[appid]['ratio_edited_positive'])
        y1.append(useful_examples[appid]['ratio_soon_after_positive'])
    else:
        x2.append(useful_examples[appid]['ratio_edited_positive'])
        y2.append(useful_examples[appid]['ratio_soon_after_positive'])
        size.append(np.log(temp_df[temp_df['appid']==appid]['metacritic_score']/3))

plt.plot(x1,y1,'o')
for i in range(len(x2)):
    plt.plot(x2[i],y2[i],'o',c='r',ms=size[i])1930
time_series_folder=os.path.join(parent_folder,"bulk","csv_no_text","time_series_by_month")

for i,appid in enumerate(useful_examples,1):
    df=pd.read_csv(os.path.join(time_series_folder,str(appid)+'_month_time_series.csv'))
    df['pos_reviews']=df.apply(lambda x: x['ratio']*x['reviews'],axis=1)
    useful_examples[appid]['overall_ratio']=df.pos_reviews.sum()/df.reviews.sum()
    if df['month'][1]==1 and df['reviews'][0] < 0.5*df['reviews'][1]:
        useful_examples[appid]['original_ratio']=(df['pos_reviews'][0]+df['pos_reviews'][1])/(df['reviews'][0]+df['reviews'][1])
    else:
        useful_examples[appid]['original_ratio']=df['ratio'][0]
    release_month=df[df['early_access']==False]['month'].min()
    last_ea_month=df[df['early_access']==True]['month'].max()
    halfpoint_month=df[df['month']<(release_month/2.0)]['month'].max()
    if release_month==0:
        halfpoint_month=0
    useful_examples[appid]['releasepoint_ratio']=float(df[df['month']==last_ea_month]['pos_reviews'])/float(df[df['month']==last_ea_month]['reviews'])
    useful_examples[appid]['prerelease_ratio']=df[df['early_access']==True].pos_reviews.sum()/df[df['early_access']==True].reviews.sum()
    useful_examples[appid]['halfpoint_ratio']=float(df[df['month']==halfpoint_month]['pos_reviews'])/float(df[df['month']==halfpoint_month]['reviews'])
    useful_examples[appid]['halfrelease_ratio']=df[df['month']<(release_month/2.0)].pos_reviews.sum()/df[df['month']<(release_month/2.0)].reviews.sum()
    clear_output(wait=True)
    print(str(i)+"/"+str(len(useful_examples)))1930/1930
x=[]
y=[]
size=[]
for appid in useful_examples:
    x.append(useful_examples[appid]['original_ratio'])
    y.append(useful_examples[appid]['ratio_soon_after_positive'])
plt.plot(x,y,'o')
plt.show()
x=[]
y=[]
for appid in useful_examples:
    x.append(useful_examples[appid]['halfpoint_ratio'])
    y.append(useful_examples[appid]['ratio_soon_after_positive'])
plt.plot(x,y,'o')
plt.show()
x=[]
y=[]
for appid in useful_examples:
    x.append(useful_examples[appid]['halfrelease_ratio'])
    y.append(useful_examples[appid]['ratio_soon_after_positive'])
plt.plot(x,y,'o')
plt.show()
x=[]
y=[]
for appid in useful_examples:
    x.append(useful_examples[appid]['prerelease_ratio'])
    y.append(useful_examples[appid]['ratio_soon_after_positive'])
plt.plot(x,y,'o')
plt.show()_____no_output_____x=[]
y=[]
size=[]
for appid in useful_examples:
    x.append(useful_examples[appid]['original_ratio'])
    y.append(useful_examples[appid]['ratio_soon_after_positive'])
    size.append(np.log(useful_examples[appid]['num_reviews_soon_after_early_access']+1.5))
for i in range(len(x)):
    plt.plot(x[i],y[i],'o',ms=size[i])
plt.xlabel("% positive reviews in first month of reviews")
plt.ylabel("% positive reviews following release")
plt.show()
x=[]
y=[]
for appid in useful_examples:
    x.append(useful_examples[appid]['halfpoint_ratio'])
    y.append(useful_examples[appid]['ratio_soon_after_positive'])
for i in range(len(x)):
    plt.plot(x[i],y[i],'o',ms=size[i])
plt.xlabel("% positive reviews in month halfway through Early Access")
plt.ylabel("% positive reviews following release")
plt.show()
x=[]
y=[]
for appid in useful_examples:
    x.append(useful_examples[appid]['halfrelease_ratio'])
    y.append(useful_examples[appid]['ratio_soon_after_positive'])
for i in range(len(x)):
    plt.plot(x[i],y[i],'o',ms=size[i])
plt.xlabel("% positive reviews during first half of Early Access")
plt.ylabel("% positive reviews following release")
plt.show()
x=[]
y=[]
for appid in useful_examples:
    x.append(useful_examples[appid]['prerelease_ratio'])
    y.append(useful_examples[appid]['ratio_soon_after_positive'])
for i in range(len(x)):
    plt.plot(x[i],y[i],'o',ms=size[i])
plt.xlabel("% positive reviews during Early Access")
plt.ylabel("% positive reviews following release")
plt.show()_____no_output_____x=[]
y=[]
for appid in useful_examples:
    x.append(useful_examples[appid]['halfrelease_ratio'])
    y.append(useful_examples[appid]['ratio_soon_after_positive']/useful_examples[appid]['halfrelease_ratio'])
for i in range(len(x)):
    plt.plot(x[i],y[i],'o',ms=size[i])
plt.xlabel("% positive reviews during first half of Early Access")
plt.ylabel("Review positivity ratio (after release over first half of early access period)")
plt.ylim(0,2)
plt.show()
x=[]
y=[]
for appid in useful_examples:
    x.append(useful_examples[appid]['original_ratio'])
    y.append(useful_examples[appid]['ratio_soon_after_positive']/useful_examples[appid]['original_ratio'])
for i in range(len(x)):
    plt.plot(x[i],y[i],'o',ms=size[i])
plt.xlabel("% positive reviews during first month of Early Access")
plt.ylabel("Review positivity ratio (after release over first month of early access)")
plt.ylim(0,2)
plt.show()

x=[]
y=[]
for appid in useful_examples:
    x.append(useful_examples[appid]['halfrelease_ratio'])
    y.append(useful_examples[appid]['ratio_soon_after_positive']-useful_examples[appid]['halfrelease_ratio'])
for i in range(len(x)):
    plt.plot(x[i],y[i],'o',ms=size[i])
plt.xlabel("% positive reviews during first half of Early Access")
#plt.ylabel("Review positivity ratio (after release over first half of early access period)")
plt.ylim(-1,1)
plt.show()
x=[]
y=[]
for appid in useful_examples:
    x.append(useful_examples[appid]['original_ratio'])
    y.append(useful_examples[appid]['ratio_soon_after_positive']-useful_examples[appid]['original_ratio'])
for i in range(len(x)):
    plt.plot(x[i],y[i],'o',ms=size[i])
plt.xlabel("% positive reviews during first month of Early Access")
#plt.ylabel("Review positivity ratio difference (after release over first month of early access)")
plt.ylim(-1,1)
plt.show()                _____no_output_____x=[]
y=[]
size=[]
for appid in useful_examples:
    x.append(useful_examples[appid]['original_ratio'])
    y.append(useful_examples[appid]['releasepoint_ratio'])
plt.plot(x,y,'o')
plt.show()_____no_output_____# Make training examples of games with early access reviews, non-early access reviews, and at least
# three months between the first early access review and the first non-early access review

csv_folder=os.path.join(parent_folder,"bulk","csv_no_text","early_access_standardized")
out_folder=os.path.join(parent_folder,"bulk","csv_no_text","time_series_by_month_from_")
if not os.path.exists(out_folder):
    os.makedirs(out_folder)

early_access_exit_time={}
normal_review_start_time={}
time_midpoint={}

for i,appid in enumerate(former_ea_games,1):
    temp_time_series=[]
    df=pd.read_csv(os.path.join(csv_folder,str(appid)+'_reviews_all.csv'))
    if len(df[df['written_during_early_access']==True])>0:
        num_reviews[int(appid)]=len(df)
        early_access_exit_time[int(appid)]=df[df['written_during_early_access']==True]['timestamp_updated'].max()
        earliest_time=df['timestamp_updated'].min()
        latest_time=df['timestamp_updated'].max()
        #print(len(df[df['written_during_early_access']==True]))
        if(len(df[df['written_during_early_access']==False]))>0:
            normal_review_start_time[int(appid)]=df[df['written_during_early_access']==False]['timestamp_updated'].min()
            midpoint=(early_access_exit_time[int(appid)]+normal_review_start_time[int(appid)])/2
            num_months_after=int(np.ceil(float(latest_time-midpoint)/(30.0*24.0*60.0*60.0)))
        else:
            midpoint=early_access_exit_time[int(appid)]+60
            num_months_after=0
        time_midpoint[int(appid)]=midpoint
        #print(appid)
        #print(float(early_access_exit_time[int(appid)]))
        #print(float(normal_review_start_time[int(appid)]))
        num_months_before=int(np.ceil(float(time_midpoint[int(appid)]-earliest_time)/(30.0*24.0*60.0*60.0)))
        for j in range(num_months_before):
            num_pos=float(len(df[(df['timestamp_updated']>=midpoint-(num_months_before-j)*30.0*24.0*60.0*60.0) &
                        (df['timestamp_updated']<midpoint-(num_months_before-(j+1))*30.0*24.0*60.0*60.0) &
                          (df['voted_up']==True)]))
            num_neg=float(len(df[(df['timestamp_updated']>=midpoint-(num_months_before-j)*30.0*24.0*60.0*60.0) &
                        (df['timestamp_updated']<midpoint-(num_months_before-(j+1))*30.0*24.0*60.0*60.0) &
                          (df['voted_up']==False)]))
            if num_neg>0 or num_pos>0:
                temp_time_series.append({'early_access':True,'month':j,'reviews':int(num_pos+num_neg),'ratio':num_pos/(num_neg+num_pos)})
        for j in range(num_months_after):
            num_pos=float(len(df[(df['timestamp_updated']>=midpoint+(j)*30.0*24.0*60.0*60.0) &
                        (df['timestamp_updated']<midpoint+(j+1)*30.0*24.0*60.0*60.0) &
                          (df['voted_up']==True)]))
            num_neg=float(len(df[(df['timestamp_updated']>=midpoint+(j)*30.0*24.0*60.0*60.0) &
                        (df['timestamp_updated']<midpoint+(j+1)*30.0*24.0*60.0*60.0) &
                          (df['voted_up']==False)]))
            if num_neg>0 or num_pos>0:
                temp_time_series.append({'early_access':False,'month':num_months_before+j,'reviews':int(num_pos+num_neg),'ratio':num_pos/(num_neg+num_pos)})
        pd.DataFrame(temp_time_series).to_csv(os.path.join(out_folder,str(appid)+'_month_time_series.csv'))
    clear_output(wait=True)
    print(str(i)+"/"+str(len(former_ea_games)))_____no_output_____j=0
feature_rows=[]

def extract_month_data(month,df,vec,k):
    ms=df['month']
    if month in df.month.unique():
        vec[0][k]=int(ms[ms==])

for i,appid in enumerate(useful_examples,1):
    df=pd.read_csv(os.path.join(time_series_folder,str(appid)+'_month_time_series.csv'))
    #df['pos_reviews']=df.apply(lambda x: x['ratio']*x['reviews'],axis=1)
    last_ea_month=df[df['early_access']==True]['month'].max()
    for j in range(last_ea_month,2,-1):
        feature_rows.append(np.zeros((1,8)))
        feature_rows[0,0]=last_ea_month[last_ea_month==]
        #feature_rows[-1][0][0]=df[]

    #clear_output(wait=True)
    #print(str(i)+"/"+str(len(useful_examples)))
print("Done: "+str(j)+"/"+str(len(useful_examples)))7    7
Name: month, dtype: int64
7
7    7
Name: month, dtype: int64
7
6    7
Name: month, dtype: int64
7
7    7
Name: month, dtype: int64
7
7    7
Name: month, dtype: int64
7
7    7
Name: month, dtype: int64
7
7    7
Name: month, dtype: int64
7
7    7
Name: month, dtype: int64
7
7    7
Name: month, dtype: int64
7
Series([], Name: month, dtype: int64)

x=[]
y=[]
for appid in useful_examples:
    if useful_examples[appid]['original_ratio'] >0.5:
        x.append(useful_examples[appid]['original_ratio'])
        y.append(useful_examples[appid]['ratio_soon_after_positive']/useful_examples[appid]['original_ratio'])
print(np.mean(y))
print(np.std(y))0.7172828275684046
0.25978298559147395


# Notebook from gillislab/MetaNeighbor-Protocol
Path: anticipated_results.ipynb

# MetaNeighbor anticipated results

Because MetaNeighbor is non-parametric, there is no fine-tuning to be done for any of the procedures presented here. Over time, we have identified two sources of potential error: bad highly variable gene selection and coding or formatting errors, which can be easily diagnosed by looking at AUROC heatmaps. We expect AUROCs to correctly represent global relationships between cell types and generalize across studies.

## Bad gene set selection

The most common problem is to forget to select a set of highly variable genes, which is expected to dampen the impact of technical variability on neighbor voting. First, we present an example of a correct analysis, where we load the BICCN GABAergic neurons, select highly variable genes, and compute cluster similarities (see Procedure 1 for more details).
_____no_output_____

library(MetaNeighbor)
biccn_data = readRDS("biccn_gaba.rds")
biccn_hvgs = variableGenes(biccn_data, exp_labels = biccn_data$study_id)

# GOOD
aurocs = MetaNeighborUS(var_genes = biccn_hvgs,
                        dat = biccn_data,
                        study_id = biccn_data$study_id,
                        cell_type = biccn_data$joint_subclass_label,
                        fast_version = TRUE)
plotHeatmap(aurocs, cex = 0.5)

# BAD
aurocs = MetaNeighborUS(var_genes = sample(rownames(biccn_data), length(biccn_hvgs)),
                        dat = biccn_data,
                        study_id = biccn_data$study_id,
                        cell_type = biccn_data$joint_subclass_label,
                        fast_version = TRUE)
plotHeatmap(aurocs, cex = 0.5)_____no_output_____

We recognize strong replicability structure, evidenced by the presence of dark red blocks. When we repeat the analysis with random genes, the replicability structure is still present, but we recognize two signatures of bad gene set selection: (a) AUROCs are low overall (shift to light red and orange), (b) within red blocks, there is a clear gradient structure. In our experience, there are 3 scenarios that lead to bad gene selection: errors in gene symbol conversion, errors when genes are stored as factors in R (that are implicitly converted to numerical values during indexing), forgetting to select highly variable genes altogether._____no_output_____## No overlap between datasets

The second problem occurs when there is no overlap between datasets. We illustate this problem with the data from Procedure 2, where we expect all cell types from the Tasic dataset to be present in the pre-trained BICCN model. According to our expectations, all cell types have strong hits with BICCN clusters and we see a hierarchical structure that is consistent with prior biological knowledge: lighter red blocks corresponding to MGE and CGE-derived inhibitory neurons. We compare with the same block of code, where we mistakenly keep non-neurons from the BICCN taxonomy instead of inhibitory neurons.
The lack of biological overlap can be deduced from 3 factors: (a) low AUROC values overall, (b) almost no strong hits (contrary to expectations), (c) lack of expected hierarchical structure (MGE and CGE derived interneurons)._____no_output_____

library(scRNAseq)
tasic = TasicBrainData(ensembl = FALSE)
tasic$study_id = "tasic"

biccn_subclasses = read.table("pretrained_biccn_subclasses.txt", check.names = FALSE)
global_aurocs = MetaNeighborUS(
  trained_model = biccn_subclasses, dat = tasic,
  study_id = tasic$study_id, cell_type = tasic$primary_type,
  fast_version = TRUE
)
gabaergic_tasic = splitTestClusters(global_aurocs, k = 4)[[2]]

# GOOD
gabaergic_biccn = splitTrainClusters(global_aurocs[gabaergic_tasic,], k = 4)[[4]]
keep_cell = makeClusterName(tasic$study_id, tasic$primary_type) %in% gabaergic_tasic
tasic_subdata = tasic[, keep_cell]
aurocs = MetaNeighborUS(
  trained_model = biccn_subclasses[, gabaergic_biccn],
  dat = tasic_subdata, study_id = tasic_subdata$study_id,
  cell_type = tasic_subdata$primary_type, fast_version = TRUE
)
plotHeatmapPretrained(aurocs, cex = 0.7)

# BAD: non-neurons instead of GABAergic neurons
gabaergic_biccn = splitTrainClusters(global_aurocs, k = 5)[[1]]
keep_cell = makeClusterName(tasic$study_id, tasic$primary_type) %in% gabaergic_tasic
tasic_subdata = tasic[, keep_cell]
aurocs = MetaNeighborUS(
  trained_model = biccn_subclasses[, gabaergic_biccn],
  dat = tasic_subdata, study_id = tasic_subdata$study_id,
  cell_type = tasic_subdata$primary_type, fast_version = TRUE
)
plotHeatmapPretrained(aurocs, cex = 0.7)_____no_output_____

## Pretrained MetaNeighbor: bad name formatting

The third problem we have encountered is a mistake that occurs when loading pre-trained model and forgetting to specify "check.names = FALSE", which is essential to preserve correct formatting of cell type names. Below, we present an example of correct code based on data from Procedure 2. We obtain the expected replicability structure, with evidence of strong hits across all cell types (see Procedure 2 for further details and analyses). When we forget "check.names = FALSE", MetaNeighbor is unable to correctly recognize dataset names and cell type names in the pre-trained model, the similarity computations become meaningless, leading to AUROC values that are essentially 0.5. This problem is easy to diagnose and fix, but can be very confusing when it occurs._____no_output_____

# GOOD
biccn_subclasses = read.table("pretrained_biccn_subclasses.txt", check.names = FALSE)
aurocs = MetaNeighborUS(
  trained_model = biccn_subclasses, dat = tasic,
  study_id = tasic$study_id, cell_type = tasic$primary_type,
  fast_version = TRUE
)
plotHeatmapPretrained(aurocs)

# BAD
biccn_subclasses = read.table("pretrained_biccn_subclasses.txt")
aurocs = MetaNeighborUS(
  trained_model = biccn_subclasses, dat = tasic,
  study_id = tasic$study_id, cell_type = tasic$primary_type,
  fast_version = TRUE
)
plotHeatmapPretrained(aurocs)_____no_output_____

## Generalizable quantification of cell type similarities

In their computation, MetaNeighbor’s AUROCs are directly related to Spearman correlations. More precisely, all computations are based on average Spearman correlations between cells from two cell types, but include an additional prediction that alleviate batch effects, while keeping an interpretability power that is comparable to correlations (where AUROC = 0 maps to correlation = -1, AUROC = 0.5 maps to correlation = 0, AUROC = 1 maps to correlation = 1).

To appreciate how the additional prediction step enables us to obtain "batch free correlations", we compare MetaNeighbor’s output with a more naive similarity output, where we compute the Spearman correlation between cell type centroids. Centroid correlations display two desirable patterns: centroids cluster primarily by cell type (then by dataset), we see global hierarchical structure (MGE-derived interneurons vs CGE-derived interneurons). However, batch effects are clearly visible throughout the heatmap. For example, within each cell type, Chromium-based datasets tend to cluster on one side, SmartSeq-based datasets on the other side. In contrast, for an equivalent computation time, all the "good" patterns are made pristinely clear with MetaNeighbor AUROCs. Technical substructure has been lost: technologies mix well within cell-types, homogenous cell groupings look uniform, and biological relationships between cell types are correctly displayed._____no_output_____

cell_types = as.factor(
    makeClusterName(biccn_data$study_id, biccn_data$joint_subclass_label)
)
normalization_factor = Matrix::colSums(assay(biccn_data)) / 1000000
cpm = assay(biccn_data)
cpm@x = cpm@x / rep.int(normalization_factor, diff(cpm@p))
cpm = as.matrix(cpm[biccn_hvgs,])

centroids = sapply(levels(cell_types), function(ct) {
    matrixStats::rowMeans2(log2(cpm+1), cols = cell_types == ct)
})

centroid_cor = cor(centroids, method = "spearman")
aurocs = MetaNeighborUS(var_genes = biccn_hvgs,
                        dat = biccn_data,
                        study_id = biccn_data$study_id,
                        cell_type = biccn_data$joint_subclass_label,
                        fast_version = TRUE)
plotHeatmap((1+centroid_cor)/2, cex = 0.5)
plotHeatmap(aurocs, cex = 0.5)_____no_output_____

Compared to correlations, AUROCs have one additional "parameter": the outgroup used for the prediction task. In Procedure 1, we illustrated how the outgroup can be controled and interpreted. A deviation from AUROC = 1 can thus be interpreted as a combination of two factors: lack of similarity between cell types and choice of outgroup (difficulty of prediction task). If the outgroup is well controled, AUROC values will generalize across studies and fundamentally indicate the quality of the clustering. For a given cell type in a given background (for example Sst cells in an unbiased sample of MOp interneurons), the similarity to Sst cells in another dataset (or any other interneuron type for that matter) should be in the range of similarity observed within the BICCN.

As a robust alternative to centroid correlations, MetaNeighbor AUROCs can be applied to simple pre-processing tasks, such as identifying and selecting cell types that overlap between datasets before applying a merging framework. However, beyond the purely applicative viewpoint, we believe that MetaNeighbor-style AUROCs are a stepping stone toward a generalizable formalization of cell type similarity._____no_output_____


# Notebook from markuszm/npm-analysis
Path: jupyterlab/r2c/FunctionsEvolAggkLoCLatestVersion.ipynb

<code>
import pandas as pd
from sqlalchemy import create_engine
engine = create_engine('dbUrl')_____no_output_____# to filter for GitHub only: where repo_url LIKE '%%github%%'
population = "top_10k"
repos = pd.read_sql_query("select distinct repo_url from repo_tags_2 where population = %s", params=[population], con=engine)

# repos = repos[repos['repo_url'] != 'https://github.com/phuu/npm-release']_____no_output_____%%time
import processing
import utils

# functions = [('http', 'createServer'), ('express', ''), ('hapi', ''), ('koa', '')]
# functions = [('http', 'get'), ('http', 'request'), ('request', ''), ('axios', '')]
# functions = [('child_process', 'exec'), ('child_process', 'spawn'), ('child_process', 'fork'), ('child_process', 'spawnSync'), ('child_process', 'execFileSync'), ('child_process', 'execSync'), ('child_process', 'execFile')]
# functions = [('child_process', 'exec'), ('child_process', 'spawn')]
functions = [('', 'eval'), ('vm', 'runInThisContext'), ('', 'new Function')]
# functions = [('', 'setImmediate'), ('', 'setTimeout'), ('', 'setInterval')]
# functions = [('', 'new Promise'), ('bluebird', '')]

# WebAssembly special case
# functions = [('WebAssembly', '')]
special = False

timeToUsageMap = {}

repoUsageMap = {}

filterType = 'test'

if filterType == 'production':
    table = 'results_de_tu_darmstadt_filter_callgraph_1_1_0'
elif filterType == 'test':
    table = 'results_de_tu_darmstadt_inverse_filter_callgraph_1_1_0'
else:
    table = 'results_de_tu_darmstadt_callgraph_1_1_0'

def countFunc(commitHash, engine):
    counts = {}
    for f in functions:
        if f[0] == '':
            if f[1] == 'eval':
                query = """select count(*) as count from {} where commit_hash = %s AND extra->>'toFunction' = '{}' AND extra->>'receiver' = ''""".format(table, f[1])
            else:
                query = """select count(*) as count from {} where commit_hash = %s AND extra->>'toFunction' = '{}'""".format(table, f[1])
        else:
            if f[1] == '':
                query = """select count(extra->'modules') as count from {} where commit_hash = %s AND extra->'modules' = '[\"{}\"]'""".format(table, f[0])
            else:
                if special:
                    query = """select count(extra->'modules') as count from {} where commit_hash = %s AND extra->>'receiver' = '{}'""".format(table, f[0])
                else:
                    query = """select count(extra->'modules') as count from {} where commit_hash = %s AND extra->'modules' = '[\"{}\"]' AND extra->>'toFunction' = '{}'""".format(table, f[0], f[1])
        functionCallCount = pd.read_sql_query(query, params=[commitHash], con=engine)
        count = functionCallCount['count'][0] if not functionCallCount.empty else 0
        counts[f] = count
    return counts

def completionFunc(x):
    kLoC = x.result()[2]
    repo = x.result()[1]
    for key, value in x.result()[0].items():
        for f, c in value.items():
            if c > 0:
                if key in repoUsageMap:
                    repoUsageMap[key].append((repo, f, c))
                else:
                    repoUsageMap[key] = [(repo, f, c)]
            if key not in timeToUsageMap:
                timeToUsageMap[key] = {}
            if f in timeToUsageMap[key]:
                val = timeToUsageMap[key][f]
                timeToUsageMap[key][f] = (val[0] + c, val[1] + 1, val[2] + kLoC)
            else:
                timeToUsageMap[key][f] = (c, 1, kLoC)

utils.runInParallelCommitProcessing(processing.processCommitsLatestVersion, repos, countFunc, processing.normalizeNoop, completionFunc, filterType)Progress: 8/9988
Progress: 2/9988
Progress: 7/9988
Progress: 4/9988
Progress: 1/9988
Progress: 6/9988
Progress: 5/9988
Progress: 12/9988
Progress: 10/9988
Progress: 14/9988
Progress: 13/9988
Progress: 11/9988
Progress: 9/9988
Progress: 15/9988
Progress: 3/9988
Progress: 16/9988
Progress: 18/9988
Progress: 17/9988
Progress: 22/9988
Progress: 20/9988
Progress: 19/9988
Progress: 21/9988
Progress: 23/9988
Progress: 24/9988
Progress: 27/9988
Progress: 28/9988
Progress: 26/9988
Progress: 29/9988
Progress: 25/9988
Progress: 31/9988
Progress: 34/9988
Progress: 32/9988
Progress: 35/9988
Progress: 33/9988
Progress: 37/9988
Progress: 30/9988
Progress: 36/9988
Progress: 40/9988
Progress: 38/9988
Progress: 41/9988
Progress: 43/9988
Progress: 42/9988
Progress: 39/9988
Progress: 45/9988
Progress: 47/9988
Progress: 44/9988
Progress: 49/9988
Progress: 48/9988
Progress: 50/9988
Progress: 46/9988
Progress: 51/9988
Progress: 52/9988
Progress: 53/9988
Progress: 55/9988
Progress: 57/9988
Progress: 54/9988
Progress: 59/9988
Progress: 56/9988
Progress: 60/9988
Progress: 58/9988
Progress: 61/9988
Progress: 64/9988
Progress: 65/9988
Progress: 67/9988
Progress: 68/9988
Progress: 63/9988
Progress: 69/9988
Progress: 66/9988
Progress: 62/9988
Progress: 72/9988
Progress: 71/9988
Progress: 73/9988
Progress: 76/9988
Progress: 75/9988
Progress: 74/9988
Progress: 77/9988
Progress: 79/9988
Progress: 70/9988
Progress: 78/9988
Progress: 81/9988
Progress: 82/9988
Progress: 83/9988
Progress: 80/9988
Progress: 84/9988
Progress: 86/9988
Progress: 88/9988
Progress: 89/9988
Progress: 87/9988
Progress: 90/9988
Progress: 85/9988
Progress: 91/9988
Progress: 92/9988
Progress: 93/9988
Progress: 94/9988
Progress: 95/9988
Progress: 96/9988
Progress: 97/9988
Progress: 98/9988
Progress: 100/9988
Progress: 99/9988
Progress: 101/9988
Progress: 102/9988
Progress: 104/9988
Progress: 103/9988
Progress: 105/9988
Progress: 106/9988
Progress: 107/9988
Progress: 108/9988
Progress: 111/9988
Progress: 110/9988
Progress: 115/9988
Progress: 113/9988
Progress: 114/9988
Progress: 109/9988
Progress: 116/9988
Progress: 117/9988
Progress: 112/9988
Progress: 118/9988
Progress: 119/9988
Progress: 121/9988
Progress: 120/9988
Progress: 122/9988
Progress: 124/9988
Progress: 126/9988
Progress: 125/9988
Progress: 130/9988
Progress: 127/9988
Progress: 129/9988
Progress: 128/9988
Progress: 131/9988
Progress: 133/9988
Progress: 123/9988
Progress: 135/9988
Progress: 134/9988
Progress: 132/9988
Progress: 137/9988
Progress: 136/9988
Progress: 140/9988
Progress: 139/9988
Progress: 138/9988
Progress: 143/9988
Progress: 142/9988
Progress: 144/9988
Progress: 145/9988
Progress: 147/9988
Progress: 146/9988
Progress: 149/9988
Progress: 150/9988
Progress: 148/9988
Progress: 151/9988
Progress: 152/9988
Progress: 141/9988
Progress: 153/9988
Progress: 154/9988
Progress: 155/9988
Progress: 157/9988
Progress: 156/9988
Progress: 159/9988
Progress: 162/9988
Progress: 160/9988
Progress: 158/9988
Progress: 161/9988
Progress: 165/9988
Progress: 166/9988
Progress: 169/9988
Progress: 164/9988
Progress: 167/9988
Progress: 163/9988
Progress: 170/9988
Progress: 171/9988
Progress: 172/9988
Progress: 168/9988
Progress: 176/9988
Progress: 174/9988
Progress: 175/9988
Progress: 173/9988
Progress: 178/9988
Progress: 177/9988
Progress: 179/9988
Progress: 180/9988
Progress: 181/9988
Progress: 183/9988
Progress: 184/9988
Progress: 186/9988
Progress: 187/9988
Progress: 188/9988
Progress: 185/9988
Progress: 189/9988
Progress: 182/9988
Progress: 191/9988
Progress: 192/9988
Progress: 194/9988
Progress: 195/9988
Progress: 190/9988
Progress: 196/9988
Progress: 193/9988
Progress: 197/9988
Progress: 198/9988
Progress: 200/9988
Progress: 202/9988
Progress: 199/9988
Progress: 201/9988
Progress: 203/9988
Progress: 205/9988
Progress: 204/9988
Progress: 206/9988
Progress: 211/9988
Progress: 209/9988
Progress: 212/9988
Progress: 208/9988
Progress: 207/9988
Progress: 213/9988
Progress: 219/9988
Progress: 214/9988
Progress: 218/9988
Progress: 210/9988
Progress: 217/9988
Progress: 215/9988
Progress: 216/9988
Progress: 221/9988
Progress: 220/9988
Progress: 223/9988
Progress: 224/9988
Progress: 225/9988
Progress: 222/9988
Progress: 226/9988
Progress: 227/9988
Progress: 230/9988
Progress: 229/9988
Progress: 232/9988
Progress: 233/9988
Progress: 228/9988
Progress: 234/9988
Progress: 231/9988
Progress: 236/9988
Progress: 238/9988
Progress: 235/9988
Progress: 239/9988
Progress: 237/9988
Progress: 240/9988
Progress: 241/9988
Progress: 243/9988
Progress: 244/9988
Progress: 246/9988
Progress: 242/9988
Progress: 248/9988
Progress: 245/9988
Progress: 249/9988
Progress: 250/9988
Progress: 247/9988
Progress: 252/9988
Progress: 256/9988
Progress: 255/9988
Progress: 253/9988
Progress: 254/9988
Progress: 257/9988
Progress: 258/9988
Progress: 251/9988
Progress: 261/9988
Progress: 262/9988
Progress: 260/9988
Progress: 264/9988
Progress: 265/9988
Progress: 263/9988
Progress: 267/9988
Progress: 266/9988
Progress: 259/9988
Progress: 271/9988
Progress: 268/9988
Progress: 272/9988
Progress: 269/9988
Progress: 270/9988
Progress: 274/9988
Progress: 277/9988
Progress: 275/9988
Progress: 276/9988
Progress: 278/9988
Progress: 273/9988
Progress: 279/9988
Progress: 280/9988
Progress: 281/9988
Progress: 282/9988
Progress: 283/9988
Progress: 285/9988
Progress: 286/9988
Progress: 287/9988
Progress: 290/9988
Progress: 284/9988
Progress: 288/9988
Progress: 293/9988
Progress: 292/9988
Progress: 291/9988
Progress: 294/9988
Progress: 296/9988
Progress: 289/9988
Progress: 295/9988
Progress: 297/9988
Progress: 299/9988
Progress: 301/9988
Progress: 298/9988
Progress: 303/9988
Progress: 305/9988
Progress: 302/9988
Progress: 300/9988
Progress: 304/9988
Progress: 310/9988
Progress: 311/9988
Progress: 309/9988
Progress: 312/9988
Progress: 308/9988
Progress: 307/9988
Progress: 306/9988
Progress: 314/9988
Progress: 315/9988
Progress: 316/9988
Progress: 317/9988
Progress: 313/9988
Progress: 319/9988
Progress: 318/9988
Progress: 321/9988
Progress: 323/9988
Progress: 322/9988
Progress: 325/9988
Progress: 320/9988
Progress: 326/9988
Progress: 328/9988
Progress: 324/9988
Progress: 329/9988
Progress: 331/9988
Progress: 332/9988
Progress: 330/9988
Progress: 327/9988
Progress: 333/9988
Progress: 334/9988
Progress: 337/9988
Progress: 338/9988
Progress: 335/9988
Progress: 339/9988
Progress: 340/9988
Progress: 341/9988
Progress: 336/9988
Progress: 342/9988
Progress: 344/9988
Progress: 343/9988
Progress: 346/9988
Progress: 347/9988
Progress: 345/9988
Progress: 349/9988
Progress: 348/9988
Progress: 350/9988
Progress: 351/9988
Progress: 352/9988
Progress: 353/9988
Progress: 354/9988
Progress: 355/9988
Progress: 356/9988
Progress: 358/9988
Progress: 361/9988
Progress: 360/9988
Progress: 357/9988
Progress: 359/9988
Progress: 363/9988
Progress: 364/9988
Progress: 362/9988
Progress: 365/9988
Progress: 366/9988
Progress: 369/9988
Progress: 368/9988
Progress: 367/9988
Progress: 370/9988
Progress: 371/9988
Progress: 372/9988
Progress: 374/9988
Progress: 373/9988
Progress: 376/9988
Progress: 378/9988
Progress: 379/9988
Progress: 377/9988
Progress: 380/9988
Progress: 381/9988
Progress: 375/9988
Progress: 383/9988
Progress: 385/9988
Progress: 384/9988
Progress: 386/9988
Progress: 382/9988
Progress: 387/9988
Progress: 390/9988
Progress: 388/9988
Progress: 391/9988
Progress: 389/9988
Progress: 392/9988
Progress: 393/9988
Progress: 394/9988
Progress: 397/9988
Progress: 396/9988
Progress: 402/9988
Progress: 398/9988
Progress: 401/9988
Progress: 399/9988
Progress: 403/9988
Progress: 400/9988
Progress: 405/9988
Progress: 395/9988
Progress: 404/9988
Progress: 407/9988
Progress: 406/9988
Progress: 409/9988
Progress: 411/9988
Progress: 408/9988
Progress: 414/9988
Progress: 415/9988
Progress: 416/9988
Progress: 413/9988
Progress: 410/9988
Progress: 412/9988
Progress: 419/9988
Progress: 420/9988
Progress: 417/9988
Progress: 418/9988
Progress: 421/9988
Progress: 424/9988
Progress: 426/9988
Progress: 423/9988
Progress: 427/9988
Progress: 422/9988
Progress: 425/9988
Progress: 428/9988
Progress: 430/9988
Progress: 429/9988
Progress: 434/9988
Progress: 433/9988
Progress: 435/9988
Progress: 432/9988
Progress: 438/9988
Progress: 436/9988
Progress: 437/9988
Progress: 431/9988
Progress: 440/9988
Progress: 439/9988
Progress: 442/9988
Progress: 444/9988
Progress: 443/9988
Progress: 441/9988
Progress: 445/9988
Progress: 446/9988
Progress: 449/9988
Progress: 447/9988
Progress: 450/9988
Progress: 448/9988
Progress: 453/9988
Progress: 452/9988
Progress: 451/9988
Progress: 455/9988
Progress: 456/9988
Progress: 457/9988
Progress: 454/9988
Progress: 458/9988
Progress: 459/9988
Progress: 461/9988
Progress: 460/9988
Progress: 462/9988
Progress: 463/9988
Progress: 467/9988
Progress: 466/9988
Progress: 470/9988
Progress: 464/9988
Progress: 469/9988
Progress: 465/9988
Progress: 468/9988
Progress: 471/9988
Progress: 473/9988
Progress: 472/9988
Progress: 474/9988
Progress: 477/9988
Progress: 475/9988
Progress: 476/9988
Progress: 479/9988
Progress: 478/9988
Progress: 482/9988
Progress: 480/9988
Progress: 483/9988
Progress: 484/9988
Progress: 485/9988
Progress: 481/9988
Progress: 486/9988
Progress: 487/9988
Progress: 488/9988
Progress: 492/9988
Progress: 489/9988
Progress: 493/9988
Progress: 495/9988
Progress: 491/9988
Progress: 496/9988
Progress: 494/9988
Progress: 490/9988
Progress: 498/9988
Progress: 497/9988
Progress: 500/9988
Progress: 499/9988
Progress: 501/9988
Progress: 503/9988
Progress: 502/9988
Progress: 504/9988
Progress: 505/9988
Progress: 506/9988
Progress: 508/9988
Progress: 507/9988
Progress: 513/9988
Progress: 510/9988
Progress: 511/9988
Progress: 509/9988
Progress: 514/9988
Progress: 516/9988
Progress: 515/9988
Progress: 512/9988
Progress: 520/9988
Progress: 517/9988
Progress: 518/9988
Progress: 524/9988
Progress: 522/9988
Progress: 523/9988
Progress: 525/9988
Progress: 521/9988
Progress: 526/9988
Progress: 527/9988
Progress: 519/9988
Progress: 529/9988
Progress: 532/9988
Progress: 531/9988
Progress: 530/9988
Progress: 533/9988
Progress: 528/9988
Progress: 535/9988
Progress: 537/9988
Progress: 534/9988
Progress: 539/9988
Progress: 538/9988
Progress: 536/9988
Progress: 540/9988
Progress: 543/9988
Progress: 542/9988
Progress: 545/9988
Progress: 544/9988
Progress: 546/9988
Progress: 541/9988
Progress: 548/9988
Progress: 547/9988
Progress: 551/9988
Progress: 549/9988
Progress: 552/9988
Progress: 554/9988
Progress: 553/9988
Progress: 555/9988
Progress: 550/9988
Progress: 556/9988
Progress: 559/9988
Progress: 560/9988
Progress: 561/9988
Progress: 562/9988
Progress: 557/9988
Progress: 565/9988
Progress: 563/9988
Progress: 564/9988
Progress: 566/9988
Progress: 558/9988
Progress: 567/9988
Progress: 570/9988
Progress: 569/9988
Progress: 568/9988
Progress: 571/9988
Progress: 573/9988
Progress: 572/9988
Progress: 574/9988
Progress: 575/9988
Progress: 578/9988
Progress: 577/9988
Progress: 579/9988
Progress: 576/9988
Progress: 582/9988
Progress: 583/9988
Progress: 581/9988
Progress: 585/9988
Progress: 584/9988
Progress: 586/9988
Progress: 580/9988
Progress: 588/9988
Progress: 591/9988
Progress: 587/9988
Progress: 593/9988
Progress: 589/9988
Progress: 594/9988
Progress: 590/9988
Progress: 595/9988
Progress: 596/9988
Progress: 597/9988
Progress: 598/9988
Progress: 600/9988
Progress: 592/9988
Progress: 599/9988
Progress: 602/9988
Progress: 603/9988
Progress: 601/9988
Progress: 605/9988
Progress: 608/9988
Progress: 606/9988
Progress: 611/9988
Progress: 604/9988
Progress: 607/9988
Progress: 609/9988
Progress: 614/9988
Progress: 610/9988
Progress: 612/9988
Progress: 615/9988
Progress: 617/9988
Progress: 616/9988
Progress: 619/9988
Progress: 613/9988
Progress: 620/9988
Progress: 618/9988
Progress: 621/9988
Progress: 622/9988
Progress: 623/9988
Progress: 625/9988
Progress: 626/9988
Progress: 627/9988
Progress: 624/9988
Progress: 628/9988
Progress: 631/9988
Progress: 629/9988
Progress: 630/9988
Progress: 634/9988
Progress: 632/9988
Progress: 635/9988
Progress: 633/9988
Progress: 636/9988
Progress: 637/9988
Progress: 638/9988
Progress: 641/9988
Progress: 639/9988
Progress: 642/9988
Progress: 643/9988
Progress: 645/9988
Progress: 644/9988
Progress: 640/9988
Progress: 646/9988
Progress: 649/9988
Progress: 650/9988
Progress: 647/9988
Progress: 652/9988
Progress: 651/9988
Progress: 653/9988
Progress: 655/9988
Progress: 648/9988
Progress: 654/9988
Progress: 658/9988
Progress: 656/9988
Progress: 660/9988
Progress: 657/9988
Progress: 659/9988
Progress: 661/9988
Progress: 662/9988
Progress: 665/9988
Progress: 666/9988
Progress: 667/9988
Progress: 668/9988
Progress: 664/9988
Progress: 663/9988
Progress: 669/9988
Progress: 672/9988
Progress: 671/9988
Progress: 670/9988
Progress: 673/9988
Progress: 675/9988
Progress: 676/9988
Progress: 674/9988
Progress: 682/9988
Progress: 680/9988
Progress: 681/9988
Progress: 677/9988
Progress: 679/9988
Progress: 678/9988
Progress: 683/9988
Progress: 688/9988
Progress: 685/9988
Progress: 686/9988
Progress: 690/9988
Progress: 687/9988
Progress: 689/9988
Progress: 684/9988
Progress: 692/9988
Progress: 691/9988
Progress: 694/9988
Progress: 695/9988
Progress: 696/9988
Progress: 693/9988
Progress: 697/9988
Progress: 698/9988
Progress: 701/9988
Progress: 700/9988
Progress: 699/9988
Progress: 702/9988
Progress: 704/9988
Progress: 705/9988
Progress: 706/9988
Progress: 709/9988
Progress: 708/9988
Progress: 703/9988
Progress: 711/9988
Progress: 710/9988
Progress: 712/9988
Progress: 714/9988
Progress: 713/9988
Progress: 707/9988
Progress: 716/9988
Progress: 715/9988
Progress: 718/9988
Progress: 721/9988
Progress: 719/9988
Progress: 717/9988
Progress: 720/9988
Progress: 722/9988
Progress: 723/9988
Progress: 725/9988
Progress: 728/9988
Progress: 726/9988
Progress: 724/9988
Progress: 729/9988
Progress: 727/9988
Progress: 732/9988
Progress: 731/9988
Progress: 734/9988
Progress: 735/9988
Progress: 730/9988
Progress: 736/9988
Progress: 733/9988
Progress: 737/9988
Progress: 738/9988
Progress: 739/9988
Progress: 740/9988
Progress: 742/9988
Progress: 744/9988
Progress: 743/9988
Progress: 741/9988
Progress: 745/9988
Progress: 747/9988
Progress: 746/9988
Progress: 748/9988
Progress: 751/9988
Progress: 749/9988
Progress: 753/9988
Progress: 752/9988
Progress: 754/9988
Progress: 755/9988
Progress: 750/9988
Progress: 756/9988
Progress: 757/9988
Progress: 759/9988
Progress: 758/9988
Progress: 761/9988
Progress: 760/9988
Progress: 762/9988
Progress: 763/9988
Progress: 765/9988
Progress: 764/9988
Progress: 767/9988
Progress: 770/9988
Progress: 766/9988
Progress: 769/9988
Progress: 768/9988
Progress: 771/9988
Progress: 773/9988
Progress: 774/9988
Progress: 772/9988
Progress: 778/9988
Progress: 776/9988
Progress: 777/9988
Progress: 779/9988
Progress: 775/9988
Progress: 780/9988
Progress: 782/9988
Progress: 783/9988
Progress: 784/9988
Progress: 785/9988
Progress: 786/9988
Progress: 788/9988
Progress: 790/9988
Progress: 789/9988
Progress: 792/9988
Progress: 791/9988
Progress: 781/9988
Progress: 787/9988
Progress: 793/9988
Progress: 794/9988
Progress: 795/9988
Progress: 799/9988
Progress: 797/9988
Progress: 796/9988
Progress: 798/9988
Progress: 803/9988
Progress: 801/9988
Progress: 802/9988
Progress: 800/9988
Progress: 807/9988
Progress: 806/9988
Progress: 804/9988
Progress: 805/9988
import json
import datetime

def myconverter(o):
    if isinstance(o, datetime.datetime):
        return o.__str__()

# json.dumps(sorted(repoUsageMap.items()), default = myconverter)
for item in sorted(repoUsageMap.items()):
    repoList = item[1]
    sortedRepoList = sorted(repoList, key=lambda tup: tup[2])
    print(item[0], sortedRepoList[-1])2010-11-01 00:00:00 ('https://github.com/less/less.js', ('', 'eval'), 15)
2010-12-01 00:00:00 ('https://github.com/less/less.js', ('', 'eval'), 15)
2011-01-01 00:00:00 ('https://github.com/less/less.js', ('', 'eval'), 15)
2011-02-01 00:00:00 ('https://github.com/less/less.js', ('', 'eval'), 15)
2011-03-01 00:00:00 ('https://github.com/Automattic/mongoose', ('', 'eval'), 15)
2011-04-01 00:00:00 ('https://github.com/mapbox/carto', ('', 'eval'), 22)
2011-05-01 00:00:00 ('https://github.com/mapbox/carto', ('', 'eval'), 22)
2011-06-01 00:00:00 ('https://github.com/mapbox/carto', ('', 'eval'), 23)
2011-07-01 00:00:00 ('https://github.com/mapbox/carto', ('', 'eval'), 23)
2011-08-01 00:00:00 ('https://github.com/mapbox/carto', ('', 'eval'), 24)
2011-09-01 00:00:00 ('https://github.com/mapbox/carto', ('', 'eval'), 24)
2011-10-01 00:00:00 ('https://github.com/mapbox/carto', ('', 'eval'), 24)
2011-11-01 00:00:00 ('https://github.com/One-com/one-color', ('', 'new Function'), 32)
2011-12-01 00:00:00 ('https://github.com/less/less.js', ('', 'eval'), 320)
2012-01-01 00:00:00 ('https://github.com/less/less.js', ('', 'eval'), 321)
2012-02-01 00:00:00 ('https://github.com/less/less.js', ('', 'eval'), 321)
2012-03-01 00:00:00 ('https://github.com/less/less.js', ('', 'eval'), 321)
2012-04-01 00:00:00 ('https://github.com/less/less.js', ('', 'eval'), 638)
2012-05-01 00:00:00 ('https://github.com/less/less.js', ('', 'eval'), 638)
2012-06-01 00:00:00 ('https://github.com/less/less.js', ('', 'eval'), 638)
2012-07-01 00:00:00 ('https://github.com/less/less.js', ('', 'eval'), 638)
2012-08-01 00:00:00 ('https://github.com/less/less.js', ('', 'eval'), 638)
2012-09-01 00:00:00 ('https://github.com/less/less.js', ('', 'eval'), 638)
2012-10-01 00:00:00 ('https://github.com/less/less.js', ('', 'eval'), 638)
2012-11-01 00:00:00 ('https://github.com/less/less.js', ('', 'eval'), 744)
2012-12-01 00:00:00 ('https://github.com/less/less.js', ('', 'eval'), 744)
2013-01-01 00:00:00 ('https://github.com/kss-node/kss-node', ('', 'eval'), 64)
2013-02-01 00:00:00 ('https://github.com/kss-node/kss-node', ('', 'eval'), 64)
2013-03-01 00:00:00 ('https://github.com/kss-node/kss-node', ('', 'eval'), 64)
2013-04-01 00:00:00 ('https://github.com/less/less.js', ('', 'eval'), 1103)
2013-05-01 00:00:00 ('https://github.com/less/less.js', ('', 'eval'), 1103)
2013-06-01 00:00:00 ('https://github.com/less/less.js', ('', 'eval'), 1103)
2013-07-01 00:00:00 ('https://github.com/less/less.js', ('', 'eval'), 1191)
2013-08-01 00:00:00 ('https://github.com/less/less.js', ('', 'eval'), 1367)
2013-09-01 00:00:00 ('https://github.com/less/less.js', ('', 'eval'), 1367)
2013-10-01 00:00:00 ('https://github.com/less/less.js', ('', 'eval'), 1464)
2013-11-01 00:00:00 ('https://github.com/less/less.js', ('', 'eval'), 1558)
2013-12-01 00:00:00 ('https://github.com/less/less.js', ('', 'eval'), 1558)
2014-01-01 00:00:00 ('https://github.com/less/less.js', ('', 'eval'), 1558)
2014-02-01 00:00:00 ('https://github.com/less/less.js', ('', 'eval'), 1655)
2014-03-01 00:00:00 ('https://github.com/less/less.js', ('', 'eval'), 1852)
2014-04-01 00:00:00 ('https://github.com/less/less.js', ('', 'eval'), 1852)
2014-05-01 00:00:00 ('https://github.com/less/less.js', ('', 'eval'), 1852)
2014-06-01 00:00:00 ('https://github.com/less/less.js', ('', 'eval'), 1852)
2014-07-01 00:00:00 ('https://github.com/less/less.js', ('', 'eval'), 2005)
2014-08-01 00:00:00 ('https://github.com/uhop/dcl', ('', 'eval'), 119)
2014-09-01 00:00:00 ('https://github.com/uhop/dcl', ('', 'eval'), 119)
2014-10-01 00:00:00 ('https://github.com/rollbar/rollbar.js', ('', 'eval'), 123)
2014-11-01 00:00:00 ('https://github.com/rollbar/rollbar.js', ('', 'eval'), 129)
2014-12-01 00:00:00 ('https://github.com/rollbar/rollbar.js', ('', 'eval'), 135)
2015-01-01 00:00:00 ('https://github.com/rollbar/rollbar.js', ('', 'eval'), 147)
2015-02-01 00:00:00 ('https://github.com/rollbar/rollbar.js', ('', 'eval'), 153)
2015-03-01 00:00:00 ('https://github.com/rollbar/rollbar.js', ('', 'eval'), 165)
2015-04-01 00:00:00 ('https://github.com/rollbar/rollbar.js', ('', 'eval'), 165)
2015-05-01 00:00:00 ('https://github.com/rollbar/rollbar.js', ('', 'eval'), 165)
2015-06-01 00:00:00 ('https://github.com/rollbar/rollbar.js', ('', 'eval'), 166)
2015-07-01 00:00:00 ('https://github.com/rollbar/rollbar.js', ('', 'eval'), 190)
2015-08-01 00:00:00 ('https://github.com/rollbar/rollbar.js', ('', 'eval'), 190)
2015-09-01 00:00:00 ('https://github.com/rollbar/rollbar.js', ('', 'eval'), 198)
2015-10-01 00:00:00 ('https://github.com/rollbar/rollbar.js', ('', 'eval'), 209)
2015-11-01 00:00:00 ('https://github.com/rollbar/rollbar.js', ('', 'eval'), 215)
2015-12-01 00:00:00 ('https://github.com/rollbar/rollbar.js', ('', 'eval'), 229)
2016-01-01 00:00:00 ('https://github.com/rollbar/rollbar.js', ('', 'eval'), 237)
2016-02-01 00:00:00 ('https://github.com/rollbar/rollbar.js', ('', 'eval'), 245)
2016-03-01 00:00:00 ('https://github.com/rollbar/rollbar.js', ('', 'eval'), 245)
2016-04-01 00:00:00 ('https://github.com/rollbar/rollbar.js', ('', 'eval'), 253)
2016-05-01 00:00:00 ('https://github.com/rollbar/rollbar.js', ('', 'eval'), 261)
2016-06-01 00:00:00 ('https://github.com/rollbar/rollbar.js', ('', 'eval'), 261)
2016-07-01 00:00:00 ('https://github.com/rollbar/rollbar.js', ('', 'eval'), 261)
2016-08-01 00:00:00 ('https://github.com/rollbar/rollbar.js', ('', 'eval'), 261)
2016-09-01 00:00:00 ('https://github.com/rollbar/rollbar.js', ('', 'eval'), 261)
2016-10-01 00:00:00 ('https://github.com/rollbar/rollbar.js', ('', 'eval'), 261)
2016-11-01 00:00:00 ('https://github.com/rollbar/rollbar.js', ('', 'eval'), 265)
2016-12-01 00:00:00 ('https://github.com/rollbar/rollbar.js', ('', 'eval'), 265)
2017-01-01 00:00:00 ('https://github.com/rollbar/rollbar.js', ('', 'eval'), 265)
2017-02-01 00:00:00 ('https://github.com/rollbar/rollbar.js', ('', 'eval'), 265)
2017-03-01 00:00:00 ('https://github.com/rollbar/rollbar.js', ('', 'eval'), 269)
2017-04-01 00:00:00 ('https://github.com/less/less.js', ('', 'eval'), 104)
2017-05-01 00:00:00 ('https://github.com/less/less.js', ('', 'eval'), 104)
2017-06-01 00:00:00 ('https://github.com/less/less.js', ('', 'eval'), 104)
2017-07-01 00:00:00 ('https://github.com/uhop/dcl', ('', 'eval'), 147)
2017-08-01 00:00:00 ('https://github.com/less/less.js', ('', 'eval'), 104)
2017-09-01 00:00:00 ('https://github.com/less/less.js', ('', 'eval'), 104)
2017-10-01 00:00:00 ('https://github.com/less/less.js', ('', 'eval'), 104)
2017-11-01 00:00:00 ('https://github.com/less/less.js', ('', 'eval'), 104)
2017-12-01 00:00:00 ('https://github.com/less/less.js', ('', 'eval'), 104)
2018-01-01 00:00:00 ('https://github.com/less/less.js', ('', 'eval'), 104)
2018-02-01 00:00:00 ('https://github.com/less/less.js', ('', 'eval'), 104)
2018-03-01 00:00:00 ('https://github.com/less/less.js', ('', 'eval'), 106)
2018-04-01 00:00:00 ('https://github.com/less/less.js', ('', 'eval'), 106)
2018-05-01 00:00:00 ('https://github.com/less/less.js', ('', 'eval'), 106)
2018-06-01 00:00:00 ('https://github.com/less/less.js', ('', 'eval'), 106)
2018-07-01 00:00:00 ('https://github.com/uhop/dcl', ('', 'eval'), 172)
2018-08-01 00:00:00 ('https://github.com/uhop/dcl', ('', 'eval'), 177)
2018-09-01 00:00:00 ('https://github.com/uhop/dcl', ('', 'eval'), 177)
2018-10-01 00:00:00 ('https://github.com/uhop/dcl', ('', 'eval'), 177)
2018-11-01 00:00:00 ('https://github.com/uhop/dcl', ('', 'eval'), 177)
timeToUsageMap_____no_output_____import matplotlib.pyplot as plt
import numpy as np
import datetime

averagedTimemap = processing.normalizeOverallPackages(timeToUsageMap, repos)

titleString = ""
fileString = ""
for f in functions:
    fileString = fileString + f[1] + "_"
    if f == functions[len(functions)-1]:
        titleString = titleString + "{}.{}".format(f[0], f[1]) if f[0] != '' else titleString + f[1]
    else:
        titleString = titleString + "{}.{} vs. ".format(f[0], f[1]) if f[0] != '' else titleString + "{} vs. ".format(f[1])

if filtered:
    fileString = fileString + filterType

plt.figure(figsize=(10,10), dpi=90)
plt.xlabel("Years")
plt.ylabel('Average call count per kLoC')
plt.title("{} Usage evolution".format(titleString))

# monthly
for function in functions:
    dates = []
    values = []
    for y in range(2010,2019):
        for m in range(1,13):
            date = datetime.datetime(y,m, 1)
            if date in averagedTimemap:
                dates.append(date)
                monthSum = averagedTimemap[date][function]
                values.append(monthSum)
    plt.plot(dates, values, label="{}.{}".format(function[0], function[1]) if function[0] != '' else function[1])

plt.legend(loc='best', frameon=False)

plt.savefig('figures/function_latestversion_{}.png'.format(fileString))

plt.show()_____no_output_____import matplotlib.pyplot as plt
import numpy as np
import datetime

values = []

# yearly sum
for y in range(2010,2019):
    yearlySum = 0
    for m in range(1,13):
        date = datetime.datetime(y,m, 1)
        if date in timeToUsageMap:
            dateSum = timeToUsageMap[date]
            yearlySum = yearlySum + dateSum
    values.append(yearlySum)

plt.figure(figsize=(10,10), dpi=90)

plt.xticks([0,1,2,3,4,5,6,7,8], ['2010','2011','2012','2013','2014','2015','2016','2017', '2018'])
plt.xlabel("Years")
plt.ylabel('Sum of call count per kLoC')
plt.title("{}.{} Usage evolution".format(module,function))
plt.plot(values)

plt.show()_____no_output_____
</code>


# Notebook from xubuild/nlp-tutorial
Path: 3-2.TextLSTM/TextLSTM_Torch.ipynb

<code>
'''
  code by Tae Hwan Jung(Jeff Jung) @graykode
'''
import numpy as np
import torch
import torch.nn as nn
import torch.optim as optim
from torch.autograd import Variable

dtype = torch.FloatTensor

char_arr = [c for c in 'abcdefghijklmnopqrstuvwxyz']
word_dict = {n: i for i, n in enumerate(char_arr)}
number_dict = {i: w for i, w in enumerate(char_arr)}
n_class = len(word_dict) # number of class(=number of vocab)

seq_data = ['make', 'need', 'coal', 'word', 'love', 'hate', 'live', 'home', 'hash', 'star']_____no_output_____# TextLSTM Parameters
n_step = 3 # input length
n_hidden = 128_____no_output_____def make_batch(seq_data):
    input_batch, target_batch = [], []

    for seq in seq_data:
        input = [word_dict[n] for n in seq[:-1]] # 'm', 'a' , 'k' is input
        target = word_dict[seq[-1]] # 'e' is target
        input_batch.append(np.eye(n_class)[input])
        target_batch.append(target)

    return Variable(torch.Tensor(input_batch)), Variable(torch.LongTensor(target_batch))_____no_output_____input_batch, target_batch = make_batch(seq_data)
print(input_batch[0])
print(target_batch[0])
input_batch[0].transpose(0, 1)tensor([[0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0.],
        [1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0.],
        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0.]])
tensor(4)
class TextLSTM(nn.Module):
    def __init__(self):
        super(TextLSTM, self).__init__()

        self.lstm = nn.LSTM(input_size=n_class, hidden_size=n_hidden)
        self.W = nn.Parameter(torch.randn([n_hidden, n_class]).type(dtype))
        self.b = nn.Parameter(torch.randn([n_class]).type(dtype))

    def forward(self, X):
        input = X.transpose(0, 1)  # X : [n_step, batch_size, n_class]

        hidden_state = Variable(torch.zeros(1, len(X), n_hidden))   # [num_layers(=1) * num_directions(=1), batch_size, n_hidden]
        cell_state = Variable(torch.zeros(1, len(X), n_hidden))     # [num_layers(=1) * num_directions(=1), batch_size, n_hidden]

        outputs, (_, _) = self.lstm(input, (hidden_state, cell_state))
        outputs = outputs[-1]  # [batch_size, n_hidden]
        model = torch.mm(outputs, self.W) + self.b  # model : [batch_size, n_class]
        return model_____no_output_____model = TextLSTM()

criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=0.001)

output = model(input_batch)

# Training
for epoch in range(1000):
    optimizer.zero_grad()

    output = model(input_batch)
    loss = criterion(output, target_batch)
    if (epoch + 1) % 100 == 0:
        print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.6f}'.format(loss))

    loss.backward()
    optimizer.step()

inputs = [sen[:3] for sen in seq_data]

predict = model(input_batch).data.max(1, keepdim=True)[1]
print(inputs, '->', [number_dict[n.item()] for n in predict.squeeze()])Epoch: 0100 cost = 0.014800
Epoch: 0200 cost = 0.003217
Epoch: 0300 cost = 0.001433
Epoch: 0400 cost = 0.000814
Epoch: 0500 cost = 0.000527
Epoch: 0600 cost = 0.000369
Epoch: 0700 cost = 0.000273
Epoch: 0800 cost = 0.000210
Epoch: 0900 cost = 0.000166
Epoch: 1000 cost = 0.000135
['mak', 'nee', 'coa', 'wor', 'lov', 'hat', 'liv', 'hom', 'has', 'sta'] -> ['e', 'd', 'l', 'd', 'e', 'e', 'e', 'e', 'h', 'r']

</code>


# Notebook from qiringji/python-causality-handbook
Path: causal-inference-for-the-brave-and-true/02-Randomised-Experiments.ipynb

# 02 - Randomised Experiments

## The Golden Standard

In the previous session, we saw why and how association is different from causation. We also saw what is required to make association be causation.

$
E[Y|T=1] - E[Y|T=0] = \underbrace{E[Y_1 - Y_0|T=1]}_{ATET} + \underbrace{\{ E[Y_0|T=1] - E[Y_0|T=0] \}}_{BIAS}
$

To recap, association becomes causation if there is no bias. There will be no bias if \\(E[Y_0|T=0]=E[Y_0|T=1]\\). In words, association will be causation if the treated and control are equal, or comparable, except for the treatment they receive. Or, in more technical words, when the outcome of the untreated is equal to the counterfactual outcome of the treated. Remember that this counterfactual outcome is the outcome of the treated group if they had not received the treatment.

I think we did an OK job explaining in math terms how to make association equal to causation. But that was only in theory. Now, we look at the first tool we have to make the bias vanish: **Randomised Experiments**. Randomised experiments consist of randomly assigning individuals in a population to a treatment or to a control group. The proportion that receives the treatment doesn't have to be 50%. You could have an experiment where only 10% of your samples get the treatment.

Randomisation annihilates bias by making the potential outcomes independent of the treatment.

$
(Y_0, Y_1) \perp\!\!\!\perp T
$

This can be confusing at first (it was for me). But don't worry, my brave and true fellow, I'll explain it further. If the outcome is independent of the treatment, doesn't this also imply that the treatment has no effect? Well, yes! But notice I'm not talking about the outcomes. Rather, I'm talking about the **potential** outcomes. The potential outcomes is how the outcome **would have been** under the treatment (\\(Y_1\\)) or under the control (\\(Y_0\\)). In randomized trials, we **don't** want the outcome to be **independent** on the treatment, since we think **the treatment causes** the outcome. But we want the **potential** outcomes to be independent from the treatment.

![img](./data/img/rct/indep.png)

Saying that the potential outcomes are independent from the treatment is saying that they would be, in expectation, the same in the treatment or the control group. In simpler terms, it means that treatment and control groups are comparable. Or that knowing the treatment assignment doesn't give me any information on how the outcome was, previous to the treatment. Consequently, \\((Y_0, Y_1)\perp T\\) means that the treatment is the only thing that is generating a difference between the outcome in the treated and in the control. To see this, notice that independence implies precisely that

$
E[Y_0|T=0]=E[Y_0|T=1]=E[Y_0]
$

Which, as we've seen, makes it so that

$
E[Y|T=1] - E[Y|T=0] = E[Y_1 - Y_0]=ATE
$

So, randomization gives us a way to use a simple difference in means between treatment and control and call that the treatment effect.

## In a School Far, Far Away

In the year of 2020, the Coronavirus Pandemic forced business to adapt to social distancing. Delivery services became widespread, big corporations shifted to a remote work strategy. With schools, it wasn't different. Many started their own online repository of classes.

Four months into the crises and many are wondering if the introduced changes could be maintained. There is no question that online learning has its benefits. For once, it is cheaper, since it can save on real estate and transportation. It can also be more digital, leveraging world class content from around the globe, not just from a fixed set of teachers. In spite of all of that, we still need to answer if online learning has a negative or positive impact on the student's academic performance.

One way to answer this is to take students from schools that give mostly online classes and compare them with students from schools that give lectures in traditional classrooms. As we know by now this is not the best approach. It could be that online schools attract only the well disciplined students that do better than average even if the class were presential. In this case, we would have a positive bias, where the treated are academically better than the untreated: \\(E[Y_0|T=1] > E[Y_0|T=0]\\).

Or, on the flip side, it could be that online classes are cheaper and are composed mostly of less wealthy students, who might have to work besides studying. In this case, these students would do worse than those from the presential schools even if they took presential classes. If this was the case, we would have bias in the other direction, where the treated are academically worse than the untreated:  \\(E[Y_0|T=1] < E[Y_0|T=0]\\).

So, although we could do simple comparisons, it wouldn't be very convincing. One way or another, we could never be sure if there wasn't any bias lurking around and masking our causal effect.

![img](./data/img/rct/lurking_bias.png)

To solve that, we need to make the treated and untreated comparable \\(E[Y_0|T=1] = E[Y_0|T=0]\\). One way to force this is by randomly assigning the online and presential classes to students. If we managed to do that, the treatment and untreated would be, on average, the same, except for the treatment they receive.

Fortunately, some economists have done that for us. They've randomized not the students, but the classes. Some of them were randomly assigned to have face-to-face lectures, others, to have only online lectures and a third group, to have a blended format of both online and face-to-face lectures. At the end of the semester, they collected data on a standard exam.

Here is what the data looks like:_____no_output_____

import pandas as pd
import numpy as np

data = pd.read_csv("./data/online_classroom.csv")
print(data.shape)
data.head()(323, 10)

We can see that we have 323 samples. It's not exactly big data, but something we can work with. To estimate the causal effect, we can simply compute the mean score for each of the treatment groups._____no_output_____

(data
 .assign(class_format = np.select(
     [data["format_ol"].astype(bool), data["format_blended"].astype(bool)],
     ["online", "blended"],
     default="face_to_face"
 ))
 .groupby(["class_format"])
 .mean())_____no_output_____

Yup. It's that simple. We can see that face to face classes yield a 78.54 average score, while online classes yield a 73.63 average score. Not so good news for the proponents of online learning. The \\(ATE\\) for online class is thus -4.91. This means that **online classes cause students to perform about 5 points lower**, on average. That's it. You don't need to worry that online classes might have poorer students that can't afford face to face classes or, for that matter, you don't have to worry that the students from the different treatments are different in any way other than the treatment they received. By design, the random experiment is made to wipe out those differences.

For this reason, a good sanity check to see if the randomisation was done right (or if you are looking at the right data) is to check if the treated are equal to the untreated in pre-treatment variables. In our data, we have information on gender and ethnicity, so we can see if they are equal across groups. For the `gender`, `asian`, `hispanic` and `white` variables, we can say that they look pretty similar. The `black` variable, however, looks a little bit different. This draws attention to what happens with a small dataset. Even under randomisation, it could be that, by chance, one group is different from another. In large samples, this difference tends to disappear.

## The Ideal Experiment

Randomised experiments are the most reliable way to get causal effects. It's a ridiculously simple technique and absurdly convincing. It is so powerful that most countries have it as a requirement for showing the effectiveness of new medicine. To make a terrible analogy, you can think of RCT and Aang, from Avatar: The Last Airbender, while other techniques are more like Sokka. Sokka is cool and can pull some neat tricks here and there, but Aang can bend the four elements and connect with the spiritual world. Think of it this way, if we could, RCT would be all we would ever do to uncover causality. A well designed RCT is the dream of any scientist.

![img](./data/img/rct/science_dream.png)

Unfortunately, they tend to be either very expensive or just plain unethical. Sometimes, we simply can't control the assignment mechanism. Imagine yourself as a physician trying to estimate the effect of smoking during pregnancy on baby weight at birth. You can't simply force a random portion of moms to smoke during pregnancy. Or say you work for a big bank and you need to estimate the impact of the credit line on customer churn. It would be too expensive to give random credit lines to your customers. Or that you want to understand the impact of increasing minimum wage on unemployment. You can't simply assign countries to have one or another minimum wage. You get the point.

We will later see how to lower the randomisation cost by using conditional randomisation, but there is nothing we can do about unethical or unfeasible experiments. Still, whenever we deal with causal questions, it is worth thinking about the **ideal experiment**. Always ask yourself, if you could, **what would be the ideal experiment you would run to uncover this causal effect?** This tends to shed some light in the way of how we can uncover the causal effect even without the ideal experiment.

## The Assignment Mechanism

In a randomised experiment, the mechanism that assigns units to one treatment or the other is, well, random. As we will see later, all causal inference techniques will somehow try to identify the assignment mechanisms of the treatments. When we know for sure how this mechanism behaves, causal inference will be much more certain, even if the assignment mechanism isn't random.

Unfortunately, the assignment mechanism can't be discovered by simply looking at the data. For example, if you have a dataset where higher education correlates with wealth, you can't know for sure which one caused which by just looking at the data. You will have to use your knowledge about how the world works to argue in favor of a plausible assignment mechanism: is it the case that schools educate people, making them more productive and hence leading them to higher paying jobs. Or, if you are pessimistic about education, you can say that schools do nothing to increase productivity and this is just a spurious correlation because only wealthy families can afford to have a kid getting a higher degree.

In causal questions, we usually have the possibility to argue in both ways: that X causes Y, or that it is a third variable Z that causes both X and Y, and hence the X and Y correlation is just spurious. It is for this reason that knowing the assignment mechanism leads to a much more convincing causal answer. This is also what makes causal inference so exciting. While applied ML is usually just pressing some buttons in the right order, applied causal inference requires you to seriously think about the mechanism generating that data.

## Key Ideas

We looked at how randomised experiments are the simplest and most effective way to uncover causal impact. It does this by making the treatment and control group comparable. Unfortunately, we can't do randomised experiments all the time, but it is still useful to think about what is the ideal experiment we would do if we could.

Someone that is familiar with statistics might be protesting right now that I didn't look at the variance of my causal effect estimate. How can I know that a 4.91 points decrease is not due to chance? In other words, how can I know if the difference is statistically significant? And they would be right. Don't worry. I intend to review some statistical concepts next.

## References

I like to think of this entire book as a tribute to Joshua Angrist, Alberto Abadie and Christopher Walters for their amazing Econometrics class. Most of the ideas here are taken from their classes at the American Economic Association. Watching them is what is keeping me sane during this tough year of 2020.
* [Cross-Section Econometrics](https://www.aeaweb.org/conference/cont-ed/2017-webcasts)
* [Mastering Mostly Harmless Econometrics](https://www.aeaweb.org/conference/cont-ed/2020-webcasts)

I'll also like to reference the amazing books from Angrist. They have shown me that Econometrics, or 'Metrics as they call it, is not only extremely useful but also profoundly fun.

* [Mostly Harmless Econometrics](https://www.mostlyharmlesseconometrics.com/)
* [Mastering 'Metrics](https://www.masteringmetrics.com/)

My final reference is Miguel Hernan and Jamie Robins' book. It has been my trustworthy companion in the most thorny causal questions I had to answer.

* [Causal Inference Book](https://www.hsph.harvard.edu/miguel-hernan/causal-inference-book/)

The data used here is from a study of Alpert, William T., Kenneth A. Couch, and Oskar R. Harmon. 2016. ["A Randomized Assessment of Online Learning"](https://www.aeaweb.org/articles?id=10.1257/aer.p20161057). American Economic Review, 106 (5): 378-82.

![img](./data/img/poetry.png)

## Contribute

Causal Inference for the Brave and True is an open-source material on causal inference, the statistics of science. It uses only free software, based in Python. Its goal is to be accessible monetarily and intellectually.
If you found this book valuable and you want to support it, please go to [Patreon](https://www.patreon.com/causal_inference_for_the_brave_and_true). If you are not ready to contribute financially, you can also help by fixing typos, suggesting edits or giving feedback on passages you didn't understand. Just go to the book's repository and [open an issue](https://github.com/matheusfacure/python-causality-handbook/issues). Finally, if you liked this content, please share it with others who might find it useful and give it a [star on GitHub](https://github.com/matheusfacure/python-causality-handbook/stargazers)._____no_output_____