# Notebook from BhavyaSree/PythonClass
Path: docs/Notebooks/13.half-pyramid.ipynb

## Print downward Half-Pyramid Pattern with Star (asterisk)_____no_output_____

<code>
list(range(6,0,-1))_____no_output_____def half_pyramid(rows):

    for i in range(rows, 0, -1):
        for _ in range(0, i):
            print('*', end=' ')
        print('')_____no_output_____half_pyramid(6)* * * * * *
* * * * *
* * * *
* * *
* *
*

</code>


# Notebook from PedroBaqui/XCOVID-BR
Path: Other_ML/lg_without_symptons.ipynb

<code>
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
import seaborn as sns
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import roc_curve
from sklearn.metrics import auc
from sklearn.metrics import accuracy_score
import pickle
from sklearn.metrics import r2_score
from sklearn.linear_model import LogisticRegression
from sklearn.neural_network import MLPClassifier
from sklearn.metrics import average_precision_score
from sklearn.metrics import precision_recall_curve
from xgboost import XGBClassifier
from datetime import datetime
from haversine import haversine, Unit
from haversine import haversine_vector
from sklearn.ensemble import RandomForestClassifier
from sklearn.impute import SimpleImputer_____no_output_____# reading dataset
# https://opendatasus.saude.gov.br/dataset/bd-srag-2020

df = pd.read_csv('/home/pedro/bkp/code/dataset/INFLUD-21-09-2020.csv',sep=';',encoding = "ISO-8859-1")

# Inputing constraint in the dataset

# Positive case:

df = df[df['PCR_SARS2']==1]
#print(df.shape)

# Hospitalized people:

df = df[df['PCR_SARS2']==1][df['HOSPITAL']==1][df['NU_IDADE_N']<=110]
#print(df.shape)

# Hospitalized people with age small than 110:

df = df[df['PCR_SARS2']==1][df['HOSPITAL']==1][df['NU_IDADE_N']<=110][df['EVOLUCAO'] != 3][df['EVOLUCAO'] != 9][df['EVOLUCAO'].notnull()]
#print(df.shape)

# %%
# Latitudes and longitudes table from municipalities

df_cod = pd.read_csv('/home/pedro/bkp/code/dataset/municipios.csv', sep=',')

# %%
# Removing last number from "codenumber"

df_cod['CO_MUN_RES'] = df_cod['CO_MUN_RES'].astype(str).str[:-1].astype(np.int64)
df_cod['CO_MU_INTE'] = df_cod['CO_MU_INTE'].astype(str).str[:-1].astype(np.int64)

# %%
# To match catalogues using muninipacity code
# latitude and longitude for pacient residence

result_01 = pd.merge(df, df_cod[['CO_MUN_RES','latitude_res','longitude_res']], on='CO_MUN_RES', how="left")
result_02 = pd.merge(df, df_cod[['CO_MU_INTE','latitude_int','longitude_int']], on='CO_MU_INTE', how="left")

#print(result_01.shape)
#print(result_02.shape)

# %%
# To transforming in tuple

patient_mun_code  = result_01[['latitude_res','longitude_res']].to_numpy()
hospital_mun_code = result_02[['latitude_int','longitude_int']].to_numpy()

#print(patient_mun_code.shape)
#print(hospital_mun_code.shape)

# %%
# To calculate the distance from patient to hospital (difference between municipalities centers in km)

df['distance'] = haversine_vector(patient_mun_code, hospital_mun_code, Unit.KILOMETERS)

# %%
# overcrowded dataset
# dataset with code from hospital (cnes) with epidemiology week and the overcrowded status of hospital.

df_cod = pd.read_csv('/home/pedro/bkp/code/dataset/hospital_overcrowded.csv', sep=',')

# CO_UNI_NOT, SEM_NOT, Overcrowded

# Overload = number of hospitalization in epidemiological week for COVID-19 / 2019 sum hospital hospitalization by SARS

# %%
# To check

df = pd.merge(df, df_cod, on=['CO_UNI_NOT', 'SEM_NOT'], how="left")
print(df.shape)

# %%
# Municipalities number inicial

# patient municipality code number
#print(len(df['CO_MUN_NOT']))

# reporting health unit code number
#print(len(df['CO_UNI_NOT']))

#print(df['CO_MUN_NOT'].nunique())
#print(df['CO_MUN_RES'].nunique())

# %%
# IDHM

# Reading IBGE code for each municipalities and separating it for IDHM index

df_atlas = pd.read_excel (r'/home/pedro/bkp/code/dataset/AtlasBrasil_Consulta.xlsx')

# removind last interger in 'code' variable

df_atlas['code'] = df_atlas['code'].astype(str).str[:-1].astype(np.int64)

# Divinding IDHM in bins

IDHM_veryhigh  =  set(df_atlas['code'][df_atlas['IDHM2010']>=0.800])
#print(len(IDHM_veryhigh))

IDHM_high  =  set(df_atlas['code'][((df_atlas['IDHM2010']>=0.700)&(df_atlas['IDHM2010']<0.800))])
#print(len(IDHM_high))

IDHM_medium  =  set(df_atlas['code'][((df_atlas['IDHM2010']>=0.600)&(df_atlas['IDHM2010']<0.700))])
#print(len(IDHM_medium))

IDHM_low  =  set(df_atlas['code'][((df_atlas['IDHM2010']>=0.500)&(df_atlas['IDHM2010']<0.600))])
#print(len(IDHM_low))

IDHM_verylow  =  set(df_atlas['code'][df_atlas['IDHM2010']<0.500])
#print(len(IDHM_verylow))

df.loc[df['CO_MUN_NOT'].isin(IDHM_veryhigh) == True, 'IDHM'] = 5
df.loc[df['CO_MUN_NOT'].isin(IDHM_high) == True, 'IDHM'] = 4
df.loc[df['CO_MUN_NOT'].isin(IDHM_medium) == True, 'IDHM'] = 3
df.loc[df['CO_MUN_NOT'].isin(IDHM_low) == True, 'IDHM'] = 2
df.loc[df['CO_MUN_NOT'].isin(IDHM_verylow) == True, 'IDHM'] = 1

# %%
# Private and public hospital separation

df_hospital = pd.read_csv('/home/pedro/bkp/code/dataset/CNES_SUS.txt', sep='\t')

public   =  set(df_hospital.iloc[:,0][df_hospital.iloc[:,3]=='S'])
private  =  set(df_hospital.iloc[:,0][df_hospital.iloc[:,3]=='N'])

df.loc[df['CO_UNI_NOT'].isin(public) == True, 'HEALTH_SYSTEM'] = 1
df.loc[df['CO_UNI_NOT'].isin(private) == True, 'HEALTH_SYSTEM'] = 0

# %%
# Constraint on dataset: We only analyze people with evolution, IDHM and Health system outcomes

df = df[df['IDHM'].notnull()][(df['HEALTH_SYSTEM']==1)|(df['HEALTH_SYSTEM']==0)]
#print(df.shape)

# %%
# Municipalities number

#print(len(df['CO_MUN_NOT']))
#print(len(df['CO_MU_INTE']))

#print(df['CO_MUN_NOT'].nunique())
#print(df['CO_MU_INTE'].nunique())

# %%
# To selecting features and target

df = df[['Overload', 'distance','NU_IDADE_N','CS_SEXO','IDHM','CS_RACA','CS_ESCOL_N','SG_UF_NOT','CS_ZONA',\
'HEALTH_SYSTEM','CS_GESTANT','FEBRE','VOMITO','TOSSE','GARGANTA','DESC_RESP','DISPNEIA','DIARREIA',\
'SATURACAO','CARDIOPATI','HEPATICA','ASMA','PNEUMOPATI','RENAL','HEMATOLOGI','DIABETES',\
'OBESIDADE','NEUROLOGIC','IMUNODEPRE','EVOLUCAO']]

# %%
# adding comorbidities

df['SUM_COMORBIDITIES'] = df.iloc[:,19:-1].replace([9,2], 0).fillna(0).sum(axis=1)

# %%
# Ordering features

df = df[['Overload','distance','NU_IDADE_N','CS_SEXO','CS_RACA','IDHM','CS_ESCOL_N','SG_UF_NOT','SUM_COMORBIDITIES',\
'HEALTH_SYSTEM','CS_ZONA','CARDIOPATI','HEPATICA','ASMA','PNEUMOPATI','RENAL','HEMATOLOGI',\
'DIABETES','OBESIDADE','NEUROLOGIC','IMUNODEPRE','EVOLUCAO']]

# %%
# Pre-Processing

df = df[df['EVOLUCAO'].notnull()][df['EVOLUCAO']!=9][df['EVOLUCAO']!=3]#[df_BR['NU_IDADE_N'].notnull()]
df['CS_SEXO']=df['CS_SEXO'].replace({'M': 1, 'F':0, 'I':9, 'NaN':np.nan})

# replacing 2 by 0 (Death patients)
df.iloc[:,11:] = df.iloc[:,11:].replace(to_replace = 2.0, value =0)

df['SG_UF_NOT'] = df['SG_UF_NOT'].map({'SP': 0, 'RJ':1, 'MG': 2 , 'ES':3, \
'RS':4, 'SC': 5, 'PR': 6, 'MT': 7, 'MS': 8, 'GO':9, 'DF':10, 'RO':11,'AC':12,'AM':13,\
'RR':14,'PA':15,'AP':16,'TO':17,'MA':18,'PI':19,'BA':20,'CE':21,'RN':22,'PB':23,'PE':24,'AL':25,'SE':26})

# For missing values in comorbidities and symptoms we filled by 0.
df.iloc[:,11:-1] = df.iloc[:,11:-1].fillna(0)

# For logistic regression in special we filled by -1 the Nan numbers

imp_mean = SimpleImputer(missing_values=np.nan, strategy='mean')
df.iloc[:,:11] = imp_mean.fit_transform(df.iloc[:,:11])

# %%
# Ordering features

df = df[['Overload','distance','NU_IDADE_N','CS_SEXO','CS_RACA','IDHM','CS_ESCOL_N','SG_UF_NOT','SUM_COMORBIDITIES',\
'HEALTH_SYSTEM','CS_ZONA','CARDIOPATI','HEPATICA','ASMA','PNEUMOPATI','RENAL','HEMATOLOGI',\
'DIABETES','OBESIDADE','NEUROLOGIC','IMUNODEPRE','EVOLUCAO']]

df = df.rename(columns={'Overload':'Overload','distance':'Distance','NU_IDADE_N':'Age','CS_SEXO':'Sex','CS_RACA':'Race','IDHM':'IDHM',\
'CS_ESCOL_N':'Education','SG_UF_NOT':'State','SUM_COMORBIDITIES':'Sum Comorbidities',\
'HEALTH_SYSTEM':'Health System','CS_ZONA':'Urbanity','CARDIOPATI':'Cardiopathy','HEPATICA':'Liver',\
'ASMA':'Asthma','PNEUMOPATI':'Pneumopathy','RENAL':'Renal','HEMATOLOGI':'Hematological',\
'DIABETES':'Diabetes','OBESIDADE':'Obesity','NEUROLOGIC':'Neurological',\
'IMUNODEPRE':'Immunosuppression','EVOLUCAO':'EVOLUCAO'})

# %%
# To analyse the Nan of Distance and Overload features
#print(df.shape)
#print(df['Overload'].isna().sum())
#print(df['Distance'].isna().sum())

# %%
# feature
x = df.iloc[:,:-1]

# labels
y = df['EVOLUCAO']

# %%
# data separation
from sklearn.model_selection import train_test_split
x_train, x_test, y_train, y_test = \
    train_test_split(x, y, test_size=0.2, random_state=5)/home/pedro/.local/lib/python3.8/site-packages/IPython/core/interactiveshell.py:3146: DtypeWarning: Columns (15,60,62,63,64,92,94,108,117,121,123) have mixed types.Specify dtype option on import or set low_memory=False.
  has_raised = await self.run_ast_nodes(code_ast.body, cell_name,
<ipython-input-2-09ce09ec14c1>:16: UserWarning: Boolean Series key will be reindexed to match DataFrame index.
  df = df[df['PCR_SARS2']==1][df['HOSPITAL']==1][df['NU_IDADE_N']<=110]
<ipython-input-2-09ce09ec14c1>:21: UserWarning: Boolean Series key will be reindexed to match DataFrame index.
  df = df[df['PCR_SARS2']==1][df['HOSPITAL']==1][df['NU_IDADE_N']<=110][df['EVOLUCAO'] != 3][df['EVOLUCAO'] != 9][df['EVOLUCAO'].notnull()]
# %%
# load the model from disk
loaded_model_lg = pickle.load(open('lg_without_symptoms.sav', 'rb'))
/home/pedro/anaconda3/lib/python3.8/site-packages/sklearn/base.py:329: UserWarning: Trying to unpickle estimator LogisticRegression from version 0.23.1 when using version 0.23.2. This might lead to breaking code or invalid results. Use at your own risk.
  warnings.warn(
# CI 95% calculation  AUC

from mlxtend.evaluate import bootstrap

def auc_CI(df_test):
    x_test, y_test = df_test[:, :-1], df_test[:,[-1]]
    pred = loaded_model_lg.predict_proba(x_test)
    fpr, tpr, thresholds = roc_curve(y_test, pred[:,1], pos_label=1)
    return auc(x=fpr, y=tpr)

df_test = pd.merge(x_test, y_test, left_index=True, right_index=True)
original, std_err, ci_bounds = bootstrap(df_test.values , num_rounds=100, func = auc_CI, ci=0.95, seed=123)

print('Mean: %.3f, SE: +/- %.3f, CI95: [%.3f, %.3f]' % (original, std_err, ci_bounds[0], ci_bounds[1]))Mean: 0.763, SE: +/- 0.002, CI95: [0.759, 0.768]
# CI 95% calculation  AP Death

def ap_CI_death(df_test):
    x_test, y_test = df_test[:, :-1], df_test[:,[-1]]
    pred = loaded_model_lg.predict_proba(x_test)
    return average_precision_score(1-y_test, pred[:,0])

df_test = pd.merge(x_test, y_test, left_index=True, right_index=True)
original, std_err, ci_bounds = bootstrap(df_test.values , num_rounds=100, func = ap_CI_death, ci=0.95, seed=123)

print('Mean Death: %.3f, SE: +/- %.3f, CI95: [%.3f, %.3f]' % (original, std_err, ci_bounds[0], ci_bounds[1]))Mean Death: 0.629, SE: +/- 0.004, CI95: [0.619, 0.639]
# CI 95% calculation  AP Cure

def ap_CI_cure(df_test):
    x_test, y_test = df_test[:, :-1], df_test[:,[-1]]
    pred = loaded_model_lg.predict_proba(x_test)
    return average_precision_score(y_test, pred[:,1])

df_test = pd.merge(x_test, y_test, left_index=True, right_index=True)
original, std_err, ci_bounds = bootstrap(df_test.values , num_rounds=100, func = ap_CI_cure, ci=0.95, seed=123)

print('Mean Cure: %.3f, SE: +/- %.3f, CI95: [%.3f, %.3f]' % (original, std_err, ci_bounds[0], ci_bounds[1]))Mean Cure: 0.837, SE: +/- 0.002, CI95: [0.833, 0.842]

</code>


# Notebook from nhutnamhcmus/datacamp-playground
Path: visualizing-geospatial-data-in-python/3. GeoSeries and folium/notebook_section_3.ipynb

import pandas as pd
import geopandas as gpd
from shapely.geometry import Point
import matplotlib.pyplot as plt

neighborhoods_path = '../datasets/neighborhoods.geojson'

# Read in the neighborhoods geojson file
neighborhoods = gpd.read_file(neighborhoods_path)

school_districts_path = '../datasets/school_districts.geojson'

school_districts = gpd.read_file(school_districts_path)

art = gpd.read_file('../datasets/public_art.csv')

# Create a geometry column from lng & lat
art['geometry'] = art.apply(lambda x: Point(float(x.Longitude), float(x.Latitude)), axis=1)

# Create a GeoDataFrame from art and verify the type
art_geo = gpd.GeoDataFrame(art, crs = neighborhoods.crs, geometry = art.geometry)

# Spatially join neighborhoods with art_geo
neighborhood_art = gpd.sjoin(art_geo, neighborhoods, op = "within")

# Create urban_art from neighborhood_art where the neighborhood name is Urban Residents
urban_art = neighborhood_art.loc[neighborhood_art.name == "Urban Residents"]

# Get just the Urban Residents neighborhood polygon and save it as urban_polygon
urban_polygon = neighborhoods.loc[neighborhoods.name == "Urban Residents"]_____no_output_____

# Find the area of the Urban Residents neighborhood
How big is the Urban Residents neighborhood?_____no_output_____

# Print the head of the urban polygon
print(urban_polygon.head())

# Create a copy of the urban_polygon using EPSG:3857 and print the head
urban_poly_3857 = urban_polygon.to_crs(epsg = 3857)
print(urban_poly_3857.head())

               name                                           geometry
41  Urban Residents  MULTIPOLYGON (((-86.78122 36.16457, -86.78205 ...
               name                                           geometry
41  Urban Residents  MULTIPOLYGON (((-9660441.281 4323289.005, -966...
area = urban_poly_3857.geometry.area / 10**6_____no_output_____print('The area of the Urban Residents neighborhood is ', area.values, ' km squared')The area of the Urban Residents neighborhood is  [1.12898961]  km squared

# The center of the Urban Residents neighborhood
Now you'll find the center point of the urban_poly_3857 and plot it over the polygon._____no_output_____

# Create downtown_center from urban_poly_3857
downtown_center = urban_poly_3857.geometry.centroid

# Print the type of downtown_center
print(type(downtown_center))

# Plot the urban_poly_3857 as ax and add the center point
ax = urban_poly_3857.plot(color = 'lightgreen')
downtown_center.plot(ax = ax, color = 'black')
plt.xticks(rotation = 45)

# Show the plot
plt.show()<class 'geopandas.geoseries.GeoSeries'>

# Prepare to calculate distances
In this exercise you will prepare a GeoDataFrame called art_dist_meters with the locations of downtown art converted to meters using EPSG:3857. You will use art_dist_meters in the next exercise to calculate the distance of each artwork from the center of the Urban Residents neighborhood in meters.

The art data is in your workspace, along with urban_poly_3857 and center_point, the center point of the Urban Residents neighborhood. A geometry column called geometry that uses degrees has already been created in the art DataFrame._____no_output_____

# Import packages
from shapely.geometry import Point
import geopandas as gpd
import pandas as pd

center_point = art.geometry.centroid

# Create art_dist_meters using art and the geometry from art
art_dist_meters = gpd.GeoDataFrame(art, geometry = art.geometry, crs = {'init': 'epsg:4326'})
print(art_dist_meters.head(2))

# Set the crs of art_dist_meters to use EPSG:3857
art_dist_meters.geometry = art_dist_meters.geometry.to_crs(epsg = 3857)
print(art_dist_meters.head(2))

# Add a column to art_meters, center
art_dist_meters['center'] = center_point                             Title Last Name First Name  \
0          [Cross Country Runners]     Frost      Miley
1  [Fourth and Commerce Sculpture]    Walker        Lin

                            Location  Medium       Type Description  Latitude  \
0     4001 Harding Rd., Nashville TN  Bronze  Sculpture              36.12856
1  333 Commerce Street, Nashville TN          Sculpture              36.16234

   Longitude        Mapped Location                    geometry
0   -86.8366   (36.12856, -86.8366)  POINT (-86.83660 36.12856)
1  -86.77774  (36.16234, -86.77774)  POINT (-86.77774 36.16234)
                             Title Last Name First Name  \
0          [Cross Country Runners]     Frost      Miley
1  [Fourth and Commerce Sculpture]    Walker        Lin

                            Location  Medium       Type Description  Latitude  \
0     4001 Harding Rd., Nashville TN  Bronze  Sculpture              36.12856
1  333 Commerce Street, Nashville TN          Sculpture              36.16234

   Longitude        Mapped Location                          geometry
0   -86.8366   (36.12856, -86.8366)  POINT (-9666606.094 4318325.479)
1  -86.77774  (36.16234, -86.77774)  POINT (-9660053.829 4322982.159)

# Art distances from neighborhood center
Now that you have the center point and the art locations in the units we need to calculate distances in meters, it's time to perform that step._____no_output_____

art_dist_meters_____no_output_____# Import package for pretty printing
import pprint

# Build a dictionary of titles and distances for Urban Residents art
art_distances = {}
for row in art_dist_meters.iterrows():
    vals = row[1]
    key = vals['Title']
    ctr = vals['center']
    art_distances[key] = vals['geometry'].distance(ctr)

# Pretty print the art_distances
pprint.pprint(art_distances){'12th & Porter Mural': 10583948.500651475,
 'A Splash of Color': 10585357.160178259,
 'A Story of Nashville': 10583565.969078569,
 'Aerial Innovations Mural': 10579929.255218184,
 'Airport Sun Project': 10570311.412935875,
 'Andrew Jackson': 10583951.468665037,
 'Angel': 10588435.041707946,
 'Anticipation': 10583698.952539692,
 'April Baby': 10586916.00892881,
 'Arial Triangle [from the Gravity Series]': 10583398.52483797,
 'Ariel': 10585905.336208293,
 'Athena Parthenos': 10586065.827814262,
 'Benedictio': 10577769.979416678,
 'Bicycle Bus-Green Fleet': 10582635.68286198,
 'Blowing Bubbles': 10588435.041707946,
 'Breaking Bread': 10582634.392548595,
 'Building a Positive Community': 10582136.29114304,
 'Can-Do': 10582120.049094511,
 'Chet Atkins': 10583602.67968694,
 "Children's Chairs For The Seasons": 10583565.969078569,
 'Confederate Memorial': 10577769.979416678,
 'Confederate Private Monument': 10585868.029575413,
 'Confluence': 10585513.082652403,
 'Cool Fences': 10579074.346806614,
 'Dancing on Air': 10570372.644642536,
 'Demonbreun Hill Mural': 10583908.579902012,
 'Dragon Wall Mural': 10584123.478874767,
 'Eastside Mural': 10581245.290643545,
 'Ed and Bernice Johnson and Mary': 10583193.594816655,
 'Edward Ward Carmack': 10583921.539018078,
 'Flights of Fantasy': 10570344.379658109,
 'Foliated Scroll': 10583565.969078569,
 'Founding Fathers Cenotaph': 10584221.716760146,
 'Founding of Nashville': 10583078.144842813,
 'Furman Monument': 10577769.979416678,
 'Gold Star Monument to World War I Veterans': 10586021.858499767,
 'Gone Fishing': 10583591.376403218,
 'Happy Times at The Arcade': 10583544.984285295,
 'Headstones': 10581464.612607874,
 'Hillsboro Village Mural': 10583854.405716073,
 'History in Stained Glass': 10584221.716760146,
 'I BELIEVE IN NASHVILLE 12 South': 10582156.834978476,
 'I BELIEVE IN NASHVILLE East Nashville': 10580136.449174412,
 'I BELIEVE IN NASHVILLE North Nashville': 10585239.187925545,
 'In Concert': 10572751.806064371,
 'Industry - The TVA Group': 10584680.719262363,
 'Industry, Agriculture, Commerce, Statesmanship': 10583520.14461468,
 'Isabella': 10583485.955582885,
 'Isabelle and Calvin': 10584967.506033441,
 'John L. Nolen Monument': 10577769.979416678,
 'John W. Thomas': 10586021.858499767,
 'Johnny Cash Mural': 10582628.289268441,
 'La Storia della Terra': 10583565.969078569,
 'Lane Motor Museum': 10577496.17633061,
 'Les Paul Guitar Mural': 10582854.565917103,
 'Library Doors': 10583565.969078569,
 'Lightning Wire Fountain': 10583073.144394962,
 'M&M Furniture-Riverside Village': 10580110.247643126,
 'Major General William Bate Monument': 10577769.979416678,
 'Maybelle Carter': 10582681.28994042,
 'Migration': 10578647.940171225,
 'Music City Center - Art Collection': 10582899.568035366,
 'Nashville, A History through Maps and Aerial Views': 10583565.969078569,
 'Oh, Roy': 10583134.666506412,
 'Orpheus and Eurydice': 10582814.322708014,
 'Our Past, Your Future': 10581513.854999846,
 'Owen Bradley': 10583977.664424112,
 'Paint Waterfalls': 10586050.17255176,
 'Parthenon East Pediment': 10586039.029226098,
 'Parthenon West Pediment': 10586086.185064087,
 'Phillips Monument': 10577769.979416678,
 'Phillips Toy Mart': 10589122.922370559,
 'Phoenix': 10583193.594816655,
 'Piecing It All Together': 10583583.83412909,
 'Pioneer Family - The Mountain Group': 10583940.557855405,
 'Polarbears with Snowballs': 10582948.623516602,
 'President Andrew Johnson': 10583888.306385722,
 'Raccoons and the Magic Horseshoes': 10588449.304129982,
 'Rolf and Daughters Mural': 10585530.548435524,
 'Rolling Hills': 10567500.082015224,
 'Rose on 65th Street': 10583485.955582885,
 'Roy Orbison Tribute Statue': 10583983.27808947,
 'Safe Haven Family Shelter Mural': 10580722.202142382,
 'Sam Davis': 10587219.534638887,
 'Sam Davis of Tennessee': 10583957.687431902,
 'Samaritan': 10588435.041707946,
 'Sea Serpent': 10584599.331890862,
 'Sergeant Alvin C. York': 10583831.406240454,
 'Servant or Master': 10583295.262231264,
 'Sir Draco Star (a.k.a Eli)': 10581769.904107656,
 'Soldier and Farmer - The Equestrian Group': 10583980.483601985,
 'Something in the Air': 10583345.067374147,
 'St. George and the Maiden': 10588435.041707946,
 'St. Vincent de Paul': 10585450.35995012,
 'Statue of Dr. W. E. B. Dubois': 10586151.741414448,
 'Stay Tuned': 10582208.784466524,
 'Teacher and Student': 10587219.534638887,
 "Tennessee Confederate Women's Memorial": 10583808.163062796,
 'Tennessee Falls': 10583749.204294587,
 'Tennessee Korean War Memorial': 10583763.398102595,
 "Tennessee Vietnam Veteran's Memorial": 10583793.462008132,
 'Tennessee WWII Memorial': 10584791.513626093,
 'The Birth of Apollo': 10582837.385831947,
 'The Flutist': 10582802.090989307,
 'The Peace Monument': 10581938.152564157,
 'The Prince': 10583565.969078569,
 'The Readers': 10583393.872061776,
 'The Recording Angel': 10582755.631360933,
 'The Three Graces': 10584019.73571512,
 'The Waterfall Mural': 10581643.66630736,
 'Thomas Gleen Ryman': 10583110.04814488,
 'Threshold': 10582893.378709938,
 'Timeless': 10582989.494809307,
 'Timothy Demonbreun': 10583247.956294278,
 'Tomatoes': 10579771.420321863,
 'Tommy Owen': 10587219.534638887,
 'Two Musicians': 10581245.290643545,
 'Untitled': 10589663.659840992,
 'Vanderbilt University - Public Collection': 10584709.70990316,
 'Various': 10582656.32923829,
 'Victory': 10583866.220337586,
 'Voices Gathering': 10583459.815030094,
 'Volume': 10581771.247803846,
 'Warner Monument': 10577769.979416678,
 'Waveform: Pan American Blues': 10570399.600396495,
 'We Are Our Stories': 10587812.786960982,
 'Welcome to Flatrock': 10574716.19476862,
 'Wind Reeds': 10570407.991440946,
 'Women Suffrage Memorial': 10583921.539018078,
 'Youth Opportunity Center-STARS Nashville - Peace Jam mural': 10585108.655408468,
 '[Cross Country Runners]': 10587219.534638887,
 '[Fourth and Commerce Sculpture]': 10583138.736775193}

# Create a folium location from the urban centroid
In order to construct a folium map of the Urban Residents neighborhood, you need to build a coordinate pair location that is formatted for folium._____no_output_____

# Print the head of the urban_polygon
print(urban_polygon.head())

# Create urban_center from the urban_polygon center
urban_center = urban_polygon.to_crs(epsg=4326).centroid

# Print urban_center
print(urban_center)

# Create array for folium called urban_location
urban_location = [urban_center.y, urban_center.x]

# Print urban_location
print(urban_location)               name                                           geometry
41  Urban Residents  MULTIPOLYGON (((-86.78122 36.16457, -86.78205 ...
41    POINT (-86.77756 36.16128)
dtype: geometry
[41    36.161278
dtype: float64, 41   -86.777565
dtype: float64]

# Create a folium map of downtown Nashville
In this exercise you will create a street map of downtown Nashville using folium._____no_output_____

import folium

# Construct a folium map with urban_location
downtown_map = folium.Map(location = urban_location, zoom_start = 15)

display(downtown_map)_____no_output_____

# Folium street map of the downtown neighborhood
This time you will create the folium map of downtown and add the Urban Residents neighborhood area from urban_polygon. The urban_polygon has been printed to your console._____no_output_____

urban_polygon['center'] = urban_polygon.loc[:, 'geometry'].centroid_____no_output_____# Create array for called folium_loc from the urban_polygon center point
point = urban_polygon.center
folium_loc = [point.y, point.x]

# Construct a map from folium_loc: downtown_map
downtown_map = folium.Map(location = folium_loc, zoom_start = 15)

# Draw our neighborhood: Urban Residents
folium.GeoJson(urban_polygon.geometry).add_to(downtown_map)

# Display the map
display(downtown_map)_____no_output_____

# Adding markers for the public art
Now that you have added the polygon for the Urban Residents neighborhood to your folium street map, it's time to add the locations of the art within the neighborhood. You can do that by creating folium markers. Each marker needs a location assigned. Use iterrows() to loop through the data to grab the values you need._____no_output_____

# Iterate through the urban_art and print each part of tuple returned
for row in urban_art.iterrows():
  print('first part: ', row[0])
  print('second part: ', row[1])

# Create a location and marker with each iteration for the downtown_map
for row in urban_art.iterrows():
    row_values = row[1]
    location = [row_values['Latitude'], row_values['Longitude']]
    marker = folium.Marker(location = location)
    marker.add_to(downtown_map)

# Display the map
display(downtown_map)first part:  1
second part:  Title                  [Fourth and Commerce Sculpture]
Last Name                                       Walker
First Name                                         Lin
Location             333 Commerce Street, Nashville TN
Medium
Type                                         Sculpture
Description
Latitude                                      36.16234
Longitude                                    -86.77774
Mapped Location                  (36.16234, -86.77774)
geometry           POINT (-86.77773999999999 36.16234)
index_right                                         41
name                                   Urban Residents
Name: 1, dtype: object
first part:  4
second part:  Title                                           A Story of Nashville
Last Name                                                     Ridley
First Name                                                      Greg
Location                             615 Church Street, Nashville TN
Medium                                      Hammered copper repousse
Type                                                          Frieze
Description        Inside the Grand Reading Room, this is a serie...
Latitude                                                    36.16215
Longitude                                                  -86.78205
Mapped Location                                (36.16215, -86.78205)
geometry                                  POINT (-86.78205 36.16215)
index_right                                                       41
name                                                 Urban Residents
Name: 4, dtype: object
first part:  21
second part:  Title                                                    Chet Atkins
Last Name                                                      Faxon
First Name                                                   Russell
Location           Corner of Fifth Avenue North and Union Street,...
Medium                   Bronze stool and guitar on a granite plynth
Type                                                       Sculpture
Description        A sculpture of a young Chet Atkins seated on a...
Latitude                                                    36.16466
Longitude                                                  -86.78102
Mapped Location                                (36.16466, -86.78102)
geometry                                  POINT (-86.78102 36.16466)
index_right                                                       41
name                                                 Urban Residents
Name: 21, dtype: object
first part:  22
second part:  Title              Children's Chairs For The Seasons
Last Name                                     McGraw
First Name                                    Deloss
Location             615 Church Street, Nashville TN
Medium                  Mixed Media - wood and paint
Type                                       Furniture
Description        chairs depicting the four seasons
Latitude                                    36.16215
Longitude                                  -86.78205
Mapped Location                (36.16215, -86.78205)
geometry                  POINT (-86.78205 36.16215)
index_right                                       41
name                                 Urban Residents
Name: 22, dtype: object
first part:  34
second part:  Title                                                Foliated Scroll
Last Name                                                    Bloomer
First Name                                                      Kent
Location                             615 Church Street, Nashville TN
Medium                                                      Aluminum
Type                                                          Frieze
Description        a contemporary adaptation of a classical folia...
Latitude                                                    36.16215
Longitude                                                  -86.78205
Mapped Location                                (36.16215, -86.78205)
geometry                                  POINT (-86.78205 36.16215)
index_right                                                       41
name                                                 Urban Residents
Name: 34, dtype: object
first part:  39
second part:  Title                                                   Gone Fishing
Last Name                                                     Cooper
First Name                                                   Michael
Location                                          Church Street Park
Medium                                              Acrylic on Brick
Type                                                           Mural
Description        Just having some fun with Trompe L'oeil balconies
Latitude                                                    36.16298
Longitude                                                  -86.78184
Mapped Location                                (36.16298, -86.78184)
geometry                                  POINT (-86.78184 36.16298)
index_right                                                       41
name                                                 Urban Residents
Name: 39, dtype: object
first part:  40
second part:  Title                                  Happy Times at The Arcade
Last Name                                                 Cooper
First Name                                               Michael
Location           In the Alley between 4th and 5th off of Union
Medium                Silicate paint on brick and concrete block
Type                                                       Mural
Description         Trompe L'oeil artwork celebrating The Arcade
Latitude                                                 36.1647
Longitude                                              -86.78043
Mapped Location                             (36.1647, -86.78043)
geometry                               POINT (-86.78043 36.1647)
index_right                                                   41
name                                             Urban Residents
Name: 40, dtype: object
first part:  54
second part:  Title                                              Johnny Cash Mural
Last Name                     Bryan Deese, Audie Adams, Ryan Shrader
First Name
Location                                              300 4th Avenue
Medium                                   Latex paint and spray paint
Type                                                           Mural
Description        Classic street art by Thoughts Manifested crew...
Latitude                                                   36.158301
Longitude                                                 -86.774955
Mapped Location                              (36.158301, -86.774955)
geometry                        POINT (-86.77495500000001 36.158301)
index_right                                                       41
name                                                 Urban Residents
Name: 54, dtype: object
first part:  55
second part:  Title                                          La Storia della Terra
Last Name                                  The Kubach - Wilmsen Team
First Name                                 The Kubach - Wilmsen Team
Location                          Church St & 7th Ave N Nashville TN
Medium                                                         Stone
Type                                                       Sculpture
Description        A 20 foot tower of stone books. There are 26 b...
Latitude                                                    36.16215
Longitude                                                  -86.78205
Mapped Location                                (36.16215, -86.78205)
geometry                                  POINT (-86.78205 36.16215)
index_right                                                       41
name                                                 Urban Residents
Name: 55, dtype: object
first part:  57
second part:  Title                           Les Paul Guitar Mural
Last Name                                      Hughes
First Name                                     Milree
Location             100 Broadway, Nashville TN 37201
Medium                                       Painting
Type                                            Mural
Description
Latitude                                      36.1624
Longitude                                   -86.77491
Mapped Location                  (36.1624, -86.77491)
geometry           POINT (-86.77491000000001 36.1624)
index_right                                        41
name                                  Urban Residents
Name: 57, dtype: object
first part:  59
second part:  Title                                                  Library Doors
Last Name                                                    LeQuire
First Name                                                      Alan
Location                             615 Church Street, Nashville TN
Medium                                                        Bronze
Type                                                         Bronzes
Description        images of local people as well as Tennessee an...
Latitude                                                    36.16215
Longitude                                                  -86.78205
Mapped Location                                (36.16215, -86.78205)
geometry                                  POINT (-86.78205 36.16215)
index_right                                                       41
name                                                 Urban Residents
Name: 59, dtype: object
first part:  60
second part:  Title                        Lightning Wire Fountain
Last Name                                     Walker
First Name                                       Lin
Location           333 Commerce Street, Nashville TN
Medium
Type                              Sculpture/Fountain
Description
Latitude                                    36.16233
Longitude                                   -86.7771
Mapped Location                 (36.16233, -86.7771)
geometry                   POINT (-86.7771 36.16233)
index_right                                       41
name                                 Urban Residents
Name: 60, dtype: object
first part:  66
second part:  Title              Nashville, A History through Maps and Aerial V...
Last Name                                                       Haas
First Name                                                   Richard
Location                             615 Church Street, Nashville TN
Medium                                                 Oil on Canvas
Type                                                           Mural
Description        Eight scenes showing the development of Nashvi...
Latitude                                                    36.16215
Longitude                                                  -86.78205
Mapped Location                                (36.16215, -86.78205)
geometry                                  POINT (-86.78205 36.16215)
index_right                                                       41
name                                                 Urban Residents
Name: 66, dtype: object
first part:  67
second part:  Title                                                        Oh, Roy
Last Name                                                      Faxon
First Name                                                   Russell
Location                                 116 5th Ave N, Nashville TN
Medium                                                        Bronze
Type                                                       Sculpture
Description        Minnie Pearl and Roy Acuff seated on a bench i...
Latitude                                                    36.16151
Longitude                                                  -86.77816
Mapped Location                                (36.16151, -86.77816)
geometry                                  POINT (-86.77816 36.16151)
index_right                                                       41
name                                                 Urban Residents
Name: 67, dtype: object
first part:  68
second part:  Title                                           Orpheus and Eurydice
Last Name                                                     Kaskey
First Name                                                       Ray
Location                              1 Symphony Place, Nashville TN
Medium                                                     Limestone
Type                                                       Sculpture
Description        Relief sculpture above main entrance of Scherm...
Latitude                                                    36.16017
Longitude                                                  -86.77575
Mapped Location                                (36.16017, -86.77575)
geometry                                  POINT (-86.77575 36.16017)
index_right                                                       41
name                                                 Urban Residents
Name: 68, dtype: object
first part:  75
second part:  Title                          Piecing It All Together
Last Name                                       Cooper
First Name                                     Michael
Location               600 Church Street, Nashville TN
Medium                               Painting on Stone
Type                                             Mural
Description
Latitude                                      36.16281
Longitude                                    -86.78186
Mapped Location                  (36.16281, -86.78186)
geometry           POINT (-86.78185999999999 36.16281)
index_right                                         41
name                                   Urban Residents
Name: 75, dtype: object
first part:  104
second part:  Title                                            The Birth of Apollo
Last Name                                                   Eskridge
First Name                                                     Casey
Location                              1 Symphony Place, Nashville TN
Medium                   Bronze sculpture as centerpiece of fountain
Type                                                       Sculpture
Description        Two figures topping a fountain, one with a lyr...
Latitude                                                    36.15993
Longitude                                                  -86.77611
Mapped Location                                (36.15993, -86.77611)
geometry                                  POINT (-86.77611 36.15993)
index_right                                                       41
name                                                 Urban Residents
Name: 104, dtype: object
first part:  105
second part:  Title                                                    The Flutist
Last Name                                                       Varo
First Name                                                    Marton
Location                              1 Symphony Place, Nashville TN
Medium                                                        Marble
Type                                                       Sculpture
Description        Statue commemorating Kenneth D Schermerhorn, M...
Latitude                                                    36.15961
Longitude                                                  -86.77594
Mapped Location                                (36.15961, -86.77594)
geometry                         POINT (-86.77594000000001 36.15961)
index_right                                                       41
name                                                 Urban Residents
Name: 105, dtype: object
first part:  107
second part:  Title                                                     The Prince
Last Name                                                     Outlaw
First Name                                                  Adrienne
Location                             615 Church Street, Nashville TN
Medium             Mixed Media Use of heavy guage aluminum wire a...
Type                                                          Mobile
Description        The Little Prince\nOutlaw hung a whimsical mob...
Latitude                                                    36.16215
Longitude                                                  -86.78205
Mapped Location                                (36.16215, -86.78205)
geometry                                  POINT (-86.78205 36.16215)
index_right                                                       41
name                                                 Urban Residents
Name: 107, dtype: object
first part:  109
second part:  Title                                         The Recording Angel
Last Name                                                   Flack
First Name                                                 Audrey
Location                           1 Symphony Place, Nashville TN
Medium                                                     Bronze
Type                                                    Sculpture
Description        Bronze angel holding a fountain pen and tablet
Latitude                                                  36.1592
Longitude                                               -86.77571
Mapped Location                              (36.1592, -86.77571)
geometry                                POINT (-86.77571 36.1592)
index_right                                                    41
name                                              Urban Residents
Name: 109, dtype: object
first part:  112
second part:  Title                                             Thomas Gleen Ryman
Last Name                                                    Shields
First Name                                                     Steve
Location                                 116 5th Ave N, Nashville TN
Medium                              Bronze sculpture with brick base
Type                                                       Sculpture
Description        Full-length standing figure of Ryman, holding ...
Latitude                                                    36.16156
Longitude                                                  -86.77789
Mapped Location                                (36.16156, -86.77789)
geometry                                  POINT (-86.77789 36.16156)
index_right                                                       41
name                                                 Urban Residents
Name: 112, dtype: object
first part:  114
second part:  Title                                                       Timeless
Last Name                                                      Sorci
First Name                                                       Joe
Location              3rd Avenue North between Commerce and Broadway
Medium                               Steel and Civil War cannonballs
Type                                                       Sculpture
Description        Two nearly interlinked hemicycles with cannonb...
Latitude                                                    36.16227
Longitude                                                  -86.77631
Mapped Location                                (36.16227, -86.77631)
geometry                                  POINT (-86.77631 36.16227)
index_right                                                       41
name                                                 Urban Residents
Name: 114, dtype: object

# Troubleshooting data issues
You will be building popups for the downtown art using the title and desc columns from the urban_art DataFrame. Here, you will inspect those columns to identify and clean up any problematic values._____no_output_____

# Print the urban_art titles
print(urban_art.Title)

#Print the urban_art descriptions
print(urban_art.Description)

# Replace Nan and ' values in description
urban_art.Description.fillna('', inplace = True)
urban_art.Description = urban_art.Description.str.replace("'", "`")

#Print the urban_art descriptions again
print(urban_art.Description)1                        [Fourth and Commerce Sculpture]
4                                   A Story of Nashville
21                                           Chet Atkins
22                     Children's Chairs For The Seasons
34                                       Foliated Scroll
39                                          Gone Fishing
40                             Happy Times at The Arcade
54                                     Johnny Cash Mural
55                                 La Storia della Terra
57                                 Les Paul Guitar Mural
59                                         Library Doors
60                               Lightning Wire Fountain
66     Nashville, A History through Maps and Aerial V...
67                                               Oh, Roy
68                                  Orpheus and Eurydice
75                               Piecing It All Together
104                                  The Birth of Apollo
105                                          The Flutist
107                                           The Prince
109                                  The Recording Angel
112                                   Thomas Gleen Ryman
114                                             Timeless
Name: Title, dtype: object
1
4      Inside the Grand Reading Room, this is a serie...
21     A sculpture of a young Chet Atkins seated on a...
22                     chairs depicting the four seasons
34     a contemporary adaptation of a classical folia...
39     Just having some fun with Trompe L'oeil balconies
40          Trompe L'oeil artwork celebrating The Arcade
54     Classic street art by Thoughts Manifested crew...
55     A 20 foot tower of stone books. There are 26 b...
57
59     images of local people as well as Tennessee an...
60
66     Eight scenes showing the development of Nashvi...
67     Minnie Pearl and Roy Acuff seated on a bench i...
68     Relief sculpture above main entrance of Scherm...
75
104    Two figures topping a fountain, one with a lyr...
105    Statue commemorating Kenneth D Schermerhorn, M...
107    The Little Prince\nOutlaw hung a whimsical mob...
109       Bronze angel holding a fountain pen and tablet
112    Full-length standing figure of Ryman, holding ...
114    Two nearly interlinked hemicycles with cannonb...
Name: Description, dtype: object
1
4      Inside the Grand Reading Room, this is a serie...
21     A sculpture of a young Chet Atkins seated on a...
22                     chairs depicting the four seasons
34     a contemporary adaptation of a classical folia...
39     Just having some fun with Trompe L`oeil balconies
40          Trompe L`oeil artwork celebrating The Arcade
54     Classic street art by Thoughts Manifested crew...
55     A 20 foot tower of stone books. There are 26 b...
57
59     images of local people as well as Tennessee an...
60
66     Eight scenes showing the development of Nashvi...
67     Minnie Pearl and Roy Acuff seated on a bench i...
68     Relief sculpture above main entrance of Scherm...
75
104    Two figures topping a fountain, one with a lyr...
105    Statue commemorating Kenneth D Schermerhorn, M...
107    The Little Prince\nOutlaw hung a whimsical mob...
109       Bronze angel holding a fountain pen and tablet
112    Full-length standing figure of Ryman, holding ...
114    Two nearly interlinked hemicycles with cannonb...
Name: Description, dtype: object

# A map of downtown art
Now you will assign a popup to each marker to give information about the artwork at each location. In particular you will assign the art title and description to the popup for each marker. You will do so by creating the map object downtown_map, then add the popups, and finally use the display function to show your map.

One warning before you start: you'll need to ensure that all instances of single quotes (') are removed from the pop-up message, otherwise your plot will not render!_____no_output_____

# Construct downtown map
downtown_map = folium.Map(location=urban_location, zoom_start=15)
folium.GeoJson(urban_polygon.geometry).add_to(downtown_map)

# Create popups inside the loop you built to create the markers
for row in urban_art.iterrows():
    row_values = row[1]
    location = [row_values['Latitude'], row_values['Longitude']]
    popup = (str(row_values['Title']) + ': ' + str(row_values['Description'])[:20])

    marker = folium.Marker(location = location, popup=popup)
    marker.add_to(downtown_map)

# Display the map.
display(downtown_map)_____no_output_____


# Notebook from emanuelevivoli/2021-Master-Thesis-UNIFI
Path: notebooks/datasets/1. Datasets-Francesco.ipynb

# Datasets Francesco
---

In this notebook we'll build/implement the Dataset classes we need to work with all the dataset we have.
First we will introduce the datasets, then we will separate those based on the usage we are going to make of them, then we will use/build/implement our classes in order to manage those different datasets and tasks._____no_output_____# 0.0 Utils
---

We will be using the 🤗*Datasets* library, so first we need to install it._____no_output_____> !pip install datasets > datasets_installation.txt_____no_output_____We'll be using also the 🤗 *Tranformers* library, as we need a tokenizer and a vocab._____no_output_____> !pip install transformers > transformers_installation.txt_____no_output_____Let's define all the `imports` and `hyperparameters` in one place._____no_output_____

# Imports
import time
import os
import json
from transformers import AutoTokenizer, AutoModel, PreTrainedTokenizer, DataCollatorForLanguageModeling, Trainer, BertForMaskedLM, TrainingArguments
import torch
import transformers
from torch.utils.data import Dataset, DataLoader
from typing import Dict, List, Union
from datasets import load_dataset, DatasetDict
from enum import Enum
# from pydash.arrays import pull_at
import numpy as np
from torch.nn.utils.rnn import pad_sequence
import math_____no_output_____# ----------------------------------- #
#           Hyperparameters
# ----------------------------------- #
RUN_NAME = 'scibert-s2orc'
RUN_NUMBER = 2
RUN_ITER = 1

# --------- logging         --------- #
verbose = True # logging function description start and end
debug = False # logging element values
print_debug = False
time_debug = True

# to avoid warning messages from huggingface-tranformers you must comment this line
transformers.logging.set_verbosity_info()

# --------- preprocessing   --------- #
# in **partial_prepare_data**
remove_None_papers = True # if True, remove papers with None eather in abstract or title
remove_Unused_columns = True
# in **preprocess**
clean_None_data = False # if True, changes all the None (abstract of title) to ''
remove_None_data = False # if True (and clean_None_data set False), remove all the None abstract/title and the correspond title/abstract

# --------- paths           --------- #
# data folder path
data_base_dir = '/home/vivoli/Thesis/data'
# If you choose 'full' without setting N, the system will use the first chunk
s2orc_type = 'full'
N = None

# --------- model/tokenizer --------- #
# hugginface model/tokenizer name
MODEL_PATH = 'allenai/scibert_scivocab_uncased'
model_name_or_path = MODEL_PATH

# --------- checkpoint model -------- #
from transformers.trainer_utils import get_last_checkpoint

# seed for reproducibility of experiments
SEED = 1234_____no_output_____

# 0.1 KeyPhrase Dataset
---

These are testing datasets_____no_output_____- Keyphrase paths

```python
dataset_names = ['inspec', 'krapivin', 'nus', 'semeval', 'kp20k', 'duc', 'stackexchange']
json_base_dir = data_base_dir + '/keyphrase/json/'
```_____no_output_____- Keyphrase read json file

```python
def json_keyphrase_read( dataset_name, json_base_dir, file_name=None ):
    """
    Args:
        dataset_name (string): Directory name with the json file.
        json_base_dir (string): Path to the Dataset directory.
        file_name (string): (Optional) Json file name.

    Return:
        json_list (list of dict): List of dictionaries, each one with the fields
            - 'title' (string)
            - 'abstract' (string)
            - 'fulltext' (string | '')
            - 'keywords' (list)
    """
    if verbose: print(dataset_name)

    input_json_path = os.path.join(json_base_dir, dataset_name,
                                   '%s_test.json' % dataset_name if file_name is None else file_name)

    json_list_of_dict = []
    with open(input_json_path, 'r') as input_json:
        for json_line in input_json:
            json_dict = json.loads(json_line)

            if dataset_name == 'stackexchange':
                json_dict['abstract'] = json_dict['question']
                json_dict['keywords'] = json_dict['tags']
                del json_dict['question']
                del json_dict['tags']

            keywords = json_dict['keywords']

            if isinstance(keywords, str):
                keywords = keywords.split(';')
                json_dict['keywords'] = keywords

            json_list_of_dict.append(json_dict)

    return json_list_of_dict
```_____no_output_____This keyphrase dataset could be useful for testing some model on keyphrase task or abstract-title summarization/generation/embedding.

For now, we can avoid implementing the Dataset's and DataLoader's classes for this objects.

Although, the dataset and Dataloader would be simple as follow:_____no_output_____- Keyphrase process data -> tensor

```python
def data_keyphrase_process(json_list_of_dict, tokenizer, debug=False):
    data = []
    for json_dict in json_list_of_dict:
        title_tensor_ = torch.tensor(tokenizer.encode(json_dict['title']),
                                dtype=torch.long)
        if debug: print(title_tensor_)
        abstract_tensor_ = torch.tensor(tokenizer.encode(json_dict['abstract']),
                                dtype=torch.long)
        if debug: print(abstract_tensor_)
        fulltext_tensor_ = torch.tensor(tokenizer.encode(json_dict['fulltext']),
                                dtype=torch.long)
        if debug: print(fulltext_tensor_)
        keywords_tensor_ = torch.tensor(tokenizer(json_dict['keywords'], padding=True)['input_ids'],
                                dtype=torch.long)
        if debug: print(keywords_tensor_)

        data.append((title_tensor_, abstract_tensor_, fulltext_tensor_, keywords_tensor_))
    return data
```_____no_output_____- Defining model and vocab

```python
# we need to get `vocab` and the `tokenizer`, all comes with *AutoTokenizer*
tokenizer = AutoTokenizer.from_pretrained(MODEL_PATH)
model = AutoModel.from_pretrained(MODEL_PATH)
```

- Extracting tokenized data
```python
# now we can use them
json_list_of_dict = json_keyphrase_read( dataset_names[0], json_base_dir )
data = data_keyphrase_process( json_list_of_dict, tokenizer )
```_____no_output_____The `data` object is composed by `500 tuples`, each one composed by 4 objects:
- `title_tensor_` is the title embedding (composed by integers values)
- `abstract_tensor_` is the abstract embedding (composed by integers values)
- `fulltext_tensor_` is the fulltext embedding (composed by integers values)
- `keywords_tensor_` is the keywords embedding (composed by integers values)_____no_output_____# 0.2 S2ORC Dataset
---

To build a generic loading function we take inspiration from [here](https://discuss.huggingface.co/t/pipeline-with-custom-dataset-tokenizer-when-to-save-load-manually/1084/11)._____no_output_____

if s2orc_type == 'sample':
    metadata_filename = 'sample'
    pdf_parses_filename = 'sample'

elif s2orc_type == 'full':

    if N is None:
        print(f"[WARNING] You set 'full' but no bucket index was specified. \n \
        We'll use the index 0, so the first bucket will be used.")
        N = 0

    metadata_filename = f"metadata_{N}"
    pdf_parses_filename = f"pdf_parses_{N}"

else:
    raise NameError(f"You must select an existed S2ORC dataset \n \
                You selected {s2orc_type}, but options are ['sample' or 'full']")

meta_s2orc = data_base_dir +f'/s2orc-{s2orc_type}-20200705v1/{s2orc_type}/metadata/{metadata_filename}.jsonl'
pdfs_s2orc = data_base_dir +f'/s2orc-{s2orc_type}-20200705v1/{s2orc_type}/pdf_/{pdf_parses_filename}.jsonl'[WARNING] You set 'full' but no bucket index was specified.
         We'll use the index 0, so the first bucket will be used.

---
---
## ❌ PARTIAL PREPARE
---
---_____no_output_____

# in case you need is necessary to set output_dir
output_dir=f'./tmp_trainer/#{RUN_NUMBER}_{RUN_ITER}_{RUN_NAME}'

# tokenizer from 'allenai/scibert_scivocab_uncased' or from checkpoint
PRETRAINED = MODEL_PATH
print(f"Checkpoint DOESN'T exist, model load from scratch at {PRETRAINED}.")

tokenizer = AutoTokenizer.from_pretrained(PRETRAINED)
# model = BertForMaskedLM.from_pretrained(PRETRAINED) Checkpoint DOESN'T exist, model load from scratch at allenai/scibert_scivocab_uncased.

Model BertConfig object for

```python
Model config BertConfig {
  "attention_probs_dropout_prob": 0.1,
  "gradient_checkpointing": false,
  "hidden_act": "gelu",
  "hidden_dropout_prob": 0.1,
  "hidden_size": 768,
  "initializer_range": 0.02,
  "intermediate_size": 3072,
  "layer_norm_eps": 1e-12,
  "max_position_embeddings": 512,
  "model_type": "bert",
  "num_attention_heads": 12,
  "num_hidden_layers": 12,
  "pad_token_id": 0,
  "position_embedding_type": "absolute",
  "transformers_version": "4.3.2",
  "type_vocab_size": 2,
  "use_cache": true,
  "vocab_size": 31090
}
```_____no_output_____Max sequence length from tokenizer, model and input might be differents:_____no_output_____

max_seq_length = model.config.max_position_embeddings
print(max_seq_length)512

Here we use only the `meta_s2orc`, so we have the data that is structured such as:_____no_output_____## 2.1 Description (s2orc)
---
The `S2ORC` dataset is in the `data` path under the folder `s2orc-full-20200705v1` (where `s2orc` is the name of the dataset, `full` is the type, as there is also a sample fingerprint; and `20200705v1` is the version).
We can reach the data folder exiting by the project and entering in the data folder:_____no_output_____

DATA_PATH = '/home/vivoli/Thesis/data'
!ls $DATA_PATHkeyphrase  s2orc-full-20200705v1  s2orc-sample-20200705v1  sentence-tranformers
README.md  s2orc-mini		  scibert		   snli_1.0
custom_path = f"{DATA_PATH}/s2orc-full-20200705v1/full"
!ls $custom_pathmetadata  pdf_parses

As you can see (going into `s2orc-full-20200705v1/full/`) there are the `metadata` folder and the `pdf_parses` folder. The main difference (as we can already get it from the names) is that in the `metadata` you only have some information about the dataset (retrieved from the published metadata), while in the `pdf_parses` you get all the extensive data conteined in the paper (if the paper was present, was correctly parsed and no restriction in the paper data were applied due to limited licence permition). For some reason, the `title` of the paper is contained only in the `metadata` file, but it can get from the `paper_id` field of the paper itself.

More information about the `S2ORC` dataset can be read in the [README.md](https://github.com/allenai/s2orc/blob/master/README.md) of the project and in the [project repository](https://github.com/allenai/s2orc/)_____no_output_____### mag field
- MAG fields of study:

| class | Field of study | All papers | Full text |
|-------|----------------|------------|-----------|
|0      | Medicine       | 12.8M      | 1.8M      |
|1      | Biology        | 9.6M       | 1.6M      |
|2      | Chemistry      | 8.7M       | 484k      |
|3      | n/a            | 7.7M       | 583k      |
|4      | Engineering    | 6.3M       | 228k      |
|5      | Comp Sci       | 6.0M       | 580k      |
|6      | Physics        | 4.9M       | 838k      |
|7      | Mat Sci        | 4.6M       | 213k      |
|8      | Math           | 3.9M       | 669k      |
|9      | Psychology     | 3.4M       | 316k      |
|10     | Economics      | 2.3M       | 198k      |
|11     | Poli Sci       | 1.8M       | 69k       |
|12     | Business       | 1.8M       | 94k       |
|13     | Geology        | 1.8M       | 115k      |
|14     | Sociology      | 1.6M       | 93k       |
|15     | Geography      | 1.4M       | 58k       |
|16     | Env Sci        | 766k       | 52k       |
|17     | Art            | 700k       | 16k       |
|18     | History        | 690k       | 22k       |
|19     | Philosophy     | 384k       | 15k       |_____no_output_____## `metadata` schema

We recommend everyone work with `metadata/` as the starting point.  This is a JSONlines file (one line per paper) with the following keys:

#### Identifier fields

* `paper_id`: a `str`-valued field that is a unique identifier for each S2ORC paper.

* `arxiv_id`: a `str`-valued field for papers on [arXiv.org](https://arxiv.org).

* `acl_id`: a `str`-valued field for papers on [the ACL Anthology](https://www.aclweb.org/anthology/).

* `pmc_id`: a `str`-valued field for papers on [PubMed Central](https://www.ncbi.nlm.nih.gov/pmc/articles).

* `pubmed_id`: a `str`-valued field for papers on [PubMed](https://pubmed.ncbi.nlm.nih.gov/), which includes MEDLINE.  Also known as `pmid` on PubMed.

* `mag_id`: a `str`-valued field for papers on [Microsoft Academic](https://academic.microsoft.com).

* `doi`: a `str`-valued field for the [DOI](http://doi.org/).

Notably:

* Resolved citation links are represented by the cited paper's `paper_id`.

* The `paper_id` resolves to a Semantic Scholar paper page, which can be verified using the `s2_url` field.

* We don't always have a value for every identifier field.  When missing, they take `null` value.

#### Metadata fields

* `title`: a `str`-valued field for the paper title.  Every S2ORC paper *must* have one, though the source can be from publishers or parsed from PDFs.  We prioritize publisher-provided values over parsed values.

* `authors`: a `List[Dict]`-valued field for the paper authors.  Authors are listed in order.  Each dictionary has the keys `first`, `middle`, `last`, and `suffix` for the author name, which are all `str`-valued with exception of `middle`, which is a `List[str]`-valued field.  Every S2ORC paper *must* have at least one author.

* `venue` and `journal`: `str`-valued fields for the published venue/journal.  *Please note that there is not often agreement as to what constitutes a "venue" versus a "journal". Consolidating these fields is being considered for future releases.*

* `year`: an `int`-valued field for the published year.  If a paper is preprinted in 2019 but published in 2020, we try to ensure the `venue/journal` and `year` fields agree & prefer non-preprint published info. *We know this decision prohibits certain types of analysis like comparing preprint & published versions of a paper.  We're looking into it for future releases.*

* `abstract`: a `str`-valued field for the abstract.  These are provided directly from gold sources (not parsed from PDFs).  We preserve newline breaks in structured abstracts, which are common in medical papers, by denoting breaks with `':::'`.

* `inbound_citations`: a `List[str]`-valued field containing `paper_id` of other S2ORC papers that cite the current paper.  *Currently derived from PDF-parsed bibliographies, but may have gold sources in the future.*

* `outbound_citations`: a `List[str]`-valued field containing `paper_id` of other S2ORC papers that the current paper cites.  Same note as above.

* `has_inbound_citations`: a `bool`-valued field that is `true` if `inbound_citations` has at least one entry, and `false` otherwise.

* `has_outbound_citations` a `bool`-valued field that is `true` if `outbound_citations` has at least one entry, and `false` otherwise.

We don't always have a value for every metadata field.  When missing, `str` fields take `null` value, while `List` fields are empty lists.

#### PDF parse-related metadata fields

* `has_pdf_parse`:  a `bool`-valued field that is `true` if this paper has a corresponding entry in `pdf_parses/`, which means we had processed that paper's PDF(s) at some point.  The field is `false` otherwise.

* `has_pdf_parsed_abstract`: a `bool`-valued field that is `true` if the paper's PDF parse contains a parsed abstract, and `false` otherwise.

* `has_pdf_parsed_body_text`: a `bool`-valued field that is `true` if the paper's PDF parse contains parsed body text, and `false` otherwise.

* `has_pdf_parsed_bib_entries`: a `bool`-valued field that is `true` if the paper's PDF parse contains parsed bibliography entries, and `false` otherwise.

* `has_pdf_parsed_ref_entries`: a `bool`-valued field that is `true` if the paper's PDF parse contains parsed reference entries (e.g. tables, figures), and `false` otherwise.

Please note:

* If `has_pdf_parse = false`, the other four fields will not be present in the JSON (trivially `false`).

* If `has_pdf_parse = true` but `has_pdf_parsed_abstract`, `has_pdf_parsed_body_text`, or `has_pdf_parsed_ref_entries` are `false`, this can be because:

    * Our PDF parser failed to extract that element
    * Our PDF parser succeeded but that paper simply did not have that element (e.g. papers without abstracts)
    * Our PDF parser succeeded but that element was removed because the paper is not identified as open-access.
_____no_output_____##### metadata_CLASS
```python
{
 "paper_id": (string),
 "title": (string),
 "authors": [
     {
         "first": (string),
         "middle": [],
         "last": (string),
         "suffix": (string)
     },
     ...
   ]: **Author_Class**,
 "abstract": (string),
 "year": (int),
 "arxiv_id": null,
 "acl_id": null,
 "pmc_id": null,
 "pubmed_id": null,
 "doi": null,
 "venue": null,
 "journal": (string),
 "mag_id": (string-number),
 "mag_field_of_study": [
     "Medicine",
     "Computer Science"
   ]: **FieldOfStudy_Enum**,
 "outbound_citations": [],
 "inbound_citations": [],
 "has_outbound_citations": false,
 "has_inbound_citations": false,
 "has_pdf_parse": false,
 "s2_url": (string)
}
```_____no_output_____Here I represent Author_Class as an object of
```python
{
    "first": (string),
    "middle": [],
    "last": (string),
    "suffix": (string)
}
```
and `FieldOfStudy_Enum` as an Enum of string such as `[ "Medicine", "Computer Science", "Physics", "Mathematics", ... ]`_____no_output_____

def partial_prepare_data(dataset_f: str,
                tokenizer: PreTrainedTokenizer,
                max_seq_length: int = None,
                batch_size: int = 64,
                num_workers: int = 4,
                seed: int = SEED,
                data_field: List[str] =  ["title", "abstract"]) -> Dict[str, DataLoader]:
    """Given an input file, prepare the train, test, validation dataloaders.
    :param dataset_f: input file (format: .txt; line by line)
    :param tokenizer: pretrained tokenizer that will prepare the data, i.e. convert tokens into IDs
    :param max_seq_length: maximal sequence length. Longer sequences will be truncated
    :param batch_size: batch size for the dataloaders
    :param num_workers: number of CPU workers to use during dataloading. On Windows this must be zero
    :return: a dictionary containing train, test, validation dataloaders
    """
    print_all_debug = False
    time_debug = True
    print_some_debug = True

    ## ------------------ ##
    ## -- LOAD DATASET -- ##
    ## ------------------ ##
    if time_debug: start = time.time()
    if time_debug: start_load = time.time()

    ## execution
    max_seq_length = tokenizer.model_max_length if not max_seq_length else max_seq_length
    if print_some_debug: print(max_seq_length)
    dataset_dict = load_dataset("json", data_files=dataset_f)

    if time_debug: end_load = time.time()
    if time_debug: print(f"[TIME] load_dataset: {end_load - start_load}")

    ## ------------------ ##
    ## ---- MANAGING ---- ##
    ## ------------------ ##
    if time_debug: start_selection = time.time()

    ## execution
    dataset = dataset_dict['train']

    if time_debug: end_selection = time.time()
    if time_debug: print(f"[TIME] dataset_train selection: {end_selection - start_selection}")
    if print_all_debug: print(dataset)

    ## ------------------ ##
    ## --- REMOVE none -- ##
    ## ------------------ ##
    if time_debug: start_removing = time.time()
    # clean input removing papers with **None** as abstract/title
    if remove_None_papers:

        ## --------------------- ##
        ## --- REMOVE.indexes -- ##
        ## --------------------- ##
        if time_debug: start_removing_indexes = time.time()
        if print_all_debug: print(data_field)

        ## execution
        none_papers_indexes = {}
        for field in data_field:
            none_indexes = [ idx_s for idx_s, s in enumerate(dataset[f"{field}"]) if s is None]
            none_papers_indexes = {**none_papers_indexes, **dict.fromkeys(none_indexes , False)}

        if time_debug: end_removing_indexes = time.time()
        if time_debug: print(f"[TIME] remove.indexes: {end_removing_indexes - start_removing_indexes}")
        if print_all_debug: print(none_papers_indexes)

        ## --------------------- ##
        ## --- REMOVE.concat --- ##
        ## --------------------- ##
        if time_debug: start_removing_concat = time.time()

        ## execution
        to_remove_indexes = list(none_papers_indexes.keys())

        if time_debug: end_removing_concat = time.time()
        if time_debug: print(f"[TIME] remove.concat: {end_removing_concat - start_removing_concat}")
        if print_all_debug: print(to_remove_indexes)
        if print_all_debug: print([ dataset["abstract"][i] for i in to_remove_indexes])

        ## --------------------- ##
        ## --- REMOVE.filter --- ##
        ## --------------------- ##
        if time_debug: start_removing_filter = time.time()

        ## execution
        dataset = dataset.filter((lambda x, ids: none_papers_indexes.get(ids, True)), with_indices=True)

        if time_debug: end_removing_filter = time.time()
        if time_debug: print(f"[TIME] remove.filter: {end_removing_filter - start_removing_filter}")
        if print_all_debug: print(dataset)

    if time_debug: end_removing = time.time()
    if time_debug: print(f"[TIME] remove None fields: {end_removing - start_removing}")

    ## --------------------- ##
    ## --- REMOVE.column --- ##
    ## --------------------- ##
    if time_debug: start_remove_unused_columns = time.time()
    if remove_Unused_columns:

        for column in dataset.column_names:
            if column not in data_field:
                if debug: print(f"{column}")
                dataset.remove_columns_(column)

    if time_debug: end_remove_unused_columns = time.time()
    if time_debug: print(f"[TIME] remove.column: {end_remove_unused_columns - start_remove_unused_columns}")

    ## ------------------ ##
    ## --- SPLIT 1.    -- ##
    ## ------------------ ##
    if time_debug: start_first_split = time.time()

    # 80% (train), 20% (test + validation)
    ## execution
    train_testvalid = dataset.train_test_split(test_size=0.2, seed=SEED)

    if time_debug: end_first_split = time.time()
    if time_debug: print(f"[TIME] first [train-(test-val)] split: {end_first_split - start_first_split}")

    ## ------------------ ##
    ## --- SPLIT 2.    -- ##
    ## ------------------ ##
    if time_debug: start_second_split = time.time()

    # 10% of total (test), 10% of total (validation)
    ## execution
    test_valid = train_testvalid["test"].train_test_split(test_size=0.5, seed=SEED)

    if time_debug: end_second_split = time.time()
    if time_debug: print(f"[TIME] second [test-val] split: {end_second_split - start_second_split}")

    ## execution
    dataset = DatasetDict({"train": train_testvalid["train"],
                          "test": test_valid["test"],
                          "valid": test_valid["train"]})
    if time_debug: end = time.time()
    if time_debug: print(f"[TIME] TOTAL: {end - start}")
    return dataset_____no_output_____%time

dictionary_input = { "data": ["abstract"], "target": ["title"], "classes": ["mag_field_of_study"]}
dictionary_columns = sum(dictionary_input.values(), [])

# here we use meta_s2orc for speed,
dataset = partial_prepare_data(meta_s2orc, tokenizer, data_field=dictionary_columns, max_seq_length=max_seq_length)CPU times: user 2 µs, sys: 1 µs, total: 3 µs
Wall time: 4.05 µs
512
print(dataset)DatasetDict({
    train: Dataset({
        features: ['title', 'abstract', 'mag_field_of_study'],
        num_rows: 609048
    })
    test: Dataset({
        features: ['title', 'abstract', 'mag_field_of_study'],
        num_rows: 76132
    })
    valid: Dataset({
        features: ['title', 'abstract', 'mag_field_of_study'],
        num_rows: 76131
    })
})
def preprocess(*sentences_by_column, data, target, classes):
    """Preprocess the raw input sentences from the text file.
    :param sentences: a list of sentences (strings)
    :return: a dictionary of "input_ids"
    """
    print_all_debug = False
    time_debug = False
    print_some_debug = False

    if debug: print(f"[INFO-START] Preprocess on data: {data}, target: {target}")

    assert data == ['abstract'], "data should be ['abstract']"
    if debug: print(data)
    assert target == ['title'], "target should be ['title']"
    if debug: print(target)

    data_columns_len = len(data)
    target_columns_len = len(target)
    columns_len = data_columns_len + target_columns_len

    assert data_columns_len == 1, "data length should be 1"
    if debug: print(data_columns_len)
    assert target_columns_len == 1, "target length should be 1"
    if debug: print(target_columns_len)

    sentences_by_column = np.asarray(sentences_by_column)
    input_columns_len = len(sentences_by_column)

    if debug: print(f'all sentences (len {input_columns_len}): {sentences_by_column}')

    if target_columns_len == 0:
        raise NameError("No target variable selected, \
                    are you sure you don't want any target?")

    data_sentences = sentences_by_column[0]
    target_sentences = sentences_by_column[1] # if columns_len == input_columns_len else sentences_by_column[data_columns_len:-1]

    if debug: print(data_sentences)
    if debug: print(target_sentences)

    """
    # clean input removing **None**, converting them to **''**
    if clean_None_data:
        data_sentences = np.asarray([ s if s is not None else '' for s in data_sentences])
        target_sentences = np.asarray([ s if s is not None else '' for s in target_sentences])

    # clean input removing papers with **None** as abstract/title
    elif remove_None_data:
        none_data_indexes = np.asarray([ idx_s for idx_s, s in enumerate(data_sentences) if s is None])
        none_target_indexes = np.asarray([ idx_s for idx_s, s in enumerate(target_sentences) if s is None])

        if debug: print(none_data_indexes)
        if debug: print(none_target_indexes)

        to_removed_indexes = np.unique(none_data_indexes, none_target_indexes)

        if debug: print(to_removed_indexes)

        data_sentences = np.delete(data_sentences, to_removed_indexes)
        target_sentences = np.delete(target_sentences, to_removed_indexes)

    if debug: print(data_sentences)
    if debug: print(target_sentences)
    """

    # sentences = [s for s in sentences if s is not None]
    # tokens = [s.strip().split() for s in sentences]
    # tokens = [t[:max_seq_length - 1] + [tokenizer.eos_token] for t in tokens]

    # The sequences are not padded here. we leave that to the dataloader in a collate_fn
    # ----------------------------------------------- #
    # -------- TODO include the `collate_fn` -------- #
    # ----------------------------------------------- #
    # That means: a bit slower processing, but a smaller saved dataset size
    if print_some_debug: print(max_seq_length)

    data_encoded_d = tokenizer(
                        text=data_sentences.tolist(),
                        # add_special_tokens=False,
                        # is_pretokenized=True,
                        padding=True, truncation=True, max_length=max_seq_length,
                        return_token_type_ids=False,
                        return_attention_mask=False,
                        # We use this option because DataCollatorForLanguageModeling (see below) is more efficient when it
                        # receives the `special_tokens_mask`.
                        return_special_tokens_mask=True,
                        return_tensors='np'
    )

    target_encoded_d = tokenizer(
                        text=target_sentences.tolist(),
                        # add_special_tokens=False,
                        # is_pretokenized=True,
                        padding=True, truncation=True, max_length=max_seq_length,
                        return_token_type_ids=False,
                        return_attention_mask=False,
                        # We use this option because DataCollatorForLanguageModeling (see below) is more efficient when it
                        # receives the `special_tokens_mask`.
                        return_special_tokens_mask=True,
                        return_tensors='np'
    )

    if debug: print(data_encoded_d["input_ids"].shape)
    if debug: print(target_encoded_d["input_ids"].shape)
    # return encoded_d

    return {"data_input_ids": data_encoded_d["input_ids"], "target_input_ids": target_encoded_d["input_ids"]}
    # return {"input_ids": sum(encoded_d['input_ids'], [])} _____no_output_____

print an example
```python
print(dataset['train'][:10]['title'], dataset['train'][:10]['abstract'])
```_____no_output_____

vocab = tokenizer.get_vocab()
print(f"[PAD]: {vocab['[PAD]']}")
print(f"[UNK]: {vocab['[UNK]']}")
print(f"[SEP]: {vocab['[SEP]']}")
print(f"[CLS]: {vocab['[CLS]']}")
print(f"0: {tokenizer.convert_ids_to_tokens(0)}")
print(f"1: {tokenizer.convert_ids_to_tokens(1)}")
print(f"2: {tokenizer.convert_ids_to_tokens(2)}")
print(f"99: {tokenizer.convert_ids_to_tokens(99)}")
print(f"100: {tokenizer.convert_ids_to_tokens(100)}")
print(f"101: {tokenizer.convert_ids_to_tokens(101)}")[PAD]: 0
[UNK]: 101
[SEP]: 103
[CLS]: 102
0: [PAD]
1: [unused0]
2: [unused1]
99: [unused98]
100: [unused99]
101: [UNK]
tokenizer_____no_output_____

Finally, I found [this](https://huggingface.co/docs/datasets/package_reference/main_classes.html?highlight=datasetdict#datasets.DatasetDict.map) documentation for the function `DatasetDict.map` from the `dataset` library._____no_output_____

dataset_map = dataset.map(preprocess, input_columns= dictionary_columns, fn_kwargs= dictionary_input, batched=True)/home/vivoli/miniconda3/envs/arxiv-manipulation/lib/python3.7/site-packages/numpy/core/_asarray.py:83: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray
  return array(a, dtype, copy=False, order=order)
dataset_map_____no_output_____

##### adding also the `mag_index`: the  `mag_field_of_study`  index present in the table_____no_output_____

mag_field_dict: Dict = {
    "Medicine":    0,
    "Biology":     1,
    "Chemistry":   2,
    "Engineering": 4,
    "Computer Science":    5,
    "Physics":     6,
    "Materials Science":     7,
    "Mathematics":        8,
    "Psychology":  9,
    "Economics":   10,
    "Political Science":    11,
    "Business":    12,
    "Geology":     13,
    "Sociology":   14,
    "Geography":   15,
    "Environmental Science":     16,
    "Art":         17,
    "History":     18,
    "Philosophy":  19
    # "null":         3,
}

# The key null is actually null, not "null":str
#
#      real_mag_field_value = paper_metadata['mag_field_of_study']
#
# so we could return the id 3 if it not contained as key of dictionary
#
#      mag_field_dict.get(real_mag_field_value, 3)
#_____no_output_____def mag_preprocessing(*mags):
    """Preprocess the raw input sentences from the text file.
    :param sentences: a list of sentences (strings)
    :return: a dictionary of "input_ids"
    """
    debug = False

    if debug: print(f"[INFO-START] Mag Preprocess")

    mag_field = np.array(mags)
    input_columns_len = mag_field.shape
    if debug: print(f'pre flatten (len {input_columns_len}): {mag_field}')
    if debug: print(f'pre types: {[type(ele) for ele in mag_field]}')
    if debug: print(f'pre types: {type(mag_field)}')

    mag_field = mag_field.flatten()
    input_columns_len = mag_field.shape
    if debug: print(f'after flatten (len {input_columns_len}): {mag_field}')
    if debug: print(f'after types: {[type(ele) for ele in mag_field]}')
    if debug: print(f'after types: {type(mag_field)}')

    mag_field = np.array([ele if type(ele) == str else list(ele)[0] for ele in mag_field])

    if input_columns_len == 0:
        raise NameError("No mag variable selected, \
                    are you sure you don't want any target?")

    if debug: print(mag_field)
    if debug: print(mag_field_dict)
    if debug: print([mag_field_dict.get(real_mag_field_value, 3) for real_mag_field_value in mag_field])

    mag_index = np.asarray([mag_field_dict.get(real_mag_field_value, 3) for real_mag_field_value in mag_field])

    if debug: print(mag_index)

    return {"mag_index": mag_index}
    # return_____no_output_____dataset_mag_map = dataset_map.map(mag_preprocessing, input_columns= dictionary_input['classes'], batched=True)
_____no_output_____dataset_mag_map_____no_output_____

# Rename it as you want
---

- `dataset_map.rename_column` ,method for renaming
- `dataset_map.set_format`, method for define what columns need to be returned_____no_output_____

dataset_map = dataset_map.rename_column("data_input_ids", "input_ids")_____no_output_____dataset_map.set_format("torch", columns=["input_ids"])_____no_output_____print(dataset_map['train'][1]['input_ids'].size())_____no_output_____

Then, if you want to store it, it will be stored in the conda environment you are_____no_output_____

%store dataset_map_____no_output_____

---
---
## ❌ FAKE PIPELINE for train BERT-based NETS
---
---_____no_output_____

%store -r dataset_map_____no_output_____dataset_map_____no_output_____# tokenizer: we already have it
# model: we already have it

# If you print some element from `dataset_map['train'][element_index]['input_ids']` you'll see that lots of element
vect = [ele[ele.nonzero()].size(0) for ele in dataset_map['train'][:]['input_ids']]_____no_output_____max_vect = max(vect)
min_vect = min(vect)
sum_vect = sum(vect)
len_vect = len(vect)

print(f" max: {max_vect} \n min: {min_vect} \n avg: {sum_vect/len_vect}") max: 512
 min: 2
 avg: 204.6087681514113

From [here](https://github.com/huggingface/transformers/blob/master/examples/language-modeling/run_mlm.py) you can get an idea from were the code has been borrowed._____no_output_____