# Notebook from arodriguezca/NLP-dataset
Path: glove-bilstm.ipynb

<code>
#Libraries

import warnings
warnings.filterwarnings('ignore')
import pandas as pd
import numpy as np
import os
import re
import json
import string
import matplotlib.pyplot as plt
%matplotlib inline
import plotly.express as px
import plotly.graph_objects as go
from tqdm.autonotebook import tqdm
from functools import partial
import torch
!pip install transformers
from transformers import BertTokenizer, BertModel
import spacyCollecting transformers
  Downloading https://files.pythonhosted.org/packages/d8/b2/57495b5309f09fa501866e225c84532d1fd89536ea62406b2181933fb418/transformers-4.5.1-py3-none-any.whl (2.1MB)
     |████████████████████████████████| 2.1MB 7.9MB/s
Collecting sacremoses
  Downloading https://files.pythonhosted.org/packages/75/ee/67241dc87f266093c533a2d4d3d69438e57d7a90abb216fa076e7d475d4a/sacremoses-0.0.45-py3-none-any.whl (895kB)
     |████████████████████████████████| 901kB 36.9MB/s
Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.19.5)
Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)
Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.0.12)
Collecting tokenizers<0.11,>=0.10.1
  Downloading https://files.pythonhosted.org/packages/ae/04/5b870f26a858552025a62f1649c20d29d2672c02ff3c3fb4c688ca46467a/tokenizers-0.10.2-cp37-cp37m-manylinux2010_x86_64.whl (3.3MB)
     |████████████████████████████████| 3.3MB 54.2MB/s
Requirement already satisfied: importlib-metadata; python_version < "3.8" in /usr/local/lib/python3.7/dist-packages (from transformers) (3.10.1)
Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.41.1)
Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)
Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from transformers) (20.9)
Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.0.1)
Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (7.1.2)
Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.15.0)
Requirement already satisfied: typing-extensions>=3.6.4; python_version < "3.8" in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < "3.8"->transformers) (3.7.4.3)
Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < "3.8"->transformers) (3.4.1)
Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2020.12.5)
Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)
Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)
Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)
Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->transformers) (2.4.7)
Installing collected packages: sacremoses, tokenizers, transformers
Successfully installed sacremoses-0.0.45 tokenizers-0.10.2 transformers-4.5.1
gpu_info = !nvidia-smi
gpu_info = '\n'.join(gpu_info)
if gpu_info.find('failed') >= 0:
  print('Select the Runtime > "Change runtime type" menu to enable a GPU accelerator, ')
  print('and then re-execute this cell.')
else:
  print(gpu_info)

print(f'GPU available: {torch.cuda.is_available()}')Wed Apr 28 19:38:10 2021
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 465.19.01    Driver Version: 460.32.03    CUDA Version: 11.2     |
|-------------------------------+----------------------+----------------------+
| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
|                               |                      |               MIG M. |
|===============================+======================+======================|
|   0  Tesla T4            Off  | 00000000:00:04.0 Off |                    0 |
| N/A   37C    P8     9W /  70W |      0MiB / 15109MiB |      0%      Default |
|                               |                      |                  N/A |
+-------------------------------+----------------------+----------------------+

+-----------------------------------------------------------------------------+
| Processes:                                                                  |
|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |
|        ID   ID                                                   Usage      |
|=============================================================================|
|  No running processes found                                                 |
+-----------------------------------------------------------------------------+
GPU available: True
#data = pd.read_csv("data.csv")
data = pd.read_csv("data.csv")
data.head()
print(len(data))4155
def read_GloVe(filename):
 embeddings = {}
 for line in open(filename).readlines():

    fields = line.strip().split(" ")
    word = fields[0]
    embeddings[word] = [float(x) for x in fields[1:]]
 return embeddings

GloVe = read_GloVe("glove.840B.300d.conll_filtered.txt")_____no_output_____data.head()_____no_output_____import re
paragraphs = []
for i in range(len(data)):
  t = data.loc[i][6]
  t = t.lower()
  t = re.findall(r"[\w']+|[.,!?;]", t)
  paragraphs.append(t)
_____no_output_____#Create mappings between tokens and indices.

from collections import Counter
import random

#Will need this later to remove 50% of words that only appear once in the training data from the vocabulary (and don't have GloVe embeddings).
wordCounts = Counter([w for l in paragraphs for w in l])
singletons = set([w for (w,c) in wordCounts.items() if c == 1 and not w in GloVe.keys()])

#Build dictionaries to map from words, characters to indices and vice versa.
#Save first two words in the vocabulary for padding and "UNK" token.
word2i = {w:i+2 for i,w in enumerate(set([w for l in paragraphs for w in l] + list(GloVe.keys())))}

i2word = {i:w for w,i in word2i.items()}

#When training, randomly replace singletons with UNK tokens sometimes to simulate situation at test time.
def getDictionaryRandomUnk(w, dictionary, train=False):
  if train and (w in singletons and random.random() > 0.5):
    return 1
  else:
    return dictionary.get(w, 1)

#Map a list of sentences from words to indices.
def sentences2indices(words, dictionary, train=False):
  #1.0 => UNK
  return [[getDictionaryRandomUnk(w,dictionary, train=train) for w in l] for l in words]

X       = sentences2indices(paragraphs, word2i, train=True)

print(i2word[253])

#Print out some examples of what the dev inputs will look like
for i in range(10):
  print(" ".join([i2word.get(w,'UNK') for w in X[i]]))
def prepare_input(X_list):
  X_padded = torch.nn.utils.rnn.pad_sequence([torch.as_tensor(l) for l in X_list], batch_first=True).type(torch.LongTensor)
  X_mask   = torch.nn.utils.rnn.pad_sequence([torch.as_tensor([1.0] * len(l)) for l in X_list], batch_first=True).type(torch.FloatTensor)
  return (X_padded, X_mask)marulis
this article investigates an important factor in student achievement parental involvement . using data from the national education longitudinal study nels , we estimate a value added education production function that includes parental effort as an input . parental effort equations are also estimated as a function of child , parent , household , and school characteristics . our results suggest that parental effort has a strong positive effect on achievement that is large relative to the effect of school resources and is not captured by family background variables . parents appear to reduce their effort in response to increased school resources , suggesting potential ''crowding out'' of school resources .
this study examines the wage gender gap of young adults in the 1970s , 1980s , and 2000 in the us . using quantile regression we estimate the gender gap across the entire wage distribution . we also study the importance of high school characteristics in predicting future labor market performance . we conduct analyses for three major racial ethnic groups in the us whites , blacks , and hispanics , employing data from two rich longitudinal studies nls and nels . our results indicate that while some school characteristics are positive and significant predictors of future wages for whites , they are less so for the two minority groups . we find significant wage gender disparities favoring men across all three surveys in the 1970s , 1980s , and 2000 . the wage gender gap is more pronounced in higher paid jobs 90th quantile for all groups , indicating the presence of a persistent and alarming ''glass ceiling . '' ó 2007 elsevier inc . all rights reserved . jel codes j16 ; j24 ; j31 the existence of gender differences in labor market outcomes , such as wages , has gained ample attention in economics and the social sciences . gender differences in wages have been researched and documented , and frequently debated in the literature . it is an established fact that males earn substantially higher wages than females . 0049 there is some empirical evidence , however , that while the gender gap is decreasing over time due to women's increased labor force participation and experience , it remains strong across the entire wage distribution . the quality of the empirical evidence on gender differences in wages has not always been very strong for two main reasons . first , typically , the samples of various studies on gender differences in wages are not representative of a well defined population . many studies have used convenient samples , and it is plausible that the results obtained from such selected samples are biased positively or negatively , and hence very different from their ''true'' population parameters . second , various previous studies on gender differences in wages have typically examined and reported group differences in means the central tendency of the distribution of wages . gender differences in the extremes e . g . , upper and lower tails of the wage distribution are only recently documented in the literature . since it is likely that gender differences in the tails of the wage distribution may be different qualitatively than differences in the middle of the distribution , examining gender differences across the entire distribution of wages is important and provides a more accurate picture of the gender gap . for example , males may be over represented in the top 10 percent of the wage distribution compared to females , a byproduct of over concentration of men in highly paid jobs . this difference may not necessarily be similar to gender differences observed in the middle or the lower tail of the wage distribution . in this study we employ base years and follow up data of national probability samples of high school students in the us , namely the national longitudinal study nls of the high school class of 1972 base year , fourth and fifth follow up and the national education longitudinal study nels of the eighth grade class of 1988 second and fourth follow up . the main advantage of these datasets is that we can link important characteristics of high schools attended in the base year to wages from follow up years , and thus , examine how high school characteristics affect wage differentials . in addition , these rich data allow us to examine the labor market performance of similarly aged individuals 7 , 8 , and 14 years after high school graduation , and hence , likely avoiding transitional labor market effects . because of the use of national probability samples our results are more likely to have higher external validity , and be more resilient to threats of selection bias . we examine gender differences in hourly wages for young adults in the late 1970s , mid 1980s , and 2000 across the entire distribution of wages , thus covering three important time spells . our estimation technique is to employ quantile regressions while adjusting for selection biases in the labor force . because the gender gap may be declining on average , but may be remaining strong in the upper or lower tails , affecting women disproportionately , we examine the following quantiles of the wage distribution 10th , 25th , 50th , 75th , and 90th quantile . we conduct separate analyses for three major race ethnic groups in the us whites , blacks , and hispanics . this permits us to determine whether the gender gap differs by race ethnic group and whether it is changing over time . an equally important objective of the present study is to investigate the link between high school characteristics and hourly wages . gauging the effects of high school characteristics on the wage gender gap is of great importance because school effects have differential and enduring effects on the earnings of individuals who attend different schools net of individual and family background characteristics constant and konstantopoulos , 2003 . previous work on school effects has yielded mixed and inconsistent findings with respect to the importance of schooling on school outputs such as academic achievement . some researchers have concluded that there is little or no evidence of school effects hanushek , 1986 , while others report that the impact of school factors may be substantial greenwald et al . , 1996 . in this study , we examine school effects on labor market outcomes and ask the question , can high school characteristics predict future wages of young adults , net of the effects of individual characteristics ? if so , then which school characteristics matter more for the economic performance of young workers , and for which ethnic groups ?
despite the striking reversal of the gender gap in education , women pursue science , technology , engineering , and mathematics stem degrees at much lower rates than those of their male peers . this study extends existing explanations for these gender differences and examines the role of the high school context for plans to major in stem fields . building on recent gender theories , we argue that widely shared and hegemonic gender beliefs manifest differently across schools so that the gender specific formation of study plans is shaped by the local environment of high schools . using the national education longitudinal study , we first show large variations between high schools in the ability to attract students to stem fields conditional on a large set of pre high school measures . schools that are successful in attracting students to these fields reduce the gender gap by 25 percent or more . as a first step toward understanding what matters about schools , we then estimate the effect of two concrete high school characteristics on plans to major in stem fields in college a high school's curriculum in stem and gender segregation of extracurricular activities . these factors have a substantial effect on the gender gap in plans to major in stem a finding that is reaffirmed in a number of sensitivity analyses . our focus on the high school context opens concrete avenues for policy intervention and is of central theoretical importance to understand the gender gap in orientations toward stem fields .
abstract past research has shown that locus of control plays an important role in a wide range of behaviors , such as academic achievement and positive social behaviors . however , little is known about whether locus of control plays the same role in minority adolescents' peer relationships . the current study examined ethnic differences in the associations between locus of control and peer relationships in early adolescence using samples from the early childhood longitudinal study ecls k 5 , 612 caucasian , 1 , 562 hispanic , 507 asian , and 908 africanamerican adolescents and the national education longitudinal study nels 8 , 484 caucasian , 1 , 604 hispanic , and 860 asian , and 1 , 228 african american adolescents . gender was approximately evenly split in both samples . the results from the two datasets were highly consistent . significant interactions between ethnicity and locus of control indicated that having a more internal locus of control was particularly important for caucasian students' peer relationships ecls k and social status nels , but less so for asian , hispanic , and african american students . our findings suggest that the role of locus of control in peer relationship is contingent upon culture . locus of control , a concept developed by rotter 1954 , focuses on the degree to which individuals generally believe that they , rather than other people or uncontrollable factors such as ''fate , '' are responsible for the outcomes of events in their lives . rotter 1954 conceptualized locus of control as occurring on a continuum from internal to external control , rather than as a dichotomous variable . among the most replicable findings , internal locus of control has been associated with better academic performance e . g . , chang et al . 2007 ; strayhorn 2010 , healthrelated outcomes e . g . , jose and weir 2013 ; sturmer et al . 2006 , job performance see review by ng et al . 2006 , and athletic performance e . g . , denny and steiner 2009 , whereas lacking a sense of control has been associated with negative outcomes , such as anxiety weems et al . 2003 and depression muris et al . 2004 . in short , decades of research have shown that locus of control plays an important role in a wide range of behaviors . most relevant to the current study , several studies also have shown that locus of control is associated with social relationships . specifically , studies with adult subjects demonstrated that those individuals who reported greater internal locus of control were more willing to communicate with out groups lam and mizerski 2005 and characterized themselves as being more sociable and open to new experiences mühlig versen et al . 2012 ; rubin 1993 . similarly , research has shown that children and adolescents with a more internal locus of control are more engaged in their classrooms you and sharkey 2009 , are less shy crozier 2011 , and display greater social maturity nelson and mathia 1995 . it also appears that having an external locus of control orientation is associated with aggressive tendencies and bullying behaviors osterman et al . 1999 , which can contribute to difficulties developing or maintaining friendships . for example , in a study of 5 7 year old australian elementary school children , slee 1993 found that children who were identified as bullies through self and teacher reports showed a more external locus of control . students who were identified as victims did not differ in locus of control and students who were not identified as either bullies or victims reported a more internal locus of control . in sum , there is considerable evidence for a significant role of locus of control in peer relationships . the above conclusion , however , is based on studies of mostly caucasian participants . little is known about whether locus of control plays the same role in minority adolescents' peer relationships . thus far , cross cultural and cross ethnic research on locus of control has focused mostly on either general locus of control or in the domain of academic achievement and the results have been inconsistent . a few earlier studies documented that asians , asian americans , and mexicanamericans had a more internal locus of control than their caucasian counterparts brown et al . 2007 ; hamilton et al . 1989 ; si et al . 1995 . a number of studies , however , showed that ethnic minority groups such as african , asian , and mexican americans tended to hold a more external locus of control leung 2001 ; o'hea et al . 2009 ; okeke et al . 1999 ; ramirez 1988 than european americans . an orientation towards external locus of control seems consistent with fatalistic beliefs that prevail in many asian cultures norenzayan and lee 2010 ; yeh et al . 2006 and in mexican and other cultures with a strong catholic tradition mccabe et al . 2008 . in african american cultures , having a more external locus of control orientation may be a way to deal with the disadvantages of poverty , unemployment , and racial discrimination threlfall et al . 2013 . in the domain of academic achievement , asians and asian americans report greater internal control than caucasian americans regarding their academic performance . asians and asian americans have been found to exceed caucasian americans in linking their successes and failures to their own efforts rather than to external circumstances e . g . , si et al . 1995 ; stevenson and stigler 1992 . bartal et al . 1980 found that perceived locus of control was positively associated with academic achievement , but more so for students of asian or african origin than for students of european , american , or israeli origin . however , gardner 2007 found that african american students tended to blame poor academic performance on external factors . in sum , previous research has shown cross cultural differences although not always consistently in general locus of control orientation as well as in the domain of academic achievement . cultural values can also influence locus of control in the domain of peer relationships , although little research has been conducted in this area . european american culture is characterized by an emphasis on the personal responsibility of individuals or ''agency'' to enter and to maintain relationships shweder et al . 2006 . children are encouraged to retain their sense of autonomy and freedom , even when interacting with peers rubin et al . 2002 . in contrast , asian and latino cultures emphasize cooperation and interdependence in the establishment and maintenance of peer relationships chen 2000 ; shweder et al . 2006 ; way 2006 . for example , one must be flexible when interacting with others to avoid embarrassing or hurting others' feelings . as a result , these groups' cultural emphasis on interdependence and maintenance of social harmony may lead them to see peers' agentic attitudes and behavior as a threat to positive relationships in the peer group . to put it succinctly , an internal locus of control in peer relationships may conflict with their cultural value of collectivism . given these differing cultural values in peer relationships , internal locus of control should be more congruent with european american children's peer relationships than with those of ethnic minority children .
the black white gap in achievement , as measured by performance on standardized tests , has received considerable attention from researchers in the past five years . claude steele's stereotype threat and disidentification mechanism is perhaps the most heralded of the new explanations for residual racial differences that persist after adjustments for social background are performed . analyzing data from the national education longitudinal study , we found qualified support for portions of the disidentification explanation . black students' academic selfevaluations are more weakly associated with their measured academic performances , a difference that could stem from stereotype threat or a belief that the evaluations are racially biased . but this discounting of performance evaluations does not seem to provoke a more complete disidentification with the schooling process or with academic achievement in general . the findings suggest that there is no clear path from being stereotyped to disidentifying , and in conclusion we discuss alternative explanations for why it may be so .
this study examines two important and related dimensions of the persisting gender gap in science , technology , engineering , and mathematics stem bachelor degrees first , the life course timing of a stable gender gap in stem orientation , and second , variations in the gender gap across high schools . we build on existing psychological and sociological gender theories to develop a theoretical argument about the development of stem orientations during adolescence and the potential influence of the local high school environment on the formation of stem orientations by females and males . using the national education longitudinal study nels , we then decompose the gender gap in stem bachelor degrees and show that the solidification of the gender gap in stem orientations is largely a process that occurs during the high school years . far from being a fixed attribute of adolescent development , however , we find that the size of the gender gap in stem orientation is quite sensitive to local high school influences ; going to school at a high school that is supportive of a positive orientation by females towards math and science can reduce the gender gap in stem bachelor degrees by 25 or more . when then harvard president lawrence summers pointed at innate differences between men and women as a possible explanation for women's under representation in high level science positions , he sparked an intense public controversy that mirrors a continuing debate in the scientific community . despite the striking reversal of the gender gap in educational attainment buchmann and diprete 2006 and the near gender parity in math performance hyde et al . 2008 , women still pursue science , technology , engineering , and mathematics stem degrees at much lower rates than their male peers do . figure 1 illustrates these trends . it shows , on the one hand , how women have made impressive gains in college attainment compared to men and now clearly outnumber men among college graduates in recent decades . on the other hand , women continue to lag behind in terms of bachelor degrees awarded in the 'quantitative' sciences illustrated in the graph for different stem sub fields . 1 this persistent pattern of gender inequality in college science majors and the implications for later career choices and labor market earnings has been a major concern for scientist and policy makers alike . such differences are not only relevant for the representation of women in high income and prestigious jobs and as such for gender equality in general but also for the supply of qualified labor in science oriented jobs , which is often regarded as a linchpin for the future of the us economy in an increasingly competitive global environment . in this paper , we explore two important and related dimensions of the persisting gender gap in stem degrees . the first dimension is the timing of the emergence of a gender gap in orientation towards stem fields . our analysis of the middle school through college phase of the educational life course reveals that the high school years play a major role in shaping gendered orientations toward science and engineering . the second dimension concerns the impact of the high school environment on the development of these orientations during the decisive high school years . in particular , we use multilevel models to document how the gender gap in stem orientation in twelfth grade varies across high schools , and we estimate the causal effect of the high school curriculum on the gender gap in stem orientation . far from being a fixed attribute of adolescent development , we find that the size of the gender gap in stem orientation is quite sensitive to local high school influences ; going to school in a high school that is supportive of a positive orientation by females towards math and science can reduce the gender gap in stem bachelor degrees by 25 or more . together these two dimensions of timing and local environmental influence extend existing theories and open concrete avenues for policy intervention . 1 exceptions to this trend are the biological , biomedical and life sciences , in which women today outnumber men . table 268 , 299 , 303 , 305 , 312 and 313 we begin by developing a theoretical argument about the development of stem orientations during adolescence , and the potential influence of the local environment on the formation of stem orientations by females and males . while acknowledging the importance of the global environment , we argue that local environments centrally affect the strength and salience of gender stereotypes about math and science . their influence operates mainly through two important processes . first , peers , parents and teachers all important actors in the local environment support and encourage certain career paths for boys and girls while disparaging others . second , the influence of gender stereotypes about stem occupations depends on the level of exposure to stem academic courses and to information about stem fields and stem occupations . both arguments imply that the size of the gender gap in stem bachelor degrees is shaped to a considerable extent by the character of the high school environment . variations in this environment will therefore affect the size of the gender gap in stem orientations . we then turn to an empirical examination of the timing and the local variation across high schools . first , we decompose the gender gap in stem bachelor degrees into various pathways to examine the emergence and solidification of gender differences in the orientation towards science and engineering in the adolescent life course . in particular , we use the national education longitudinal study nels to follow the 1988 cohort of eighth grade students through adolescence and young adulthood , and observe how orientations towards stem fields emerge and change from eighth grade through college . we find that the substantial gender gap in eighth grade orientation is relatively inconsequential for the persisting gender gap in stem degrees at the completion of college . instead , the high school years play a major role in shaping gendered orientations towards science and engineering . second , we examine the role of the high school for this decisive period . in particular , we use multilevel models to document how the gender gap in stem orientation at the end of high school varies across high schools , and we estimate the causal effect of high school curriculum on the gender gap in stem orientation . the results show substantial variation in the gender gap in stem orientation across schools , which supports our argument that the local environment plays a major role in shaping these orientations among boys and girls . the significant effect of high school curriculum on the gender gap in stem orientation provides the beginning of an understanding about the source of the high school effect . our results imply that the gender gap could be considerably reduced if high school environments were reshaped to model those high schools that are most supportive of female interest and competency in science and mathematics .
abstract this paper analyses the impact of teacher gender towards students' test results in a blinded math test administered to students in catalonia spain . the data for this analysis are drawn from a sample of secondary school students who participated in an international blind test known as the mathematical kangaroo in 2008 . the estimation considers a two stage procedure since participation on the test leads to the presence of sample selection . results show a correlation between female teacher gender and student results . moreover , students with female teachers have a higher probability of participating in the kangaroo test in this case , the effect being more marked among male students . jel codes i28 , j16 a number of countries , including australia , canada , finland , new zealand , united kingdom and the united states have implemented policy initiatives to increase the number of male teachers in primary education where they have been in a clear minority to improve the results of male pupils , since it is assumed that , among other reasons , they are related to the lower presence of male teachers see a review in klein , 2004 ; carrington et al . , 2007 ; younger and warrington , 2008 ; skelton , 2009 . these policies have been implemented although there is ongoing debate as to whether students' results can be correlated with the gender of their teachers . thus , some studies report that students perform better if they have a same gender teacher , others point out that it is better to have a female teacher , while the third group of studies indicate that there is no gender effect at all . the reasons to support the first type of results relate to the fact that teachers might prefer teaching students of their own gender or that gender stereotypes may influence teacher evaluations of their students . in addition , teachers may act as role models for their students see , among others , carrington and skelton , 2003 ; gray and leith , 2004 ; holmlund and sund , 2008 . in regards to the second kind of results , students assigned to female teachers perform better since female teachers tend to be more supportive , provide a more positive classroom atmosphere , and tend to use a more student oriented style of teaching stressing the importance of motivation see stake and katz , 1982 ; singer , 1996 ; krieg , 2005 ; nelson laird , 2011 . below we summarize some of the existing evidence for primary and secondary education middle and high school . in primary education , the studies reviewed conclude that teacher gender is either irrelevant or that female teachers improve both girls' and boys' performance . thus , in a large scale analysis 8 , 978 eleven year old pupils and 413 teachers in 113 primary schools year 6 in england , during the 1997 98 academic year , carrington et al . 2008 conclude that having a same gender teacher has no impact on student performance measured through tests , either that of males or females , in mathematics , reading and science . moreover , with a sample of students and teachers in 19 primary and secondary schools in australia , lingard et al . 2002 show that teacher gender is not a significant factor in determining positive outcomes for students in literacy and mathematical tests . this result is also found in sokal et al . 2005 in a study with 6 to 8 yearold children in a school in canada regards to reading performance , as well as in driessen 2007 with a large scale sample of dutch primary schools , in relation to language and maths tests' results , which included 5 , 181 grade eight pupils , 251 teachers and 163 schools . krieg 2005 also concludes that having a teacher of the same gender is not relevant . however , the study shows that male and female students in grade three 8 year olds assigned to female teachers obtain higher marks on a standardized test the washington assessment of student learning . controlling for student ability as well as for school and district fixed effects , krieg shows that the students of female teachers are more likely to score well on the maths , reading and writing sections of the test i . e . , they obtain higher results and present a higher probability of passing . in developing countries , chudgar and sankar 2008 consider a sample of grade four and six students ages 9 to 11 , in 300 public schools in india , and conclude that being in a female teacher's classroom is advantageous for language learning but teacher gender has no effect on mathematics learning . in secondary education we find numerous evidence teacher gender is irrelevant regards to students' results ; female teachers increase students' outcomes ; and or students assigned to a same gender teacher have better results . thus , in an analysis of upper secondary education students 16 to 18 year olds in 69 schools in stockholm sweden , holmlund and sund 2008 find no evidence to show that teacher gender improves student outcomes . in this study , for each student and school year , they are able to identify both the student outcome the final overall grade as well as the grades obtained in several individual subjects and the gender of the students and teachers . the authors argue that the gender performance differential in favor of female students is greater in subjects in which the share of female teachers is higher but this effect is not causal , since it is not observed when the analysis controls for teacher turnover and student mobility nor when the assumption of random student teacher matching within a subject holds . the same kind of evidence is provided by ehrenberg et al . 1995 for the usa . using data from the national education longitudinal study of 1988 , which comprises nearly 25 , 000 eighth graders lower secondary education as well as two of each of the student's teachers , they find that a teacher's gender is not correlated with the achievement test scores in mathematics and science of students although , in some cases , teachers' subjective evaluations about their students are . however , dee 2007 shows that that assignment to a same gender teacher significantly improves the achievement of both girls and boys in terms of their test scores in mathematics , science , reading and history . data comes from the national education longitudinal study of 1988 and he controls for student traits , classroom and teacher characteristics as well as including student , class and teacher fixed effects . finally , considering 3 , 446 pupils from 110 public schools in israel , klein 2004 concludes that male and female pupils get higher scores , defined as end of the year grades in subjects related to humanities , such as literature and history , and science mathematics , chemistry and physics with female teachers . thus , the available empirical evidence does not allow an accurate determination of the correlation between teacher's gender and student's academic performance in primary and secondary education . in this context , this paper analyses the impact of teacher gender on student marks , considering student results in a mathematical blind test named kangaroo . the issue is of importance since , as holmlund and sund 2008 point out , the gender gap may have both educational and economic consequences in terms of an efficiency loss whereby students with higher ability might obtain lower marks and face greater difficulties in accessing higher levels of education and furthering their professional careers . in addition , as mentioned before , educational policies have been implemented in a number of countries to regulate this issue with labor consequences for the teachers . three specific aspects of this paper are worth highlighting . first , in this study we include the characteristics of the agents that might have an impact on student results , such as pupils , teachers , and schools . second , we include fixed effects to control for student and school characteristics not observed in the analysis unobservables , as the recent literature recommends dee , 2007 ; hoffman and oreopoulos , 2009 ; holmlund and sund , 2008 . finally , to the best of our knowledge , this is the first analysis conducted in spain or in any similar country in southern europe .
today , social network is very important to many people particularly for the youths in their everyday lives , integrating their online and offline experience and becoming one of the primary means of connecting and socially interacting worldwide , whether among individuals , students , business people and even governments . the purpose of this study is to propose a conceptual framework on the relationship between social network usage especially facebook with academic performance . according to hussain 2005 , facebook is the most popular social media worldwide , and its trend indicated that university students used social media for self enjoyment and friendship . however , they preferred to share their study experiences research projects , educational events , information , and developing networks . social networking sites such as facebook , twitter , youtube , skype and blog are widely used in the exchange of different information and in communicating with various cultures worldwide o'reilly , 2007 . in the current 2000s , online social networking sites have gained popularity among college students . facebook is one of the highly popular social media worldwide . facebook provides an opportunity for users particularly college students to create their profiles , make personal information known to other users , and upload and display pictures . moreover , users are able to look at the profiles of other users , gather and make more friends online , and connect with those friends via messages , gifts , and photo tagging kalpidou , dan costin , jessica , 2011 . the increasing connection of college students to these worldwide online communities has constituted an anxiety to both parents and educators to the extent of stimulating research in this area to investigate the phenomenon . some researchers have pointed out that students using social media learn in new ways and for this reason educators should welcome these new developments ito et al . , 2009 ; jenkins , 2006 . park , kee and valenzuela 2009 , found that facebook could negatively affect the overall academic performance of the students . according to khan 2009 , facebook users often experience poor performance academically . moreover , many researchers were focused on various demographic variables such as age and gender on academic performance . the result of these researches found the significance of gender in student's academic achievement for example ; they found a significant difference in gpa grade between male and female students in a class jabor et al . , 2011 . similar with an analysis of data from the national education longitudinal study nels ; lee and burka 1996 , found a large advantage for males on the physical sciences and the modest advantage for females on the life science . however , coley 2001 , studied gender difference within ethnic groups of varying ages , and it revealed more similarities than differences . in conclusion , more research on age and gender differences in academic performance is needed to make conclusive implication of the effect of age , gender , and facebook intensity on the academic performance of university students . this study will investigate whether students' age , gender , and facebook intensity may make a difference in their academic performance .
a study investigated the college application behaviors of students from different racial ethnic groups whites , african americans , asian americans , hispanic americans to understand differences in the college search and choice process . data we . re drawn from two large national longitudinal studies , the national education longitudinal study 1988 and the beginning postsecondary studentn longitudinal study . analysis revealed significant group differentes in college application behavior number of colleges applied to , time of submission of application , first choice of institution , and tuition cost . substantial data tables showing analyses are included . asian americans were most likely to follow assumptions underlying traditional college choice models . latino students were the least prepared regarding knowledge about college and least likely to fit traditional college choice models . it is concluded that the findings suggest a need for campuses to evaluate the potential effects of policy decisions that may affect student choice for different applicant populations . implications for institutional research needs are also noted . contains 7 tables , 5 appendixes , and 18 references . mse access and equity have long been central goals of american higher education , as reflections of both egalitarian and pragmatic interests . most often , measures of enrollment and persistence have been used to track overall participation rates and to gauge the success of various groups in securing equal levels of opportunity . there is fairly wide agreement that throughout the 1960s and '70s , minority men and women of all ethnic groups achieved ever increasing levels of representation at american two and four year institutions , and that disparities between socioeconomic , racial ethnic , and gender groups decreased alexander , pallas holupka , 1987 ; orfield , 1990 ; paul , 1990 . there is less agreement regarding the cease at these gains some researchers credit the vast increases in public and private student aid expenditures during this period astin , 1982 cited in nora and horvath , 1989 , and others claim no evidence exists to suggest that financial aid improves access hanson , 1980 cited in nora and horvath , 1989 ; zemsky , 1988 cited in orf eld , 1990 . there is also deep disagreement over whether racial and ethnic groups and those of lower socioeconomic status have gained or lost ground since the retrenchment of the 1980s . alexander , et al . 1987 found that for a cohort of 1980 high school seniors , within individual socio economic status ses levels , minority youth consistently showed higher participation rates than white students , yet low ses was nonetheless strongly associated with less participation . paul 1990 cites the failure of some researchers to take into account the increasing number of minority high school graduates when they claim advances in higher education representation of minorities . instead , she contends that when minority enrollment in higher education is considered as a percentage of minority high school graduates , both african americans and latinos lost considerable ground between the mid 1970s and the mid 1980s . at the same time , however , critics of affirmative action in admissions suggest that such programs and policies are either no longer necessary or that they provide an unfair advantage to choice among racial ethnic groups p . 2 racial ethnic groups over white applicants to college . these differing points of view suggest that it is time to reexamine the progress and barriers to progress in terms of access to higher education for different racial ethnic groups . yet united states higher education is not a monolith of similar institutions evenly dispersed throughout the land . there is a great variety of institution types , from large , prestigious research institutions producing bachelor's degrees through doctorates , to small two year community colleges offering associates degrees and vocational training . cost , availability of financial support , and entrance requirements all differ among institutions , affecting access in a number of ways pascarella terenzini , 1991 . likewise , degrees confer individual benefits of economic opportunity and prestige and increase human resources to society in amounts which differ from one institution to the next . therefore , it is important when judging equality of access to higher education and return on investment to consider the distribution of students among institutions of different types . using the theoretical model established by hossler and associates 1984 , we investigated the college application behaviors of various racial ethnic groups in order to understand differences in the college search and choice processes . hossler and gallagher 1987 posit three phases of the college choice process the predisposition , search , and choice phase when students' backgrounds , attributes , activities , and institutional characteristics interact to influence the decision making process . the first stage is the predisposition phase when family background , ability , and students' early preferences predispose students to aspire to specific degree attainments and seek information about colleges . during the next phase , both the student and institutions engage in search activities . while students seek information about and make decisions concerning the types of institutions they will consider applying to , institutions typically also provide information to students they are interested in recruiting . in the third and last phase of the college choice process , students narrow the range of schools they are considering to a choice set composed of two or more schools , and colleges engage in courtship activities ranging from invitations for campus visits to the offering of financial aid packages . choice among racial' fanic groups p . 3 , because college pricing , financial aid , and other factors are critical to understanding this process , we set out to explore continuing differences in groups both at the senior year of high school and once in college . student demographics , preferences , academic ability , and income levels were taken into account in our analyses . erdman 1983 examined factors that influenced high school seniors' applications to specific colleges and found traditional age students rank the following factors from . most influential to least academic programs , reputation , location , size , parent recommendation , counselor recommendation , cost , and alumni contact . erdman concludes that the reputation of a particular institution in the mind of students , the location of that institution , and its size are powerful forces in the selection process , outweighing other factors examined , including cost p . 6 . in contrast , other work on nontraditional students consisting mainly of adults students suggest that these students are more sensitive to tuition cost than recent high school graduates bishop van dyk , 1977 . moreover , recent studies have shown that the typical models for college choice are less effective in predicting nontraditional or delayed entry students' search and choice processes than they are of traditional aged students bers and smith , 1987 ; hurtado , kurotsuchi , and sharp , 1996 . we examine these issues across racial ethnic groups in order to determine key differences in college choice and access .
in this paper , we develop a new direct measure of state anti smoking sentiment and merge it with micro data on youth smoking in 1992 and 2000 . the empirical results from the cross sectional models show two consistent patterns after controlling for differences in state anti smoking sentiment , the price of cigarettes has a weak and statistically insignificant influence on smoking participation ; and state anti smoking sentiment appears to be a potentially important influence on youth smoking participation . the cross sectional results are corroborated by results from discrete time hazard models of smoking initiation that include state fixed effects . however , there is evidence of price responsiveness in the conditional cigarette demand by youth and young adult smokers . despite mainly encouraging trends , public health policy makers continue to be concerned about youth smoking . although the prevalence of smoking among high school seniors in the u . s . has fallen from its 1997 peak of 36 . 5 percent to about 23 percent , it still has a ways to go to reach the healthy people 2010 national health objective of 16 percent johnston 2005 , usdhss 2004 . and the most recent data suggest the rate of decline in smoking among u . s . youth may have stopped or even reversed johnston 2005 , cdc 2006 . the most recent trend data led the president of the american legacy foundation the anti smoking organization established by the 1998 legal settlement with the tobacco industry to conclude that today's news from the cdc is a warning sign act now to support tobacco prevention work across our nation , and we can drive down youth smoking rates . if we wait , the percentage of youth smokers will continue to rise . . . . american legacy foundation 2006 . higher cigarette prices are widely seen as one of the most effective ways to reduce youth smoking usdhss 2000 . in fact , another healthy people 2010 objective is for combined federal and state excise taxes to average 2 . 00 per pack . as of november 1 , 2005 , combined federal and state excise taxes reached or almost reached 2 . 00 per pack in only 13 states orzechowski and walker 2005 . some research calls into doubt the policy prescription that higher prices are an effective way to reduce youth smoking wasserman et al . 1991 , douglas 1994 , douglas and hariharan 1998 , decicca , kenkel and mathios 2001 , 2002 . most research that supports the policy prescription uses variation across states in cigarette taxes as a natural experiment to identify the price responsiveness of youth smoking e . g . , lewit , coate , and grossman 1981 , chaloupka and grossman 1996 , harris and chan 1999 , ross and chaloupka 2003 . however , tax rates are not randomly set , but result from the political process which reflects public sentiment towards smoking . warner 1982 , p . 483 concludes that the fluctuations in new cigarette tax activity do not appear to have occurred randomly . to the contrary , they correspond closely to the evolution of public concerns about the link between cigarette smoking and illness . . . . as evidence supporting this hypothesis , he points out that from 1921 1952 tobacco producing states and other states taxed cigarettes similarly ; but as public awareness of the smoking andhealth issue grew , other states were much more likely to increase cigarette taxes than were the tobacco states . similarly , based on their econometric analysis of 1946 1989 data , hunter and nelson 1992 conclude that public policy concerns . . . contribute positively to the level of tobacco excise taxation in a state . these findings suggest that in the political process taxes are seen as an anti smoking policy and hence probably reflect anti smoking sentiment . if antismoking sentiment is itself an important determinant of youth smoking , failing to control for differences in anti smoking sentiment across states will yield biased estimates of the priceresponsiveness of youth smoking . in this paper , we extend previous work decicca , kenkel , and mathios 2002 to explore in much greater depth the role of state anti smoking sentiment in empirical models of youth and young adult smoking . in particular , in section 2 we analyze data from the tobacco use supplements of the current population survey tus cps to develop a new direct measure of state anti smoking sentiment during the 1990s . section 3 describes the national education longitudinal study nels , a nationally representative micro data set that contains youth smoking information . in section 4 we use the 1992 and 2000 waves of nels to re examine the impact of prices in models of youth smoking that include the new measure of anti smoking sentiment . section 5 explores whether our main results are robust to several sensitivity checks . in section 6 we explore other approaches to control for anti smoking sentiment in cross sectional data , and we also estimate hazard models of smoking initiation over time that include state fixed effects . section 7 concludes .
import torch.nn as nn
num_words = len(word2i) + 2
weights = torch.Tensor(np.zeros((num_words, 300)))
E = nn.Embedding(num_words, 300)
for i in range(0,num_words):
  if i in i2word.keys():
    if i2word[i] in GloVe.keys():
      weights[i] = torch.Tensor(GloVe[i2word[i]])
    else:

      weights[i] = E.weight[i]
  else:
    weights[i] = E.weight[i]
E.weight.data = weights

_____no_output_____gold_seq = []
count = 0
for i in range(len(data)):
  words = paragraphs[i]
  label = data.loc[i][4]
  t = label.lower()
  t = re.findall(r"[\w']+|[.,!?;]", t)
  seq = np.zeros(len(words))
  for j in range(len(words)):
    if words[j:j+len(t)] == t:
      index = (j, j+len(t))
      break
    else:
      index = False
  seq = np.zeros(len(words))
  if index:
    for j in range(index[0], index[1]):
      seq[j] = 1
  if all(v == 0 for v in seq):
    count+= 1
    for j in range(len(words)):
      for z in t:
        if words[j] in z:
          seq[j] = 1
          t.remove(z)
          break
  gold_seq.append(seq)

_____no_output__________no_output_____
class BasicLSTMtagger(nn.Module):
    def __init__(self, DIM_EMB=10, DIM_HID=10):
        super(BasicLSTMtagger, self).__init__()

        self.E = nn.Embedding(num_words, 300)
        self.init_glove(GloVe)
        self.L = nn.LSTM(DIM_EMB, DIM_HID, bidirectional=True, batch_first=True)
        self.W = nn.Linear(DIM_HID*2, 2)
        self.logSoftmax = nn.LogSoftmax(dim=2)
        #TODO: initialize parameters - embedding layer, nn.LSTM, nn.Linear and nn.LogSoftmax

    def forward(self, X, train=False):
        #TODO: Implement the forward computation.
        X = X.cuda()

        x = self.E(X)

        x, hidden = self.L(x)

        x = self.W(x)
        return self.logSoftmax(x)

    def init_glove(self, GloVe):
      weights = torch.Tensor(np.zeros((num_words, 300)))
      for i in range(0,num_words):
        if i in i2word.keys():
          if i2word[i] in GloVe.keys():
            weights[i] = torch.Tensor(GloVe[i2word[i]])
          else:

            weights[i] = self.E.weight[i]
        else:
          weights[i] = self.E.weight[i]
      self.E.weight.data = weights
    def inference(self, sentences):

      pred = self.forward(sentences).argmax(dim=2)
      return pred

#The following code will initialize a model and test that your forward computation runs without errors.
lstm   = BasicLSTMtagger(DIM_HID=7, DIM_EMB=300)
lstm = lstm.cuda()
print(X[0])
lstm_output = lstm.forward(torch.LongTensor(X[0]).reshape((1,len(X[0])))).cuda()

#Check the shape of the lstm_output and one-hot label tensors.

[43864, 67965, 7832, 66689, 12325, 11515, 50548, 40974, 31024, 48933, 10550, 50319, 69259, 42497, 65289, 35347, 32453, 66685, 50354, 61569, 53847, 47309, 44876, 23181, 65043, 57913, 27971, 66685, 59965, 52553, 59159, 49246, 48933, 71558, 58334, 66689, 39539, 50319, 48933, 71558, 3388, 30718, 11232, 34424, 58334, 65043, 52553, 13429, 50854, 47309, 63205, 47309, 67929, 47309, 22361, 1186, 60168, 50319, 14866, 16278, 39327, 59159, 48933, 71558, 71610, 65043, 62668, 1006, 36358, 18148, 31024, 59159, 47160, 6423, 33089, 16007, 35347, 36358, 13429, 1186, 23347, 22361, 47160, 57645, 12796, 12044, 1396, 66884, 28827, 50319, 20084, 5420, 16007, 1862, 24133, 71558, 50548, 59024, 16007, 52164, 1186, 23347, 47309, 46859, 56526, 21657, 14446, 13429, 1186, 23347, 50319]
#Training

from random import sample
import tqdm
import os
import subprocess
import random
import torch.optim as optim
def shuffle_sentences(sentences, tags):
  shuffled_sentences = []
  shuffled_tags      = []
  indices = list(range(len(sentences)))
  random.shuffle(indices)
  for i in indices:
    #print(len(sentences[i]), len(tags[i]))
    shuffled_sentences.append(sentences[i])
    shuffled_tags.append(tags[i])
  return (shuffled_sentences, shuffled_tags)
nEpochs = 7

def train_basic_lstm(sentences, tags, lstm):
  optimizer = optim.Adadelta(lstm.parameters(), lr=0.1)
  #TODO: initialize optimizer

  batchSize = 10
  for epoch in range(nEpochs):
    totalLoss = 0.0
    l = nn.CrossEntropyLoss(reduction="sum", ignore_index=10).cuda()
    (shuffled_sentences, shuffled_tags) = shuffle_sentences(sentences, tags)
    X       = sentences2indices(shuffled_sentences, word2i, train=True)
    for batch in tqdm.notebook.tqdm(range(0, len(sentences) - batchSize, batchSize), leave=False):
      lstm.zero_grad()
      #TODO: Impelement gradient update.

      input = prepare_input(X[batch:batch+batchSize-1])[0]
      Y = []
      for j in range(batch, batch+batchSize-1):
        Y.append(torch.LongTensor(shuffled_tags[j]))
      Y_padded = torch.nn.utils.rnn.pad_sequence(Y, batch_first=True,padding_value=10)
      lstm_output = lstm.forward(input)
      loss = l(torch.flatten(lstm_output, start_dim=0, end_dim = 1), torch.flatten(Y_padded).cuda())
      loss.backward()
      totalLoss += loss

      optimizer.step()
      torch.cuda.empty_cache()

    print(f"loss on epoch {epoch} = {totalLoss}")

lstm = BasicLSTMtagger(DIM_HID=500, DIM_EMB=300).cuda()
train_basic_lstm(paragraphs, gold_seq, lstm)_____no_output_____torch.save(lstm.state_dict(), "lstm.pt")_____no_output_____for j in range(0,len(data), 50):
  z = torch.LongTensor(X[j])

  result = lstm.inference(z.reshape((1,len(X[j])))).tolist()
  result = np.array(result[0])
  index = np.where(result == 1)[0]
  print(index)
  for i in index:
    print(i2word[X[j][i]])
  print(data.loc[j])
[16 17 18 19]
national
education
longitudinal
study
Unnamed: 0                                                       4
Id                            c754dec7-c5a3-4337-9892-c02158475064
pub_title        Parental Effort, School Resources, and Student...
dataset_title                National Education Longitudinal Study
dataset_label                National Education Longitudinal Study
cleaned_label                national education longitudinal study
text              This article investigates an important factor...
mask                                                          True
Name: 0, dtype: object
[817 818]
slosh
sea
Unnamed: 0                                                     347
Id                            07ce992d-e2f3-49cb-bedb-49616a19405f
pub_title        U.S. IOOS coastal and ocean modeling testbed: ...
dataset_title       Sea, Lake, and Overland Surges from Hurricanes
dataset_label                                          SLOSH model
cleaned_label                                          slosh model
text              [1] A Gulf of Mexico performance evaluation a...
mask                                                          True
Name: 50, dtype: object
[202 203 204 205 206 207]
baltimore
longitudinal
study
of
aging
blsa
Unnamed: 0                                                     556
Id                            12b7418c-3114-4ffa-92bf-3468187ee885
pub_title        Stride variability measures derived from wrist...
dataset_title         Baltimore Longitudinal Study of Aging (BLSA)
dataset_label         Baltimore Longitudinal Study of Aging (BLSA)
cleaned_label          baltimore longitudinal study of aging blsa
text              Many epidemiological and clinical studies use...
mask                                                          True
Name: 100, dtype: object
[401 402 403 404 405 406]
baltimore
longitudinal
study
of
aging
blsa
Unnamed: 0                                                     740
Id                            ae17c19b-811f-4a21-b39a-5d580809a4b1
pub_title        Absence of Age-Related Increase in Central Art...
dataset_title         Baltimore Longitudinal Study of Aging (BLSA)
dataset_label         Baltimore Longitudinal Study of Aging (BLSA)
cleaned_label          baltimore longitudinal study of aging blsa
text              Abstract-Increased arterial stiffness is thou...
mask                                                          True
Name: 150, dtype: object
[122 123 124 125]
agricultural
resource
management
survey
Unnamed: 0                                                    1211
Id                            c5b3794e-3577-44a0-ba69-0787765746be
pub_title        A Comparison of Data Collected through Farm Ma...
dataset_title              Agricultural Resource Management Survey
dataset_label              Agricultural Resource Management Survey
cleaned_label              agricultural resource management survey
text              Purpose: This study compares the characterist...
mask                                                          True
Name: 200, dtype: object
[]
Unnamed: 0                                                    1407
Id                            642e187d-44dc-4ea7-92dd-64f3d73c206d
pub_title        An Analysis of the Effect of Global Warming on...
dataset_title    International Best Track Archive for Climate S...
dataset_label    International Best Track Archive for Climate S...
cleaned_label    international best track archive for climate s...
text              A statistical intensity adjustment is utilize...
mask                                                          True
Name: 250, dtype: object
[67 68 69 70 71]
baltimore
longitudinal
study
of
aging
Unnamed: 0                                                    1727
Id                            bcbc988a-086f-47ce-aa0c-d468b0f318ab
pub_title        Glucose and Insulin Measurements From the Oral...
dataset_title         Baltimore Longitudinal Study of Aging (BLSA)
dataset_label                Baltimore Longitudinal Study of Aging
cleaned_label                baltimore longitudinal study of aging
text              Background. Diabetes is associated with decre...
mask                                                          True
Name: 300, dtype: object
[141 142 143 144 145]
baltimore
longitudinal
study
of
aging
Unnamed: 0                                                    1842
Id                            afd0a2dc-4cc3-4578-91b6-41a9e05f28fd
pub_title        Association of periodontitis and metabolic syn...
dataset_title         Baltimore Longitudinal Study of Aging (BLSA)
dataset_label                Baltimore Longitudinal Study of Aging
cleaned_label                baltimore longitudinal study of aging
text              Background and aims: Metabolic syndrome (MetS...
mask                                                          True
Name: 350, dtype: object
[58 59 60 61 62]
baltimore
longitudinal
study
of
aging
Unnamed: 0                                                    2012
Id                            91b42587-e2d7-4cdf-853d-0406423fcf00
pub_title        Association Between Non-Iron-Deficient Anemia ...
dataset_title         Baltimore Longitudinal Study of Aging (BLSA)
dataset_label                Baltimore Longitudinal Study of Aging
cleaned_label                baltimore longitudinal study of aging
text              Background: Anemia is associated with poorer ...
mask                                                          True
Name: 400, dtype: object
[650 651 652 653 654]
baltimore
longitudinal
study
of
aging
Unnamed: 0                                                    2218
Id                            70a3b684-ddfb-4ca1-b5ba-27ca42218c7c
pub_title        Age and gender affect ventricular-vascular cou...
dataset_title         Baltimore Longitudinal Study of Aging (BLSA)
dataset_label                Baltimore Longitudinal Study of Aging
cleaned_label                baltimore longitudinal study of aging
text              The goal of this study was to examine the age...
mask                                                          True
Name: 450, dtype: object
[75 76 77 78 79]
baltimore
longitudinal
study
of
aging
Unnamed: 0                                                    2387
Id                            f36f2051-878f-4336-ad06-548d7f7657fa
pub_title        Basal body temperature as a biomarker of healt...
dataset_title         Baltimore Longitudinal Study of Aging (BLSA)
dataset_label                Baltimore Longitudinal Study of Aging
cleaned_label                baltimore longitudinal study of aging
text              Abstract Scattered evidence indicates that a ...
mask                                                          True
Name: 500, dtype: object
[83 84 85 86 87]
baltimore
longitudinal
study
of
aging
Unnamed: 0                                                    2536
Id                            e80d9f54-df77-4e2a-9492-6edfb6d3e280
pub_title        Impact of age on the cardiovascular response t...
dataset_title         Baltimore Longitudinal Study of Aging (BLSA)
dataset_label                Baltimore Longitudinal Study of Aging
cleaned_label                baltimore longitudinal study of aging
text              Impact of age on the cardiovascular response ...
mask                                                          True
Name: 550, dtype: object
[41 42 43 44]
early
childhood
longitudinal
study
Unnamed: 0                                                    2824
Id                            bc1ad101-6fc7-415a-b8c4-66ec47716016
pub_title        Etiological Subgroups of Small-for-Gestational...
dataset_title                   Early Childhood Longitudinal Study
dataset_label                   Early Childhood Longitudinal Study
cleaned_label                   early childhood longitudinal study
text              Abstract\nIt remains unclear why substantial ...
mask                                                          True
Name: 600, dtype: object
[39 40 41 42]
early
childhood
longitudinal
study
Unnamed: 0                                                    2967
Id                            9fc40e2a-0d43-4a14-a75e-02326be7be10
pub_title        Associations between maternal prepregnancy bod...
dataset_title                   Early Childhood Longitudinal Study
dataset_label                   Early Childhood Longitudinal Study
cleaned_label                   early childhood longitudinal study
text              OBJECTIVE: Both underweight and obese mothers...
mask                                                          True
Name: 650, dtype: object
[43 44 45 46]
early
childhood
longitudinal
study
Unnamed: 0                                                    3118
Id                            ec5170c4-d184-4ab5-bbde-7114eebf5799
pub_title        Concurrent Trajectories of Female Drinking and...
dataset_title                   Early Childhood Longitudinal Study
dataset_label                   Early Childhood Longitudinal Study
cleaned_label                   early childhood longitudinal study
text              Abstract The purpose of this longitudinal stu...
mask                                                          True
Name: 700, dtype: object
[66 67 68 69]
early
childhood
longitudinal
study
Unnamed: 0                                                    3316
Id                            19093f76-a34f-410f-9c2e-a3dbb53ddee6
pub_title        Family Complexity, Siblings, and Children’s Ag...
dataset_title                   Early Childhood Longitudinal Study
dataset_label                   Early Childhood Longitudinal Study
cleaned_label                   early childhood longitudinal study
text              Abstract As family structure in the United St...
mask                                                          True
Name: 750, dtype: object
[174]
adni
Unnamed: 0                                                    3458
Id                            9e3af1ac-ae73-4145-a8d5-fecc0263b854
pub_title        Common folate gene variant, MTHFR C677T, is as...
dataset_title    Alzheimer's Disease Neuroimaging Initiative (A...
dataset_label                                                 ADNI
cleaned_label                                                 adni
text              A commonly carried C677T polymorphism in a fo...
mask                                                          True
Name: 800, dtype: object
[843]
adni
Unnamed: 0                                                    3517
Id                            88d12a7d-03ed-44fd-adc4-a060cbbecb24
pub_title        Identifying amyloid pathology–related cerebros...
dataset_title    Alzheimer's Disease Neuroimaging Initiative (A...
dataset_label                                                 ADNI
cleaned_label                                                 adni
text              Introduction: The dynamic range of cerebrospi...
mask                                                          True
Name: 850, dtype: object
[80]
adni
Unnamed: 0                                                    3584
Id                            30c0f4eb-e24a-434e-baca-27306565cef6
pub_title        Effect of EPHA1 Genetic Variation on Cerebrosp...
dataset_title    Alzheimer's Disease Neuroimaging Initiative (A...
dataset_label                                                 ADNI
cleaned_label                                                 adni
text              Ephrin type-A receptor 1 (EPHA1) (11771145) w...
mask                                                          True
Name: 900, dtype: object
[43]
adni
Unnamed: 0                                                    3657
Id                            22382206-5c16-47f6-9d61-e1bdabb24248
pub_title        Treatment With Cholinesterase Inhibitors and M...
dataset_title    Alzheimer's Disease Neuroimaging Initiative (A...
dataset_label                                                 ADNI
cleaned_label                                                 adni
text              To assess the clinical characteristics and co...
mask                                                          True
Name: 950, dtype: object
[2 5]
disease
adni
Unnamed: 0                                                    3742
Id                            7060dd90-b569-41c0-ad0f-c1cc9ac0e4de
pub_title        Does MRI scan acceleration affect power to tra...
dataset_title    Alzheimer's Disease Neuroimaging Initiative (A...
dataset_label                                                 ADNI
cleaned_label                                                 adni
text              The Alzheimer's Disease Neuroimaging Initiati...
mask                                                          True
Name: 1000, dtype: object
[640]
adni
Unnamed: 0                                                    3811
Id                            ee2c450b-ff76-4356-be60-f09b870e815a
pub_title        Prediction of Conversion from Mild Cognitive I...
dataset_title    Alzheimer's Disease Neuroimaging Initiative (A...
dataset_label                                                 ADNI
cleaned_label                                                 adni
text              Prediction of disease progress is of great im...
mask                                                          True
Name: 1050, dtype: object
[93]
adni
Unnamed: 0                                                    3879
Id                            052d8f0d-be4c-425e-93a8-ea6a494ed9e3
pub_title        Benchmarking machine learning models for late-...
dataset_title    Alzheimer's Disease Neuroimaging Initiative (A...
dataset_label                                                 ADNI
cleaned_label                                                 adni
text              Background: Late-Onset Alzheimer's Disease (L...
mask                                                          True
Name: 1100, dtype: object
[191]
adni
Unnamed: 0                                                    3944
Id                            c8cb7cbe-28f8-4dac-abdd-9411c00735dc
pub_title        TREML2 Mutation Mediate Alzheimer’s Disease Ri...
dataset_title    Alzheimer's Disease Neuroimaging Initiative (A...
dataset_label                                                 ADNI
cleaned_label                                                 adni
text              A coding missense mutation (rs3747742) in tri...
mask                                                          True
Name: 1150, dtype: object
[1131]
adni
Unnamed: 0                                                    4020
Id                            2729433a-91b0-4198-ad89-851f1e4273f4
pub_title        A genome-wide scan for common variants affecti...
dataset_title    Alzheimer's Disease Neuroimaging Initiative (A...
dataset_label                                                 ADNI
cleaned_label                                                 adni
text              Age-related cognitive decline is likely promo...
mask                                                          True
Name: 1200, dtype: object
[264]
adni
Unnamed: 0                                                    4108
Id                            75b35fd6-a326-4cf6-8b8d-f3c64fa45eb5
pub_title        Multi-Modal MultiTask Learning for Joint Predi...
dataset_title    Alzheimer's Disease Neuroimaging Initiative (A...
dataset_label                                                 ADNI
cleaned_label                                                 adni
text              Many machine learning and pattern classificat...
mask                                                          True
Name: 1250, dtype: object
[84]
adni
Unnamed: 0                                                    4200
Id                            d29a0494-7270-45db-b2ad-edfd79df96b2
pub_title        Measurement of Longitudinal -Amyloid Change wi...
dataset_title    Alzheimer's Disease Neuroimaging Initiative (A...
dataset_label                                                 ADNI
cleaned_label                                                 adni
text              The accurate measurement of β-amyloid (Aβ) ch...
mask                                                          True
Name: 1300, dtype: object
[349 350]
initiative
adni
Unnamed: 0                                                    4284
Id                            3ac7b472-8357-44e1-a87d-8d8fe8af2115
pub_title        A multivariate metabolic imaging marker for be...
dataset_title    Alzheimer's Disease Neuroimaging Initiative (A...
dataset_label                                                 ADNI
cleaned_label                                                 adni
text              Introduction: The heterogeneity of behavioral...
mask                                                          True
Name: 1350, dtype: object
[38]
adni
Unnamed: 0                                                    4368
Id                            0efe99c0-9b51-4060-986e-d62d3fc3eb9e
pub_title        Testing for association with multiple traits i...
dataset_title    Alzheimer's Disease Neuroimaging Initiative (A...
dataset_label                                                 ADNI
cleaned_label                                                 adni
text              There is an increasing need to develop and ap...
mask                                                          True
Name: 1400, dtype: object
[106]
adni
Unnamed: 0                                                    4460
Id                            600585a2-708f-4231-85f9-8284b524f441
pub_title        Cortical thickness and semantic fluency in Alz...
dataset_title    Alzheimer's Disease Neuroimaging Initiative (A...
dataset_label                                                 ADNI
cleaned_label                                                 adni
text              The hallmark of Alzheimer's disease (AD) is d...
mask                                                          True
Name: 1450, dtype: object
[615]
adni
Unnamed: 0                                                    4544
Id                            da859d6b-2c18-4f88-a4bd-8364d5b04b9e
pub_title        A Longitudinal Imaging Genetics Study of Neuro...
dataset_title    Alzheimer's Disease Neuroimaging Initiative (A...
dataset_label                                                 ADNI
cleaned_label                                                 adni
text              Background-Neuroanatomical asymmetries have r...
mask                                                          True
Name: 1500, dtype: object
[117]
adni
Unnamed: 0                                                    4650
Id                            b5582deb-3e3c-4977-b206-07b2d5399b11
pub_title        Network-Guided Sparse Learning for Predicting ...
dataset_title    Alzheimer's Disease Neuroimaging Initiative (A...
dataset_label                                                 ADNI
cleaned_label                                                 adni
text              Alzheimer's disease (AD) is characterized by ...
mask                                                          True
Name: 1550, dtype: object
[44]
adni
Unnamed: 0                                                    4761
Id                            44846ee2-7530-4370-b8f1-917829d81888
pub_title            A Bayesian Spatial Model for Imaging Genetics
dataset_title    Alzheimer's Disease Neuroimaging Initiative (A...
dataset_label                                                 ADNI
cleaned_label                                                 adni
text              In this paper we develop a Bayesian bivariate...
mask                                                          True
Name: 1600, dtype: object
[87]
adni
Unnamed: 0                                                    4887
Id                            779388a3-3523-42dc-8767-9a055215151b
pub_title        Structural Similarity based Anatomical and Fun...
dataset_title    Alzheimer's Disease Neuroimaging Initiative (A...
dataset_label                                                 ADNI
cleaned_label                                                 adni
text              Abstract. Multimodal medical image fusion hel...
mask                                                          True
Name: 1650, dtype: object
[98]
adni
Unnamed: 0                                                    5011
Id                            a2914670-0692-497c-a238-3bb064b80269
pub_title        Combining regional metrics for disease-related...
dataset_title    Alzheimer's Disease Neuroimaging Initiative (A...
dataset_label                                                 ADNI
cleaned_label                                                 adni
text              In this paper, we present a new metric combin...
mask                                                          True
Name: 1700, dtype: object
[130]
adni
Unnamed: 0                                                    5168
Id                            3e7c19cd-e15d-4aea-b64f-31f5bb875c7b
pub_title        Simultaneous segmentation and grading of anato...
dataset_title    Alzheimer's Disease Neuroimaging Initiative (A...
dataset_label                                                 ADNI
cleaned_label                                                 adni
text              In this paper, we propose an innovative appro...
mask                                                          True
Name: 1750, dtype: object
[235]
adni
Unnamed: 0                                                    5300
Id                            5ff6a7a6-ae78-49eb-8a4c-caaaa9bb605a
pub_title        A learning-based wrapper method to correct sys...
dataset_title    Alzheimer's Disease Neuroimaging Initiative (A...
dataset_label                                                 ADNI
cleaned_label                                                 adni
text              We propose a simple but generally applicable ...
mask                                                          True
Name: 1800, dtype: object
[117]
adni
Unnamed: 0                                                    5444
Id                            9faf1abd-6474-4d8f-b4d3-af5fae754542
pub_title        Can measuring hippocampal atrophy with a fully...
dataset_title    Alzheimer's Disease Neuroimaging Initiative (A...
dataset_label                                                 ADNI
cleaned_label                                                 adni
text              To quantify the "segmentation noise" of sever...
mask                                                          True
Name: 1850, dtype: object
[499]
adni
Unnamed: 0                                                    5645
Id                            5e5ce4c3-4a7d-4dff-8a85-2ac3ee6a7cdd
pub_title        Subjective Cognitive Impairment Is a Predomina...
dataset_title    Alzheimer's Disease Neuroimaging Initiative (A...
dataset_label                                                 ADNI
cleaned_label                                                 adni
text              Subjective cognitive impairment · Subjective ...
mask                                                          True
Name: 1900, dtype: object
[352]
adni
Unnamed: 0                                                    5886
Id                            180dae50-baa4-4707-a21d-a0202c8fed05
pub_title        Quantitative Amyloid Imaging in Autosomal Domi...
dataset_title    Alzheimer's Disease Neuroimaging Initiative (A...
dataset_label                                                 ADNI
cleaned_label                                                 adni
text              Amyloid imaging plays an important role in th...
mask                                                          True
Name: 1950, dtype: object
[553]
adni
Unnamed: 0                                                    6194
Id                            b3900cd6-6a1f-436a-89a4-85791932b83c
pub_title        A Review on Image- and Network-based Brain Dat...
dataset_title    Alzheimer's Disease Neuroimaging Initiative (A...
dataset_label                                                 ADNI
cleaned_label                                                 adni
text              Abstract. Unveiling pathological brain change...
mask                                                          True
Name: 2000, dtype: object
[617]
adni
Unnamed: 0                                                    6723
Id                            94cac787-fcdb-43c1-ab65-33a3784c8965
pub_title        Analysis of biased PET images caused by inaccu...
dataset_title    Alzheimer's Disease Neuroimaging Initiative (A...
dataset_label                                                 ADNI
cleaned_label                                                 adni
text              PET scanners with an elongated axial field of...
mask                                                          True
Name: 2050, dtype: object
[192 193 194 195 196 197 198]
trends
in
international
mathematics
and
science
study
Unnamed: 0                                                    7137
Id                            b6051caa-9eea-4569-b3e8-90ff50668cf7
pub_title        The interplay of g and mathematical abilities ...
dataset_title    Trends in International Mathematics and Scienc...
dataset_label    Trends in International Mathematics and Scienc...
cleaned_label    trends in international mathematics and scienc...
text              This study investigates the interplay between...
mask                                                          True
Name: 2100, dtype: object
[240 241 242 243 244 245 246]
trends
in
international
mathematics
and
science
study
Unnamed: 0                                                    7290
Id                            8baa76b3-06e8-40a4-b6e9-cef667bb7854
pub_title        LEADERSHIP PRACTICES AMONGST HEADMASTERS OF C ...
dataset_title    Trends in International Mathematics and Scienc...
dataset_label    Trends in International Mathematics and Scienc...
cleaned_label    trends in international mathematics and scienc...
text              Abstract: This research intended to investiga...
mask                                                          True
Name: 2150, dtype: object
[262 263 264 265 266 267 268]
trends
in
international
mathematics
and
science
study
Unnamed: 0                                                    7439
Id                            53e978f6-6a75-40eb-9281-3c3235f7c6ed
pub_title        Early Learning Experiences, School Entry Skill...
dataset_title    Trends in International Mathematics and Scienc...
dataset_label    Trends in International Mathematics and Scienc...
cleaned_label    trends in international mathematics and scienc...
text              Early childhood is an opportune period during...
mask                                                          True
Name: 2200, dtype: object
[208 209 210 211 212 213 214]
trends
in
international
mathematics
and
science
study
Unnamed: 0                                                    7599
Id                            163ef835-2dcf-4d14-b34c-9210851771cf
pub_title        What happens when econometrics and psychometri...
dataset_title    Trends in International Mathematics and Scienc...
dataset_label    Trends in International Mathematics and Scienc...
cleaned_label    trends in international mathematics and scienc...
text              International large-scale assessments such as...
mask                                                          True
Name: 2250, dtype: object
[338 339 340 341 342 343 344]
trends
in
international
mathematics
and
science
study
Unnamed: 0                                                    7801
Id                            ef03f007-1bbd-47b2-b29f-51f5b3c0b89d
pub_title        Gendered study choice: a literature review. A ...
dataset_title    Trends in International Mathematics and Scienc...
dataset_label    Trends in International Mathematics and Scienc...
cleaned_label    trends in international mathematics and scienc...
text              Female participation in higher tertiary educa...
mask                                                          True
Name: 2300, dtype: object
[151 152 153 154 155 156 157]
trends
in
international
mathematics
and
science
study
Unnamed: 0                                                    8027
Id                            bef5b889-616b-4d8d-ae6c-a8e57e172d42
pub_title        Qatar’s Educational Reform: The Experiences an...
dataset_title    Trends in International Mathematics and Scienc...
dataset_label    Trends in International Mathematics and Scienc...
cleaned_label    trends in international mathematics and scienc...
text              This study describes the implications of the ...
mask                                                          True
Name: 2350, dtype: object
[35 36 37]
education
longitudinal
study
Unnamed: 0                                                    8233
Id                            c0f3aec2-4635-402a-8613-f85c1a1bc2a5
pub_title        Special Education Services Received by Student...
dataset_title                         Education Longitudinal Study
dataset_label                         Education Longitudinal Study
cleaned_label                         education longitudinal study
text              Little is known about how special education s...
mask                                                          True
Name: 2400, dtype: object
[283]
adni
Unnamed: 0                                                    8665
Id                            16d394ca-1064-4e48-a600-1097423e8118
pub_title        Modeling disease progression via fused sparse ...
dataset_title    Alzheimer's Disease Neuroimaging Initiative (A...
dataset_label    Alzheimer's Disease Neuroimaging Initiative (A...
cleaned_label    alzheimer s disease neuroimaging initiative adni
text              Alzheimer's Disease (AD) is the most common n...
mask                                                          True
Name: 2450, dtype: object
[118]
adni
Unnamed: 0                                                    8782
Id                            6b4c82f4-f74b-4e7d-8aa4-a56653a5afaf
pub_title        Quantification of Structural Brain Connectivit...
dataset_title    Alzheimer's Disease Neuroimaging Initiative (A...
dataset_label    Alzheimer's Disease Neuroimaging Initiative (A...
cleaned_label    alzheimer s disease neuroimaging initiative adni
text              Connectomics has proved promising in quantify...
mask                                                          True
Name: 2500, dtype: object
[67]
adni
Unnamed: 0                                                    8892
Id                            bd7bc179-b246-4dd8-b6a9-e3a27b083a48
pub_title        ABCA7 genotype altered Aβ levels in cerebrospi...
dataset_title    Alzheimer's Disease Neuroimaging Initiative (A...
dataset_label    Alzheimer's Disease Neuroimaging Initiative (A...
cleaned_label    alzheimer s disease neuroimaging initiative adni
text              Background: ATP-binding cassette transporter ...
mask                                                          True
Name: 2550, dtype: object
[159]
adni
Unnamed: 0                                                    9030
Id                            1a768bde-fe42-4091-90fe-39e88601eb08
pub_title        Altered regional brain volumes in elderly carr...
dataset_title    Alzheimer's Disease Neuroimaging Initiative (A...
dataset_label    Alzheimer's Disease Neuroimaging Initiative (A...
cleaned_label    alzheimer s disease neuroimaging initiative adni
text              Abstract Dopamine D2 receptors mediate the re...
mask                                                          True
Name: 2600, dtype: object
[326]
adni
Unnamed: 0                                                    9174
Id                            ea9dcd2f-6394-4b90-b7c3-57b21957fe97
pub_title        Effect of apolipoprotein E phenotype on the as...
dataset_title    Alzheimer's Disease Neuroimaging Initiative (A...
dataset_label    Alzheimer's Disease Neuroimaging Initiative (A...
cleaned_label    alzheimer s disease neuroimaging initiative adni
text              Introduction: The plasma concentration of bet...
mask                                                          True
Name: 2650, dtype: object
[82]
adni
Unnamed: 0                                                    9336
Id                            f9dc2971-3075-4705-9a9d-9ee3fa22b780
pub_title        Feature Selection and Classification for High-...
dataset_title    Alzheimer's Disease Neuroimaging Initiative (A...
dataset_label    Alzheimer's Disease Neuroimaging Initiative (A...
cleaned_label    alzheimer s disease neuroimaging initiative adni
text              Due to missing values, incomplete dataset is ...
mask                                                          True
Name: 2700, dtype: object
[1126]
adni
Unnamed: 0                                                    9478
Id                            0046e50c-6d19-4f6b-a7c5-4fe935d3b6f8
pub_title        Bayesian Generalized Low Rank Regression Model...
dataset_title    Alzheimer's Disease Neuroimaging Initiative (A...
dataset_label    Alzheimer's Disease Neuroimaging Initiative (A...
cleaned_label    alzheimer s disease neuroimaging initiative adni
text              The emergence of high-dimensional data in gen...
mask                                                          True
Name: 2750, dtype: object
[76]
adni
Unnamed: 0                                                    9621
Id                            38391dde-bc08-4794-982c-36c5c0356ff8
pub_title        A peripheral neutrophil-related inflammatory f...
dataset_title    Alzheimer's Disease Neuroimaging Initiative (A...
dataset_label    Alzheimer's Disease Neuroimaging Initiative (A...
cleaned_label    alzheimer s disease neuroimaging initiative adni
text              Background: Studies suggest a role of the inn...
mask                                                          True
Name: 2800, dtype: object
[316]
adni
Unnamed: 0                                                    9787
Id                            71346804-3538-4a46-b28c-fe2cccc1a7ca
pub_title        Predicting progression from mild cognitive imp...
dataset_title    Alzheimer's Disease Neuroimaging Initiative (A...
dataset_label    Alzheimer's Disease Neuroimaging Initiative (A...
cleaned_label    alzheimer s disease neuroimaging initiative adni
text              Introduction: We investigate whether longitud...
mask                                                          True
Name: 2850, dtype: object
[67]
adni
Unnamed: 0                                                    9989
Id                            a805f81b-f512-4ade-86ce-64064379d785
pub_title        Multivariate genotypic analyses that identify ...
dataset_title    Alzheimer's Disease Neuroimaging Initiative (A...
dataset_label    Alzheimer's Disease Neuroimaging Initiative (A...
cleaned_label    alzheimer s disease neuroimaging initiative adni
text              INTRODUCTION: Genetic contributions to Alzhei...
mask                                                          True
Name: 2900, dtype: object
[87]
adni
Unnamed: 0                                                   10161
Id                            17a38798-9c48-4d76-a5cb-ce606e4571f3
pub_title        Multimodal MRI-based imputation of the Aβ+ in ...
dataset_title    Alzheimer's Disease Neuroimaging Initiative (A...
dataset_label    Alzheimer's Disease Neuroimaging Initiative (A...
cleaned_label    alzheimer s disease neuroimaging initiative adni
text              Objective: The primary goal of this study was...
mask                                                          True
Name: 2950, dtype: object
[355]
adni
Unnamed: 0                                                   10363
Id                            a9fa6cfa-b348-4471-863f-c54e6388ec1b
pub_title        Bayesian hidden Markov models for delineating ...
dataset_title    Alzheimer's Disease Neuroimaging Initiative (A...
dataset_label    Alzheimer's Disease Neuroimaging Initiative (A...
cleaned_label    alzheimer s disease neuroimaging initiative adni
text              Alzheimer's disease is a firmly incurable and...
mask                                                          True
Name: 3000, dtype: object
[183]
adni
Unnamed: 0                                                   10511
Id                            797d094e-fb32-43c1-9b45-9631ffcff806
pub_title        Discriminative self-representation sparse regr...
dataset_title    Alzheimer's Disease Neuroimaging Initiative (A...
dataset_label    Alzheimer's Disease Neuroimaging Initiative (A...
cleaned_label    alzheimer s disease neuroimaging initiative adni
text              In this paper, we propose a novel feature sel...
mask                                                          True
Name: 3050, dtype: object
[98]
adni
Unnamed: 0                                                   10724
Id                            d8a9b202-c858-4b1f-ad6e-5936a1427149
pub_title        Scoring by nonlocal image patch estimator for ...
dataset_title    Alzheimer's Disease Neuroimaging Initiative (A...
dataset_label    Alzheimer's Disease Neuroimaging Initiative (A...
cleaned_label    alzheimer s disease neuroimaging initiative adni
text              Detection of Alzheimer's disease (AD) at the ...
mask                                                          True
Name: 3100, dtype: object
[566 567]
initiative
adni
Unnamed: 0                                                   10922
Id                            30920707-a7d6-44df-97da-be13ae88239b
pub_title        A hierarchical independent component analysis ...
dataset_title    Alzheimer's Disease Neuroimaging Initiative (A...
dataset_label    Alzheimer's Disease Neuroimaging Initiative (A...
cleaned_label    alzheimer s disease neuroimaging initiative adni
text              In recent years, longitudinal neuroimaging st...
mask                                                          True
Name: 3150, dtype: object
[53]
ibtracs
Unnamed: 0                                                   11161
Id                            0ec716b4-f0e6-470b-81dd-047382ecb8ec
pub_title        ENSO Effect on East Asian Tropical Cyclone Lan...
dataset_title    International Best Track Archive for Climate S...
dataset_label                                              IBTrACS
cleaned_label                                              ibtracs
text              Improvements on a statistical tropical cyclon...
mask                                                          True
Name: 3200, dtype: object
[146 147 148 149 150 151 152 153 154 155]
survey
of
graduate
students
and
postdoctorates
in
science
and
engineering
Unnamed: 0                                                   11582
Id                            4194c421-b580-4824-9e88-8d1acbc5c8e8
pub_title        Monitoring the compliance of the academic ente...
dataset_title    Survey of Graduate Students and Postdoctorates...
dataset_label    NSF Survey of Graduate Students and Postdoctor...
cleaned_label    nsf survey of graduate students and postdoctor...
text              On December 1 2016, the Fair Labor Standards ...
mask                                                          True
Name: 3250, dtype: object
[703 704 705 706]
national
education
longitudinal
study
Unnamed: 0                                                   12479
Id                            f5480aeb-c638-4145-a6f8-773d67fe5300
pub_title        Parental Divorce and Students' Performance: Ev...
dataset_title                         Education Longitudinal Study
dataset_label                         Education Longitudinal Study
cleaned_label                         education longitudinal study
text              Establishing whether parental divorce has a c...
mask                                                          True
Name: 3300, dtype: object
[721 722 723 724 725]
beginning
postsecondary
students
longitudinal
study
Unnamed: 0                                                   12760
Id                            c98c3268-3df1-4b1c-8f95-1337e78868e7
pub_title        First-Generation Students: Undergraduates Whos...
dataset_title                      Beginning Postsecondary Student
dataset_label                     Beginning Postsecondary Students
cleaned_label                     beginning postsecondary students
text              Over the past 25 years, the value of a colleg...
mask                                                          True
Name: 3350, dtype: object
[76 77 78]
beginning
postsecondary
students
Unnamed: 0                                                   13034
Id                            c5757662-9b3c-4086-b50e-e91c56d909c5
pub_title        2004 National Postsecondary Student Aid Study ...
dataset_title                      Beginning Postsecondary Student
dataset_label                     Beginning Postsecondary Students
cleaned_label                     beginning postsecondary students
text              The 2004 National Postsecondary Student Aid S...
mask                                                          True
Name: 3400, dtype: object
[3 4 5 6 7]
national
teacher
and
principal
survey
Unnamed: 0                                                   13400
Id                            49740dae-3bf1-4862-9cdd-92ec0807cf40
pub_title        Characteristics of Public and Private Elementa...
dataset_title                National Teacher and Principal Survey
dataset_label                National Teacher and Principal Survey
cleaned_label                national teacher and principal survey
text              The 2017-18 National Teacher and Principal Su...
mask                                                          True
Name: 3450, dtype: object
[25 26 27 28]
early
childhood
longitudinal
study
Unnamed: 0                                                   13838
Id                            6be99b24-61c1-4da6-9200-8af1ab7d703a
pub_title        Rural Disparities in Baseline Data of the Earl...
dataset_title                   Early Childhood Longitudinal Study
dataset_label                   Early Childhood Longitudinal Study
cleaned_label                   early childhood longitudinal study
text              This report introduces the results from an an...
mask                                                          True
Name: 3500, dtype: object
[]
Unnamed: 0                                                   14142
Id                            a189ca34-070a-4093-9a26-4053def34109
pub_title         Geospatial Analysis of Potential Flooding fro...
dataset_title    International Best Track Archive for Climate S...
dataset_label                                              IBTrACS
cleaned_label                                              ibtracs
text              In this chapter, we present a spatial approac...
mask                                                          True
Name: 3550, dtype: object
[ 177  178  179  180 1253]
survey
of
earned
doctorates
survey
Unnamed: 0                                                   15020
Id                            e2b14e5e-d021-49f2-adf1-f09005485bf3
pub_title         Interdisciplinary research among US doctoral ...
dataset_title                          Survey of Earned Doctorates
dataset_label                          Survey of Earned Doctorates
cleaned_label                          survey of earned doctorates
text              In recent years, the scientific and academic ...
mask                                                          True
Name: 3600, dtype: object
[199 200 201 202]
agricultural
resource
management
survey
Unnamed: 0                                                   15561
Id                            6df9c4a6-f988-4074-b283-bf04a2440768
pub_title        Direct marketing channel choices among US farm...
dataset_title              Agricultural Resource Management Survey
dataset_label              Agricultural Resource Management Survey
cleaned_label              agricultural resource management survey
text              After a period of increasing consumer interes...
mask                                                          True
Name: 3650, dtype: object
[359 360 361 362]
agricultural
resource
management
survey
Unnamed: 0                                                   16302
Id                            af890dcc-3da4-456b-bf6c-0e3d9d0dfb86
pub_title         Certified organic field crop profitability In...
dataset_title              Agricultural Resource Management Survey
dataset_label              Agricultural Resource Management Survey
cleaned_label              agricultural resource management survey
text              U.S. crop acres under USDA certified organic ...
mask                                                          True
Name: 3700, dtype: object
[615 616 617 618]
survey
of
doctorate
recipients
Unnamed: 0                                                   16637
Id                            2b374669-6f54-45a1-9952-18049f654eb8
pub_title        The Diffusion of it in Higher Education: Publi...
dataset_title                       Survey of Doctorate Recipients
dataset_label                       Survey of Doctorate Recipients
cleaned_label                       survey of doctorate recipients
text              This paper investigates widening access to th...
mask                                                          True
Name: 3750, dtype: object
[3316 3317 3318]
census
of
agriculture
Unnamed: 0                                                   17139
Id                            599e89b9-ba7f-441e-baa6-8117e4cd037e
pub_title        Crop Supply Dynamics and the Illusion of Parti...
dataset_title                                Census of Agriculture
dataset_label                                Census of Agriculture
cleaned_label                                census of agriculture
text              We use field-level data to estimate the respo...
mask                                                          True
Name: 3800, dtype: object
[516 517 518 519 520]
north
american
breeding
bird
survey
Unnamed: 0                                                   17683
Id                            39107c70-e57e-4475-8292-f1cb4c070d0f
pub_title        A Comparison of White-Winged Dove Zenaida asia...
dataset_title            North American Breeding Bird Survey (BBS)
dataset_label                  North American Breeding Bird Survey
cleaned_label                  north american breeding bird survey
text              Surveying bird populations through visual obs...
mask                                                          True
Name: 3850, dtype: object
[68 69 70 71 72 73]
north
american
breeding
bird
survey
bbs
Unnamed: 0                                                   17877
Id                            085ff51a-fae9-4b2e-90eb-7decf85e11ec
pub_title        Geostatistics as a Tool for Examining Hypothes...
dataset_title            North American Breeding Bird Survey (BBS)
dataset_label            North American Breeding Bird Survey (BBS)
cleaned_label             north american breeding bird survey bbs
text              JSTOR is a not-for-profit service that helps ...
mask                                                          True
Name: 3900, dtype: object
[161 162 163 164 165 573]
north
american
breeding
bird
survey
north
Unnamed: 0                                                   18114
Id                            8bc4076a-c685-42d0-9bc3-fe5ff75d1815
pub_title        The relationship between species detection pro...
dataset_title            North American Breeding Bird Survey (BBS)
dataset_label                  North American Breeding Bird Survey
cleaned_label                  north american breeding bird survey
text              Abstract In community-level ecological studie...
mask                                                          True
Name: 3950, dtype: object
[595 596 597 598 599]
covid
19
open
research
dataset
Unnamed: 0                                                   18448
Id                            8a23c0a4-9344-41e4-ae55-cab3bf6fc080
pub_title        Knowledge synthesis from 100 million biomedica...
dataset_title             COVID-19 Open Research Dataset (CORD-19)
dataset_label                       COVID-19 Open Research Dataset
cleaned_label                       covid 19 open research dataset
text              Since December 2019, the SARS-CoV-2 virus has...
mask                                                          True
Name: 4000, dtype: object
[36 37 38 39 40 54 55 56 57 58]
sars
cov
2
genome
sequences
sars
cov
2
genome
sequence
Unnamed: 0                                                   18901
Id                            32d45b92-01be-48b6-b7fe-fb57ab43fac9
pub_title        Comprehensive variant and haplotype landscapes...
dataset_title                           SARS-CoV-2 genome sequence
dataset_label                          SARS-CoV-2 genome sequences
cleaned_label                          sars cov 2 genome sequences
text              Comprehensive analysis of SARS-CoV-2 genomes ...
mask                                                          True
Name: 4050, dtype: object
[433 434 435 436 437 438]
genome
sequences
of
sars
cov
2
Unnamed: 0                                                   19327
Id                            f1b680dc-b2d1-41ee-a081-1c980933c24e
pub_title        Molecular targets for COVID-19 drug developmen...
dataset_title                           SARS-CoV-2 genome sequence
dataset_label                       genome sequences of SARS-CoV-2
cleaned_label                       genome sequences of sars cov 2
text              There is little or no research initiated to e...
mask                                                          True
Name: 4100, dtype: object
[112 113 114 115 116]
genome
sequences
of
2019
ncov
Unnamed: 0                                                   19613
Id                            d90a010d-8dba-4ffa-a2f1-473a45ddb8c7
pub_title        Identification of potential inhibitors of SARS...
dataset_title                           SARS-CoV-2 genome sequence
dataset_label                        genome sequences of 2019-nCoV
cleaned_label                        genome sequences of 2019 ncov
text              A case of unidentified pneumonia was reported...
mask                                                          True
Name: 4150, dtype: object
from google.colab import files
files.download("lstm.pt")_____no_output__________no_output_____
</code>

Stopped here_____no_output_____

<code>
model = BertModel.from_pretrained('bert-base-uncased')

_____no_output_____tokens = BertTokenizer.from_pretrained('bert-base-uncased')_____no_output_____#filtered = []
#for d in range(len(data)):
 # sample = data.loc[d][6]
#  t = tokens(sample)["input_ids"][1:-1]
  #chunks = []
#  for i in range(0, len(t), 500):
   # chunks.append(tokens.decode(t[i:i+500]))
 # j = 0
 # for i in range(len(chunks)):
  #  if data.loc[d][5] in chunks[i]:
   #   j = i
   #   break
  #filtered.append(chunks[j])
_____no_output_____#data["filtered"] = filtered
#data.to_csv("data_filtered.csv")
_____no_output_____data = pd.read_csv("data_filtered.csv")_____no_output_____
</code>

Crashes here trying to find hidden states_____no_output_____

<code>
states = []
for i in range(len(data)):
  if i /50 == 0:
    print(i)

  t = tokens(data.loc[i][-1], return_tensors="pt")

  states.append(model(**t).last_hidden_state)_____no_output_____sample = data.loc[0][6]
count = 0
print(len(t))
token_col = []
len_col = []
for i in range(len(data)):
  t = tokens(data.loc[i][6], return_tensors="pt")
  token_col.append(t)
  len_col.append(t["input_ids"].shape)
  if t["input_ids"].shape[1] > 512:
    count +=1
print(count)
print(sample)
t = tokens(sample, return_tensors="pt", max_length=512)
print(t["input_ids"].shape[1])
outputs = model(**t)
last_hidden_layers = outputs.last_hidden_state
print(last_hidden_layers.shape)_____no_output_____
changed_text = []
t = tokens(data.loc[0][6])
print(tokens.decode(t["input_ids"]))
print(data.loc[0][6])
for i in range(len(data)):
  t = data.loc[i][6].split(' ')_____no_output_____print(len_col)_____no_output_____data = data[data["token_len"] < 512]
print(len(data))_____no_output_____from spacy.lang.en import English
spacy_sent = spacy.load('en_core_web_sm')
sentences = spacy_sent(data.loc[0][6]).sents
base = torch.zeros((1,0,768))
print(data.loc[0][6])
for i in sentences:
   print(i.text)
   #t = tokens(i.text, return_tensors="pt")
   #outputs = model(**t)
   #last_hidden_layers = outputs.last_hidden_state
   #base = torch.cat((base, last_hidden_layers), dim=1)
_____no_output_____print(base.shape)_____no_output_____t = tokens(data.loc[0][6], return_tensors="pt")
print(t["input_ids"].shape)_____no_output__________no_output_____
</code>