# Notebook from laurebonfils/toolkitten
Path: summer-of-code/week-04/wk4-homework-submissions/soc_wk4_cert_Laure_Bonfils/soc_wk4_cert_Laure_Bonfils.ipynb

#### Day 1 ####_____no_output_____## Intermediate homework for certification:
## Exercise: calculate the prediction for 100% pos, and 100% neg review_____no_output_____%matplotlib inline
datadir = '/Users/Laure/' # The datadir fixture allows test functions and methods to access resources in so-called “data directories”.

import urllib.request, os, gzip

def download_data(dataset_name, datadir):
    filename = 'reviews_%s_5.json' % dataset_name
    filepath = os.path.join(datadir, filename)
    if os.path.exists(filepath):
        print('Dataset %s has already been downloaded to %s' % (dataset_name, datadir))
    else:
        url = 'http://snap.stanford.edu/data/amazon/productGraph/categoryFiles/%s.gz' %filename
        urllib.request.urlretrieve(url, filepath + ".gz")
        with gzip.open(filepath + ".gz", 'rb') as fin:
            with open(filepath, 'wb') as fout:
                fout.write(fin.read())
            print('Downloaded dataset %s and saved it to %s' % (dataset_name, datadir))

dataset = 'Baby'
download_data(dataset, datadir)Dataset Baby has already been downloaded to /Users/Laure/
import json

def load_data(dataset_name, datadir):
    filepath = os.path.join(datadir, 'reviews_%s_5.json' % dataset_name)
    if not os.path.exists(filepath):
        download_data(dataset_name, datadir)
    data = []
    with open(filepath, 'r') as f:
            for line in f:
                item_hash = hash(line)
                item = json.loads(line)
                item['hash'] = item_hash
                data.append(item)
    print("Loaded %d data for dataset %s" % (len(data), dataset_name))
    return data

baby = load_data(dataset, datadir)

print(baby[9426])Loaded 160792 data for dataset Baby
{'reviewerID': 'A2H4QWDVXARPAU', 'asin': 'B0000TYHD2', 'reviewerName': 'Erin White "Erin"', 'helpful': [7, 8], 'reviewText': "I bought this pump for my new baby because it just as others below have said it looks more comfortable than others and it is! Including Medela. With my other child I encountered breastfeeding problems and had a horrible cheap pump. Now with my new baby she was born with a heart problem (she is fine now after a long road to recovery) and had to stay in the hospital for an extended length of time. Meanwhile I had other children at home and we live 6 hours away from our family and so I had no choice but to divide my time between the hospital and home, which meant I needed a hospital grade pump originally I rented one from the hospital (Medela) and it made my breasts hurt really bad. On the way home from the hospital I stopped in at Babiesrus and bought this pump because it was on our registry (we studied and found it to be more comfortable and with coupons it was much cheaper than Medela,) and started using it and I have been so happy with it that sometimes I still use it so my husband can give her a bottle! I love that all I have to do is pump it and add a nipple and its ready to go and its also easier to store than regular breastmilk bags from medela and its also less expensive also. I love the quiet operation I can pump while were watching tv and we don't have to turn the tv up!", 'overall': 5.0, 'summary': 'Has more comforts than Medela!', 'unixReviewTime': 1116547200, 'reviewTime': '05 20, 2005', 'hash': -3384608529151427007}
def partition_train_validation_test(data):
    data_train = [item for item in data if item['hash']%10<=5]
    data_valid = [item for item in data if item['hash']%10 in [6,7]]
    data_test = [item for item in data if item['hash']%10 in [8,9]]
    return data_train, data_valid, data_test

baby_train, baby_valid, baby_test = partition_train_validation_test(baby)

print('Now we have', len(baby_train), 'training eamples', len(baby_valid), 'validation examples and', len(baby_test), "test examples")Now we have 96337 training eamples 32226 validation examples and 32229 test examples
from nltk.corpus import opinion_lexicon
positive_words = set(opinion_lexicon.positive())
negative_words = set(opinion_lexicon.negative())_____no_output_____from nltk.corpus import stopwords
from nltk.tokenize import sent_tokenize, word_tokenize

eng_stopwords = set(stopwords.words('english'))

def my_tokenize(text):
    tokens = []
    for sentence in sent_tokenize(text):
        tokens.extend(x for x in word_tokenize(sentence.lower())
            if x not in eng_stopwords and any(i.isalpha() for i in x))
    return tokens

def pos_neg_fraction(text):
    tokens = my_tokenize(text)
    count_pos, count_neg = 0,0
    for t in tokens:
        if t in positive_words:
            count_pos +=1
        if t in negative_words:
            count_neg +=1
    count_all = len(tokens)
    if count_all !=0:
        return count_pos/count_all, count_neg/count_all
    else:
        return 0., 0.

pos_example = 'This is a good, great, fantastic, amazing, wonderful, super product!!!'
neg_example = 'This is a bad, atrocious, terrible, dreadful, awful, abysmal product!!!'
print(pos_neg_fraction(pos_example))
print(pos_neg_fraction(neg_example))(0.8571428571428571, 0.0)
(0.0, 0.8571428571428571)
import numpy

def dataset_to_matrix(data):
    return numpy.array([list(pos_neg_fraction(item['reviewText'])) for item in data])

X_train = dataset_to_matrix(baby_train)
most_pos, most_neg = numpy.argmax(X_train, axis=0)

print("We found a fraction of %f %% positive words for example %d" % (100.*X_train[most_pos, 0], most_pos))
print(baby_train[most_pos])
print("We found a fraction of %f %% negative words for example %d" % (100.*X_train[most_neg, 1], most_neg))
print(baby_train[most_neg])We found a fraction of 100.000000 % positive words for example 5911
{'reviewerID': 'AKU0ZZ3IEZW42', 'asin': 'B0001BUKA8', 'helpful': [0, 0], 'reviewText': 'useful peace', 'overall': 5.0, 'summary': 'Five Stars', 'unixReviewTime': 1405382400, 'reviewTime': '07 15, 2014', 'hash': 5820922656776727582}
We found a fraction of 100.000000 % negative words for example 15741
{'reviewerID': 'A2416HDN71TOGG', 'asin': 'B000HZEQSU', 'reviewerName': 'Elizabeth Evans', 'helpful': [0, 0], 'reviewText': 'uncomfortable', 'overall': 1.0, 'summary': 'One Star', 'unixReviewTime': 1404691200, 'reviewTime': '07 7, 2014', 'hash': 5454254189285250170}
def dataset_to_targets(data):
    return numpy.array([item['overall'] for item in data])

Y_train = dataset_to_targets(baby_train)

print("Our feature matrix is two-dimensional and has shape", X_train.shape)
print("Our target vector is one-dimensional and has shape", Y_train.shape)Our feature matrix is two-dimensional and has shape (96337, 2)
Our target vector is one-dimensional and has shape (96337,)
import pandas

pandas.Series(Y_train).value_counts().plot.bar()_____no_output_____df = pandas.DataFrame(data=X_train, columns = ['fpos', 'fneg'])
df.plot.kde(xlim=(-0.05, 0.5))_____no_output_____from sklearn.linear_model import LinearRegression

lreg = LinearRegression().fit(X_train, Y_train)

print("The coefficient for the fpos variable is", lreg.coef_[0])
print("The coefficient for the fneg variable is", lreg.coef_[1])
print("The intercept is", lreg.intercept_)The coefficient for the fpos variable is 3.2145066730293066
The coefficient for the fneg variable is -5.652362626053202
The intercept is 4.013629000068324
features = [[0.2, 0]]
expected_rating_A = lreg.predict(features)[0]
print("The expected rating is %f stars" % expected_rating_A)
# we can also compute this explicitly:
expected_rating_B = lreg.intercept_ + 0.2*lreg.coef_[0] + 0*lreg.coef_[1]
print("This is the same as %f stars" % expected_rating_B)The expected rating is 4.656530 stars
This is the same as 4.656530 stars
features = [[1, 0]]
expected_rating_A = lreg.predict(features)[0]
print("The expected rating is %f stars" % expected_rating_A)
# we can also compute this explicitly:
expected_rating_B = lreg.intercept_ + 1*lreg.coef_[0] + 0*lreg.coef_[1]
print("This is the same as %f stars" % expected_rating_B)The expected rating is 7.228136 stars
This is the same as 7.228136 stars
features = [[0, 1]]
expected_rating_A = lreg.predict(features)[0]
print("The expected rating is %f stars" % expected_rating_A)
# we can also compute this explicitly:
expected_rating_B = lreg.intercept_ + 0*lreg.coef_[0] + 1*lreg.coef_[1]
print("This is the same as %f stars" % expected_rating_B)The expected rating is -1.638734 stars
This is the same as -1.638734 stars

As expected, we get a very odd prediction. The issue lies on the fact that since we have not told our predictor
that all ratings must lie between 1 and 5 stars, it became a bit overenthusiastic in its predictions for these
extreme examples. The end of course aims to achieve that with predict_lreg._____no_output_____

## Exercise: repeat this same process for "Apps for Android" dataset_____no_output_____dataset2 = 'Apps_for_Android'

# Download and save data

download_data(dataset, datadir)Dataset Apps_for_Android has already been downloaded to /Users/Laure/
# Load data

apps = load_data(dataset, datadir)

print(apps[9426])Loaded 752937 data for dataset Apps_for_Android
{'reviewerID': 'A37ZP4RA4ERTG2', 'asin': 'B004GYYQLA', 'reviewerName': 'Cookie', 'helpful': [7, 8], 'reviewText': "I don't mind ads as much as others so that wasn't a big deal, but it asked for all of this information about you, like your e-mail and zip code, so I deleted it. This ad is not for people who dislike giving out their information.", 'overall': 1.0, 'summary': 'Asks for your info', 'unixReviewTime': 1325289600, 'reviewTime': '12 31, 2011', 'hash': -7479212533070807264}
#  Partition data for training, validating and testing the analyser

apps_train, apps_valid, apps_test = partition_train_validation_test(apps)

print('Now we have', len(apps_train), 'training examples', len(apps_valid), ', validation examples and', len(apps_test), "test examples.")Now we have 451816 training examples 150623 , validation examples and 150498 test examples.
# Define X_train, list of fraction of pos/neg words in section "reviewText", for each review.

X_train = dataset_to_matrix(apps_train)
most_pos, most_neg = numpy.argmax(X_train, axis=0)

print("We found a fraction of %f %% positive words for example %d" % (100.*X_train[most_pos, 0], most_pos))
print(apps_train[most_pos])
print("We found a fraction of %f %% negative words for example %d" % (100.*X_train[most_neg, 1], most_neg))
print(apps_train[most_neg])We found a fraction of 100.000000 % positive words for example 850
{'reviewerID': 'AWUDE6LOH5Y2Q', 'asin': 'B004DLPXAO', 'reviewerName': 'Amazon Customer', 'helpful': [0, 0], 'reviewText': 'Love it!', 'overall': 5.0, 'summary': 'Take you Kindle Library along whenever you have your phone!', 'unixReviewTime': 1404691200, 'reviewTime': '07 7, 2014', 'hash': -7311770520111658757}
We found a fraction of 100.000000 % negative words for example 6427
{'reviewerID': 'A1HZYCWWJ53TS5', 'asin': 'B004HE5TAG', 'helpful': [0, 0], 'reviewText': 'boring', 'overall': 2.0, 'summary': 'Two Stars', 'unixReviewTime': 1405382400, 'reviewTime': '07 15, 2014', 'hash': 508774179190454234}
# Define Y_train, the number of starts overall atrributed by the reviewer, for each review.

Y_train = dataset_to_targets(apps_train)

print("Our feature matrix is two-dimensional and has shape", X_train.shape)
print("Our target vector is one-dimensional and has shape", Y_train.shape)Our feature matrix is two-dimensional and has shape (451816, 2)
Our target vector is one-dimensional and has shape (451816,)
# Plots

pandas.Series(Y_train).value_counts().plot.bar()_____no_output_____df = pandas.DataFrame(data=X_train, columns = ['fpos', 'fneg'])
df.plot.kde(xlim=(-0.05, 0.5))_____no_output_____# Predictions_____no_output_____from sklearn.linear_model import LinearRegression

lreg = LinearRegression().fit(X_train, Y_train)

print("The coefficient for the fpos variable is", lreg.coef_[0])
print("The coefficient for the fneg variable is", lreg.coef_[1])
print("The intercept is", lreg.intercept_)The coefficient for the fpos variable is 3.7653261633820923
The coefficient for the fneg variable is -3.9136953513070405
The intercept is 3.5486485002103008

The star rating we would expect for a review that contains neither positive nor negative words (fpos==0 and fneg==0)
is 3.5._____no_output_____

features = [[0.25, 0]]
expected_rating_A = lreg.predict(features)[0]
print("The expected rating is %f stars" % expected_rating_A)
# we can also compute this explicitly:
expected_rating_B = lreg.intercept_ + 0.25*lreg.coef_[0] + 0*lreg.coef_[1]
print("This is the same as %f stars" % expected_rating_B)The expected rating is 4.489980 stars
This is the same as 4.489980 stars
def predict_lreg(features):
    expected_rating = lreg.predict(features)
    expected_rating[expected_rating > 5.0] = 5.0
    expected_rating[expected_rating < 1.0] = 1.0
    return expected_rating

pred_train = predict_lreg(X_train)_____no_output_____def analyze_training_example(i):
    print("Training example number", i)
    print("True rating = %f stars, expected rating = %f stars" % (Y_train[i],
                                                                  pred_train[i]))
    print("Features = %f / %f" % (X_train[i,0], X_train[i,1]))
    print("Review text = %s" % apps_train[i]['reviewText'])

for i in [0, 10000, 20000, 30000, 50000]:
    analyze_training_example(i)Training example number 0
True rating = 5.000000 stars, expected rating = 4.113447 stars
Features = 0.150000 / 0.000000
Review text = My 1 year old goes back to this game over and over again.  It is simple but very easy for a toddler to use.  I've even caught my 5 year old playing it as well.  Good app for little guys!
Training example number 10000
True rating = 4.000000 stars, expected rating = 4.000556 stars
Features = 0.151515 / 0.030303
Review text = I have some phone pics that were taken in dark settings and this app did a very good job of brightening the image without washing the picture out. I also like the simplicity of the slider settings. I'll definitely keep this around to enhance some pics. I agree that this should probably be a $1-$2 app. $5 seems a bit much. Glad I got it for free.
Training example number 20000
True rating = 1.000000 stars, expected rating = 3.269099 stars
Features = 0.000000 / 0.071429
Review text = Never got to use it. First time, my tablet started acting up. Has a Trojan Horse per McAfee, dumped it ASAP.
Training example number 30000
True rating = 2.000000 stars, expected rating = 3.771601 stars
Features = 0.080000 / 0.020000
Review text = greetings fellow amazonians got out of the Nashville city jail last night after being given time served and a year of probation. a word to the wise never streak through the grand ol opry if you don't want to be thrown in the clink.anyhoo I checked out the faotd to see what todays offering was and I was excited because I could put a reminder to help me pay my probation officer. unfortunately it didn't work and now they are putting me back in jail for violating my probation. thanks amazon for screwing up my life.!
Training example number 50000
True rating = 4.000000 stars, expected rating = 4.254647 stars
Features = 0.187500 / 0.000000
Review text = the game is tons of fun. typical popcap quality although there is a bit of slowdown in some spots (usually only after picking up the end-of-level reward).highly recommend!

Examples number 2000 didn't work.
Examples number 1000 and 5000 are quite well predicted.
Examples number 0 and 3000 are a bit off._____no_output_____

from sklearn.metrics import mean_absolute_error
mae_train = mean_absolute_error(pred_train, Y_train)
print("The mean absolute error on the training data is %f stars" % mae_train)The mean absolute error on the training data is 0.990830 stars

Mean absolute error is pretty high, so the prediction is pretty accurate!_____no_output_____

#### Day 2 ####_____no_output_____## Exercise (Intermediate HW)
# Change the treshold from 0.5 to 0.2, and rerun the code._____no_output_____"The following is done for the last example, i.e. for 'Apps for Android'"_____no_output_____def discretize_targets(Y):
    return Y<=3.0

D_train = discretize_targets(Y_train)
print("The training data contains %f %% dissatisfied customers" % (100.*D_train.mean()))
print(D_train[:10])The training data contains 27.635586 % dissatisfied customers
[False False False False False  True False  True False  True]
from sklearn.linear_model import LogisticRegression

logreg = LogisticRegression().fit(X_train, D_train)

prob2_train = logreg.predict_proba(X_train)[:,1]
pred2_train = prob2_train > 0.5
max_prob2 = numpy.argmax(prob2_train)
min_prob2 = numpy.argmin(prob2_train)

def analyze_training_example_2(i):
    print("Training example number", i)
    print("True rating = %f stars" % Y_train[i])
    print("Expected to be dissatisfied:", pred2_train[i])
    print("Expected probability of being dissatisfied : %f" % prob2_train[i])
    print("Features = %f / %f" % (X_train[i,0], X_train[i,1]))
    print("Review text = %s" % apps_train[i]['reviewText'])

analyze_training_example_2(max_prob2)Training example number 6427
True rating = 2.000000 stars
Expected to be dissatisfied: True
Expected probability of being dissatisfied : 0.996447
Features = 0.000000 / 1.000000
Review text = boring
analyze_training_example_2(min_prob2)Training example number 850
True rating = 5.000000 stars
Expected to be dissatisfied: False
Expected probability of being dissatisfied : 0.000189
Features = 1.000000 / 0.000000
Review text = Love it!
from sklearn.metrics import precision_score, recall_score

precision2 = precision_score(D_train, pred2_train)

recall2 = recall_score(D_train, pred2_train)

print("For the default threshold (0.5) we get precision = %f "
      "and recall = %f" % (precision2, recall2))For the default threshold (0.5) we get precision = 0.652477 and recall = 0.228332
pred2_train = prob2_train > 0.2

precision2 = precision_score(D_train, pred2_train)

recall2 = recall_score(D_train, pred2_train)

print("For the default threshold (0.5) we get precision = %f "
      "and recall = %f" % (precision2, recall2))For the default threshold (0.5) we get precision = 0.372590 and recall = 0.840128
# Give a commentary in plain English about how that changed precision and recall.
# What does that mean? What is now included that wasn't before?
# What part of it is good? What is bad from our Task perspective.
# Remember: our task was to identify Dissatisfied reviews._____no_output_____

Precision: how many selected items are relevant?
Recall: how many relevant items are selected?

Treshold = 50% -> precision = 0.65 and recall = 0.23
Treshold = 20% -> precision = 0.37 and recall = 0.84

So for the case of a treshold of 50%(20%), it means:
Recall = 0.23 (0.84): We only found 23% (84%) of the relevant results, so 77% (16%) are false negatives, many reviewers are actually dissatisfied but we didn't manage to catch them.
Precision = 0.65 (0.37): out of those that we have identified, 65% are actually correctly identified. So anything that we have flagged as dissatisfied customers, only 65% (37%) are actually dissatisfied, and 35% (63%) are actually ok/satisfied but we thought they weren't.

So, what does it actually means changing the treshold from 0.5 to 0.2:
"Treshold = 0.5" means:  if we are 50% sure that the person is dissatisfied we will classify him as dissatisfied.
"Treshold = 0.2" means: if we think that a reviewer might be dissatisfied, there's actually only 20% chance that he might be dissatisfied, but we are still going to flag him as a dissatisfied reviewer because we don't want to miss him out in case he actually is.
Therefore, when we lower the treshold, the recall increases i.e. we catch more of the actual dissatisfied people, which was our Task perspective, but the precision drops because we accept more and more reviewers._____no_output_____

#### Day 3 ####_____no_output_____## Exercise: Add two features:
#     (1) length of review (in thousands of character) - truncate at 2,500
#     (2) percentage of exclamation marks (in %)_____no_output_____

Included in the next exercise._____no_output_____

## Exercise: Refactor day_3's code._____no_output_____import urllib.request, os, gzip
import numpy
import json
%matplotlib inline

from sklearn.linear_model import LinearRegression
from sklearn.metrics import mean_absolute_error
from sklearn.ensemble import RandomForestRegressor

from nltk.corpus import opinion_lexicon
from nltk.corpus import stopwords
from nltk.tokenize import sent_tokenize, word_tokenize
from nltk.sentiment.vader import SentimentIntensityAnalyzer
from nltk.sentiment import SentimentAnalyzer
from nltk.sentiment.util import mark_negation

sia = SentimentIntensityAnalyzer()/Users/Laure/anaconda3/lib/python3.6/site-packages/nltk/twitter/__init__.py:20: UserWarning: The twython library has not been installed. Some functionality from the twitter package will not be available.
  warnings.warn("The twython library has not been installed. "
def download_data(dataset_name, datadir):
    filename = 'reviews_%s_5.json' % dataset_name
    filepath = os.path.join(datadir, filename)
    if os.path.exists(filepath):
        print('Dataset %s has already been downloaded to %s' % (dataset_name, datadir))
    else:
        url = 'http://snap.stanford.edu/data/amazon/productGraph/categoryFiles/%s.gz' %filename
        urllib.request.urlretrieve(url, filepath + ".gz")
        with gzip.open(filepath + ".gz", 'rb') as fin:
            with open(filepath, 'wb') as fout:
                fout.write(fin.read())
            print('Downloaded dataset %s and saved it to %s' % (dataset_name, datadir))_____no_output_____def load_data(dataset_name, datadir):
    filepath = os.path.join(datadir, 'reviews_%s_5.json' % dataset_name)
    if not os.path.exists(filepath):
        download_data(dataset_name, datadir)
    data = []
    with open(filepath, 'r') as f:
        for line in f:                            # read file line by line
            item_hash = hash(line)                # we will use this later for partitioning our data
            item = json.loads(line)               # convert JSON string to Python dict
            item['hash'] = item_hash              # add hash for identification purposes
            data.append(item)
    print("Loaded %d data for dataset %s" % (len(data), dataset_name))
    return data_____no_output_____def partition_train_validation_test(data):
    # 60% : modulus is 0, 1, 2, 3, 4, or 5
    data_train = [item for item in data if item['hash']%10<=5]
    # 20% : modulus is 6 or 7
    data_valid = [item for item in data if item['hash']%10 in [6,7]]
    # 20% : modulus is 8 or 9
    data_test  = [item for item in data if item['hash']%10 in [8,9]]
    return data_train, data_valid, data_test_____no_output_____positive_words = set(opinion_lexicon.positive())
negative_words = set(opinion_lexicon.negative())
eng_stopwords = set(stopwords.words('english'))
negated_stopwords = set(x+"_NEG" for x in eng_stopwords)
all_stopwords = eng_stopwords.union(negated_stopwords)        # set union
all_positive_words = positive_words.union({x+"_NEG" for x in negative_words})
all_negative_words = negative_words.union({x+"_NEG" for x in positive_words})_____no_output_____def tokenize_with_negation(text):
    # split text into lower-case tokens, removing all-punctuation tokens and stopwords
    tokens = []
    for sentence in sent_tokenize(text):
        pretokens = word_tokenize(sentence.lower())
        pretokens = [x for x in pretokens if any(i.isalpha() for i in x)]
        pretokens = mark_negation(pretokens)
        tokens.extend(x for x in pretokens if x not in all_stopwords)
    return tokens_____no_output_____def pos_neg_fraction_with_negation(text):
    tokens = tokenize_with_negation(text)
    # count how many positive and negative words occur in the text
    count_pos, count_neg = 0, 0
    for t in tokens:
        if t in all_positive_words:
            count_pos += 1
        if t in all_negative_words:
            count_neg += 1
    count_all = len(tokens)
    if count_all != 0:
        return count_pos/count_all, count_neg/count_all
    else:  # avoid division by zero
        return 0., 0._____no_output_____def dataset_to_matrix_with_neg(data):
    return numpy.array([list(pos_neg_fraction_with_negation(item['reviewText'])) for item in data])_____no_output_____def dataset_to_targets(data):
    return numpy.array([item['overall'] for item in data])_____no_output_____def sia_features(dataset):
    """For each review text in the dataset, extract:
       (1) the mean positive sentiment over all sentences
       (2) the mean neutral sentiment over all sentences
       (3) the mean negative sentiment over all sentences
       (4) the maximum positive sentiment over all sentences
       (5) the maximum neutral sentiment over all sentences
       (6) the maximum negative sentiment over all sentences"""
    feat_matrix = numpy.empty((len(dataset), 6))
    for i in range(len(dataset)):
        sentences = sent_tokenize(dataset[i]['reviewText'])
        nsent = len(sentences)
        if nsent:
            sentence_polarities = numpy.empty((nsent, 3))
            for j in range(nsent):
                polarity = sia.polarity_scores(sentences[j])
                sentence_polarities[j, 0] = polarity['pos']
                sentence_polarities[j, 1] = polarity['neu']
                sentence_polarities[j, 2] = polarity['neg']
            feat_matrix[i, 0:3] = numpy.mean(sentence_polarities, axis=0) # mean over the columns
            feat_matrix[i, 3:6] = numpy.max(sentence_polarities, axis=0) # maximum over the columns
        else:
            feat_matrix[i, 0:6] = 0.0
    return feat_matrix_____no_output_____def len_features(dataset):
    """Add two features:
       (1) length of review (in thousands of characters) - truncate at 2,500
       (2) percentage of exclamation marks (in %)"""
    feat_matrix = numpy.empty((len(dataset), 2))
    for i in range(len(dataset)):
        text = dataset[i]['reviewText']
        feat_matrix[i, 0] = len(text) / 1000.
        if text:
            feat_matrix[i, 1] = 100. * text.count('!') / len(text)
        else:
            feat_matrix[i, 1] = 0.0
    feat_matrix[feat_matrix>2.5] = 2.5
    return feat_matrix_____no_output_____def error(an_input):
    X_neg = dataset_to_matrix_with_neg(an_input)
    Y = dataset_to_targets(an_input)
    sia_data = sia_features(an_input)
    len_data = len_features(an_input)
    X_augmented = numpy.concatenate((X_neg, sia_data, len_data), axis=1) # stack horizontally

    lreg_augmented = LinearRegression().fit(X_augmented, Y)
    pred_lreg_augmented = lreg_augmented.predict(X_augmented)
    mae_lreg_augmented = mean_absolute_error(pred_lreg_augmented, Y)

    rf_augmented = RandomForestRegressor().fit(X_augmented, Y)
    pred_rf_augmented = rf_augmented.predict(X_augmented)
    mae_rf_augmented = mean_absolute_error(pred_rf_augmented, Y)

    print("On this set, we get %f error for the linear regression" % mae_lreg_augmented)
    print("And %f for the random forest regression" % mae_rf_augmented)_____no_output_____def grabAndPartition(set_name, datadir):
    # load the data...
    a_set = load_data(set_name, datadir)

    # partition
    set_train, set_valid, set_test = partition_train_validation_test(a_set)

    return set_train, set_valid, set_test   _____no_output_____datadir = '/Users/Laure/' # The datadir fixture allows test functions and methods to access resources in so-called “data directories”.

train, valid, test = grabAndPartition('Baby', datadir)Loaded 160792 data for dataset Baby
error(train)On this set, we get 0.756192 error for the linear regression
And 0.282817 for the random forest regression
error(valid)On this set, we get 0.756387 error for the linear regression
And 0.281893 for the random forest regression
error(test)On this set, we get 0.751177 error for the linear regression
And 0.279483 for the random forest regression


# Notebook from shiwei23/Chromatin_Analysis_Scripts
Path: Library_design/CTP-07/merge_probes.ipynb

# Merge probes from my source_____no_output_____

<code>
%run "E:\Users\puzheng\Documents\Startup_py3.py"
sys.path.append(r"E:\Users\puzheng\Documents")

import ImageAnalysis3
from ImageAnalysis3 import get_img_info, visual_tools, corrections, library_tools

from ImageAnalysis3.library_tools import LibraryDesigner as ld
from ImageAnalysis3.library_tools import LibraryTools

%matplotlib notebook
print(os.getpid())

# biopython for SeqRecord
from Bio import SeqIO
from Bio.Seq import Seq
from Bio.SeqRecord import SeqRecord
from Bio.Alphabet import IUPAC

# blast
from Bio.Blast.Applications import NcbiblastnCommandline
from Bio.Blast import NCBIXML
import ImageAnalysis3.library_tools.quality_check as qc135200
pool_folder = r'X:\Libraries\CTP-07'
fds = glob.glob(os.path.join(pool_folder, 'chr21_*'))
_____no_output_____files = [
    r'X:\Libraries\CTP-07\chr21_intronic\final_probes\extra_filtered_full_probes.fasta',
    r'X:\Libraries\CTP-07\chr21_5x10_2k\probes_new_primer_2_9.fasta',
    r'X:\Libraries\CTP-07\chr21_5x10_5k\filtered_probes_n_50_size_5000.fasta',
    r'X:\Libraries\CTP-07\chr21_9x36_2k\probes_new_primer_4_3.fasta',
    r'X:\Libraries\CTP-07\chr21_9x36_5k\probes_new_primer_0_1.fasta',
]

all_records = []
for _fl in files:
    with open(_fl, 'r') as _handle:
        ct = 0
        for _record in SeqIO.parse(_handle, "fasta"):
            all_records.append(_record)
            ct += 1
    print(f"- num probes in {_fl}: {ct}")
print(len(all_records))- num probes in X:\Libraries\CTP-07\chr21_intronic\final_probes\extra_filtered_full_probes.fasta: 5637
- num probes in X:\Libraries\CTP-07\chr21_5x10_2k\probes_new_primer_2_9.fasta: 3768
- num probes in X:\Libraries\CTP-07\chr21_5x10_5k\filtered_probes_n_50_size_5000.fasta: 9216
- num probes in X:\Libraries\CTP-07\chr21_9x36_2k\probes_new_primer_4_3.fasta: 1827
- num probes in X:\Libraries\CTP-07\chr21_9x36_5k\probes_new_primer_0_1.fasta: 4411
24859
save_filename = os.path.join(pool_folder, 'merged_chr21.fasta')
with open(save_filename, 'w') as _output_handle:
    SeqIO.write(all_records, _output_handle, "fasta")_____no_output_____
</code>


# Notebook from chemaar/python-programming-course
Path: intro-programming/7_Resource__Management.ipynb

# Chapter 7: Resource Management_____no_output_____##7.1 Introduction

A software development process must carefully consider the resources required to run a program. Usually, at the very first stages of development, the use of resources, the performance, the throughput of a software are not always critical. These are non-functional requirements that are, many times, taken into account when the program is already providing the desired functionalities and it is closed to be transferred to an operational environment (dealing with real workloads).

Physical resources are a cornerstone to ensure that the functionality is properly delivered. Flexibility, scalability and other non-functional aspects are not always tested but they are critical when our software must evolve.

There are many factors that can potentially impact the use of resources in our software: the architecture, the users, the operational environment, etc. Most of them must be, theoretically, stated in the non-functional requirements, designed using the proper notation in UML (e.g. physical deployment diagram) and implemented through the proper technologies (e.g. gateways, protocols, etc.) and programming models (e.g. push, event-oriented, etc.). Although non-functional requirements are very relevant to the success of a software program, the implementation of such requirements is sometimes not prioritized, delegating them to a secondary position. However, the history has demonstrated that these non-functional aspects of a system are, at least, as important as the functionalities. If a capability is delivered but it cannot be properly used, what are we creating?

As examples of how systems have evolved overtime considering non-functional aspects, we can list some interesting well-known social network plataforms such as Twitter, Linkedin or Whatsapp. They basically applied the principle: "**Think big, start small and keep growing**". The functionality was almost the same but, the sucess of their platforms implied a great effort on the non-functional aspects driving them to a success. For more details, in the software architecture evolution, you can take a look to the following links:

* https://engineering.linkedin.com/architecture/brief-history-scaling-linkedin
* https://blog.twitter.com/engineering/en_us/topics/infrastructure/2017/the-infrastructure-behind-twitter-scale.html
* http://highscalability.com/blog/2014/2/26/the-whatsapp-architecture-facebook-bought-for-19-billion.html

However, in this chapter, we focus more on the low-level aspects of resource management. Those that can be managed at the programming level. Any software platform (a process) will consume physical resources: processing units, memory, input/output devices (e.g. network). In this context, programming, as an engineering technique, must consider the proper use of these three categories of resources. Good programming techniques will help us to make a proper use of computer resources. Then, other engineering methods such as software design and software architecture methods will also address the problem from other high-level perspectives.

The experience has demonstrated that bad coding practices or small issues in a program can lead us to further and unexpected huge problems in a larger setting. We all have experienced these situations:

* Too much time to run some program because the CPU is completely busy running another process. (Processor).
* Lack of memory because a program is allocating more and more memory while it is running. (Memory)
* Fail to save a file because it is being used by another program, although the program is not anymore running. (Input/Output system)

These situations lead us to think in the origin of such issues. In general, if we assume that the program does not have any special requirement (e.g. CPU, memory or input/output), then, there is a potential problem in the implementation, at the coding level: a memory leak, a zombie process, a file that has not been closed, a bad use of the programming language, etc.

That is why, in this chapter, the focus relies on understanding the resources a program can use, identifying potential points of interest and implementing Python programs more efficiently.

To do so, the chapter is divided into two main subsections:
* Process management.
* Memory management.
* Input/Output system. This last topic will be also presented in the next chapter._____no_output_____##7.2 Process management

A process is a program in execution that is consuming computer resources. The source code is loaded in the main memory, there is a stack to pass parameters to a function and to leave the result, a heap to create new objects, etc. At the same time, the code is executed consuming CPU cycles, new processes and threads can be created to parallelize the processing and the input/output system is used to gather inputs from an user or an external service and to persist results. When a program is running, resources are consumed and CPU time is one of them.

###**Problem**

CPU time is a high-valuable resource. It allows us to execute programs that will provide some functionalities. If we run many programs at the same time, we will potentially face time execution issues. If our program is not properly coded and abuses of the CPU time, other programs will be affected. So, here the question is if we can understand and prevent a bad use of the CPU time by applying small improvements in our code.

###**Concept**

Firstly, it is necessary to understand how a program is assigned to a CPU slice. There are some strategies to schedule programs (e.g. First Come, First Served) and parameters to prioritize one process over another. To do so, the operating system keeps a process table with metainformation of each process under execution and plans the schedule according to different parameters.

From our programing perspective, it is interesting to know this metainformation about our processes to profile our programs and detect potential issues in the time that our code is executing.

To do so, there are APIs and profiling tools that can help us to detect bottlenecks.

Obviously, process management is an advanced topic in the field of Operating Systems so, the intention is to show what we can know from a coding perspective to, at least, get some awareness of the decisions we are making when implementing a program.

###**Application**

The main application of a proper process management is to identify those parts of code that can be taken more time than necessary due to a bad programming practice (e.g. a loop that iterates over a full list when it could finish before).

###**Python implementation**

In Python, like in other interpreted programming languages, the virtualized execution of programs is a relevant aspect. Everything is actually managed by the interpreter that can make decisions about our program, e.g. code optimization. However, it is important to remark some modules and functionalities that can help us to explore our programs and to run profiling tools.

* `psutils`: https://psutil.readthedocs.io/en/latest/
* `dis`: https://docs.python.org/3/library/dis.html
* `profiler`: https://docs.python.org/3/library/profile.html

_____no_output_____##7.3 Memory management

Memory is one of the key computer resources. Depending on the programming language and the type of execution, the memory will be managed at different levels: program, interpreter or operating system.

###**Problem statement**

When developing a software system, the proper use of memory is critical to avoid problems related to memory leaks (the use of memory is growing due to a bad management of resources, the memory that is allocated is not properly released). This situation is specially relevant in programming languages such as C, C++, Pascal or Fortran where dynamic memory must be managed by the developer.

In this sense, it is possible classify memory according to different aspects. From a developer point of view, the implementation of a software system shall have a clear view of the type of memory that is in use:

* Static memory, containing the program code, functions, constants or variables.
* Dynamic memory, containing the space to allocate new objects, etc.

Again, depending on the programming language and the execution type, the strategies to manage the memory will dramatically change. That is why, it is convenient to be able to solve the following questions:

* What is memory?
* Which are the types of memory?
* Why do we need memory?
* What happened with variables?
  * When are they created?
  * Whey are they deleted?
* How memory is allocated?
* How memory is released?

###**Concept**

Memory management is the process of ensuring the proper allocation and release memory blocks during the execution of program.

As it has been previously introduced, the memory management will depend in many factors:

* Type of programming language: static vs dynamic typing.
* Type of execution: native vs interpreted.
* Type of programming language paradigm: procedural vs objects.

###**Application**

The main application of memory management lies on the proper allocation and release of program resources. When a program is running (a process), the memory is used to first load the program including all static configuration:

* Main program and functions code.
* Data: constants and variables (static) with different data types.
* Stack: memory space for function calls.
* Heap: memory space to dynamically allocate space for new data items.

Depending on the programming language, it is very important to understand how the memory is managed (specially the stack and heap spaces).

###**Python implementation**

####Memory architecture

In Python, the memory management process depends on the implementation of the interpreter. In our context, every behavior refers to the default implementation of the Python interpreter in C: **CPython**. Other interpreters, such as Jython, IronPython, etc. could change some of the strategies to execute Python programs. However, the notions of memory management will remain similar.

Firstly, it is interesting to see the stack of memory elements in CPython.

>![alt text](https://rushter.com/static/uploads/img/memory_layers.svg)

>Figure: Memory layers in Python (source: https://rushter.com/blog/python-memory-managment/)

This diagram comes from the real implementation, see the next link:

* https://github.com/python/cpython/blob/ad051cbce1360ad3055a048506c09bc2a5442474/Objects/obmalloc.c#L534

Here, it is convenient to describe the different layers:
* Object-specific memory: memory blocks allocated for the program variables (with different data types).
* Python core: memory blocks for the elements of the Python standard library.
* Python object allocator: creation and deletion of objects (the "magic").
* Python's raw memory allocator: interpreter management of memory.
* Underlying general-purpose allocator: implementation in the C programming language of the memory allocation and release.
* OS-specific Virtual Memory Manager: memory management of the operating system: different strategies to bring memory pages, etc.
* Physical memory: hardware in which everything is stored.

####Memory structure

In the CPython interpreter, the memory is structured according to three elements:

* **Block**: a portion of memory that can keep only one Python object of a fixed size.
  * *The size of the block can vary from 8 to 512 bytes and must be a multiple of eight (i.e., use 8-byte alignment).*

* **Pool**: a composition of blocks of the same size. The size of a pool is commonly equals to the memory page (4KB). See following, the pool implementation in the C programming language.

```
/* Pool for small blocks. */
struct pool_header {
    union { block *_padding;
            uint count; } ref;          /* number of allocated blocks    */
    block *freeblock;                   /* pool's free list head         */
    struct pool_header *nextpool;       /* next pool of this size class  */
    struct pool_header *prevpool;       /* previous pool       ""        */
    uint arenaindex;                    /* index into arenas of base adr */
    uint szidx;                         /* block size class index        */
    uint nextoffset;                    /* bytes to virgin block         */
    uint maxnextoffset;                 /* largest valid nextoffset      */
};
```

Each pool has three states:

* used — partially used, neither empty nor full.
* full — all the pool's blocks are currently allocated.
* empty — all the pool's blocks are currently available for allocation.

* **Arena**: a chunk of 256kB memory allocated on the heap, which provides memory for 64 pools. Arenas are organized into a doubly linked list called `usable_arenas`. Arenas are sorted according to the number of free pools. This notion of "free" is at Python level (no operating system).

####The Python object: GIL (Global Interpreter Lock) and GC (Garbage Collector)

**Everything in Python is an object.**

At the low level, in the CPython interpreter there is a struct (a record) which is used by every other object in CPython.
  * `ob_refcnt`: reference count. This counter is used for the garbage collection.
  * `ob_type`: pointer to another type. The type of of tha Python object (e.g. int, str, dict, etc.)

There are two elements that have a very important impact in the memory management:

* **The Global Interpreter Lock (GIL)**. It is a solution to protect the access to shared resources (e.g. memory). For instance, two threads tryin to rewrite a variable value will be locked and the requests will be serialized. The impact of the GIL is that actually there is no parallelization at the thread level (processes are **single-thread**). This is a major topic of discussion: how to manage race condition in Python.

* **The Garbage Collector (GC)**. It is a functionality that basically counts the number of references of an object to decide whether the memory can be released (reference count is 0). There are many ways of implementing garbage collection strategies (e.g. generations). In our context, it is interesting to know when an object increases the number of refences:
     * Assign it to another variable.
     * Pass the object as an argument.
     * Include the object in a list.

Finally, there are some lessons or keypoints about the Python memory management strategy:
 * Python completely abstracts the memory management. Developers can work at a very high-level abstraction (e.g. no need to specify the bytes to allocate or release).
 * The memory release is done by the GC. However, the implementation of this functionality cannot be easily configured (it will depend on the interpreter).

In general, it is necessary to know how things work behind the scenes. However, unless we work on compilers, program optimization or interpreters, we do not really need to manage these low-level features.
_____no_output_____##7.4 Input/Output system

This topic will be extensively presented in the next chapter._____no_output_____##Relevant references

* https://cython.readthedocs.io/en/latest/src/tutorial/memory_allocation.html
* "Fluent Python": http://shop.oreilly.com/product/0636920032519.do
* https://rushter.com/blog/python-memory-managment/
* https://rushter.com/blog/python-garbage-collector/
* https://realpython.com/python-memory-management/
* https://towardsdatascience.com/memory-management-in-python-6bea0c8aecc9
_____no_output_____


# Notebook from anuprulez/single_cell_analysis
Path: data_format_conversion.ipynb

<code>
import os
import csv
import scanpy.api as sc
_____no_output_____dataset_dir = "data/"
data_file = "matrix.mtx"
var_names_file = "genes.tsv"
obs_names_file = "barcodes.tsv"
output_h5ad_file = "68kPBMCs.h5ad"

data_path = os.path.join(dataset_dir,data_file)
var_names_path = os.path.join(dataset_dir,var_names_file)
obs_names_path = os.path.join(dataset_dir,obs_names_file)
output_h5ad_path = os.path.join(dataset_dir,output_h5ad_file)_____no_output_____with open(var_names_path, "r") as var_file:
    var_read = csv.reader(var_file, delimiter='\t')
    var_names = []
    for row in var_read:
        var_names.append(row[1])_____no_output_____with open(obs_names_path, "r") as obs_file:
    obs_read = csv.reader(obs_file, delimiter='\t')
    obs_names = []
    for row in obs_read:
        obs_names.append(row[0])_____no_output_____andata = sc.read(data_path)
andata = andata.transpose()_____no_output_____andata.var_names = var_names
andata.var_names_make_unique()
andata.obs_names = obs_names
andata.obs_names_make_unique()

andata.write(filename=output_h5ad_path)Variable names are not unique. To make them unique, call `.var_names_make_unique`.
# filtering

min_cells = 3,
min_genes = 10

sc_raw = andata

sc.pp.filter_cells(sc_raw, min_genes=min_genes, copy=False)
print("Filtering of the raw data is done with minimum "
      "%d genes per cell." % min_genes)

sc.pp.filter_genes(sc_raw, min_cells=min_cells, copy=False)
print("Filtering of the raw data is done with minimum"
      " %d cells per gene." % min_cells)

cells_count = sc_raw.shape[0]
genes_count = sc_raw.shape[1]

print("Cells number is %d , with %d genes per cell."
      % (cells_count, genes_count))Filtering of the raw data is done with minimum 10 genes per cell.
Filtering of the raw data is done with minimum 3 cells per gene.
Cells number is 68579 , with 17789 genes per cell.
# scaling
scale = "normalize_per_cell_LS_20000"

if "normalize_per_cell_LS_" in str(scale):

    lib_size = int(scale.split('_')[-1])
    sc.pp.normalize_per_cell(sc_raw,
                             counts_per_cell_after=lib_size)
    scale = {"scaling": 'normalize_per_cell_LS',
                  "scale_value": lib_size}

else:

    warnings.warn("The scaling of the data is unknown, library size "
                  "library size normalization with 20k will be applied")

    lib_size = int(sscale.split('_')[-1])
    sc.pp.normalize_per_cell(sc_raw,
                             counts_per_cell_after=lib_size)
    self.scale = {"scaling": 'normalize_per_cell_LS',
                  "scale_value": lib_size}

print("Scaling of the data is done using " + scale["scaling"]
      + " with " + str(scale["scale_value"]))Scaling of the data is done using normalize_per_cell_LS with 20000
output_h5ad_processed_file = "68kPBMCs_processed.h5ad"
output_h5ad_processed_path = os.path.join(dataset_dir, output_h5ad_processed_file)
sc_raw.write(output_h5ad_processed_path)_____no_output_____import collections
import numpy as np

sct = collections.namedtuple('sc', ('barcode', 'count_no', 'genes_no'))
exp_share = 0.2
n_exp_data = int(exp_share * sc_raw.shape[0])
sc_raw_exp = sc_raw[:n_exp_data]

train_data = np.zeros((sc_raw_exp.shape[0], sc_raw_exp.shape[1]))

def process_line(line):
    scmd = sct(barcode=line.obs_names[0],
               count_no=int(np.sum(line.X)),
               genes_no=line.obs['n_genes'][0]
              )
    return line.X, scmd

for i, line in enumerate(sc_raw_exp):
    sc_genes, d = process_line(line)
    print(sc_genes)
    train_data[i] = sc_genes
    break[ 0.        0.        0.       ...  0.       16.447369  0.      ]
sc_raw.shape_____no_output_____train_data.shape_____no_output_____for i, line in enumerate(sc_raw_exp):
    sc_genes, d = process_line(line)
    print(sc_genes.shape)
    print(type(sc_genes))
    train_data[i] = sc_genes
    break(17789,)
<class 'numpy.ndarray'>
train_data_____no_output_____
</code>


# Notebook from haormj/ai
Path: 07.Tensorflow/08.me.ipynb

<code>
# 通过对tensorflow的学习，了解了基本的使用，不过学习和实践还是有区别的
# 在实际生活中，我们有很多的图片，为了方便我自己学习我打算进行一个实验
# 1. 自己准备原材料，通过软件绘制5*5px的png图片，其中主要是图形，然后给定标签
# 2. 在tensorflow建立模型，并对其进行训练，然后输入新的内容，看是否可以识别
# 如果上面的实验完成后，我们就有了一个解决图像识别的利器，自己手动标注，通过机器学习，后面的数据进行分类

# 实践
# 创建 3*3px的简单图片作为并添加类别到dataset中，并标注类别
# 当然测试集和数据集都是一样的，只是一个用户训练分一部分作为训练
_____no_output_____# 遍历目录读取所有图片，并将文件名作为其标签, 准备数据
import os
import png
import numpy as np

dir = './08.resource'
fs = os.listdir(dir)

X_list = []
y_list = []

for f in fs:
    label = f.split('.')[0].split('-')[0]
    p = os.path.join(dir,f)
    _,_,rows,_ = png.Reader(p).read_flat()
    X_list.append(rows.tolist())
    y_list.append(label)

X_raw = np.array(X_list, dtype=np.float64).reshape(12,100)
y_raw = np.array(y_list, dtype=np.str)
y_raw_____no_output_____from sklearn.preprocessing import LabelEncoder
le = LabelEncoder()
y = le.fit_transform(y_raw)
X = X_raw
y_____no_output_____# 分割数据集
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2,stratify=y)

# 数据标准化
from sklearn.preprocessing import StandardScaler
ss = StandardScaler()
ss.fit(X_train)
X_train_std = ss.transform(X_train)
X_test_std = ss.transform(X_test)

from sklearn.linear_model import LogisticRegression
lr = LogisticRegression(C=50.0)
lr.fit(X_train_std, y_train)
lr.score(X_test_std, y_test)_____no_output_____# 预测矩形
r = png.Reader('./08.resource/rect-test.png')
_,_,rows,_ = r.read_flat()
X_rect_raw = np.array(rows, dtype=np.float64).reshape(1,100)
y_rect_raw = np.array(['rect'])

X_rect_std = ss.transform(X_rect_raw)
y_rect = le.transform(y_rect_raw)

y_pred = lr.predict(X_rect_std)
print(y_rect == y_pred)[ True]
# 预测圆
r = png.Reader('./08.resource/circle-test.png')
_,_,rows,_ = r.read_flat()
X_circle_raw = np.array(rows, dtype=np.float64).reshape(1,100)
y_circle_raw = np.array(['circle'])

X_circle_std = ss.transform(X_rect_raw)
y_circle = le.transform(y_rect_raw)

y_pred = lr.predict(X_rect_std)
print(y_circle == y_pred)[ True]
# 预测星
r = png.Reader('./08.resource/star-test.png')
_,_,rows,_ = r.read_flat()
X_star_raw = np.array(rows, dtype=np.float64).reshape(1,100)
y_star_raw = np.array(['star'])

X_star_std = ss.transform(X_star_raw)
y_star = le.transform(y_star_raw)

y_pred = lr.predict(X_star_std)
print(y_star == y_pred)[ True]
# 增加难度，直接给出不是正方形
r = png.Reader('./08.resource/rect-test2.png')
_,_,rows,_ = r.read_flat()
X_rect_raw = np.array(rows, dtype=np.float64).reshape(1,100)
y_rect_raw = np.array(['rect'])

X_rect_std = ss.transform(X_rect_raw)
y_rect = le.transform(y_rect_raw)

y_pred = lr.predict(X_rect_std)
print(y_rect == y_pred)
print(y_pred)
# 通过结果我们可以看到，他已经不能够识别了，不过因为测试数据中是,直接认为我给出的数据是圆，实际上给出的是长方形
# 之所以出现这种结果其实也是可以理解的，是因为数据集很小，从而导致无法准确的识别，有了这个基础之后，我其实有一个问题
# 所有的分类模型我都可以用于识别图片，只是纠结的其实是如何将图片转化为之前训练的数据，最后其实就是将图片的数据直接拉平，
# 其中tensorflow按照我现在的看法，他那边主要是实现了神经网络，[False]
[0]

</code>


# Notebook from Nikoleta-v3/bibliometric-study-of-the-prisoners-dilemma
Path: src/nbs/LDA on abstracts PD trials.ipynb

**Packages**_____no_output_____

import re
import numpy as np
from pprint import pprint

import spacy_____no_output_____import gensim
import gensim.corpora as corpora
from gensim.utils import simple_preprocess
from gensim.models import CoherenceModel_____no_output_____import pyLDAvis
import pyLDAvis.gensim
import matplotlib.pyplot as plt
%matplotlib inline_____no_output_____import imp

tools = imp.load_source('tools', '../lda_tools.py')_____no_output_____import matplotlib
import matplotlib.pyplot as plt
from matplotlib.lines import Line2D

import matplotlib as mpl
mpl.rcParams['legend.numpoints'] = 1

%matplotlib inline_____no_output_____font = {"size": 10, "weight": "bold"}
matplotlib.rc("font", **font)_____no_output_____from sklearn.manifold import TSNE
from bokeh.plotting import figure, output_file, show
from bokeh.models import Label
from bokeh.io import output_notebook_____no_output_____import numpy as np
import matplotlib.pyplot as plt
from scipy.spatial import Voronoi_____no_output_____

**The data set**_____no_output_____

import pandas as pd_____no_output_____df = pd.read_csv('../data/prisoners_dilemma_articles_meta_data_clean.csv')_____no_output_____data = df[['abstract', 'unique_key', 'title', 'date']]_____no_output_____data = data.drop_duplicates()_____no_output_____data = data.reset_index(drop=True)_____no_output_____data.head()_____no_output_____len(data)_____no_output_____data.describe()_____no_output_____data.iloc[1645]['title']_____no_output_____data.iloc[1969]['title']_____no_output_____data.iloc[660]['title']_____no_output_____data.iloc[1183]['title']_____no_output_____data.iloc[2115]['title']_____no_output_____data.iloc[2144]['title']_____no_output_____data.iloc[1777]['title']_____no_output_____

**other side**_____no_output_____

data.iloc[932]['title']_____no_output_____data.iloc[1309]['title']_____no_output_____data.iloc[1199]['title']_____no_output_____data.iloc[1364]['title']_____no_output_____data.iloc[868]['title']_____no_output_____

**Sentences to words**_____no_output_____

words = list(tools.sentences_to_words(data['abstract'].values))_____no_output_____

**Cleaning data set**_____no_output_____

from nltk.corpus import stopwords_____no_output_____stop_words = stopwords.words('english')_____no_output_____# Build the bigram and trigram models
bigram = gensim.models.Phrases(words, min_count=5, threshold=100)
trigram = gensim.models.Phrases(bigram[words], threshold=100)

# Faster way to get a sentence clubbed as a trigram/bigram
bigram_mod = gensim.models.phrases.Phraser(bigram)
trigram_mod = gensim.models.phrases.Phraser(trigram)_____no_output_____lemmatized_words = tools.clean_words(words, stop_words)_____no_output_____len(lemmatized_words)_____no_output_____len(words)_____no_output_____

**Create corpus**_____no_output_____

id2word = corpora.Dictionary(lemmatized_words)

texts = lemmatized_words

corpus = [id2word.doc2bow(text) for text in texts]_____no_output_____

**Create model and select number of topics**_____no_output_____

mallet_path = '/Users/storm/rsc/mallet-2.0.8/bin/mallet'_____no_output_____limit = 7
start = 5
step = 1_____no_output_____model_list, coherence_values = tools.compute_coherence_values(limit=limit,
                                                              step=step,
                                                              start=start,
                                                              mallet_path=mallet_path,
                                                              dictionary=id2word,
                                                              corpus=corpus,
                                                              texts=lemmatized_words)_____no_output_____# fig = plt.subplots()

# num_of_topics = range(2, limit, 1)
# plt.plot(num_of_topics, coherence_values, '-o', color='black')

# plt.xlabel("number of topics")
# plt.ylabel("Coherence score");

# plt.xticks(num_of_topics)

# plt.savefig("../../assets/images/coherence_values.pdf",
#                 bbox_inches="tight")_____no_output_____num_of_topics = range(start, limit, step)_____no_output_____for num, cv in zip(num_of_topics, coherence_values):
    print("Num Topics =", num, " has Coherence Value of", round(cv, 4))Num Topics = 5  has Coherence Value of 0.4065
Num Topics = 6  has Coherence Value of 0.418
max_coherence_value = max(coherence_values)
max_index = coherence_values.index(max_coherence_value)_____no_output_____num_of_topics[max_index]_____no_output_____pprint(model_list[max_index].print_topics())[(0,
  '0.039*"population" + 0.028*"evolutionary" + 0.026*"dynamic" + 0.021*"model" '
  '+ 0.014*"selection" + 0.013*"result" + 0.011*"evolution" + 0.011*"evolve" + '
  '0.011*"show" + 0.010*"process"'),
 (1,
  '0.105*"game" + 0.084*"strategy" + 0.053*"player" + 0.041*"agent" + '
  '0.021*"play" + 0.018*"dilemma" + 0.016*"state" + 0.014*"prisoner" + '
  '0.014*"payoff" + 0.013*"equilibrium"'),
 (2,
  '0.024*"behavior" + 0.022*"social" + 0.019*"human" + 0.018*"decision" + '
  '0.016*"study" + 0.016*"experiment" + 0.013*"make" + 0.013*"suggest" + '
  '0.011*"result" + 0.011*"behaviour"'),
 (3,
  '0.090*"cooperation" + 0.055*"network" + 0.024*"interaction" + '
  '0.021*"structure" + 0.021*"study" + 0.019*"evolution" + 0.016*"find" + '
  '0.014*"behavior" + 0.014*"cooperative" + 0.014*"simulation"'),
 (4,
  '0.033*"model" + 0.027*"theory" + 0.017*"system" + 0.015*"base" + '
  '0.015*"paper" + 0.014*"problem" + 0.013*"propose" + 0.011*"present" + '
  '0.011*"approach" + 0.011*"provide"'),
 (5,
  '0.048*"individual" + 0.030*"group" + 0.018*"good" + 0.017*"social" + '
  '0.014*"punishment" + 0.013*"level" + 0.013*"cost" + 0.013*"mechanism" + '
  '0.012*"dilemma" + 0.012*"cooperative"')]
n_topics = 6 #num_of_topics[max_index] #num_of_topics[0]_____no_output_____lda_model = gensim.models.ldamodel.LdaModel(corpus=corpus,
                                           id2word=id2word,
                                           num_topics=n_topics,
                                           random_state=100,
                                           update_every=1,
                                           chunksize=100,
                                           passes=10,
                                           alpha='auto',
                                           per_word_topics=True)_____no_output_____pyLDAvis.enable_notebook()
vis = pyLDAvis.gensim.prepare(lda_model, corpus, id2word)
vis_____no_output_____n_topics = num_of_topics[max_index] #num_of_topics[0]_____no_output_____topic_weights = []
for i, row_list in enumerate(lda_model[corpus]):
    topic_weights.append([w for i, w in row_list[0]])_____no_output_____# Array of topic weights
arr = pd.DataFrame(topic_weights).fillna(0).values

# Keep the well separated points (optional)
arr = arr[np.amax(arr, axis=1) > 0.35]

# Dominant topic number in each doc
topic_num = np.argmax(arr, axis=1)_____no_output_____# T
tsne_model = TSNE(n_components=2, verbose=1, random_state=0, angle=.99, init='pca')
tsne_lda = tsne_model.fit_transform(arr)[t-SNE] Computing 91 nearest neighbors...
[t-SNE] Indexed 2228 samples in 0.006s...
[t-SNE] Computed neighbors for 2228 samples in 0.063s...
[t-SNE] Computed conditional probabilities for sample 1000 / 2228
[t-SNE] Computed conditional probabilities for sample 2000 / 2228
[t-SNE] Computed conditional probabilities for sample 2228 / 2228
[t-SNE] Mean sigma: 0.000000
[t-SNE] KL divergence after 250 iterations with early exaggeration: 67.729424
[t-SNE] KL divergence after 1000 iterations: 0.861782
 def voronoi_finite_polygons_2d(vor, radius=None):
    """
    Reconstruct infinite voronoi regions in a 2D diagram to finite
    regions.

    Parameters
    ----------
    vor : Voronoi
        Input diagram
    radius : float, optional
        Distance to 'points at infinity'.

    Returns
    -------
    regions : list of tuples
        Indices of vertices in each revised Voronoi regions.
    vertices : list of tuples
        Coordinates for revised Voronoi vertices. Same as coordinates
        of input vertices, with 'points at infinity' appended to the
        end.

    """

    if vor.points.shape[1] != 2:
        raise ValueError("Requires 2D input")

    new_regions = []
    new_vertices = vor.vertices.tolist()

    center = vor.points.mean(axis=0)
    if radius is None:
        radius = vor.points.ptp().max()

    # Construct a map containing all ridges for a given point
    all_ridges = {}
    for (p1, p2), (v1, v2) in zip(vor.ridge_points, vor.ridge_vertices):
        all_ridges.setdefault(p1, []).append((p2, v1, v2))
        all_ridges.setdefault(p2, []).append((p1, v1, v2))

    # Reconstruct infinite regions
    for p1, region in enumerate(vor.point_region):
        vertices = vor.regions[region]

        if all(v >= 0 for v in vertices):
            # finite region
            new_regions.append(vertices)
            continue

        # reconstruct a non-finite region
        ridges = all_ridges[p1]
        new_region = [v for v in vertices if v >= 0]

        for p2, v1, v2 in ridges:
            if v2 < 0:
                v1, v2 = v2, v1
            if v1 >= 0:
                # finite ridge: already in the region
                continue

            # Compute the missing endpoint of an infinite ridge

            t = vor.points[p2] - vor.points[p1] # tangent
            t /= np.linalg.norm(t)
            n = np.array([-t[1], t[0]])  # normal

            midpoint = vor.points[[p1, p2]].mean(axis=0)
            direction = np.sign(np.dot(midpoint - center, n)) * n
            far_point = vor.vertices[v2] + direction * radius

            new_region.append(len(new_vertices))
            new_vertices.append(far_point.tolist())

        # sort region counterclockwise
        vs = np.asarray([new_vertices[v] for v in new_region])
        c = vs.mean(axis=0)
        angles = np.arctan2(vs[:,1] - c[1], vs[:,0] - c[0])
        new_region = np.array(new_region)[np.argsort(angles)]

        # finish
        new_regions.append(new_region.tolist())

    return new_regions, np.asarray(new_vertices)_____no_output_____vor = Voronoi(list(zip(tsne_lda[:,0], tsne_lda[:,1])))_____no_output_____regions, vertices = voronoi_finite_polygons_2d(vor)_____no_output_____colors = np.array([matplotlib.cm.tab20b(3 * i) for i in range(2, 5)] + [matplotlib.cm.tab20c(3 * i) for i in range(1, 4)])_____no_output_____colors_points = np.array([matplotlib.cm.tab20b(i * i) for i in range(2, 5)] + [matplotlib.cm.tab20c(i * i) for i in range(1, 4)])_____no_output_____colors_points[5] = matplotlib.cm.Paired(2)
colors_points[1] = matplotlib.cm.Paired(6)
colors_points[2] = matplotlib.cm.Paired(8)_____no_output_____legend_elements = [Line2D([0], [0], marker='o',
                          color='w',
                          label=f'Topic {i + 1}',
                          markersize=10,
                          markerfacecolor=c) for i, c in enumerate(colors_points)]_____no_output_____colors_____no_output_____fig, axes = plt.subplots(figsize=(10, 6))

for region, color in zip(regions, colors[topic_num]):
    polygon = vertices[region]
    plt.fill(*zip(*polygon), color=color, edgecolor='none')

plt.scatter(x=tsne_lda[:,0],
            y=tsne_lda[:,1],
            s=10,
            zorder=100,
            color=colors_points[topic_num])

plt.xlim(vor.min_bound[0] - 0.1, vor.max_bound[0] + 0.1)
plt.ylim(vor.min_bound[1] - 0.1, vor.max_bound[1] + 0.1)

axes.legend(handles=legend_elements,
            bbox_to_anchor=(.57, -0.10),
            ncol=3);

fig.savefig(f"../../assets/images/topics_scatter_plot_{n_topics}.pdf",
                bbox_inches="tight")_____no_output_____fig, axes = plt.subplots(figsize=(10, 6))
polygons = []
for region, topic in zip(regions, topic_num):
    if topic == 0:
        polygon = vertices[region]
        polygons.append(polygon)
        plt.fill(*zip(*polygon), color=colors[0], edgecolor='none', alpha=0.4)_____no_output_____import shapely.geometry as sg
import shapely.ops as so_____no_output_____len(polygons)_____no_output_____merged = []_____no_output_____indices = [0, 7, 1, 11, 2, 4, 3, 8, 5, 6, 10, 13, 12, 21, 14, 23, 15, 24, 16, 22, 17, 33, 18, 36, 19, 20, 25, 31, 26, 29, 27, 28, 32, 34,
          35, 38, 37, 40]_____no_output_____indices.sort()_____no_output_____indices[-10:]_____no_output_____len(indices)_____no_output_____individual = [9, 30, 39, 41]_____no_output_____len(merged)_____no_output_____try_merge = [sg.Polygon(merged[i]) for i in [17, 18]]_____no_output_____new_shape = so.cascaded_union(try_merge)_____no_output_____xs, ys = new_shape.exterior.xy_____no_output_____squized = []

s_indices = [0, 3, 1, 6, 2, 17, 4, 8, 5, 14, 7, 10, 11, 15, 12, 14]
s = len(s_indices)
for i in range(0, s, 2):

    merge = [merged[s_indices[i]], merged[s_indices[i + 1]]]
    new_shape = so.cascaded_union(merge)
    squized.append(new_shape)_____no_output_____13, 16, 17, 18_____no_output_____merged.append(new_shape)_____no_output_____len(indices), len(merged)_____no_output_____merged = []
for i in range(0, 38, 2):
    merge = [sg.Polygon(polygons[indices[i]]), sg.Polygon(polygons[indices[i + 1]])]
    new_shape = so.cascaded_union(merge)
    merged.append(new_shape)_____no_output_____fig, axes = plt.subplots()
for pol in squized:
    xs, ys = pol.exterior.xy
    axes.fill(xs, ys, color='red', edgecolor='none', alpha=0.2)

for j in [13, 16, 17, 18]:
    xs, ys = merged[j].exterior.xy
    axes.fill(xs, ys, color='red', edgecolor='none', alpha=0.2)

for j in individual:
    polygon = polygons[j]
    plt.fill(*zip(*polygon), color='red', edgecolor='none', alpha=0.2);_____no_output_____for polygon in polygons[:40]:
    plt.fill(*zip(*polygon), color='red', edgecolor='none', alpha=0.2);
    _____no_output_____import itertools_____no_output_____merged_polygons = []
indices = [0]

index = 0
starting_polygon = polygons[index]
r1 = sg.Polygon(starting_polygon)_____no_output_____for i, polygon in enumerate(polygons[index:]):

    r2 = sg.Polygon(polygon)
    new_shape = so.cascaded_union([r1, r2])
    try:
        xs, ys = new_shape.exterior.xy
        merged_polygons.append(new_shape)
        indices.append(i + 1)
    except AttributeError:
        pass_____no_output_____indices_____no_output_____plot_polygons = []
new_indices = []
for polygon, index in zip(merged_polygons, indices):
    r1 = sg.Polygon(polygon)
    for i, other_polygon in enumerate(polygons[index:]):
        r2 = sg.Polygon(polygon)
        new_shape = so.cascaded_union([r1, r2])
        try:
            xs, ys = new_shape.exterior.xy
            plot_polygons.append(new_shape)
            new_indices.append(i + 1 + index)
        except AttributeError:
            print(index)_____no_output_____# merged_polygons = []
# indices = []
# for index, _ in enumerate(polygons):
#     if index not in indices:
#         starting_polygon = polygons[index]
#         r1 = sg.Polygon(starting_polygon)

#         for i, polygon in enumerate(polygons[index:]):
#             if i + index + 1 not in indices:
#                 r2 = sg.Polygon(polygon)
#                 new_shape = so.cascaded_union([r1, r2])
#                 try:
#                     xs, ys = new_shape.exterior.xy
#                     merged_polygons.append(new_shape)
#                     indices.append(i + index)
#                 except AttributeError:
#                     pass
                _____no_output_____indices.sort()_____no_output_____fig, axes = plt.subplots()
# for polygon in [elem for i, elem in enumerate(polygons) if i not in indices]:
#     plt.fill(*zip(*polygon), color='red', edgecolor='none', alpha=0.2);

for pol in plot_polygons:

    xs, ys = pol.exterior.xy

    axes.fill(xs, ys, color='red', edgecolor='none', alpha=0.2)_____no_output_____stuff = range(len(polygons))_____no_output_____possible_combinations = list(itertools.combinations(stuff, 2))_____no_output_____merged_combinations = []
for combination in possible_combinations:

    r1 = sg.Polygon(polygons[combination[0]])
    r2 = sg.Polygon(polygons[combination[1]])

    new_shape = so.cascaded_union([r1,r2])

    try:
        xs, ys = new_shape.exterior.xy
        merged_combinations.append(combination)
    except AttributeError:
        pass
_____no_output_____merged_polygons = []
merged_combinations[0][0]_____no_output_____merged_combinations[:7]_____no_output_____fig, axs = plt.subplots()
for pol in plot_stuff:

    xs, ys = pol.exterior.xy

    axs.fill(xs, ys, alpha=0.2, fc='r', ec='none')_____no_output_____for polygon in [polygons[5], polygons[6]]:
    plt.fill(*zip(*polygon), color='red', edgecolor='none', alpha=0.2);_____no_output_____import shapely.geometry as sg
import shapely.ops as so
import matplotlib.pyplot as plt_____no_output_____r1.exterior_____no_output_____r2.exterior_____no_output_____new_shape_____no_output_____#constructing the first rect as a polygon
r1 = sg.Polygon(polygons[5])

#a shortcut for constructing a rectangular polygon
r2 = sg.Polygon(polygons[7])

#cascaded union can work on a list of shapes
new_shape = so.cascaded_union([r1,r2])

#exterior coordinates split into two arrays, xs and ys
# which is how matplotlib will need for plotting
xs, ys = new_shape.exterior.xy

#plot it
fig, axs = plt.subplots()
axs.fill(xs, ys, alpha=0.2, fc='r', ec='none')
plt.show() #if not interactive_____no_output_____plt.scatter(x=tsne_lda[:,0],
            y=tsne_lda[:,1],
            color=colors[topic_num]);_____no_output_____import numpy as np
import matplotlib.pyplot as plt
from shapely.ops import polygonize,unary_union
from shapely.geometry import LineString, MultiPolygon, MultiPoint, Point
from scipy.spatial import Voronoi_____no_output_____
points = [[-30.0, 30.370371], [-27.777777, 35.925926], [-34.444443, 58.51852], [-2.9629631, 57.777779], [-17.777779, 75.185181], [-29.25926, 58.148151], [-11.111112, 33.703705], [-11.481482, 40.0], [-27.037037, 40.0], [-7.7777777, 94.444443], [-2.2222223, 122.22222], [-20.370371, 106.66667], [1.1111112, 125.18518], [-6.2962961, 128.88889], [6.666667, 133.7037], [11.851852, 136.2963], [8.5185184, 140.74074], [20.370371, 92.962959], [17.777779, 114.81482], [12.962962, 97.037041], [13.333334, 127.77778], [22.592592, 120.37037], [16.296295, 127.77778], [11.851852, 50.740742], [20.370371, 54.814816], [19.25926, 47.40741], [32.59259, 122.96296], [20.74074, 130.0], [24.814816, 84.814819], [26.296295, 91.111107], [56.296295, 131.48149], [60.0, 141.85185], [32.222221, 136.66667], [53.703705, 147.03703], [87.40741, 196.2963], [34.074074, 159.62964], [34.444443, -2.5925925], [36.666668, -1.8518518], [34.074074, -7.4074073], [35.555557, -18.888889], [76.666664, -39.629627], [35.185184, -37.777779], [25.185184, 14.074074], [42.962959, 32.962963], [35.925926, 9.2592592], [52.222221, 77.777779], [57.777779, 92.222221], [47.037041, 92.59259], [82.222221, 54.074074], [48.888889, 24.444445], [35.925926, 47.777779], [50.740742, 69.259254], [51.111111, 51.851849], [56.666664, -12.222222], [117.40741, -4.4444447], [59.629631, -5.9259262], [66.666664, 134.07408], [91.481483, 127.40741], [66.666664, 141.48149], [53.703705, 4.0740738], [85.185181, 11.851852], [69.629631, 0.37037039], [68.518517, 99.259262], [75.185181, 100.0], [70.370369, 113.7037], [74.444443, 82.59259], [82.222221, 93.703697], [72.222221, 84.444443], [77.777779, 167.03703], [88.888893, 168.88889], [73.703705, 178.88889], [87.037041, 123.7037], [78.518517, 97.037041], [95.555557, 52.962959], [85.555557, 57.037041], [90.370369, 23.333332], [100.0, 28.51852], [88.888893, 37.037037], [87.037041, -42.962959], [89.259262, -24.814816], [93.333328, 7.4074073], [98.518517, 5.185185], [92.59259, 1.4814816], [85.925919, 153.7037], [95.555557, 154.44444], [92.962959, 150.0], [97.037041, 95.925919], [106.66667, 115.55556], [92.962959, 114.81482], [108.88889, 56.296295], [97.777779, 50.740742], [94.074081, 89.259262], [96.666672, 91.851852], [102.22222, 77.777779], [107.40741, 40.370369], [105.92592, 29.629629], [105.55556, -46.296295], [118.51852, -47.777779], [112.22222, -43.333336], [112.59259, 25.185184], [115.92592, 27.777777], [112.59259, 31.851852], [107.03704, -36.666668], [118.88889, -32.59259], [114.07408, -25.555555], [115.92592, 85.185181], [105.92592, 18.888889], [121.11111, 14.444445], [129.25926, -28.51852], [127.03704, -18.518518], [139.25926, -12.222222], [141.48149, 3.7037036], [137.03703, -4.814815], [153.7037, -26.666668], [-2.2222223, 5.5555558], [0.0, 9.6296301], [10.74074, 20.74074], [2.2222223, 54.074074], [4.0740738, 50.740742], [34.444443, 46.296295], [11.481482, 1.4814816], [24.074076, -2.9629631], [74.814819, 79.259254], [67.777779, 152.22223], [57.037041, 127.03704], [89.259262, 12.222222]]
points = np.array(points)
vor = Voronoi(points)
lines = [
    LineString(vor.vertices[line])
    for line in vor.ridge_vertices if -1 not in line
]

convex_hull = MultiPoint([Point(i) for i in points]).convex_hull.buffer(2)
result = MultiPolygon(
    [poly.intersection(convex_hull) for poly in polygonize(lines)])
result = MultiPolygon(
    [p for p in result]
    + [p for p in convex_hull.difference(unary_union(result))])

plt.plot(points[:,0], points[:,1], 'ko')
for r in result:
    plt.fill(*zip(*np.array(list(
        zip(r.boundary.coords.xy[0][:-1], r.boundary.coords.xy[1][:-1])))),
        alpha=0.4)
plt.show()_____no_output_____fig, axes = plt.subplots(figsize=(10, 6))

for i, topic in enumerate(topic_num):
    axes.scatter(x=tsne_lda[i,0],
                 y=tsne_lda[i,1],
                 color=colors[topic],
                marker=markers[topic],
               );
axes.legend(handles=legend_elements);

fig.savefig(f"../../assets/images/topics_scatter_plot_{n_topics}.pdf",
                bbox_inches="tight")_____no_output_____# legend_elements = [Line2D([0], [0], marker='o',
#                           color='w',
#                           label=f'Topic {i}',
#                           markersize=10,
#                           markerfacecolor=c) for i, c in enumerate(colors)]

# fig, axes = plt.subplots(figsize=(10, 6.5))
# n_topics = 6

# plt.scatter(x=tsne_lda[:,0],
#             y=tsne_lda[:,1],
#             color=colors[topic_num],
#             s=10
#            );

# axes.legend(handles=legend_elements);

# fig.savefig("../../assets/images/topics_scatter_plot.pdf",
#                 bbox_inches="tight")_____no_output_____tsne_lda[:,0]_____no_output_____pprint(model_list[max_index].print_topics())_____no_output_____lda_model = gensim.models.ldamodel.LdaModel(corpus=corpus,
                                           id2word=id2word,
                                           num_topics=4,
                                           random_state=100,
                                           update_every=1,
                                           chunksize=100,
                                           passes=10,
                                           alpha='auto',
                                           per_word_topics=True)_____no_output_____pyLDAvis.enable_notebook()
vis = pyLDAvis.gensim.prepare(lda_model, corpus, id2word)
vis//anaconda3/envs/literature/lib/python3.6/site-packages/pyLDAvis/_prepare.py:257: FutureWarning: Sorting because non-concatenation axis is not aligned. A future version
of pandas will change to not sort by default.

To accept the future behavior, pass 'sort=False'.

To retain the current behavior and silence the warning, pass 'sort=True'.

  return pd.concat([default_term_info] + list(topic_dfs))
pprint(model_list[2].print_topics())[(0,
  '0.040*"model" + 0.038*"network" + 0.029*"agent" + 0.018*"system" + '
  '0.018*"base" + 0.016*"study" + 0.012*"structure" + 0.010*"dynamic" + '
  '0.009*"simulation" + 0.009*"propose"'),
 (1,
  '0.027*"social" + 0.019*"group" + 0.017*"behavior" + 0.014*"human" + '
  '0.012*"decision" + 0.011*"make" + 0.011*"experiment" + 0.009*"punishment" + '
  '0.008*"study" + 0.008*"suggest"'),
 (2,
  '0.059*"cooperation" + 0.030*"individual" + 0.025*"population" + '
  '0.020*"evolution" + 0.019*"evolutionary" + 0.015*"interaction" + '
  '0.013*"cooperator" + 0.012*"mechanism" + 0.012*"cooperative" + '
  '0.011*"level"'),
 (3,
  '0.075*"game" + 0.060*"strategy" + 0.038*"player" + 0.021*"dilemma" + '
  '0.016*"theory" + 0.016*"play" + 0.012*"payoff" + 0.012*"state" + '
  '0.010*"prisoner" + 0.010*"show"')]
lda_model = gensim.models.ldamodel.LdaModel(corpus=corpus,
                                           id2word=id2word,
                                           num_topics=5,
                                           random_state=100,
                                           update_every=1,
                                           chunksize=100,
                                           passes=10,
                                           alpha='auto',
                                           per_word_topics=True)_____no_output_____pyLDAvis.enable_notebook()
vis = pyLDAvis.gensim.prepare(lda_model, corpus, id2word)
vis//anaconda3/envs/literature/lib/python3.6/site-packages/pyLDAvis/_prepare.py:257: FutureWarning: Sorting because non-concatenation axis is not aligned. A future version
of pandas will change to not sort by default.

To accept the future behavior, pass 'sort=False'.

To retain the current behavior and silence the warning, pass 'sort=True'.

  return pd.concat([default_term_info] + list(topic_dfs))
pprint(model_list[3].print_topics())[(0,
  '0.040*"social" + 0.031*"behavior" + 0.018*"human" + 0.016*"study" + '
  '0.013*"experiment" + 0.012*"cooperative" + 0.011*"cooperation" + '
  '0.010*"suggest" + 0.009*"find" + 0.009*"behaviour"'),
 (1,
  '0.027*"individual" + 0.025*"group" + 0.015*"good" + 0.012*"show" + '
  '0.012*"high" + 0.012*"increase" + 0.011*"punishment" + 0.011*"cost" + '
  '0.011*"result" + 0.009*"benefit"'),
 (2,
  '0.092*"game" + 0.073*"strategy" + 0.047*"player" + 0.036*"agent" + '
  '0.025*"dilemma" + 0.020*"play" + 0.015*"payoff" + 0.015*"state" + '
  '0.012*"prisoner" + 0.012*"equilibrium"'),
 (3,
  '0.061*"cooperation" + 0.043*"network" + 0.030*"population" + '
  '0.026*"evolutionary" + 0.024*"evolution" + 0.022*"interaction" + '
  '0.020*"dynamic" + 0.017*"structure" + 0.016*"cooperator" + 0.015*"study"'),
 (4,
  '0.034*"model" + 0.023*"theory" + 0.019*"base" + 0.016*"system" + '
  '0.012*"problem" + 0.012*"paper" + 0.011*"propose" + 0.011*"information" + '
  '0.010*"provide" + 0.009*"approach"')]
lda_model = gensim.models.ldamodel.LdaModel(corpus=corpus,
                                           id2word=id2word,
                                           num_topics=4,
                                           random_state=100,
                                           update_every=1,
                                           chunksize=100,
                                           passes=10,
                                           alpha='auto',
                                           per_word_topics=True)_____no_output_____pyLDAvis.enable_notebook()
vis = pyLDAvis.gensim.prepare(lda_model, corpus, id2word)
vis//anaconda3/envs/literature/lib/python3.6/site-packages/pyLDAvis/_prepare.py:257: FutureWarning: Sorting because non-concatenation axis is not aligned. A future version
of pandas will change to not sort by default.

To accept the future behavior, pass 'sort=False'.

To retain the current behavior and silence the warning, pass 'sort=True'.

  return pd.concat([default_term_info] + list(topic_dfs))
pprint(model_list[2].print_topics())[(0,
  '0.030*"model" + 0.019*"theory" + 0.014*"system" + 0.010*"problem" + '
  '0.010*"provide" + 0.009*"propose" + 0.009*"base" + 0.009*"information" + '
  '0.008*"approach" + 0.008*"work"'),
 (1,
  '0.059*"cooperation" + 0.036*"network" + 0.023*"population" + '
  '0.021*"evolutionary" + 0.020*"interaction" + 0.020*"evolution" + '
  '0.017*"dynamic" + 0.014*"individual" + 0.013*"cooperator" + '
  '0.012*"structure"'),
 (2,
  '0.076*"game" + 0.060*"strategy" + 0.039*"player" + 0.029*"agent" + '
  '0.018*"dilemma" + 0.016*"play" + 0.012*"payoff" + 0.011*"show" + '
  '0.011*"state" + 0.011*"study"'),
 (3,
  '0.033*"social" + 0.022*"behavior" + 0.020*"group" + 0.017*"individual" + '
  '0.014*"human" + 0.011*"study" + 0.011*"experiment" + 0.010*"result" + '
  '0.009*"cooperative" + 0.009*"suggest"')]

**Choose num = 4**_____no_output_____

optimal_model = model_list[3]
model_topics = optimal_model.show_topics(formatted=False)_____no_output_____def format_topics_sentences(ldamodel,
                            corpus,
                            data_frame):
    data_with_topics = pd.DataFrame()

    for i, row in enumerate(ldamodel[corpus]):
        row = sorted(row, key=lambda x: (x[1]), reverse=True)
        for j, (topic_num, prop_topic) in enumerate(row):
            if j == 0:
                wp = ldamodel.show_topic(topic_num)
                topic_keywords = ", ".join([word for word, prop in wp])
                data_with_topics = data_with_topics.append(pd.Series([int(topic_num),
                                                                  round(prop_topic,4),
                                                                  topic_keywords]),
                                                         ignore_index=True)
            else:
                break
    data_with_topics.columns = ['Dominant_Topic', 'Perc_Contribution', 'Topic_Keywords']

    data_with_topics = pd.concat([data_with_topics, data_frame], axis=1)
    return data_with_topics_____no_output_____data_with_topics = format_topics_sentences(ldamodel=optimal_model,
                                           corpus=corpus,
                                           data_frame=data)_____no_output_____len(data_with_topics), len(df)_____no_output_____data_with_topics = data_with_topics[["Dominant_Topic", "Perc_Contribution", "Topic_Keywords", 'unique_key']]_____no_output_____len(data_with_topics['unique_key'].unique()), len(df['unique_key'].unique())_____no_output_____merged = df.merge(data_with_topics, on='unique_key')_____no_output_____len(merged), len(merged['unique_key'].unique())_____no_output_____merged.to_csv('../data/prisoners_dilemma_meta_data_with_topics.csv')_____no_output_____