# Notebook from ksun0/data-visualizations
Path: text-project/text-project.ipynb

import json
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

from sklearn import metrics
from sklearn.linear_model import SGDClassifier
from sklearn.pipeline import Pipeline
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.cross_validation import train_test_split
from sklearn.model_selection import cross_val_score

import nltk
from nltk import *
from nltk.corpus import PlaintextCorpusReader
from nltk.tokenize import sent_tokenize, word_tokenize
from nltk.corpus import stopwords
from nltk.sentiment.vader import SentimentIntensityAnalyzer

import sys
import codecs
import string
import random
import re

import requests
import urllib
from bs4 import BeautifulSoup
from readability.readability import Document # https://github.com/buriy/python-readability. Tried Goose, Newspaper (python libraries on Github). Bad results.
from http.cookiejar import CookieJar # _____no_output_____

# Data Exploration
First, load in science article news data, which contains basic information of science articles including URL and description. I previously mined the articles from Stat News, Science Daily, and Scientific American, but they are missing a lot of crucial information for the goal of this text project._____no_output_____

news_df = pd.read_json("data/news.json")
print('Shape of data (rows, columns):', news_df.shape, '\n')
print('Column Names: ', news_df.columns) # flagged means whether or not the article is incomplete or inaccurateShape of data (rows, columns): (4000, 12)

Column Names:  Index(['articleUrl', 'author', 'categories', 'createdAt', 'date',
       'description', 'flagged', 'imageUrl', 'slug', 'source', 'sourceUrl',
       'title'],
      dtype='object')

I have a couple of goals in mind for this text data: exploring some interesting features about the articles and extracting keywords. Although there is also a "categories" column in this data, it would be nice to create a model to categorize into categories -- for example, "Physics" or "Artificial Intelligence" -- to be able to compare these articles perhaps with new ones without the "categories" label. Also, it would be cool to see if article writers sometimes "incorrectly" categorizes an article.

To clean up this data a bit, let's also remove a few unnecesary columns. For example, since "description" is unclean and can be replaced by the full "article, this column can be deleted._____no_output_____

# new article features to be populated
news_df['article'] = '' # full article
news_df['num_words_not_in_dictionary'] = 0 # number of words not in English dictionary (super scientific words)
news_df['stopwords2words'] = 0 # ratio of stopwords to total words
news_df['readability_score'] = 0 # readability score probably based on Flesch–Kincaid
news_df['sentiment_polarity'] = 0 # the sentiment polarity within an article (range of emotions)
news_df = news_df.drop(['author', 'createdAt', 'imageUrl', 'slug', 'date', 'description'], axis=1)
news_df = news_df.replace(np.nan, 0) # replace NaN with 0 for "flagged" column
news_df.head()_____no_output_____

# Scraping the Full Article Texts
First, populate the "article" column bye the whole article from the URL (will clean those full texts up later).
 NOTE: The following cell will take a long time, so it is commented. The cell after it will directly load in the result of the cell._____no_output_____

# for i in range(len(news_df['articleUrl'])):
#     try:
#         url = news_df['articleUrl'][i] # Bad: 2841, 3013, 3015, 3077, 3091, 3092
#         opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor)
#         opener.addheaders = [('User-agent', 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.77 Safari/535.7')]
#         html =  opener.open(url).read().decode('utf-8')
#         readable_article = Document(html).summary()
#         soup = BeautifulSoup(readable_article, "lxml")
#         text = soup.get_text()
#         news_df.loc[i, 'article'] = text
#         print(i, " ", end="")
#         break
#     except:
#         print(str(i) + "error", " ", end="")_____no_output_____# Run this to skip above steps
news_df = pd.read_json("data/news_with_articles.json")
news_df = news_df.sort_index(ascending=True)
news_df = news_df[['articleUrl', 'categories', 'flagged', 'source', 'sourceUrl', 'title', 'article',
             'num_words_not_in_dictionary', 'stopwords2words', 'readability_score', 'sentiment_polarity']]
news_df.head()_____no_output_____

Web scraping is very messy, so we have to clean the "article" column. We will keep the '\n' characters to be able to split the text into paragraphs (without the use of NLTK).

On closer ispection of the articles from "Science Daily," many articles did not scrape correctly. I will remove them now. This is the downsides of webscraping: so much noise in data.

Let's also save the list of articles to use as a corpus._____no_output_____

news_df = news_df.replace({'\xa0': ' '}, regex=True) # clean these empty null characters.
news_df['article'][19]_____no_output_____for i in range(len(news_df)):
    if len(news_df['article'][i]) < 10:
        news_df = news_df.drop(i)
news_df = news_df.reset_index(drop=True)
news_df_____no_output_____articles = [str(s) for s in news_df['article']] # create a corpus of all articles
articles[0]_____no_output_____

### Stopwords, stemming, and tokenization to clean the text even more
tokenize_and_stem: tokenizes and also stems each token.

tokenize_only: tokenizes the text only to be able to convert stemmed tokens back to the full word.
_____no_output_____

# load nltk's English stopwords
stopwords = nltk.corpus.stopwords.words('english')
# load nltk's SnowballStemmer
stemmer = SnowballStemmer("english")

# Define a tokenizer and stemmer that returns the set of stems of the text
def tokenize_and_stem(text):
    # first tokenize by sentence, then by word to ensure that punctuation is caught as it's own token
    tokens = [word for sent in nltk.sent_tokenize(text) for word in nltk.word_tokenize(sent)]
    filtered_tokens = []
    # filter out any tokens not containing letters (e.g., numeric tokens, raw punctuation)
    for token in tokens:
        if len(token) <= 1: # remove one character words, "a" apostraphe s.
            continue
        if token.isalpha():
            filtered_tokens.append(token)
    stems = [stemmer.stem(t) for t in filtered_tokens]
    return stems

def tokenize_only(text):
    # first tokenize by sentence, then by word to ensure that punctuation is caught as it's own token
    tokens = [word.lower() for sent in nltk.sent_tokenize(text) for word in nltk.word_tokenize(sent)]
    filtered_tokens = []
    # filter out any tokens not containing letters (e.g., numeric tokens, raw punctuation)
    for token in tokens:
        if len(token) <= 1: # remove one character words, "a" apostraphe s.
            continue
        if token.isalpha():
            filtered_tokens.append(token)
    return filtered_tokens_____no_output_____

Use stemming + tokenizing and tokenizing only functions to create two vocabulary lists.

These two lists create a Pandas DataFrame with the stemmed vocabulary as the index and the tokenized words as the column (to convert stem into its first full word/token, could be many, but take first for simplicity).

Stemming is not the best... especially when it comes to Proper Nouns and other special words._____no_output_____

totalvocab_stemmed = []
totalvocab_tokenized = []
for i in articles:
    allwords_stemmed = tokenize_and_stem(i) #for each article in "articles", tokenize/stem
    totalvocab_stemmed.extend(allwords_stemmed) #extend the 'totalvocab_stemmed' list

    allwords_tokenized = tokenize_only(i)
    totalvocab_tokenized.extend(allwords_tokenized)

vocab_df = pd.DataFrame({'words': totalvocab_tokenized}, index = totalvocab_stemmed) # map stem to first actual word for simplicity
print('Shape of vocab_df (rows, columns):', vocab_df.shape, '\n')
vocab_df.head()Shape of vocab_df (rows, columns): (1662526, 1)

### Also, let's start filling out the other columns since we have the full article texts now
_____no_output_____

stopwords = set(w.lower() for w in nltk.corpus.stopwords.words('english'))

def stopwords_to_words(s):
    raw = nltk.word_tokenize(s)
    text = nltk.Text(raw)
    text_vocab = set(w.lower() for w in text if w.isalpha())
    return len(text_vocab.intersection(stopwords)) / len(text_vocab)

for i in range(len(news_df['articleUrl'])):
    try:
        news_df.loc[i, 'stopwords2words'] = stopwords_to_words(news_df['article'][i])
        print(i, ':', news_df['articleUrl'][i], ":", stopwords_to_words(news_df['article'][i]))
    except:
        pass # keeps empty articles 0.00 : https://www.statnews.com/2017/02/09/dana-farber-mar-lago/ : 0.25
1 : https://www.statnews.com/2017/02/09/antibiotics-resistance-superbugs/ : 0.17032967032967034
2 : https://www.statnews.com/2017/02/07/scientist-march-trump/ : 0.20891364902506965
3 : https://www.statnews.com/2017/02/09/twitter-hospitals-medicine/ : 0.14606741573033707
4 : https://www.statnews.com/2017/02/09/obamacare-repeal-preexisting-conditions/ : 0.18032786885245902
5 : https://www.statnews.com/2017/02/08/drug-names-process/ : 0.11202830188679246
6 : https://www.statnews.com/2017/02/09/alzheimers-disease-statins/ : 0.18518518518518517
7 : https://www.statnews.com/2016/03/28/google-life-sciences-exodus/ : 0.08673894912427023
8 : https://www.statnews.com/2017/02/06/mental-health-college-students/ : 0.13237063778580024
9 : https://www.statnews.com/2017/02/09/doctors-training-social-change/ : 0.14678899082568808
10 : https://www.statnews.com/2017/02/09/injury-states-unusually-common/ : 0.22522522522522523
11 : https://www.statnews.com/2017/02/08/social-media-compulsion-mental-illness/ : 0.17567567567567569
12 : https://www.statnews.com/2017/02/09/harvard-reunion-sugar-study/ : 0.24761904761904763
13 : https://www.statnews.com/2017/02/06/health-insurance-high-deductible-experiment/ : 0.21535181236673773
14 : https://www.statnews.com/2017/02/09/margaret-chan-who-fda/ : 0.1827956989247312
15 : https://www.statnews.com/2017/02/07/trump-iraq-refugee-doctor/ : 0.12383177570093458
16 : https://www.statnews.com/2017/02/08/toledo-alzheimers-suits/ : 0.20386266094420602
17 : https://www.statnews.com/2017/01/30/trump-mental-health/ : 0.1262135922330097
18 : https://www.statnews.com/2017/01/30/fda-regulations-executive-order-trump/ : 0.19333333333333333
19 : http://neurosciencenews.com/balloon-hearing-test-6027/ : 0.20320855614973263
20 : https://www.statnews.com/2017/02/06/trump-biopharma/ : 0.35294117647058826
21 : https://www.statnews.com/pharmalot/2017/02/08/kaleo-naloxone-drug-prices/ : 0.2653061224489796
22 : https://www.statnews.com/pharmalot/2017/02/07/general-counsel-whistleblower/ : 0.3333333333333333
23 : https://www.statnews.com/pharmalot/2017/02/06/mylan-subpoena-epipen/ : 0.3253012048192771
24 : https://www.statnews.com/pharmalot/2017/02/03/winners-losers-2/ : 0.2827586206896552
25 : https://www.statnews.com/pharmalot/2017/02/03/jobs-sanofi-shire-merck/ : 0.3448275862068966
26 : https://www.statnews.com/pharmalot/2017/02/02/drug-prices-generics-vouchers/ : 0.2636363636363636
27 : https://www.statnews.com/pharmalot/2017/02/09/india-patents-hiv-opioids-drug-prices/ : 0.2803030303030303
28 : https://www.statnews.com/pharmalot/2017/02/08/immigration-sanofi-drug-prices-teva/ : 0.2627118644067797
29 : https://www.statnews.com/pharmalot/2017/02/06/trump-open-letter-drug-pricing/ : 0.2727272727272727
30 : https://www.statnews.com/pharmalot/2017/02/03/california-coupons-drug-prices/ : 0.27058823529411763
31 : https://www.statnews.com/pharmalot/2017/02/02/vertex-ireland-cystic-fibrosis/ : 0.3106796116504854
32 : https://www.statnews.com/pharmalot/2017/02/09/ted-cruz-fda-drugs-2/ : 0.30927835051546393
33 : https://www.statnews.com/pharmalot/2017/02/08/merrimack-insider-trading-akebia/ : 0.33783783783783783
34 : https://www.statnews.com/pharmalot/2017/02/08/sanofi-regeneron-amgen-cholesterol/ : 0.2727272727272727
35 : https://www.statnews.com/2017/02/09/samsung-biologics-trump-trade/ : 0.2638888888888889
36 : https://www.statnews.com/pharmalot/2017/02/07/china-bribes-glaxo-fda-teva/ : 0.24324324324324326
37 : https://www.statnews.com/pharmalot/2017/02/07/fda-advisory-committee-redactions/ : 0.26666666666666666
38 : https://www.statnews.com/pharmalot/2017/02/07/ftc-citizens-generics/ : 0.2631578947368421
39 : https://www.statnews.com/2017/02/06/germany-pharma-trump/ : 0.32894736842105265
40 : https://www.statnews.com/pharmalot/2017/02/06/teva-vigodman/ : 0.30303030303030304
41 : https://www.statnews.com/pharmalot/2017/02/06/novartis-bribes-breast-implants-generics/ : 0.22302158273381295
42 : https://www.statnews.com/2017/02/06/week-feb-6-watch-biopharma/ : 0.3411764705882353
43 : https://www.statnews.com/pharmalot/2017/02/03/lilly-alzheimers-layoffs/ : 0.26666666666666666
44 : https://www.statnews.com/2017/02/03/trump-pharma/ : 0.2875
45 : https://www.statnews.com/pharmalot/2017/02/03/opioid-lawsuit-oxycontin/ : 0.25892857142857145
46 : https://www.statnews.com/pharmalot/2017/02/02/amgen-cholesterol-sanofi-regeneron/ : 0.26956521739130435
47 : https://www.statnews.com/pharmalot/2017/02/03/biogen-drug-prices/ : 0.24161073825503357
48 : https://www.statnews.com/2017/02/02/implant-opioid-addiction-ipo/ : 0.35555555555555557
49 : https://www.scientificamerican.com/podcast/episode/frog-spit-behaves-like-bug-catching-ketchup/ : 0.2894736842105263
50 : https://www.scientificamerican.com/video/soft-sticky-frog-tongues-slurp-supper/ : 0.42857142857142855
51 : https://www.scientificamerican.com/article/tangled-up-in-spacetime1/ : 0.2948717948717949
52 : https://www.scientificamerican.com/podcast/episode/knot-not-easy-to-knot/ : 0.29591836734693877
53 : https://www.scientificamerican.com/article/in-case-you-missed-it-the-first-crispr-clinical-trial-a-new-shield-for-chernobyl-mdash-and-more/ : 0.19597989949748743
54 : https://www.scientificamerican.com/article/florida-has-seen-bad-effects-from-trump-like-climate-gag-orders/ : 0.16666666666666666
55 : https://www.scientificamerican.com/article/scientists-protest-trump-order-with-boycotts-of-journals-conferences/ : 0.2581818181818182
56 : https://www.scientificamerican.com/article/obama-science-adviser-trump-immigration-ban-ldquo-an-abomination-rdquo/ : 0.2222222222222222
57 : https://www.scientificamerican.com/article/plug-a-leaky-bottle-with-the-power-of-air/ : 0.22273781902552203
58 : https://www.scientificamerican.com/article/stay-warm-with-thermal-insulation/ : 0.1941031941031941
59 : https://www.scientificamerican.com/article/doubts-cloud-claims-of-metallic-hydrogen/ : 0.20612813370473537
60 : https://www.scientificamerican.com/article/icecube-closes-in-on-mysterious-nature-of-neutrinos/ : 0.17906976744186046
61 : https://www.scientificamerican.com/article/the-epa-has-started-to-remove-obama-era-information/ : 0.1615720524017467
62 : https://www.scientificamerican.com/article/house-science-panel-to-hold-hearing-on-ldquo-making-the-epa-great-again-rdquo/ : 0.20418848167539266
63 : https://www.scientificamerican.com/article/u-s-government-takes-animal-welfare-data-offline/ : 0.1839080459770115
64 : https://www.scientificamerican.com/article/ligo-rsquo-s-underdog-cousin-ready-to-enhance-gravitational-wave-hunt/ : 0.15
65 : https://www.scientificamerican.com/article/build-an-irrigation-system/ : 0.20869565217391303
66 : https://www.scientificamerican.com/article/trump-immigration-ban-can-worsen-u-s-doctor-shortage-hurt-hospitals/ : 0.18683651804670912
67 : https://www.scientificamerican.com/article/e-u-looks-to-china-for-climate-leadership/ : 0.1640625
68 : https://www.scientificamerican.com/article/build-a-2-stage-balloon-rocket/ : 0.20802005012531327
69 : https://www.scientificamerican.com/article/facebook-google-and-others-launch-drive-against-fake-news-in-france/ : 0.22857142857142856
70 : https://www.scientificamerican.com/article/aquatic-robot-braves-volcanoes-and-typhoons-to-detect-tsunamis/ : 0.163727959697733
71 : https://www.scientificamerican.com/article/a-breakthrough-in-the-search-for-alpha-centauris-planets/ : 0.1227080394922426
72 : https://www.scientificamerican.com/article/immigrants-do-not-increase-crime-research-shows/ : 0.17302052785923755
73 : https://www.scientificamerican.com/article/trump-rsquo-s-executive-order-strands-brooklyn-doctor-in-sudan/ : 0.1960352422907489
74 : https://www.scientificamerican.com/article/future-of-paris-accord-uncertain-as-tillerson-becomes-secretary-of-state/ : 0.1348122866894198
75 : https://www.scientificamerican.com/article/are-we-living-in-a-computer-simulation1/ : 0.32727272727272727
76 : https://www.scientificamerican.com/article/cosmic-test-bolsters-einsteins-ldquo-spooky-action-at-a-distance-rdquo/ : 0.15938864628820962
77 : https://www.scientificamerican.com/article/u-k-s-brexit-plans-call-for-leaving-e-u-nuclear-agency/ : 0.17403314917127072
78 : https://www.scientificamerican.com/article/to-treat-primates-more-humanely-transparency/ : 0.23893805309734514
79 : https://www.scientificamerican.com/article/hunting-dark-matter-between-the-ticks-of-an-atomic-clock/ : 0.1710843373493976
80 : https://www.scientificamerican.com/article/are-wormholes-a-dead-end-for-faster-than-light-travel/ : 0.16274089935760172
81 : https://www.scientificamerican.com/article/science-and-the-u-s-supreme-court-the-cases-to-watch-in-2017/ : 0.1425339366515837
82 : https://www.scientificamerican.com/article/can-empirical-entertainment-rescue-science-in-an-alt-fact-world/ : 0.20726495726495728
83 : https://www.scientificamerican.com/article/trump-supreme-court-nominee-would-put-agencies-on-short-leash/ : 0.13996316758747698
84 : https://www.scientificamerican.com/article/newfound-source-of-mysterious-cosmic-bursts-poses-deeper-enigmas/ : 0.11275964391691394
85 : https://www.scientificamerican.com/podcast/episode/rapid-response-vaccines-for-epidemic-outbreaks/ : 0.22580645161290322
86 : https://www.scientificamerican.com/article/did-a-changing-climate-wipe-out-the-giant-kangaroo/ : 0.1986970684039088
87 : https://www.scientificamerican.com/article/the-epa-has-started-to-remove-obama-era-information/ : 0.1615720524017467
88 : https://www.scientificamerican.com/article/a-tax-on-carbon-pollution-can-benefit-business/ : 0.16923076923076924
# WARNING: Takes a long time. Might not be accurate but relatively should determine which article has more complex words.

# def num_unusual_words(s):
#     raw = nltk.word_tokenize(s)
#     text = nltk.Text(raw)
#     text_vocab = set(w.lower() for w in text if w.isalpha())
#     english_vocab = set(w.lower() for w in nltk.corpus.words.words())
#     unusual = text_vocab - english_vocab
#     return len(sorted(unusual))

# for i in range(len(news_df['articleUrl'])):
#     try:
#         news_df.loc[i, 'num_words_not_in_dictionary'] = num_unusual_words(news_df['article'][i])
#         print(i, ':', news_df['articleUrl'][i], ":", num_unusual_words(news_df['article'][i]))
#     except:
#         pass # keeps empty articles 0.00 : https://www.statnews.com/2017/02/09/dana-farber-mar-lago/ : 34
1 : https://www.statnews.com/2017/02/09/antibiotics-resistance-superbugs/ : 116
2 : https://www.statnews.com/2017/02/07/scientist-march-trump/ : 76
3 : https://www.statnews.com/2017/02/09/twitter-hospitals-medicine/ : 105
4 : https://www.statnews.com/2017/02/09/obamacare-repeal-preexisting-conditions/ : 84
5 : https://www.statnews.com/2017/02/08/drug-names-process/ : 231
6 : https://www.statnews.com/2017/02/09/alzheimers-disease-statins/ : 85
7 : https://www.statnews.com/2016/03/28/google-life-sciences-exodus/ : 311
8 : https://www.statnews.com/2017/02/06/mental-health-college-students/ : 192
9 : https://www.statnews.com/2017/02/09/doctors-training-social-change/ : 102
10 : https://www.statnews.com/2017/02/09/injury-states-unusually-common/ : 27
11 : https://www.statnews.com/2017/02/08/social-media-compulsion-mental-illness/ : 130
12 : https://www.statnews.com/2017/02/09/harvard-reunion-sugar-study/ : 72
13 : https://www.statnews.com/2017/02/06/health-insurance-high-deductible-experiment/ : 84
14 : https://www.statnews.com/2017/02/09/margaret-chan-who-fda/ : 64
15 : https://www.statnews.com/2017/02/07/trump-iraq-refugee-doctor/ : 215
16 : https://www.statnews.com/2017/02/08/toledo-alzheimers-suits/ : 88
17 : https://www.statnews.com/2017/01/30/trump-mental-health/ : 180
18 : https://www.statnews.com/2017/01/30/fda-regulations-executive-order-trump/ : 78
19 : http://neurosciencenews.com/balloon-hearing-test-6027/ : 82
20 : https://www.statnews.com/2017/02/06/trump-biopharma/ : 18
21 : https://www.statnews.com/pharmalot/2017/02/08/kaleo-naloxone-drug-prices/ : 21
22 : https://www.statnews.com/pharmalot/2017/02/07/general-counsel-whistleblower/ : 17
23 : https://www.statnews.com/pharmalot/2017/02/06/mylan-subpoena-epipen/ : 17
24 : https://www.statnews.com/pharmalot/2017/02/03/winners-losers-2/ : 28
25 : https://www.statnews.com/pharmalot/2017/02/03/jobs-sanofi-shire-merck/ : 14
26 : https://www.statnews.com/pharmalot/2017/02/02/drug-prices-generics-vouchers/ : 18
27 : https://www.statnews.com/pharmalot/2017/02/09/india-patents-hiv-opioids-drug-prices/ : 19
28 : https://www.statnews.com/pharmalot/2017/02/08/immigration-sanofi-drug-prices-teva/ : 27
29 : https://www.statnews.com/pharmalot/2017/02/06/trump-open-letter-drug-pricing/ : 14
30 : https://www.statnews.com/pharmalot/2017/02/03/california-coupons-drug-prices/ : 12
31 : https://www.statnews.com/pharmalot/2017/02/02/vertex-ireland-cystic-fibrosis/ : 14
32 : https://www.statnews.com/pharmalot/2017/02/09/ted-cruz-fda-drugs-2/ : 17
33 : https://www.statnews.com/pharmalot/2017/02/08/merrimack-insider-trading-akebia/ : 13
34 : https://www.statnews.com/pharmalot/2017/02/08/sanofi-regeneron-amgen-cholesterol/ : 19
35 : https://www.statnews.com/2017/02/09/samsung-biologics-trump-trade/ : 15
36 : https://www.statnews.com/pharmalot/2017/02/07/china-bribes-glaxo-fda-teva/ : 20
37 : https://www.statnews.com/pharmalot/2017/02/07/fda-advisory-committee-redactions/ : 17
38 : https://www.statnews.com/pharmalot/2017/02/07/ftc-citizens-generics/ : 17
39 : https://www.statnews.com/2017/02/06/germany-pharma-trump/ : 12
40 : https://www.statnews.com/pharmalot/2017/02/06/teva-vigodman/ : 23
41 : https://www.statnews.com/pharmalot/2017/02/06/novartis-bribes-breast-implants-generics/ : 34
42 : https://www.statnews.com/2017/02/06/week-feb-6-watch-biopharma/ : 15
43 : https://www.statnews.com/pharmalot/2017/02/03/lilly-alzheimers-layoffs/ : 14
44 : https://www.statnews.com/2017/02/03/trump-pharma/ : 14
45 : https://www.statnews.com/pharmalot/2017/02/03/opioid-lawsuit-oxycontin/ : 31
46 : https://www.statnews.com/pharmalot/2017/02/02/amgen-cholesterol-sanofi-regeneron/ : 21
47 : https://www.statnews.com/pharmalot/2017/02/03/biogen-drug-prices/ : 21
48 : https://www.statnews.com/2017/02/02/implant-opioid-addiction-ipo/ : 18
49 : https://www.scientificamerican.com/podcast/episode/frog-spit-behaves-like-bug-catching-ketchup/ : 32
50 : https://www.scientificamerican.com/video/soft-sticky-frog-tongues-slurp-supper/ : 2
51 : https://www.scientificamerican.com/article/tangled-up-in-spacetime1/ : 11
52 : https://www.scientificamerican.com/podcast/episode/knot-not-easy-to-knot/ : 35
53 : https://www.scientificamerican.com/article/in-case-you-missed-it-the-first-crispr-clinical-trial-a-new-shield-for-chernobyl-mdash-and-more/ : 54
54 : https://www.scientificamerican.com/article/florida-has-seen-bad-effects-from-trump-like-climate-gag-orders/ : 102
55 : https://www.scientificamerican.com/article/scientists-protest-trump-order-with-boycotts-of-journals-conferences/ : 57
56 : https://www.scientificamerican.com/article/obama-science-adviser-trump-immigration-ban-ldquo-an-abomination-rdquo/ : 62
57 : https://www.scientificamerican.com/article/plug-a-leaky-bottle-with-the-power-of-air/ : 66
58 : https://www.scientificamerican.com/article/stay-warm-with-thermal-insulation/ : 54
59 : https://www.scientificamerican.com/article/doubts-cloud-claims-of-metallic-hydrogen/ : 76
60 : https://www.scientificamerican.com/article/icecube-closes-in-on-mysterious-nature-of-neutrinos/ : 87
61 : https://www.scientificamerican.com/article/the-epa-has-started-to-remove-obama-era-information/ : 117
62 : https://www.scientificamerican.com/article/house-science-panel-to-hold-hearing-on-ldquo-making-the-epa-great-again-rdquo/ : 37
63 : https://www.scientificamerican.com/article/u-s-government-takes-animal-welfare-data-offline/ : 93
64 : https://www.scientificamerican.com/article/ligo-rsquo-s-underdog-cousin-ready-to-enhance-gravitational-wave-hunt/ : 123
65 : https://www.scientificamerican.com/article/build-an-irrigation-system/ : 60
66 : https://www.scientificamerican.com/article/trump-immigration-ban-can-worsen-u-s-doctor-shortage-hurt-hospitals/ : 117
67 : https://www.scientificamerican.com/article/e-u-looks-to-china-for-climate-leadership/ : 74
68 : https://www.scientificamerican.com/article/build-a-2-stage-balloon-rocket/ : 61
69 : https://www.scientificamerican.com/article/facebook-google-and-others-launch-drive-against-fake-news-in-france/ : 40
70 : https://www.scientificamerican.com/article/aquatic-robot-braves-volcanoes-and-typhoons-to-detect-tsunamis/ : 92
71 : https://www.scientificamerican.com/article/a-breakthrough-in-the-search-for-alpha-centauris-planets/ : 159
72 : https://www.scientificamerican.com/article/immigrants-do-not-increase-crime-research-shows/ : 85
73 : https://www.scientificamerican.com/article/trump-rsquo-s-executive-order-strands-brooklyn-doctor-in-sudan/ : 106
74 : https://www.scientificamerican.com/article/future-of-paris-accord-uncertain-as-tillerson-becomes-secretary-of-state/ : 137
75 : https://www.scientificamerican.com/article/are-we-living-in-a-computer-simulation1/ : 5
76 : https://www.scientificamerican.com/article/cosmic-test-bolsters-einsteins-ldquo-spooky-action-at-a-distance-rdquo/ : 114
77 : https://www.scientificamerican.com/article/u-k-s-brexit-plans-call-for-leaving-e-u-nuclear-agency/ : 79
78 : https://www.scientificamerican.com/article/to-treat-primates-more-humanely-transparency/ : 45
79 : https://www.scientificamerican.com/article/hunting-dark-matter-between-the-ticks-of-an-atomic-clock/ : 95
80 : https://www.scientificamerican.com/article/are-wormholes-a-dead-end-for-faster-than-light-travel/ : 79
81 : https://www.scientificamerican.com/article/science-and-the-u-s-supreme-court-the-cases-to-watch-in-2017/ : 107
82 : https://www.scientificamerican.com/article/can-empirical-entertainment-rescue-science-in-an-alt-fact-world/ : 98
83 : https://www.scientificamerican.com/article/trump-supreme-court-nominee-would-put-agencies-on-short-leash/ : 126
84 : https://www.scientificamerican.com/article/newfound-source-of-mysterious-cosmic-bursts-poses-deeper-enigmas/ : 188
85 : https://www.scientificamerican.com/podcast/episode/rapid-response-vaccines-for-epidemic-outbreaks/ : 11
86 : https://www.scientificamerican.com/article/did-a-changing-climate-wipe-out-the-giant-kangaroo/ : 68
87 : https://www.scientificamerican.com/article/the-epa-has-started-to-remove-obama-era-information/ : 117
88 : https://www.scientificamerican.com/article/a-tax-on-carbon-pollution-can-benefit-business/ : 80

"readability_score" is not super important for this project, but here are some interesting leads for it: https://datawarrior.wordpress.com/2016/03/29/flesch-kincaid-readability-measure/_____no_output_____

# From: http://www.nltk.org/howto/sentiment.html
def sentiment_polarity(s):
    paragraphs = re.split('\n' , s)
    paragraphs = [p for p in paragraphs if p != '']
    sid = SentimentIntensityAnalyzer()
    neu = []
    compound = []
    for sentence in paragraphs:
        ss = sid.polarity_scores(sentence)
        neu.append(ss['neu'])
    return np.var(neu)

for i in range(len(news_df['articleUrl'])):
    try:
        news_df.loc[i, 'sentiment_polarity'] = sentiment_polarity(news_df['article'][i])
        print(i, ':', news_df['articleUrl'][i], ":", sentiment_polarity(news_df['article'][i]))
    except:
        pass # keeps empty articles 0.00 : https://www.statnews.com/2017/02/09/dana-farber-mar-lago/ : 0.0704219183673
1 : https://www.statnews.com/2017/02/09/antibiotics-resistance-superbugs/ : 0.0353557191358
2 : https://www.statnews.com/2017/02/07/scientist-march-trump/ : 0.0554019555556
3 : https://www.statnews.com/2017/02/09/twitter-hospitals-medicine/ : 0.04819336
4 : https://www.statnews.com/2017/02/09/obamacare-repeal-preexisting-conditions/ : 0.040396102071
5 : https://www.statnews.com/2017/02/08/drug-names-process/ : 0.0216973525377
6 : https://www.statnews.com/2017/02/09/alzheimers-disease-statins/ : 0.0433910495868
7 : https://www.statnews.com/2016/03/28/google-life-sciences-exodus/ : 0.0387522992879
8 : https://www.statnews.com/2017/02/06/mental-health-college-students/ : 0.0254864799899
9 : https://www.statnews.com/2017/02/09/doctors-training-social-change/ : 0.0498242376543
10 : https://www.statnews.com/2017/02/09/injury-states-unusually-common/ : 0.14703521
11 : https://www.statnews.com/2017/02/08/social-media-compulsion-mental-illness/ : 0.0316238751148
12 : https://www.statnews.com/2017/02/09/harvard-reunion-sugar-study/ : 0.0406530763889
13 : https://www.statnews.com/2017/02/06/health-insurance-high-deductible-experiment/ : 0.0388978784
14 : https://www.statnews.com/2017/02/09/margaret-chan-who-fda/ : 0.0452888480726
15 : https://www.statnews.com/2017/02/07/trump-iraq-refugee-doctor/ : 0.0272495459556
16 : https://www.statnews.com/2017/02/08/toledo-alzheimers-suits/ : 0.0316790971429
17 : https://www.statnews.com/2017/01/30/trump-mental-health/ : 0.0275825876
18 : https://www.statnews.com/2017/01/30/fda-regulations-executive-order-trump/ : 0.0499074182825
19 : http://neurosciencenews.com/balloon-hearing-test-6027/ : 0.0
20 : https://www.statnews.com/2017/02/06/trump-biopharma/ : 0.00124504
21 : https://www.statnews.com/pharmalot/2017/02/08/kaleo-naloxone-drug-prices/ : 0.104810359375
22 : https://www.statnews.com/pharmalot/2017/02/07/general-counsel-whistleblower/ : 0.1054106875
23 : https://www.statnews.com/pharmalot/2017/02/06/mylan-subpoena-epipen/ : 0.10606825
24 : https://www.statnews.com/pharmalot/2017/02/03/winners-losers-2/ : 0.106728734375
25 : https://www.statnews.com/pharmalot/2017/02/03/jobs-sanofi-shire-merck/ : 0.0257175510204
26 : https://www.statnews.com/pharmalot/2017/02/02/drug-prices-generics-vouchers/ : 0.104847
27 : https://www.statnews.com/pharmalot/2017/02/09/india-patents-hiv-opioids-drug-prices/ : 0.1053661875
28 : https://www.statnews.com/pharmalot/2017/02/08/immigration-sanofi-drug-prices-teva/ : 0.105705984375
29 : https://www.statnews.com/pharmalot/2017/02/06/trump-open-letter-drug-pricing/ : 0.04020625
30 : https://www.statnews.com/pharmalot/2017/02/03/california-coupons-drug-prices/ : 0.104548234375
31 : https://www.statnews.com/pharmalot/2017/02/02/vertex-ireland-cystic-fibrosis/ : 0.1042796875
32 : https://www.statnews.com/pharmalot/2017/02/09/ted-cruz-fda-drugs-2/ : 0.1053699375
33 : https://www.statnews.com/pharmalot/2017/02/08/merrimack-insider-trading-akebia/ : 0.107837609375
34 : https://www.statnews.com/pharmalot/2017/02/08/sanofi-regeneron-amgen-cholesterol/ : 0.105055609375
35 : https://www.statnews.com/2017/02/09/samsung-biologics-trump-trade/ : 0.15254016
36 : https://www.statnews.com/pharmalot/2017/02/07/china-bribes-glaxo-fda-teva/ : 0.108853234375
37 : https://www.statnews.com/pharmalot/2017/02/07/fda-advisory-committee-redactions/ : 0.1044286875
38 : https://www.statnews.com/pharmalot/2017/02/07/ftc-citizens-generics/ : 0.104625484375
39 : https://www.statnews.com/2017/02/06/germany-pharma-trump/ : 0.125099333333
40 : https://www.statnews.com/pharmalot/2017/02/06/teva-vigodman/ : 0.108025984375
41 : https://www.statnews.com/pharmalot/2017/02/06/novartis-bribes-breast-implants-generics/ : 0.104402234375
42 : https://www.statnews.com/2017/02/06/week-feb-6-watch-biopharma/ : 0.100543555556
43 : https://www.statnews.com/pharmalot/2017/02/03/lilly-alzheimers-layoffs/ : 0.1071679375
44 : https://www.statnews.com/2017/02/03/trump-pharma/ : 0.14238656
45 : https://www.statnews.com/pharmalot/2017/02/03/opioid-lawsuit-oxycontin/ : 0.104595109375
46 : https://www.statnews.com/pharmalot/2017/02/02/amgen-cholesterol-sanofi-regeneron/ : 0.1052914375
47 : https://www.statnews.com/pharmalot/2017/02/03/biogen-drug-prices/ : 0.10481825
48 : https://www.statnews.com/2017/02/02/implant-opioid-addiction-ipo/ : 0.107484109375
49 : https://www.scientificamerican.com/podcast/episode/frog-spit-behaves-like-bug-catching-ketchup/ : 0.00277422222222
50 : https://www.scientificamerican.com/video/soft-sticky-frog-tongues-slurp-supper/ : 0.0
51 : https://www.scientificamerican.com/article/tangled-up-in-spacetime1/ : 0.0
52 : https://www.scientificamerican.com/podcast/episode/knot-not-easy-to-knot/ : 0.0843984
53 : https://www.scientificamerican.com/article/in-case-you-missed-it-the-first-crispr-clinical-trial-a-new-shield-for-chernobyl-mdash-and-more/ : 0.00298602083333
54 : https://www.scientificamerican.com/article/florida-has-seen-bad-effects-from-trump-like-climate-gag-orders/ : 0.0651872430556
55 : https://www.scientificamerican.com/article/scientists-protest-trump-order-with-boycotts-of-journals-conferences/ : 0.00572976
56 : https://www.scientificamerican.com/article/obama-science-adviser-trump-immigration-ban-ldquo-an-abomination-rdquo/ : 0.0118877222222
57 : https://www.scientificamerican.com/article/plug-a-leaky-bottle-with-the-power-of-air/ : 0.0126398206361
58 : https://www.scientificamerican.com/article/stay-warm-with-thermal-insulation/ : 0.0072810451843
59 : https://www.scientificamerican.com/article/doubts-cloud-claims-of-metallic-hydrogen/ : 0.0235070277778
60 : https://www.scientificamerican.com/article/icecube-closes-in-on-mysterious-nature-of-neutrinos/ : 0.00273996694215
61 : https://www.scientificamerican.com/article/the-epa-has-started-to-remove-obama-era-information/ : 0.00388298765432
62 : https://www.scientificamerican.com/article/house-science-panel-to-hold-hearing-on-ldquo-making-the-epa-great-again-rdquo/ : 0.00335888888889
63 : https://www.scientificamerican.com/article/u-s-government-takes-animal-welfare-data-offline/ : 0.00529042666667
64 : https://www.scientificamerican.com/article/ligo-rsquo-s-underdog-cousin-ready-to-enhance-gravitational-wave-hunt/ : 0.00180469135802
65 : https://www.scientificamerican.com/article/build-an-irrigation-system/ : 0.0337265123967
66 : https://www.scientificamerican.com/article/trump-immigration-ban-can-worsen-u-s-doctor-shortage-hurt-hospitals/ : 0.00398290972222
67 : https://www.scientificamerican.com/article/e-u-looks-to-china-for-climate-leadership/ : 0.00975990972222
68 : https://www.scientificamerican.com/article/build-a-2-stage-balloon-rocket/ : 0.0266634034763
69 : https://www.scientificamerican.com/article/facebook-google-and-others-launch-drive-against-fake-news-in-france/ : 0.0058605
70 : https://www.scientificamerican.com/article/aquatic-robot-braves-volcanoes-and-typhoons-to-detect-tsunamis/ : 0.00270753719008
71 : https://www.scientificamerican.com/article/a-breakthrough-in-the-search-for-alpha-centauris-planets/ : 0.00281235743802
72 : https://www.scientificamerican.com/article/immigrants-do-not-increase-crime-research-shows/ : 0.0187399363905
73 : https://www.scientificamerican.com/article/trump-rsquo-s-executive-order-strands-brooklyn-doctor-in-sudan/ : 0.00507142355372
74 : https://www.scientificamerican.com/article/future-of-paris-accord-uncertain-as-tillerson-becomes-secretary-of-state/ : 0.00611655113636
75 : https://www.scientificamerican.com/article/are-we-living-in-a-computer-simulation1/ : 0.0
76 : https://www.scientificamerican.com/article/cosmic-test-bolsters-einsteins-ldquo-spooky-action-at-a-distance-rdquo/ : 0.0203399501134
77 : https://www.scientificamerican.com/article/u-k-s-brexit-plans-call-for-leaving-e-u-nuclear-agency/ : 0.00322491
78 : https://www.scientificamerican.com/article/to-treat-primates-more-humanely-transparency/ : 0.00188176
79 : https://www.scientificamerican.com/article/hunting-dark-matter-between-the-ticks-of-an-atomic-clock/ : 0.00310223140496
80 : https://www.scientificamerican.com/article/are-wormholes-a-dead-end-for-faster-than-light-travel/ : 0.0550378349609
81 : https://www.scientificamerican.com/article/science-and-the-u-s-supreme-court-the-cases-to-watch-in-2017/ : 0.0119779017013

Let's create a checkpoint file to save our progress!_____no_output_____

# news_df.to_json('data/news_checkpoint1.json')
# checkpoint_df = pd.read_json('data/news_checkpoint.json')
# checkpoint_df = checkpoint_df.sort_index(ascending=True)
# checkpoint_df = checkpoint_df[['articleUrl', 'categories', 'flagged', 'source', 'sourceUrl', 'title', 'article',
#                    'num_words_not_in_dictionary', 'stopwords2words', 'readability_score', 'sentiment_polarity']]
# news_df = checkpoint_df_____no_output_____

# Explore TFIDF
### So far we have 3 important variables: news_df for all the news, articles for list/corpus of all articles, and vocab_df for the vocabularies of the articles
Now, I will explore TFIDF to extract keywords and try to match them to a categorizing model. This mapping can perhaps compare the results of the two algorithms. Many of the columns I populated above are not used, but they are interesting to look at and can be features in future models!

Quoted directly from http://brandonrose.org/clustering#Stopwords,-stemming,-and-tokenizingHere.

"I define term frequency-inverse document frequency (tf-idf) vectorizer parameters and then convert the synopses list into a tf-idf matrix. To get a Tf-idf matrix, first count word occurrences by document. This is transformed into a document-term matrix (dtm). This is also just called a term frequency matrix. An example of a dtm is here at right. Then apply the term frequency-inverse document frequency weighting: words that occur frequently within a document but not frequently within the corpus receive a higher weighting as these words are assumed to contain more meaning in relation to the document. A couple things to note about the parameters I define below:
-- max_df: this is the maximum frequency within the documents a given feature can have to be used in the tfi-idf matrix. If the term is in greater than 80% of the documents it probably cares little meanining (in the context of film synopses)
-- min_idf: this could be an integer (e.g. 5) and the term would have to be in at least 5 of the documents to be considered. Here I pass 0.2; the term must be in at least 20% of the document. I found that if I allowed a lower min_df I ended up basing clustering on names--for example "Michael" or "Tom" are names found in several of the movies and the synopses use these names frequently, but the names carry no real meaning.
-- ngram_range: this just means I'll look at unigrams, bigrams and trigrams."_____no_output_____

# Define vectorizer parameters using Gridsearch (later in this notebook)
tfidf_vectorizer = TfidfVectorizer(stop_words='english', min_df=0.01, max_df = 0.8, use_idf=True, ngram_range=(1,1),
                                   tokenizer=tokenize_and_stem)

tfidf_matrix = tfidf_vectorizer.fit_transform(articles) #fit the vectorizer to descriptions
terms = tfidf_vectorizer.get_feature_names() # terms is just a list of the features used in the tf-idf matrix.
print(tfidf_matrix.shape)
terms(2150, 3235)

### Clustering attempt to extract common keywords
Using the tf-idf matrix, I can now cluster the articles with k-means clustering. Since this a ML model that often converges to a "local optimum", sometimes these cells need to be run many times._____no_output_____

from sklearn.cluster import KMeans
num_clusters = 5 # played around with this number, and decided on 6 clusters / 6 main categories of the articles
km = KMeans(n_clusters=num_clusters)
km.fit(tfidf_matrix)
clusters = km.labels_.tolist()_____no_output_____from sklearn.externals import joblib
# joblib.dump(km, 'data/km.pkl')
km = joblib.load('data/km.pkl') _____no_output_____urls = [str(t) for t in news_df['articleUrl']]
categories = [c for c in news_df['categories']]# just to compare the cluster and the actual category
news_articles = { 'url': urls, 'article': articles, 'cluster': clusters, 'categories': categories}
cluster_df = pd.DataFrame(news_articles, index = [clusters] , columns = ['url', 'article', 'cluster', 'categories'])
print(cluster_df['cluster'].value_counts()) #number of films per cluster (clusters from 0 to 5)
cluster_df.head()0    573
1    508
2    435
3    377
4    257
Name: cluster, dtype: int64
#sort cluster centers by proximity to centroid
order_centroids = km.cluster_centers_.argsort()[:, ::-1]

for i in range(num_clusters):
    print("Cluster", i, "top words:", end='')
    for ind in order_centroids[i, :10]: # 10 words per cluster
#         print(terms[ind].split(' '))
#         print(vocab_df.ix[terms[ind].split(' ')].values.tolist())
        print(vocab_df.ix[terms[ind].split(' ')].values.tolist()[0][0].encode('utf-8', 'ignore'), end=', ')
    print() #add whitespace
    print() #add whitespace

    # USE STEM

#     print("Cluster", i, "titles:", end='')
#     for title in cluster_df.ix[i]['title'].values.tolist():
#         print(title, end='')
#     print() #add whitespace
#     print() #add whitespaceCluster 0 top words:b'said', b'health', b'patients', b'care', b'trump', b'medical', b'climate', b'hospital', b'drugs', b'states',

Cluster 1 top words:b'brain', b'research', b'studies', b'imaging', b'neurons', b'http', b'neurosciencenews', b'sleeping', b'active', b'february',

Cluster 2 top words:b'ai', b'science', b'research', b'machine', b'data', b'use', b'learn', b'say', b'computers', b'like',

Cluster 3 top words:b'plus', b'stat', b'subscribe', b'logging', b'biotechs', b'drugs', b'reads', b'story', b'article', b'available',

Cluster 4 top words:b'cell', b'cancer', b'gene', b'immune', b'research', b'tumors', b'protein', b'patients', b'diseases', b'neurons',

Looks like Cluster 0 represents science policy/environmental science, Cluster 1 represents neuroscience, Cluster 2 represents artificial intelligence, Cluster 3 sort of represents pharmaceuticals, and Cluster 4 represnts immunology. This clustering will be the first D3 visualization!_____no_output_____# Visualization
"dist is defined as 1 - the cosine similarity of each document. Cosine similarity is measured against the tf-idf matrix and can be used to generate a measure of similarity between each document and the other documents in the corpus (each synopsis among the synopses). Subtracting it from 1 provides cosine distance which I will use for plotting on a euclidean (2-dimensional) plane.
Note that with dist it is possible to evaluate the similarity of any two or more synopses."_____no_output_____

from sklearn.metrics.pairwise import cosine_similarity
dist = 1 - cosine_similarity(tfidf_matrix)
dist_____no_output_____from sklearn.manifold import MDS
MDS()
# convert two components as we're plotting points in a two-dimensional plane
# "precomputed" because we provide a distance matrix
# we will also specify `random_state` so the plot is reproducible.
mds = MDS(n_components=2, dissimilarity="precomputed", random_state=1)
pos = mds.fit_transform(dist)  # shape (n_components, n_samples)
xs, ys = pos[:, 0], pos[:, 1]
xs, ys_____no_output_____#set up colors per clusters using a dict
cluster_colors = {0: '#1b9e77', 1: '#d95f02', 2: '#7570b3', 3: '#e7298a', 4: '#66a61e'}

#set up cluster names using a dict
cluster_names = {0: 'Sci Policy/Env Sci: trump, climate, states',
                 1: 'Neuroscience: brain, imaging, neurons',
                 2: 'Artificial Intelligence: ai, machine, data',
                 3: 'Pharmaceuticals: biotechs, dugs',
                 4: 'Immunology: cell, cancer, gene, immune, tumors, protein, parient, disease'}

#create data frame that has the result of the MDS plus the cluster numbers and titles
mds_df = pd.DataFrame(dict(x=xs, y=ys, cluster=clusters, categories=categories, url=urls))
mds_df.to_csv('data/mds_df.csv', index=False)
mds_df.head()_____no_output_____

To wrap up the visualization, let's also generate a word cloud for all the vocab in vocab_df._____no_output_____

full_text = [w for w in vocab_df['words'] if w not in stopwords and len(w) >= 5]
fdist = nltk.FreqDist(full_text)_____no_output_____df = pd.DataFrame(fdist.most_common(50), columns=["word", "count"])
df.to_csv("data/word_cloud.csv", index=False)
df_____no_output_____


# Notebook from ka-rocha/LSSTC-DSFP-Sessions
Path: Sessions/Session01/Day4/IntroToMachineLearning.ipynb

from __future__ import division, print_function, absolute_import, unicode_literals_____no_output_____

#Introduction to Machine Learning:
Examples of Unsupervised and Supervised Machine-Learning Algorithms
========

#####Version 0.1

Broadly speaking, machine-learning methods constitute a diverse collection of data-driven algorithms designed to classify/characterize/analyze sources in multi-dimensional spaces. The topics and studies that fall under the umbrella of machine learning is growing, and there is no good catch-all definition. The number (and variation) of algorithms is vast, and beyond the scope of these exercises. While we will discuss a few specific algorithms today, more importantly, we will explore the scope of the two general methods: unsupervised learning and supervised learning and introduce the powerful (and dangerous?) Python package [`scikit-learn`](http://scikit-learn.org/stable/).

***
By AA Miller (Jet Propulsion Laboratory, California Institute of Technology.)

(c) 2016 California Institute of Technology. Government sponsorship acknowledged.
_____no_output_____

import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline_____no_output_____

##Problem 1) Introduction to `scikit-learn`

At the most basic level, `scikit-learn` makes machine learning extremely easy within Python. By way of example, here is a short piece of code that builds a complex, non-linear model to classify sources in the Iris data set that we learned about yesterday:

    from sklearn import datasets
    from sklearn.ensemble import RandomForestClassifier
    iris = datasets.load_iris()
    RFclf = RandomForestClassifier().fit(iris.data, iris.target)

Those 4 lines of code have constructed a model that is superior to any system of hard cuts that we could have encoded while looking at the multidimensional space. This can be fast as well: execute the dummy code in the cell below to see how "easy" machine-learning is with `scikit-learn`._____no_output_____

# execute dummy code here

from sklearn import datasets
from sklearn.ensemble import RandomForestClassifier
iris = datasets.load_iris()
RFclf = RandomForestClassifier().fit(iris.data, iris.target)_____no_output_____

Generally speaking, the procedure for `scikit-learn` is uniform across all machine-learning algorithms. Models are accessed via the various modules (`ensemble`, `SVM`, `neighbors`, etc), with user-defined tuning parameters. The features (or data) for the models are stored in a 2D array, `X`, with rows representing individual sources and columns representing the corresponding feature values. [In a minority of cases, `X`, represents a similarity or distance matrix where each entry represents the distance to every other source in the data set.] In cases where there is a known classification or scalar value (typically supervised methods), this information is stored in a 1D array `y`.

Unsupervised models are fit by calling `.fit(X)` and supervised models are fit by calling `.fit(X, y)`. In both cases, predictions for new observations, `Xnew`, can be obtained by calling `.predict(Xnew)`. Those are the basics and beyond that, the details are algorithm specific, but the documentation for essentially everything within `scikit-learn` is excellent, so read the docs.

To further develop our intuition, we will now explore the Iris dataset a little further.

**Problem 1a** What is the pythonic type of `iris`?_____no_output_____You likely haven't encountered a `scikit-learn Bunch` before. It's functionality is essentially the same as a dictionary.

**Problem 1b** What are the keys of iris?_____no_output_____Most importantly, iris contains `data` and `target` values. These are all you need for `scikit-learn`, though the feature and target names and description are useful.

**Problem 1c** What is the shape and content of the `iris` data?_____no_output_____

print(np.shape( # complete
print( # complete_____no_output_____

**Problem 1d** What is the shape and content of the `iris` target?_____no_output_____

print(np.shape( # complete
print( # complete_____no_output_____

Finally, as a baseline for the exercises that follow, we will now make a simple 2D plot showing the separation of the 3 classes in the iris dataset. This plot will serve as the reference for examining the quality of the clustering algorithms.

**Problem 1e** Make a scatter plot showing sepal length vs. sepal width for the iris data set. Color the points according to their respective classes.

*Hint - determine which columns of data correspond to sepal length and sepal width.*_____no_output_____

plt.scatter( # complete
plt.xlabel('sepal length')
plt.ylabel('sepal width')_____no_output_____

##Problem 2) Unsupervised Machine Learning

Unsupervised machine learning, sometimes referred to as clustering or data mining, aims to group or classify sources in the multidimensional feature space. The "unsupervised" comes from the fact that there are no target labels provided to the algorithm, so the machine is asked to cluster the data "on its own." The lack of labels means there is no (simple) method for validating the accuracy of the solution provided by the machine (though sometimes simple examination can show the results are **terrible**).

For this reason [*note* - this is my (AAM) opinion and there many be many others who disagree], unsupervised methods are not particularly useful for astronomy. Supposing one did find some useful clustering structure, an adversarial researcher could always claim that the current feature space does not accurately capture the physics of the system and as such the clustering result is not interesting or, worse, erroneous. The one potentially powerful exception to this broad statement is outlier detection, which can be a branch of both unsupervised and supervised learning. Finding weirdo objects is an astronomical pastime, and there are unsupervised methods that may help in that regard in the LSST era.

To begin today we will examine one of the most famous, and simple, clustering algorithms: [$k$-means](https://en.wikipedia.org/wiki/K-means_clustering). $k$-means clustering looks to identify $k$ convex clusters, where $k$ is a user defined number. And here-in lies the rub: if we truly knew the number of clusters in advance, we likely wouldn't need to perform any clustering in the first place. This is the major downside to $k$-means. Operationally, pseudocode for the algorithm can be summarized as the following:

    initiate search by identifying k points (i.e. the cluster centers)
    loop
        assign each point in the data set to the closest cluster center
        calculate new cluster centers based on mean position of all cluster points
        if diff(new center - old center) < threshold:
            stop (i.e. clusters are defined)

The threshold is defined by the user, though in some cases the total number of iterations is also An advantage of $k$-means is that the solution will always converge, though the solution may only be a local minimum. Disadvantages include the assumption of convexity, i.e. difficult to capture complex geometry, and the curse of dimensionality (though you can combat that with dimensionality reduction after yesterday).

In `scikit-learn` the [`KMeans`](http://scikit-learn.org/stable/modules/generated/sklearn.cluster.KMeans.html#sklearn.cluster.KMeans) algorithm is implemented as part of the [`sklearn.cluster`](http://scikit-learn.org/stable/modules/classes.html#module-sklearn.cluster) module.

**Problem 2a** Fit two different $k$-means models to the iris data, one with 2 clusters and one with 3 clusters. Plot the resulting clusters in the sepal length-sepal width plane (same plot as above). How do the results compare to the true classifications?

*Hint - the `.labels_` attribute of the `KMeans` object will return the clusters measured by the algorithm.*_____no_output_____

from sklearn.cluster import KMeans

Kcluster = KMeans( # complete
# complete

plt.figure()
plt.scatter( # complete
plt.xlabel('sepal length')
plt.ylabel('sepal width')

Kcluster = KMeans( # complete
# complete

plt.figure()
plt.scatter( # complete
plt.xlabel('sepal length')
plt.ylabel('sepal width')_____no_output_____

With 3 clusters the algorithm does a good job of separating the three classes. However, without the a priori knowledge that there are 3 different types of iris, the 2 cluster solution would appear to be superior.

**Problem 2b** How do the results change if the 3 cluster model is called with `n_init = 1` and `init = 'random'` options? Use `rs` for the random state [this allows me to cheat in service of making a point].

*Note - the respective defaults for these two parameters are 10 and `k-means++`, respectively. Read the docs to see why these choices are, likely, better than those in 2b. _____no_output_____

rs = 14
Kcluster1 = KMeans(# complete

plt.figure()
plt.scatter(# complete
plt.xlabel('sepal length')
plt.ylabel('sepal width')_____no_output_____

**A random aside that is not particularly relevant here**

$k$-means evaluates the Euclidean distance between individual sources and cluster centers, thus, the magnitude of the individual features has a strong effect on the final clustering outcome.

**Problem 2c** Calculate the mean, standard deviation, min, and max of each feature in the iris data set. Based on these summaries, which feature is most important for clustering? _____no_output_____

print("feature\t\t\tmean\tstd\tmin\tmax")
for featnum, feat in enumerate(iris.feature_names):
    print("{:s}\t{:.2f}\t{:.2f}\t{:.2f}\t{:.2f}".format( # complete_____no_output_____

Petal length has the largest range and standard deviation, thus, it will have the most "weight" when determining the $k$ clusters. _____no_output_____The truth is that the iris data set is fairly small and straightfoward. Nevertheless, we will now examine the clustering results after re-scaling the features. [Some algorithms, *cough* Support Vector Machines *cough*, are notoriously sensitive to the feature scaling, so it is important to know about this step.] Imagine you are classifying stellar light curves: the data set will include contact binaries with periods of $\sim 0.1 \; \mathrm{d}$ and Mira variables with periods of $\gg 100 \; \mathrm{d}$. Without re-scaling, this feature that covers 4 orders of magnitude may dominate all others in the final model projections.

The two most common forms of re-scaling are to rescale to a guassian with mean $= 0$ and variance $= 1$, or to rescale the min and max of the feature to $[0, 1]$. The best normalization is problem dependent. The [`sklearn.preprocessing`](http://scikit-learn.org/stable/modules/classes.html#module-sklearn.preprocessing) module makes it easy to re-scale the feature set. **It is essential that the same scaling used for the training set be used for all other data run through the model.** The testing, validation, and field observations cannot be re-scaled independently. This would result in meaningless final classifications/predictions.

**Problem 2d** Re-scale the features to normal distributions, and perform $k$-means clustering on the iris data. How do the results compare to those obtained earlier?

*Hint - you may find [`'StandardScaler()'`](http://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.StandardScaler.html#sklearn.preprocessing.StandardScaler) within the `sklearn.preprocessing` module useful.*_____no_output_____

from sklearn.preprocessing import StandardScaler

scaler = StandardScaler().fit( # complete

Kcluster = KMeans( # complete

plt.figure()
plt.scatter( # complete
plt.xlabel('sepal length')
plt.ylabel('sepal width')_____no_output_____

These results are almost identical to those obtained without scaling. This is due to the simplicity of the iris data set. _____no_output_____**How do I test the accuracy of my clusters?**

Essentially - you don't. There are some methods that are available, but they essentially compare clusters to labeled samples, and if the samples are labeled it is likely that supervised learning is more useful anyway. If you are curious, `scikit-learn` does provide some [built-in functions for analyzing clustering](http://scikit-learn.org/stable/modules/clustering.html#clustering-performance-evaluation), but again, it is difficult to evaluate the validity of any newly discovered clusters.

**What if I don't know how many clusters are present in the data?**

An excellent question, as you will almost never know this a priori. Many algorithms, like $k$-means, do require the number of clusters to be specified, but some other methods do not. As an example [`DBSCAN`](https://en.wikipedia.org/wiki/DBSCAN). In brief, `DBSCAN` requires two parameters: `minPts`, the minimum number of points necessary for a cluster, and $\epsilon$, a distance measure. Clusters are grown by identifying *core points*, objects that have at least `minPts` located within a distance $\epsilon$. *Reachable points* are those within a distance $\epsilon$ of at least one *core point* but less than `minPts` *core points*. Identically, these points define the outskirts of the clusters. Finally, there are also *outliers* which are points that are $> \epsilon$ away from any *core points*. Thus, `DBSCAN` naturally identifies clusters, does not assume clusters are convex, and even provides a notion of outliers. The downsides to the algorithm are that the results are highly dependent on the two tuning parameters, and that clusters of highly different densities can be difficult to recover (because $\epsilon$ and `minPts` is specified for all clusters.

In `scitkit-learn` the
[`DBSCAN`](http://scikit-learn.org/stable/modules/generated/sklearn.cluster.DBSCAN.html#sklearn.cluster.DBSCAN) algorithm is part of the `sklearn.cluster` module. $\epsilon$ and `minPts` are set by `eps` and `min_samples`, respectively.

**Problem 2e** Cluster the iris data using `DBSCAN`. Play around with the tuning parameters to see how they affect the final clustering results. How does the use of `DBSCAN` compare to $k$-means? Can you obtain 3 clusters with `DBSCAN`? If not, given the knowledge that the iris dataset has 3 classes - does this invalidate `DBSCAN` as a viable algorithm?

*Note - DBSCAN labels outliers as $-1$, and thus, `plt.scatter()`, will plot all these points as the same color.*
_____no_output_____

from sklearn.cluster import DBSCAN

dbs = DBSCAN( # complete
dbs.fit( # complete

dbs_outliers = # complete

plt.figure()
plt.scatter( # complete
plt.scatter( # complete

plt.xlabel('sepal length')
plt.ylabel('sepal width')_____no_output_____

I was unable to obtain 3 clusters with `DBSCAN`. While these results are, on the surface, worse than what we got with $k$-means, my suspicion is that the 4 features do not adequately separate the 3 classes. [See - a nayseyer can always make that argument.] This is not a problem for `DBSCAN` as an algorithm, but rather, evidence that no single algorithm works well in all cases. _____no_output_____##Challenge Problem) Cluster SDSS Galaxy Data

The following query will select 10k likely galaxies from the SDSS database and return the results of that query into an [`astropy.Table`](http://docs.astropy.org/en/stable/table/) object. (For now, if you are not familiar with the SDSS DB schema, don't worry about this query, just know that it returns a bunch of photometric features.)_____no_output_____

from astroquery.sdss import SDSS  # enables direct queries to the SDSS database

GALquery = """SELECT TOP 10000
             p.dered_u - p.dered_g as ug, p.dered_g - p.dered_r as gr,
             p.dered_g - p.dered_i as gi, p.dered_g - p.dered_z as gz,
             p.petroRad_i, p.petroR50_i, p.deVAB_i
             FROM PhotoObjAll AS p JOIN specObjAll s ON s.bestobjid = p.objid
             WHERE p.mode = 1 AND s.sciencePrimary = 1 AND p.clean = 1 AND p.type = 3
               """
SDSSgals = SDSS.query_sql(GALquery)
SDSSgals_____no_output_____

I have used my own domain knowledge to specificly choose features that may be useful when clustering galaxies. If you know a bit about SDSS and can think of other features that may be useful feel free to add them to the query.

One nice feature of `astropy` tables is that they can readily be turned into `pandas DataFrames`, which can in turn easily be turned into a `sklearn X` array with `NumPy`. For example:

    X = np.array(SDSSgals.to_pandas())

And you are ready to go.

**Challenge Problem** Using the SDSS dataset above, identify interesting clusters within the data [this is intentionally very open ended, if you uncover anything especially exciting you'll have a chance to share it with the group]. Feel free to use the algorithms discussed above, or any other packages available via `sklearn`. Can you make sense of the clusters in the context of galaxy evolution?

*Hint - don't fret if you know nothing about galaxy evolution (neither do I!). Just take a critical look at the clusters that are identified*_____no_output_____

# complete_____no_output_____

Note - the above solution seems to separate out elliptical galaxies from blue star forming galaxies, however, the results are highly, highly dependent upon the tuning parameters. _____no_output_____##Problem 3) Supervised Machine Learning

Supervised machine learning, on the other hand, aims to predict a target class or produce a regression result based on the location of labelled sources (i.e. the training set) in the multidimensional feature space. The "supervised" comes from the fact that we are specifying the allowed outputs from the model. As there are labels available for the training set, it is possible to estimate the accuracy of the model (though there are generally important caveats about generalization, which we will explore in further detail later).

The details and machinations of supervised learning will be explored further during the following break-out session. Here, we will simply introduce some of the basics as a point of comparison to unsupervised machine learning.

We will begin with a simple, but nevertheless, elegant algorithm for classification and regression: [$k$-nearest-neighbors](https://en.wikipedia.org/wiki/K-nearest_neighbors_algorithm) ($k$NN). In brief, the classification or regression output is determined by examining the $k$ nearest neighbors in the training set, where $k$ is a user defined number. Typically, though not always, distances between sources are Euclidean, and the final classification is assigned to whichever class has a plurality within the $k$ nearest neighbors (in the case of regression, the average of the $k$ neighbors is the output from the model). We will experiment with the steps necessary to optimize $k$, and other tuning parameters, in the detailed break-out problem.

In `scikit-learn` the [`KNeighborsClassifer`](http://scikit-learn.org/stable/modules/generated/sklearn.neighbors.KNeighborsClassifier.html) algorithm is implemented as part of the [`sklearn.neighbors`](http://scikit-learn.org/stable/modules/classes.html#module-sklearn.neighbors) module.

**Problem 3a** Fit two different $k$NN models to the iris data, one with 3 neighbors and one with 10 neighbors. Plot the resulting class predictions in the sepal length-sepal width plane (same plot as above). How do the results compare to the true classifications? Is there any reason to be suspect of this procedure?

*Hint - after you have constructed the model, it is possible to obtain model predictions using the `.predict()` method, which requires a feature array, same features and order as the training set, as input.*

*Hint that isn't essential, but is worth thinking about - should the features be re-scaled in any way?*_____no_output_____

from sklearn.neighbors import KNeighborsClassifier

KNNclf = KNeighborsClassifier( # complete
preds = # complete
plt.figure()
plt.scatter( # complete

KNNclf = KNeighborsClassifier(# complete
preds = # complete
plt.figure()
plt.scatter( # complete_____no_output_____

These results are almost identical to the training classifications. However, we have cheated! In this case we are evaluating the accuracy of the model (98% in this case) using the same data that defines the model. Thus, what we have really evaluated here is the training error. The relevant parameter, however, is the generalization error: how accurate are the model predictions on new data?

Without going into too much detail, we will test this using cross validation (CV), which will be explored in more detail later. In brief, CV provides predictions on the training set using a subset of the data to generate a model that predicts the class of the remaining sources. Using [`cross_val_predict`](http://scikit-learn.org/stable/modules/generated/sklearn.cross_validation.cross_val_predict.html), we can get a better sense of the model accuracy. Predictions from `cross_val_predict` are produced in the following manner:

    from sklearn.cross_validation import cross_val_predict
    CVpreds = cross_val_predict(sklearn.model(), X, y)

where `sklearn.model()` is the desired model, `X` is the feature array, and `y` is the label array.

**Problem 3b** Produce cross-validation predictions for the iris dataset and a $k$NN with 5 neighbors. Plot the resulting classifications, as above, and estimate the accuracy of the model as applied to new data. How does this accuracy compare to a $k$NN with 50 neighbors?_____no_output_____

from sklearn.cross_validation import cross_val_predict

CVpreds = cross_val_predict( # complete
plt.figure()
plt.scatter( # complete
print("The accuracy of the kNN = 5 model is ~{:.4}".format(  # complete

CVpreds50 = cross_val_predict( # complete
print("The accuracy of the kNN = 50 model is ~{:.4}".format(  # complete_____no_output_____

While it is useful to understand the overall accuracy of the model, it is even more useful to understand the nature of the misclassifications that occur.

**Problem 3c** Calculate the accuracy for each class in the iris set, as determined via CV for the $k$NN = 50 model._____no_output_____

# complete_____no_output_____

We just found that the classifier does a much better job classifying setosa and versicolor than it does for virginica. The main reason for this is some viginica flowers lie far outside the main virginica locus, and within predominantly versicolor "neighborhoods". In addition to knowing the accuracy for the individual classes, it is also useful to know class predictions for the misclassified sources, or in other words where there is "confusion" for the classifier. The best way to summarize this information is with a confusion matrix. In a confusion matrix, one axis shows the true class and the other shows the predicted class. For a perfect classifier all of the power will be along the diagonal, while confusion is represented by off-diagonal signal.

Like almost everything else we have encountered during this exercise, `scikit-learn` makes it easy to compute a confusion matrix. This can be accomplished with the following:

    from sklearn.metrics import confusion_matrix
    cm = confusion_matrix(y_test, y_prep)

**Problem 3d** Calculate the confusion matrix for the iris training set and the $k$NN = 50 model._____no_output_____

from sklearn.metrics import confusion_matrix
cm = confusion_matrix( # complete
print(cm)_____no_output_____

From this representation, we see right away that most of the virginica that are being misclassifed are being scattered into the versicolor class. However, this representation could still be improved: it'd be helpful to normalize each value relative to the total number of sources in each class, and better still, it'd be good to have a visual representation of the confusion matrix. This visual representation will be readily digestible. Now let's normalize the confusion matrix.

**Problem 3e** Calculate the normalized confusion matrix. Be careful, you have to sum along one axis, and then divide along the other.

*Anti-hint: This operation is actually straightforward using some array manipulation that we have not covered up to this point. Thus, we have performed the necessary operations for you below. If you have extra time, you should try to develop an alternate way to arrive at the same normalization.*_____no_output_____

normalized_cm = cm.astype('float')/cm.sum(axis = 1)[:,np.newaxis]

normalized_cm_____no_output_____

The normalization makes it easier to compare the classes, since each class has a different number of sources. Now we can procede with a visual representation of the confusion matrix. This is best done using `imshow()` within pyplot. You will also need to plot a colorbar, and labeling the axes will also be helpful.

**Problem C3** Plot the confusion matrix. Be sure to label each of the axeses.

*Hint - you might find the [`sklearn` confusion matrix tutorial](http://scikit-learn.org/stable/auto_examples/model_selection/plot_confusion_matrix.html#example-model-selection-plot-confusion-matrix-py) helpful for making a nice plot.*_____no_output_____

plt.imshow( # complete_____no_output_____

Now it is straight-forward to see that virginica and versicolor flowers are the most likely to be confused, which we could intuit from the very first plot in this notebook, but this exercise becomes far more important for large data sets with many, many classes.

Thus concludes our introduction to `scikit-learn` and supervised and unsupervised learning. _____no_output_____


# Notebook from oliverphilcox/ChempyMulti
Path: Chempy_tutorials/4-Simple_stellar_population.ipynb

## Simple Stellar Population (SSP)
Here we will explain the central routine of **Chempy**. The observational counterpart of an SSP is the open cluster. All stars are approximately born at the same time, and from the same ISM abundances. Therefore the SSP is characterized by its total **mass**, its birth **age** and its **initial elemental composition**. When assuming **stellar lifetimes**, an **IMF** and **nucleosynthetic feedback** we can calculate its feedback over time.
You will see how this is realized in Chempy._____no_output_____

%pylab inlinePopulating the interactive namespace from numpy and matplotlib
# Chempy has to be called from within the /source directory. We load the default parameters

from Chempy.parameter import ModelParameters
a = ModelParameters()_____no_output_____

First we load the default values of:
- IMF
- SN2, AGB and SNIa feedback_____no_output_____

# Load the IMF

from Chempy.imf import IMF
basic_imf = IMF(a.mmin,a.mmax,a.mass_steps)
getattr(basic_imf, a.imf_type_name)((a.chabrier_para1,a.chabrier_para2,a.chabrier_para3,a.high_mass_slope))

# Load the yields of the default yield set

from Chempy.yields import SN2_feedback, AGB_feedback, SN1a_feedback
basic_sn2 = SN2_feedback()
getattr(basic_sn2, "Nomoto2013")()
basic_1a = SN1a_feedback()
getattr(basic_1a, "Seitenzahl")()
basic_agb = AGB_feedback()
getattr(basic_agb, "Karakas_net_yield")()_____no_output_____

We check how many elements are traced by our yield set_____no_output_____

# Print all supported elements

elements_to_trace = list(np.unique(basic_agb.elements+basic_sn2.elements+basic_1a.elements))
print(elements_to_trace)['Al', 'Ar', 'B', 'Be', 'C', 'Ca', 'Cl', 'Co', 'Cr', 'Cu', 'F', 'Fe', 'Ga', 'Ge', 'H', 'He', 'K', 'Li', 'Mg', 'Mn', 'N', 'Na', 'Ne', 'Ni', 'O', 'P', 'S', 'Sc', 'Si', 'Ti', 'V', 'Zn']

When initialising the SSP class, we have to make the following choices(in brackets our choices are given):

- Which metallicity does it have (Solar)
- How long will the SSP live and at which timesteps do we want the feedback to be evaluated (Delta t = 0.025 for 13.5 Gyr)
- What is the lifetime rroutine (Argast+ 2000)
- How should the interpolation in metallicity from the yield tables be (logarithmic)
- Do we want to safe the net yields as well (No)
_____no_output_____

# Load solar abundances

from Chempy.solar_abundance import solar_abundances
basic_solar = solar_abundances()
getattr(basic_solar, 'Asplund09')()

# Initialise the SSP class with time-steps

time_steps = np.linspace(0.,13.5,541)
from Chempy.weighted_yield import SSP
basic_ssp = SSP(False, np.copy(basic_solar.z), np.copy(basic_imf.x), np.copy(basic_imf.dm), np.copy(basic_imf.dn), np.copy(time_steps), list(elements_to_trace), 'Argast_2000', 'logarithmic', False) _____no_output_____

This initialises the SSP class. It calculates the inverse IMF (stars of which minimal mass will be dead until each time-step) and it also initialises the feedback table, which need to be filled by the feedback from each subroutine, representing a specific nucleosynthetic process (CC-SN, AGB star, SN Ia). For the inverse IMF's first time-steps we see that only after the first and second time-step CC-SN will contribute (only stars smaller than 8Msun survive longer...)._____no_output_____

# Plotting the inverse IMF

plt.plot(time_steps[1:],basic_ssp.inverse_imf[1:])
plt.ylabel('Mass of stars dying')
plt.xlabel('Time in Gyr')
plt.ylim((0.7,11))
for i in range(5):
    print(time_steps[i], ' Gyr -->', basic_ssp.inverse_imf[i], ' Msun')0.0  Gyr --> 100.0  Msun
0.025  Gyr --> 10.4384857378  Msun
0.05  Gyr --> 7.16435126426  Msun
0.075  Gyr --> 5.87896782591  Msun
0.1  Gyr --> 5.14928794088  Msun

In order to calculate the feedback for the CC-SN and AGB star we will need to provide the elemental abundances at birth, for which we will use solar abundances for now. (This is the case for net yields, gross yields already include the initial stellar abundance that will be expelled.)_____no_output_____

# Producing the SSP birth elemental fractions (here we use solar)

solar_fractions = []
elements = np.hstack(basic_solar.all_elements)
for item in elements_to_trace:
    solar_fractions.append(float(basic_solar.fractions[np.where(elements==item)]))_____no_output_____# Each nucleosynthetic process has its own method on the SSP class and adds its feedback into the final table

basic_ssp.sn2_feedback(list(basic_sn2.elements), dict(basic_sn2.table), np.copy(basic_sn2.metallicities), float(a.sn2mmin), float(a.sn2mmax),list(solar_fractions))
basic_ssp.agb_feedback(list(basic_agb.elements), dict(basic_agb.table), list(basic_agb.metallicities), float(a.agbmmin), float(a.agbmmax),list(solar_fractions))
basic_ssp.sn1a_feedback(list(basic_1a.elements), list(basic_1a.metallicities), dict(basic_1a.table), str(a.time_delay_functional_form), float(a.sn1ammin), float(a.sn1ammax), [a.N_0,a.sn1a_time_delay,a.sn1a_exponent,a.dummy],float(a.total_mass), a.stochastic_IMF)_____no_output_____

### Elemental feedback from an SSP over time_____no_output_____

# Now we can plot the feedback of an SSP over time for a few elements

plt.plot(time_steps,np.cumsum(basic_ssp.table['H']), label = 'H')
plt.plot(time_steps,np.cumsum(basic_ssp.table['O']), label = 'O')
plt.plot(time_steps,np.cumsum(basic_ssp.table['C']), label = 'C')
plt.plot(time_steps,np.cumsum(basic_ssp.table['Fe']), label = 'Fe')
plt.yscale('log')
plt.xscale('log')
plt.ylabel('Mass fraction of Element expelled')
plt.title('Total feedback of an SSP of 1Msun')
plt.xlabel('Time in Gyr')
plt.legend()_____no_output_____

### The difference between different yield sets_____no_output_____

# Loading an SSP which uses the alternative yield set

basic_ssp_alternative = SSP(False, np.copy(basic_solar.z), np.copy(basic_imf.x), np.copy(basic_imf.dm), np.copy(basic_imf.dn), np.copy(time_steps), list(elements_to_trace), 'Argast_2000', 'logarithmic', False)
basic_sn2_alternative = SN2_feedback()
getattr(basic_sn2_alternative, "chieffi04")()
basic_1a_alternative = SN1a_feedback()
getattr(basic_1a_alternative, "Thielemann")()
basic_agb_alternative = AGB_feedback()
getattr(basic_agb_alternative, "Ventura")()
basic_ssp_alternative.sn2_feedback(list(basic_sn2_alternative.elements), dict(basic_sn2_alternative.table), np.copy(basic_sn2_alternative.metallicities), float(a.sn2mmin), float(a.sn2mmax),list(solar_fractions))
basic_ssp_alternative.agb_feedback(list(basic_agb_alternative.elements), dict(basic_agb_alternative.table), list(basic_agb_alternative.metallicities), float(a.agbmmin), float(a.agbmmax),list(solar_fractions))
basic_ssp_alternative.sn1a_feedback(list(basic_1a_alternative.elements), list(basic_1a_alternative.metallicities), dict(basic_1a_alternative.table), str(a.time_delay_functional_form), float(a.sn1ammin), float(a.sn1ammax), [a.N_0,a.sn1a_time_delay,a.sn1a_exponent,a.dummy],float(a.total_mass), a.stochastic_IMF)

# Plotting the difference

plt.plot(time_steps,np.cumsum(basic_ssp.table['H']), label = 'H', color = 'b')
plt.plot(time_steps,np.cumsum(basic_ssp.table['O']), label = 'O', color = 'orange')
plt.plot(time_steps,np.cumsum(basic_ssp.table['C']), label = 'C', color = 'g')
plt.plot(time_steps,np.cumsum(basic_ssp.table['Fe']), label = 'Fe', color = 'r')
plt.plot(time_steps,np.cumsum(basic_ssp_alternative.table['H']), linestyle = '--', color = 'b')
plt.plot(time_steps,np.cumsum(basic_ssp_alternative.table['O']), linestyle = '--', color = 'orange')
plt.plot(time_steps,np.cumsum(basic_ssp_alternative.table['C']), linestyle = '--', color = 'g')
plt.plot(time_steps,np.cumsum(basic_ssp_alternative.table['Fe']), linestyle = '--', color = 'r')
plt.yscale('log')
plt.xscale('log')
plt.ylabel('Mass fraction of Element expelled')
plt.title('default/alternative yield set in solid/dashed lines')
plt.xlabel('Time in Gyr')
plt.legend()_____no_output_____

### The contribution of different nucleosynthetic paths to a single element_____no_output_____

# The SSP class stores the individual feedback of each nucleosynthetic channel
# Here we plot the Carbon feedback over time

plt.plot(time_steps,np.cumsum(basic_ssp.table['C']), label = 'total')
plt.plot(time_steps,np.cumsum(basic_ssp.sn2_table['C']), label = 'CC-SN')
plt.plot(time_steps,np.cumsum(basic_ssp.sn1a_table['C']), label = 'SN Ia')
plt.plot(time_steps,np.cumsum(basic_ssp.agb_table['C']), label = 'AGB')
plt.yscale('log')
plt.xscale('log')
plt.ylabel('Mass fraction of Element expelled')
plt.xlabel('Time in Gyr')
plt.title('Carbon feedback per nucleosynthetic process')
plt.legend()_____no_output_____

### The number of events_____no_output_____

# The number of events is stored as well

plt.plot(time_steps,np.cumsum(basic_ssp.table['sn2']), label = 'CC-SN')
plt.plot(time_steps,np.cumsum(basic_ssp.table['sn1a']), label = 'SN Ia')
plt.plot(time_steps,np.cumsum(basic_ssp.table['pn']), label = 'AGB')
plt.yscale('log')
plt.xscale('log')
plt.ylabel('# of events')
plt.xlabel('Time in Gyr')
plt.title('Number of events per SSP of 1Msun')
plt.legend()_____no_output_____

### The mass fractions_____no_output_____

# As is the mass fraction of stars, remnants, dying stars from which the total feedback mass can be calculated

plt.plot(time_steps,basic_ssp.table['mass_in_ms_stars'], label = 'Ms stars')
plt.plot(time_steps,np.cumsum(basic_ssp.table['mass_in_remnants']), label = 'remnants')
plt.plot(time_steps,np.cumsum(basic_ssp.table['mass_of_ms_stars_dying']), label = 'dying')
plt.plot(time_steps,np.cumsum(basic_ssp.table['mass_of_ms_stars_dying']) - np.cumsum(basic_ssp.table['mass_in_remnants']), label = 'feedback')

plt.yscale('log')
plt.xscale('log')
plt.ylabel('Mass fraction')
plt.xlabel('Time in Gyr')
plt.title('Mass of stars gets transformed into remnants and feedback over time')
plt.legend(loc = 'right', bbox_to_anchor= (1.6,0.5))_____no_output_____

## IMF weighted yield of an SSP
- Depends on the timespan over which we integrate (here we use the full 13.5Gyr)
- Depends on the IMF
- On the chosen yield set
- The mass range of the nucleosynthetic process (e.g. low-mass CC-SN have only solar alpha/Fe abundances)
- etc... play around and investigate_____no_output_____

# Here we print the time-integrated yield of an SSP (feedback after 13,5Gyr)
# for different elements and also for CC-SNe feedback only

normalising_element = 'Fe'
print('alternative yield set')
print('Element, total SSP yield, CC-SN yield ([X/Fe] i.e. normalised to solar)')
for element in ['C', 'O', 'Mg', 'Ca', 'Mn', 'Ni']:
    element_ssp_sn2 = sum(basic_ssp_alternative.sn2_table[element])
    element_ssp = sum(basic_ssp_alternative.table[element])
    element_sun = basic_solar.fractions[np.where(elements == element)]
    normalising_element_ssp_sn2 = sum(basic_ssp_alternative.sn2_table[normalising_element])
    normalising_element_ssp = sum(basic_ssp_alternative.table[normalising_element])
    normalising_element_sun = basic_solar.fractions[np.where(elements == normalising_element)]
    print(element, np.log10(element_ssp/element_sun)-np.log10(normalising_element_ssp/normalising_element_sun), np.log10(element_ssp_sn2/element_sun)-np.log10(normalising_element_ssp_sn2/normalising_element_sun))
print('------------------------------------------')
print('default yield set')
print('Element, total SSP yield, CC-SN yield ([X/Fe] i.e. normalised to solar)')
for element in ['C', 'O', 'Mg', 'Ca', 'Mn', 'Ni']:
    element_ssp_sn2 = sum(basic_ssp.sn2_table[element])
    element_ssp = sum(basic_ssp.table[element])
    element_sun = basic_solar.fractions[np.where(elements == element)]
    normalising_element_ssp_sn2 = sum(basic_ssp.sn2_table[normalising_element])
    normalising_element_ssp = sum(basic_ssp.table[normalising_element])
    normalising_element_sun = basic_solar.fractions[np.where(elements == normalising_element)]
    print(element, np.log10(element_ssp/element_sun)-np.log10(normalising_element_ssp/normalising_element_sun), np.log10(element_ssp_sn2/element_sun)-np.log10(normalising_element_ssp_sn2/normalising_element_sun))alternative yield set
Element, total SSP yield, CC-SN yield ([X/Fe] i.e. normalised to solar)
C [-0.13114334] [ 0.10377963]
O [ 0.02348811] [ 0.36715726]
Mg [-0.13455906] [ 0.1718083]
Ca [-0.11044523] [ 0.09744231]
Mn [-0.00451402] [-0.01990174]
Ni [ 0.29423482] [ 0.15736106]
------------------------------------------
default yield set
Element, total SSP yield, CC-SN yield ([X/Fe] i.e. normalised to solar)
C [-0.19445142] [ 0.01555086]
O [ 0.10937471] [ 0.54719255]
Mg [-0.04278569] [ 0.35500187]
Ca [-0.03516263] [ 0.24030457]
Mn [ 0.16472003] [-0.26083037]
Ni [ 0.13933476] [-0.07825312]

### Net yield vs. gross yield
Here the difference between newly produced material (net yield) and total expelled material (gross yield) is shown for AGB and CC-SN._____no_output_____

# We can set the the additional table (e.g. agb_table) to only save the newly produced material

basic_ssp_net = SSP(False, np.copy(basic_solar.z), np.copy(basic_imf.x), np.copy(basic_imf.dm), np.copy(basic_imf.dn), np.copy(time_steps), list(elements_to_trace), 'Argast_2000', 'logarithmic', True)
basic_ssp_net.agb_feedback(list(basic_agb.elements), dict(basic_agb.table), list(basic_agb.metallicities), float(a.agbmmin), float(a.agbmmax),list(solar_fractions))

# And then compare these net yields to gross yields for C

plt.plot(time_steps,np.cumsum(basic_ssp.agb_table['C']), label = 'total')
plt.plot(time_steps,np.cumsum(basic_ssp_net.agb_table['C']), label = 'newly produced')
plt.yscale('log')
plt.xscale('log')
plt.ylabel('Carbon feedback from AGB stars')
plt.xlabel('Time in Gyr')
plt.title('Net yield vs. gross yield')
plt.legend()_____no_output_____# And show the same for O and CC-SNe

basic_ssp_net = SSP(False, basic_solar.z, basic_imf.x, basic_imf.dm, basic_imf.dn, time_steps, elements_to_trace, 'Argast_2000', 'logarithmic', True)
basic_ssp_net.sn2_feedback(list(basic_sn2.elements), dict(basic_sn2.table), np.copy(basic_sn2.metallicities), float(a.sn2mmin), float(a.sn2mmax),list(solar_fractions))
plt.plot(time_steps,np.cumsum(basic_ssp.sn2_table['He']), label = 'total')
plt.plot(time_steps,np.cumsum(basic_ssp_net.sn2_table['He']), label = 'newly produced')
plt.yscale('log')
plt.xscale('log')
plt.ylabel('Cumulative Helium feedback from CC-SN')
plt.xlabel('Time in Gyr')
plt.title('Net vs. gross yield')
plt.legend()_____no_output_____

## Stochastic IMF sampling
- The feedback and the explosion of SN Ia can also be calculated stochastically.
- The mass of the SSP needs to be provided (the feedback table is given in fractions).
- Each realisation will be new (you can check by redoing the plot)._____no_output_____

# The IMF can be sampled stochastically. First we plot the analytic version

basic_imf = IMF(a.mmin,a.mmax,a.mass_steps)
getattr(basic_imf, a.imf_type_name)((a.chabrier_para1,a.chabrier_para2,a.chabrier_para3,a.high_mass_slope))
basic_ssp = SSP(False, np.copy(basic_solar.z), np.copy(basic_imf.x), np.copy(basic_imf.dm), np.copy(basic_imf.dn), np.copy(time_steps), list(elements_to_trace), 'Argast_2000', 'logarithmic', False)
basic_ssp.sn2_feedback(list(basic_sn2.elements), dict(basic_sn2.table), np.copy(basic_sn2.metallicities), float(a.sn2mmin), float(a.sn2mmax),list(solar_fractions))
basic_ssp.agb_feedback(list(basic_agb.elements), dict(basic_agb.table), list(basic_agb.metallicities), float(a.agbmmin), float(a.agbmmax),list(solar_fractions))
basic_ssp.sn1a_feedback(list(basic_1a.elements), list(basic_1a.metallicities), dict(basic_1a.table), str(a.time_delay_functional_form), float(a.sn1ammin), float(a.sn1ammax), [a.N_0,a.sn1a_time_delay,a.sn1a_exponent,a.dummy],float(a.total_mass), True)
plt.plot(time_steps,np.cumsum(basic_ssp.table['Fe']), label = 'analytic IMF')

# Then we add the stochastic sampling for 3 different masses

for mass in [1e5,5e3,1e2]:
    basic_imf = IMF(a.mmin,a.mmax,a.mass_steps)
    getattr(basic_imf, a.imf_type_name)((a.chabrier_para1,a.chabrier_para2,a.chabrier_para3,a.high_mass_slope))
    basic_imf.stochastic_sampling(mass)
    basic_ssp = SSP(False, np.copy(basic_solar.z), np.copy(basic_imf.x), np.copy(basic_imf.dm), np.copy(basic_imf.dn), np.copy(time_steps), list(elements_to_trace), 'Argast_2000', 'logarithmic', False)
    basic_ssp.sn2_feedback(list(basic_sn2.elements), dict(basic_sn2.table), np.copy(basic_sn2.metallicities), float(a.sn2mmin), float(a.sn2mmax),list(solar_fractions))
    basic_ssp.agb_feedback(list(basic_agb.elements), dict(basic_agb.table), list(basic_agb.metallicities), float(a.agbmmin), float(a.agbmmax),list(solar_fractions))
    basic_ssp.sn1a_feedback(list(basic_1a.elements), list(basic_1a.metallicities), dict(basic_1a.table), str(a.time_delay_functional_form), float(a.sn1ammin), float(a.sn1ammax), [a.N_0,a.sn1a_time_delay,a.sn1a_exponent,a.dummy],float(a.total_mass), True)
    plt.plot(time_steps,np.cumsum(basic_ssp.table['Fe']), label = '%d Msun' %(mass))
plt.xscale('log')
plt.ylabel('Cumulative fractional iron feedback of an SSP')
plt.xlabel('Time in Gyr')
plt.legend(bbox_to_anchor = (1.5,1))_____no_output_____

## SSP wrapper
In order to query the SSP feedback faster and easier we write a little wrapper in wrapper.py and look at the imf weighted yield change with IMF. We compare the bottom-heavy Kroupa IMF with the Salpeter IMF (which has more high-mass stars). We see that the total SSP yield changes quite drastically._____no_output_____

# Here we show the functionality of the wrapper, which makes the SSP calculation easy.
# We want to show the differences of the SSP feedback for 2 IMFs and load Kroupa first

a.only_net_yields_in_process_tables = False
a.imf_type_name = 'normed_3slope'
a.imf_parameter = (-1.3,-2.2,-2.7,0.5,1.0)

# The feedback can now be calculated by just typing the next three lines

from Chempy.wrapper import SSP_wrap
basic_ssp = SSP_wrap(a)
basic_ssp.calculate_feedback(float(basic_solar.z),list(elements_to_trace),list(solar_fractions),np.copy(time_steps))

# We print the Kroupa IMF feedback

print('Kroupa IMF')
print('Element, total SSP yield, CC-SN yield ([X/Fe] i.e. normalised to solar)')
for element in ['C', 'O', 'Mg', 'Ca', 'Mn', 'Ni']:
    element_ssp_sn2 = sum(basic_ssp.sn2_table[element])
    element_ssp = sum(basic_ssp.table[element])
    element_sun = basic_solar.fractions[np.where(elements == element)]
    normalising_element_ssp_sn2 = sum(basic_ssp.sn2_table[normalising_element])
    normalising_element_ssp = sum(basic_ssp.table[normalising_element])
    normalising_element_sun = basic_solar.fractions[np.where(elements == normalising_element)]
    print(element, np.log10(element_ssp/element_sun)-np.log10(normalising_element_ssp/normalising_element_sun), np.log10(element_ssp_sn2/element_sun)-np.log10(normalising_element_ssp_sn2/normalising_element_sun))

# Change to Salpeter

a.imf_type_name = 'salpeter'
a.imf_parameter = (2.35)

# Calculate the SSP feedback

basic_ssp = SSP_wrap(a)
basic_ssp.calculate_feedback(float(basic_solar.z),list(elements_to_trace),list(solar_fractions),np.copy(time_steps))

# And print the feedback for comparison

print('Salpeter IMF')
print('Element, total SSP yield, CC-SN yield ([X/Fe] i.e. normalised to solar)')
for element in ['C', 'O', 'Mg', 'Ca', 'Mn', 'Ni']:
    element_ssp_sn2 = sum(basic_ssp.sn2_table[element])
    element_ssp = sum(basic_ssp.table[element])
    element_sun = basic_solar.fractions[np.where(elements == element)]
    normalising_element_ssp_sn2 = sum(basic_ssp.sn2_table[normalising_element])
    normalising_element_ssp = sum(basic_ssp.table[normalising_element])
    normalising_element_sun = basic_solar.fractions[np.where(elements == normalising_element)]
    print(element, np.log10(element_ssp/element_sun)-np.log10(normalising_element_ssp/normalising_element_sun), np.log10(element_ssp_sn2/element_sun)-np.log10(normalising_element_ssp_sn2/normalising_element_sun))Kroupa IMF
Element, total SSP yield, CC-SN yield ([X/Fe] i.e. normalised to solar)
C [-0.32372351] [-0.04899912]
O [-0.07550641] [ 0.49041114]
Mg [-0.20643796] [ 0.30501324]
Ca [-0.13817342] [ 0.16954995]
Mn [ 0.20630872] [-0.33799485]
Ni [ 0.17670711] [-0.02987776]
Salpeter IMF
Element, total SSP yield, CC-SN yield ([X/Fe] i.e. normalised to solar)
C [-0.31873905] [-0.01374845]
O [ 0.10167535] [ 0.58380394]
Mg [-0.07934627] [ 0.37154707]
Ca [-0.09149007] [ 0.19399203]
Mn [ 0.18548835] [-0.32610437]
Ni [ 0.17057456] [-0.01551558]

## Paper plot
Here is the code create a plot similar to figure 4 of the paper. _____no_output_____

# Loading the default parameters so that we can change them and see what happens with the SSP feedback

a = ModelParameters()
a.high_mass_slope = -2.29
a.N_0 = np.power(10,-2.75)
a.sn1a_time_delay = np.power(10,-0.8)
a.imf_parameter = (22.8978, 716.4, 0.25, a.high_mass_slope)
a.sn1a_parameter = [a.N_0, a.sn1a_time_delay, 1.12, 0.0]
a.mmax = 100
time_steps = np.linspace(0.,13.5,1401)

# Then calculating the feedback table

basic_ssp = SSP_wrap(a)
basic_ssp.calculate_feedback(float(basic_solar.z),list(a.elements_to_trace),list(solar_fractions),np.copy(time_steps))

# To prettify the plot

plt.clf()
m=plt.cm.jet
text_size = 20
cor_text = 22
plt.rc('font', family='serif',size = text_size)
plt.rc('xtick', labelsize=text_size)
plt.rc('ytick', labelsize=text_size)
plt.rc('axes', labelsize=text_size, lw=2.0)
plt.rc('lines', linewidth = 2)
plt.rcParams['ytick.major.pad']='8'
plt.rcParams['text.latex.preamble']=[r"\usepackage{libertine}"]
params = {'text.usetex' : True,
          'font.size' : 16,
          'font.family' : 'libertine',
          'text.latex.unicode': True,
          }
plt.rcParams.update(params)

alpha = 0.5
factor = 1.05

## Actual plotting

fig = plt.figure(figsize=(8.69,6.69), dpi=100)
ax = fig.add_subplot(111)
ax.plot(time_steps,np.cumsum(basic_ssp.sn2_table["Fe"]),'b', label = 'CC-SN')
ax.annotate(xy = (time_steps[-1]*factor,np.sum(basic_ssp.sn2_table["Fe"])*0.9) ,s = 'Fe',color = 'b')
ax.plot(time_steps,np.cumsum(basic_ssp.sn1a_table["Fe"]), 'r', label = 'SN Ia')
ax.annotate(xy = (time_steps[-1]*factor,np.sum(basic_ssp.sn1a_table["Fe"])) ,s = 'Fe',color = 'r')

ax.plot(time_steps,np.cumsum(basic_ssp.sn2_table["Mg"]),'b')
ax.annotate(xy = (time_steps[-1]*factor,np.sum(basic_ssp.sn2_table["Mg"])) ,s = 'Mg',color = 'b')
ax.plot(time_steps,np.cumsum(basic_ssp.sn1a_table["Mg"]),'r')
ax.annotate(xy = (time_steps[-1]*factor,np.sum(basic_ssp.sn1a_table["Mg"])) ,s = 'Mg',color = 'r')

ax.plot(time_steps,np.ones_like(time_steps)*5e-3,marker = '|', markersize = 10, linestyle = '', color = 'k', alpha = 2*alpha)#, label = 'time-steps')
ax.annotate(xy = (time_steps[1],2.7e-3),s = r'model time-steps with mass of stars dying in M$_\odot$', color = 'k', alpha = 2*alpha)
for numb in [1,2,4,11,38,117,1000]:
    if numb < len(time_steps):
        plt.annotate(xy = (time_steps[numb],3.5e-3),s = '%.f' %(basic_ssp.inverse_imf[numb]), color = 'k', alpha = 2*alpha)
ax.legend()
ax.set_ylim(2e-5,6e-3)
ax.set_xlim(7e-3,25)
ax.set_title(r'yield of SSP with mass = 1M$_\odot$ and metallicity = Z$_\odot$')
ax.set_ylabel(r"net yield in M$_\odot$")
ax.set_xlabel("time in Gyr")

ax.set_yscale('log')
ax.set_xscale('log')
plt.show()_____no_output_____

That's why Fe is a fairly good indicator for the incidence of SN Ia and Mg is a very good indicator for the incidence of CC-SN._____no_output_____


# Notebook from soniakau/final_project
Path: ee250/labML/examples/logisticregresion/perceptron.ipynb

<code>
import pandas as pd
import numpy as np

import matplotlib.pyplot as plt
plt.rcParams["figure.figsize"] = (10,7)
import seaborn as sns

from sklearn.linear_model import LogisticRegression
from sklearn.linear_model import Perceptron

from sklearn.model_selection import train_test_split_____no_output_____# This CSV file contains a dataset of 240 stars of 6 classes:

# Brown Dwarf -> Star Type = 0

# Red Dwarf -> Star Type = 1

# White Dwarf-> Star Type = 2

# Main Sequence -> Star Type = 3

# Supergiant -> Star Type = 4

# Hypergiant -> Star Type = 5

# The Luminosity and radius of each star is calculated w.r.t. that of the values of Sun.
# Lo = 3.828 x 10^26 Watts
# Ro = 6.9551 x 10^8 m

stars = pd.read_csv("stars.csv")
stars.head()_____no_output_____redwhitedwarfs = stars[ (stars["Star type"] == 1) | (stars["Star type"] == 2) ]_____no_output_____sns.scatterplot(data=redwhitedwarfs,x="Temperature(K)",y="Radius(R/Ro)", hue="Star type")_____no_output_____# test train split

X = redwhitedwarfs[["Temperature(K)","Radius(R/Ro)"]]
y = redwhitedwarfs[["Star type"]]
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)_____no_output_____clf = LogisticRegression(tol=1e-10, random_state=99,fit_intercept=True,  max_iter=10000)
#clf = Perceptron(tol=1e-10, random_state=99, eta0=1e-15,fit_intercept=True,  max_iter=10000)

clf.fit(X_train.to_numpy(), y_train.to_numpy().flatten())_____no_output_____data_test = X_test.copy()
data_test["groundtruth"] = y_test["Star type"]
data_test["predicted"]=clf.predict(X_test.to_numpy())_____no_output_____sns.scatterplot(data=data_test, x = "Temperature(K)", y = "Radius(R/Ro)", hue = "groundtruth")
#f = lambda x: (-clf.coef_[0][0]*x  - clf.intercept_[0]) / clf.coef_[0][1]

x1,x2 = -10000,10000
plt.plot([x1,x2],[f(x1),f(x2)])
plt.legend().remove()
plt.xlim(0,20000)
plt.ylim(-3,3)_____no_output_____import pickle
pickle.dump(clf,open("model.pickle",''wb))_____no_output_____clf.predict([[1,2]])[0]_____no_output_____
</code>